From: Christopher Lalancette <clalance@redhat.com>
Date: Thu, 17 Sep 2009 09:40:53 -0400
Subject: [xen] implement fully preemptible page table teardown
Message-id: <4AB20425.70103@redhat.com>
Patchwork-id: 20890
O-Subject: [RHEL5.5 PATCH v2]: Xen fully preemptible page table teardown
Bugzilla: 510037
RH-Acked-by: Andrew Jones <drjones@redhat.com>
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
RH-Acked-by: Don Dutile <ddutile@redhat.com>
All,
In RHEL-5.4, we removed a heavily contested spinlock in the guest
page-table teardown path. This greatly improved the hypervisors's
responsiveness when destroying large memory guests, but it did not completely
solve it. In particular, you can still get long delays when tearing down the
L2/L3/L4 tables of a guest.
The rest of the solution involves fully preemptible page table teardown.
With the fully preemptible teardown, the hypervisor can tear down some of the
page tables, schedule, and then resume doing the teardown where it stopped last
time.
The attached patch does exactly this. It's a backport of xen-unstable c/s
18159, 18412, 18420, 18620, 18731, 18741, 18742, 18747, and 19374. I've done
basic testing with this patch in place on both i386 and x86_64, and things seem
to be working well. I've also done targeted testing for this particular issue,
by restricting the dom0 and domU's to 1 vcpu each, then destroying a very large
(64GB) guest. Before this patch, you would see ping responses of 400-500 ms
during the domain teardown. After this patch, the ping responses during domain
teardown are essentially indistinguishable from the responses during any other
time (i.e. in the sub-millisecond range). Full details of the test are in the BZ.
This should solve BZ 510037. Please review and ACK.
v2: backport a few missing bits as pointed out by Drew Jones.
Signed-off-by: Don Zickus <dzickus@redhat.com>
diff --git a/arch/x86/domain.c b/arch/x86/domain.c
index 6cf255d..aa2dc35 100644
--- a/arch/x86/domain.c
+++ b/arch/x86/domain.c
@@ -1688,56 +1688,76 @@ static int relinquish_memory(struct domain *d, struct list_head *list,
}
if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
- put_page_and_type(page);
+ ret = put_page_and_type_preemptible(page, 1);
+ switch ( ret )
+ {
+ case 0:
+ break;
+ case -EAGAIN:
+ case -EINTR:
+ list_move(&page->list, list);
+ set_bit(_PGT_pinned, &page->u.inuse.type_info);
+ put_page(page);
+ goto out;
+ default:
+ BUG();
+ }
if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
put_page(page);
-#ifdef DOMAIN_DESTRUCT_AVOID_RECURSION
- /*
- * Forcibly drop reference counts of page tables above top most (which
- * were skipped to prevent long latencies due to deep recursion - see
- * the special treatment in free_lX_table()).
- */
- y = page->u.inuse.type_info;
- if ( (type < PGT_root_page_table) &&
- unlikely(((y + PGT_type_mask) &
- (PGT_type_mask|PGT_validated)) == type) )
- {
- BUG_ON((y & PGT_count_mask) >=
- (page->count_info & PGC_count_mask));
- while ( y & PGT_count_mask )
- {
- put_page_and_type(page);
- y = page->u.inuse.type_info;
- }
- }
-#endif
-
/*
* Forcibly invalidate top-most, still valid page tables at this point
- * to break circular 'linear page table' references. This is okay
- * because MMU structures are not shared across domains and this domain
- * is now dead. Thus top-most valid tables are not in use so a non-zero
- * count means circular reference.
+ * to break circular 'linear page table' references as well as clean up
+ * partially validated pages. This is okay because MMU structures are
+ * not shared across domains and this domain is now dead. Thus top-most
+ * valid tables are not in use so a non-zero count means circular
+ * reference or partially validated.
*/
y = page->u.inuse.type_info;
for ( ; ; )
{
x = y;
- if ( likely((x & (PGT_type_mask|PGT_validated)) !=
- (type|PGT_validated)) )
+ if ( likely((x & PGT_type_mask) != type) ||
+ likely(!(x & (PGT_validated|PGT_partial))) )
break;
- y = cmpxchg(&page->u.inuse.type_info, x, x & ~PGT_validated);
+ y = cmpxchg(&page->u.inuse.type_info, x,
+ x & ~(PGT_validated|PGT_partial));
if ( likely(y == x) )
{
- free_page_type(page, type);
+ /* No need for atomic update of type_info here: noone else updates it. */
+ switch ( ret = free_page_type(page, x, 1) )
+ {
+ case 0:
+ break;
+ case -EINTR:
+ list_move(&page->list, list);
+ page->u.inuse.type_info |= PGT_validated;
+ if ( x & PGT_partial )
+ put_page(page);
+ put_page(page);
+ ret = -EAGAIN;
+ goto out;
+ case -EAGAIN:
+ list_move(&page->list, list);
+ page->u.inuse.type_info |= PGT_partial;
+ if ( x & PGT_partial )
+ put_page(page);
+ goto out;
+ default:
+ BUG();
+ }
+ if ( x & PGT_partial )
+ {
+ page->u.inuse.type_info--;
+ put_page(page);
+ }
break;
}
}
- /* Follow the list chain and /then/ potentially free the page. */
+ /* Put the page on the list and /then/ potentially free it. */
ent = ent->next;
list_move_tail(&page->list, &d->arch.relmem_list);
put_page(page);
@@ -1749,6 +1769,7 @@ static int relinquish_memory(struct domain *d, struct list_head *list,
}
}
+ /* list is empty at this point. */
list_splice_init(&d->arch.relmem_list, list);
out:
@@ -1877,11 +1898,6 @@ int domain_relinquish_resources(struct domain *d)
/* fallthrough */
case RELMEM_done:
-#ifdef DOMAIN_DESTRUCT_AVOID_RECURSION
- ret = relinquish_memory(d, &d->page_list, PGT_l1_page_table);
- if ( ret )
- return ret;
-#endif
break;
default:
diff --git a/arch/x86/mm.c b/arch/x86/mm.c
index 452c829..4ec267e 100644
--- a/arch/x86/mm.c
+++ b/arch/x86/mm.c
@@ -465,11 +465,11 @@ static int alloc_segdesc_page(struct page_info *page)
goto fail;
unmap_domain_page(descs);
- return 1;
+ return 0;
fail:
unmap_domain_page(descs);
- return 0;
+ return -EINVAL;
}
@@ -523,20 +523,25 @@ static int get_page_from_pagenr(unsigned long page_nr, struct domain *d)
static int get_page_and_type_from_pagenr(unsigned long page_nr,
unsigned long type,
- struct domain *d)
+ struct domain *d,
+ int partial,
+ int preemptible)
{
struct page_info *page = mfn_to_page(page_nr);
+ int rc;
- if ( unlikely(!get_page_from_pagenr(page_nr, d)) )
- return 0;
+ if ( likely(partial >= 0) &&
+ unlikely(!get_page_from_pagenr(page_nr, d)) )
+ return -EINVAL;
- if ( unlikely(!get_page_type(page, type)) )
- {
+ rc = (preemptible ?
+ get_page_type_preemptible(page, type) :
+ (get_page_type(page, type) ? 0 : -EINVAL));
+
+ if ( unlikely(rc) && partial >= 0 )
put_page(page);
- return 0;
- }
- return 1;
+ return rc;
}
/*
@@ -667,12 +672,13 @@ get_page_from_l2e(
if ( unlikely((l2e_get_flags(l2e) & L2_DISALLOW_MASK)) )
{
MEM_LOG("Bad L2 flags %x", l2e_get_flags(l2e) & L2_DISALLOW_MASK);
- return 0;
+ return -EINVAL;
}
- rc = get_page_and_type_from_pagenr(l2e_get_pfn(l2e), PGT_l1_page_table, d);
- if ( unlikely(!rc) )
- rc = get_l2_linear_pagetable(l2e, pfn, d);
+ rc = get_page_and_type_from_pagenr(
+ l2e_get_pfn(l2e), PGT_l1_page_table, d, 0, 0);
+ if ( unlikely(rc == -EINVAL) && get_l2_linear_pagetable(l2e, pfn, d) )
+ rc = 0;
return rc;
}
@@ -682,7 +688,7 @@ get_page_from_l2e(
define_get_linear_pagetable(l3);
static int
get_page_from_l3e(
- l3_pgentry_t l3e, unsigned long pfn, struct domain *d)
+ l3_pgentry_t l3e, unsigned long pfn, struct domain *d, int partial, int preemptible)
{
int rc;
@@ -692,12 +698,13 @@ get_page_from_l3e(
if ( unlikely((l3e_get_flags(l3e) & l3_disallow_mask(d))) )
{
MEM_LOG("Bad L3 flags %x", l3e_get_flags(l3e) & l3_disallow_mask(d));
- return 0;
+ return -EINVAL;
}
- rc = get_page_and_type_from_pagenr(l3e_get_pfn(l3e), PGT_l2_page_table, d);
- if ( unlikely(!rc) )
- rc = get_l3_linear_pagetable(l3e, pfn, d);
+ rc = get_page_and_type_from_pagenr(
+ l3e_get_pfn(l3e), PGT_l2_page_table, d, partial, preemptible);
+ if ( unlikely(rc == -EINVAL) && get_l3_linear_pagetable(l3e, pfn, d) )
+ rc = 0;
return rc;
}
@@ -707,7 +714,7 @@ get_page_from_l3e(
define_get_linear_pagetable(l4);
static int
get_page_from_l4e(
- l4_pgentry_t l4e, unsigned long pfn, struct domain *d)
+ l4_pgentry_t l4e, unsigned long pfn, struct domain *d, int partial, int preemptible)
{
int rc;
@@ -717,12 +724,13 @@ get_page_from_l4e(
if ( unlikely((l4e_get_flags(l4e) & L4_DISALLOW_MASK)) )
{
MEM_LOG("Bad L4 flags %x", l4e_get_flags(l4e) & L4_DISALLOW_MASK);
- return 0;
+ return -EINVAL;
}
- rc = get_page_and_type_from_pagenr(l4e_get_pfn(l4e), PGT_l3_page_table, d);
- if ( unlikely(!rc) )
- rc = get_l4_linear_pagetable(l4e, pfn, d);
+ rc = get_page_and_type_from_pagenr(
+ l4e_get_pfn(l4e), PGT_l3_page_table, d, partial, preemptible);
+ if ( unlikely(rc == -EINVAL) && get_l4_linear_pagetable(l4e, pfn, d) )
+ rc = 0;
return rc;
}
@@ -857,29 +865,47 @@ void put_page_from_l1e(l1_pgentry_t l1e, struct domain *d)
* NB. Virtual address 'l2e' maps to a machine address within frame 'pfn'.
* Note also that this automatically deals correctly with linear p.t.'s.
*/
-static void put_page_from_l2e(l2_pgentry_t l2e, unsigned long pfn)
+static int put_page_from_l2e(l2_pgentry_t l2e, unsigned long pfn)
{
- if ( (l2e_get_flags(l2e) & _PAGE_PRESENT) &&
- (l2e_get_pfn(l2e) != pfn) )
+ if ( (l2e_get_flags(l2e) & _PAGE_PRESENT) &&
+ (l2e_get_pfn(l2e) != pfn) )
+ {
put_page_and_type(l2e_get_page(l2e));
+ return 0;
+ }
+ return 1;
}
#if CONFIG_PAGING_LEVELS >= 3
-static void put_page_from_l3e(l3_pgentry_t l3e, unsigned long pfn)
+static int __put_page_type(struct page_info *, int preemptible);
+
+static int put_page_from_l3e(l3_pgentry_t l3e, unsigned long pfn,
+ int partial, int preemptible)
{
- if ( (l3e_get_flags(l3e) & _PAGE_PRESENT) &&
- (l3e_get_pfn(l3e) != pfn) )
- put_page_and_type(l3e_get_page(l3e));
+ if ( (l3e_get_flags(l3e) & _PAGE_PRESENT) &&
+ (l3e_get_pfn(l3e) != pfn) )
+ {
+ if ( unlikely(partial > 0) )
+ return __put_page_type(l3e_get_page(l3e), preemptible);
+ return put_page_and_type_preemptible(l3e_get_page(l3e), preemptible);
+ }
+ return 1;
}
#endif
#if CONFIG_PAGING_LEVELS >= 4
-static void put_page_from_l4e(l4_pgentry_t l4e, unsigned long pfn)
+static int put_page_from_l4e(l4_pgentry_t l4e, unsigned long pfn,
+ int partial, int preemptible)
{
if ( (l4e_get_flags(l4e) & _PAGE_PRESENT) &&
(l4e_get_pfn(l4e) != pfn) )
- put_page_and_type(l4e_get_page(l4e));
+ {
+ if ( unlikely(partial > 0) )
+ return __put_page_type(l4e_get_page(l4e), preemptible);
+ return put_page_and_type_preemptible(l4e_get_page(l4e), preemptible);
+ }
+ return 1;
}
#endif
@@ -888,7 +914,7 @@ static int alloc_l1_table(struct page_info *page)
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l1_pgentry_t *pl1e;
- int i;
+ unsigned int i;
pl1e = map_domain_page(pfn);
@@ -902,7 +928,7 @@ static int alloc_l1_table(struct page_info *page)
}
unmap_domain_page(pl1e);
- return 1;
+ return 0;
fail:
MEM_LOG("Failure in alloc_l1_table: entry %d", i);
@@ -911,7 +937,7 @@ static int alloc_l1_table(struct page_info *page)
put_page_from_l1e(pl1e[i], d);
unmap_domain_page(pl1e);
- return 0;
+ return -EINVAL;
}
#if defined(CONFIG_X86_PAE) || defined(CONFIG_COMPAT)
@@ -1043,61 +1069,57 @@ static void pae_flush_pgd(
# define pae_flush_pgd(mfn, idx, nl3e) ((void)0)
#endif
-static int alloc_l2_table(struct page_info *page, unsigned long type)
+static int alloc_l2_table(struct page_info *page, unsigned long type,
+ int preemptible)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l2_pgentry_t *pl2e;
- int i;
+ unsigned int i;
+ int rc = 0;
pl2e = map_domain_page(pfn);
- for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
+ for ( i = page->nr_validated_ptes; i < L2_PAGETABLE_ENTRIES; i++ )
{
- if ( is_guest_l2_slot(d, type, i) &&
- unlikely(!get_page_from_l2e(pl2e[i], pfn, d)) )
- goto fail;
-
- adjust_guest_l2e(pl2e[i], d);
- }
+ if ( preemptible && i && hypercall_preempt_check() )
+ {
+ page->nr_validated_ptes = i;
+ rc = -EAGAIN;
+ break;
+ }
-#if CONFIG_PAGING_LEVELS == 2
- /* Xen private mappings. */
- memcpy(&pl2e[L2_PAGETABLE_FIRST_XEN_SLOT],
- &idle_pg_table[L2_PAGETABLE_FIRST_XEN_SLOT],
- L2_PAGETABLE_XEN_SLOTS * sizeof(l2_pgentry_t));
- pl2e[l2_table_offset(LINEAR_PT_VIRT_START)] =
- l2e_from_pfn(pfn, __PAGE_HYPERVISOR);
- for ( i = 0; i < PDPT_L2_ENTRIES; i++ )
- pl2e[l2_table_offset(PERDOMAIN_VIRT_START) + i] =
- l2e_from_page(
- virt_to_page(page_get_owner(page)->arch.mm_perdomain_pt) + i,
- __PAGE_HYPERVISOR);
-#endif
+ if ( !is_guest_l2_slot(d, type, i) ||
+ (rc = get_page_from_l2e(pl2e[i], pfn, d)) > 0 )
+ continue;
- unmap_domain_page(pl2e);
- return 1;
+ if ( rc < 0 )
+ {
+ MEM_LOG("Failure in alloc_l2_table: entry %d", i);
+ while ( i-- > 0 )
+ if ( is_guest_l2_slot(d, type, i) )
+ put_page_from_l2e(pl2e[i], pfn);
+ break;
+ }
- fail:
- MEM_LOG("Failure in alloc_l2_table: entry %d", i);
- while ( i-- > 0 )
- if ( is_guest_l2_slot(d, type, i) )
- put_page_from_l2e(pl2e[i], pfn);
+ adjust_guest_l2e(pl2e[i], d);
+ }
unmap_domain_page(pl2e);
- return 0;
+ return rc > 0 ? 0 : rc;
}
#if CONFIG_PAGING_LEVELS >= 3
-static int alloc_l3_table(struct page_info *page)
+static int alloc_l3_table(struct page_info *page, int preemptible)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l3_pgentry_t *pl3e;
- int i;
+ unsigned int i;
+ int rc = 0, partial = page->partial_pte;
-#ifdef CONFIG_X86_PAE
+#if CONFIG_PAGING_LEVELS == 3
/*
* PAE pgdirs above 4GB are unacceptable if the guest does not understand
* the weird 'extended cr3' format for dealing with high-order address
@@ -1108,7 +1130,7 @@ static int alloc_l3_table(struct page_info *page)
d->vcpu[0] && d->vcpu[0]->is_initialised )
{
MEM_LOG("PAE pgd must be below 4GB (0x%lx >= 0x100000)", pfn);
- return 0;
+ return -EINVAL;
}
#endif
@@ -1124,60 +1146,103 @@ static int alloc_l3_table(struct page_info *page)
if ( is_pv_32on64_domain(d) )
memset(pl3e + 4, 0, (L3_PAGETABLE_ENTRIES - 4) * sizeof(*pl3e));
- for ( i = 0; i < L3_PAGETABLE_ENTRIES; i++ )
+ for ( i = page->nr_validated_ptes; i < L3_PAGETABLE_ENTRIES;
+ i++, partial = 0 )
{
-#if defined(CONFIG_X86_PAE) || defined(CONFIG_COMPAT)
if ( is_pv_32bit_domain(d) && (i == 3) )
{
if ( !(l3e_get_flags(pl3e[i]) & _PAGE_PRESENT) ||
- (l3e_get_flags(pl3e[i]) & l3_disallow_mask(d)) ||
- !get_page_and_type_from_pagenr(l3e_get_pfn(pl3e[i]),
- PGT_l2_page_table |
- PGT_pae_xen_l2,
- d) )
- goto fail;
+ (l3e_get_flags(pl3e[i]) & l3_disallow_mask(d)) )
+ rc = -EINVAL;
+ else
+ rc = get_page_and_type_from_pagenr(l3e_get_pfn(pl3e[i]),
+ PGT_l2_page_table |
+ PGT_pae_xen_l2,
+ d, partial, preemptible);
}
- else
-#endif
- if ( is_guest_l3_slot(i) &&
- unlikely(!get_page_from_l3e(pl3e[i], pfn, d)) )
- goto fail;
-
- adjust_guest_l3e(pl3e[i], d);
- }
+ else if ( !is_guest_l3_slot(i) ||
+ (rc = get_page_from_l3e(pl3e[i], pfn, d,
+ partial, preemptible)) > 0 )
+ continue;
- if ( !create_pae_xen_mappings(d, pl3e) )
- goto fail;
+ if ( rc == -EAGAIN )
+ {
+ page->nr_validated_ptes = i;
+ page->partial_pte = partial ?: 1;
+ }
+ else if ( rc == -EINTR && i )
+ {
+ page->nr_validated_ptes = i;
+ page->partial_pte = 0;
+ rc = -EAGAIN;
+ }
+ if ( rc < 0 )
+ break;
- unmap_domain_page(pl3e);
- return 1;
+ adjust_guest_l3e(pl3e[i], d);
+ }
- fail:
- MEM_LOG("Failure in alloc_l3_table: entry %d", i);
- while ( i-- > 0 )
- if ( is_guest_l3_slot(i) )
- put_page_from_l3e(pl3e[i], pfn);
+ if ( rc >= 0 && !create_pae_xen_mappings(d, pl3e) )
+ rc = -EINVAL;
+ if ( rc < 0 && rc != -EAGAIN && rc != -EINTR )
+ {
+ MEM_LOG("Failure in alloc_l3_table: entry %d", i);
+ while ( i-- > 0 )
+ {
+ if ( !is_guest_l3_slot(i) )
+ continue;
+ unadjust_guest_l3e(pl3e[i], d);
+ put_page_from_l3e(pl3e[i], pfn, 0, 0);
+ }
+ }
unmap_domain_page(pl3e);
- return 0;
+ return rc > 0 ? 0 : rc;
}
#else
-#define alloc_l3_table(page) (0)
+#define alloc_l3_table(page, preemptible) (-EINVAL)
#endif
#if CONFIG_PAGING_LEVELS >= 4
-static int alloc_l4_table(struct page_info *page)
+static int alloc_l4_table(struct page_info *page, int preemptible)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l4_pgentry_t *pl4e = page_to_virt(page);
- int i;
+ unsigned int i;
+ int rc = 0, partial = page->partial_pte;
- for ( i = 0; i < L4_PAGETABLE_ENTRIES; i++ )
+ for ( i = page->nr_validated_ptes; i < L4_PAGETABLE_ENTRIES;
+ i++, partial = 0 )
{
- if ( is_guest_l4_slot(d, i) &&
- unlikely(!get_page_from_l4e(pl4e[i], pfn, d)) )
- goto fail;
+ if ( !is_guest_l4_slot(d, i) ||
+ (rc = get_page_from_l4e(pl4e[i], pfn, d,
+ partial, preemptible)) > 0 )
+ continue;
+
+ if ( rc == -EAGAIN )
+ {
+ page->nr_validated_ptes = i;
+ page->partial_pte = partial ?: 1;
+ }
+ else if ( rc == -EINTR )
+ {
+ if ( i )
+ {
+ page->nr_validated_ptes = i;
+ page->partial_pte = 0;
+ rc = -EAGAIN;
+ }
+ }
+ else if ( rc < 0 )
+ {
+ MEM_LOG("Failure in alloc_l4_table: entry %d", i);
+ while ( i-- > 0 )
+ if ( is_guest_l4_slot(d, i) )
+ put_page_from_l4e(pl4e[i], pfn, 0, 0);
+ }
+ if ( rc < 0 )
+ return rc;
adjust_guest_l4e(pl4e[i], d);
}
@@ -1191,23 +1256,11 @@ static int alloc_l4_table(struct page_info *page)
pl4e[l4_table_offset(PERDOMAIN_VIRT_START)] =
l4e_from_page(virt_to_page(d->arch.mm_perdomain_l3),
__PAGE_HYPERVISOR);
- if ( is_pv_32on64_domain(d) )
- pl4e[l4_table_offset(COMPAT_ARG_XLAT_VIRT_BASE)] =
- l4e_from_page(virt_to_page(d->arch.mm_arg_xlat_l3),
- __PAGE_HYPERVISOR);
-
- return 1;
-
- fail:
- MEM_LOG("Failure in alloc_l4_table: entry %d", i);
- while ( i-- > 0 )
- if ( is_guest_l4_slot(d, i) )
- put_page_from_l4e(pl4e[i], pfn);
- return 0;
+ return rc > 0 ? 0 : rc;
}
#else
-#define alloc_l4_table(page) (0)
+#define alloc_l4_table(page, preemptible) (-EINVAL)
#endif
@@ -1216,7 +1269,7 @@ static void free_l1_table(struct page_info *page)
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l1_pgentry_t *pl1e;
- int i;
+ unsigned int i;
pl1e = map_domain_page(pfn);
@@ -1228,74 +1281,113 @@ static void free_l1_table(struct page_info *page)
}
-static void free_l2_table(struct page_info *page)
+static int free_l2_table(struct page_info *page, int preemptible)
{
#ifdef CONFIG_COMPAT
struct domain *d = page_get_owner(page);
#endif
unsigned long pfn = page_to_mfn(page);
l2_pgentry_t *pl2e;
- int i;
+ unsigned int i = page->nr_validated_ptes - 1;
+ int err = 0;
pl2e = map_domain_page(pfn);
- for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
- if ( is_guest_l2_slot(d, page->u.inuse.type_info, i) )
- put_page_from_l2e(pl2e[i], pfn);
+ ASSERT(page->nr_validated_ptes);
+ do {
+ if ( is_guest_l2_slot(d, page->u.inuse.type_info, i) &&
+ put_page_from_l2e(pl2e[i], pfn) == 0 &&
+ preemptible && i && hypercall_preempt_check() )
+ {
+ page->nr_validated_ptes = i;
+ err = -EAGAIN;
+ }
+ } while ( !err && i-- );
unmap_domain_page(pl2e);
- page->u.inuse.type_info &= ~PGT_pae_xen_l2;
+ if ( !err )
+ page->u.inuse.type_info &= ~PGT_pae_xen_l2;
+
+ return err;
}
#if CONFIG_PAGING_LEVELS >= 3
-
-static void free_l3_table(struct page_info *page)
+static int free_l3_table(struct page_info *page, int preemptible)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l3_pgentry_t *pl3e;
- int i;
-
-#ifdef DOMAIN_DESTRUCT_AVOID_RECURSION
- if ( d->arch.relmem == RELMEM_l3 )
- return;
-#endif
+ int rc = 0, partial = page->partial_pte;
+ unsigned int i = page->nr_validated_ptes - !partial;
pl3e = map_domain_page(pfn);
- for ( i = 0; i < L3_PAGETABLE_ENTRIES; i++ )
+ do {
if ( is_guest_l3_slot(i) )
{
- put_page_from_l3e(pl3e[i], pfn);
+ rc = put_page_from_l3e(pl3e[i], pfn, partial, preemptible);
+ if ( rc < 0 )
+ break;
+ partial = 0;
+ if ( rc > 0 )
+ continue;
unadjust_guest_l3e(pl3e[i], d);
}
+ } while ( i-- );
unmap_domain_page(pl3e);
-}
+ if ( rc == -EAGAIN )
+ {
+ page->nr_validated_ptes = i;
+ page->partial_pte = partial ?: -1;
+ }
+ else if ( rc == -EINTR && i < L3_PAGETABLE_ENTRIES - 1 )
+ {
+ page->nr_validated_ptes = i + 1;
+ page->partial_pte = 0;
+ rc = -EAGAIN;
+ }
+ return rc > 0 ? 0 : rc;
+}
+#else
+#define free_l3_table(page, preemptible) (-EINVAL)
#endif
#if CONFIG_PAGING_LEVELS >= 4
-
-static void free_l4_table(struct page_info *page)
+static int free_l4_table(struct page_info *page, int preemptible)
{
struct domain *d = page_get_owner(page);
unsigned long pfn = page_to_mfn(page);
l4_pgentry_t *pl4e = page_to_virt(page);
- int i;
+ int rc = 0, partial = page->partial_pte;
+ unsigned int i = page->nr_validated_ptes - !partial;
-#ifdef DOMAIN_DESTRUCT_AVOID_RECURSION
- if ( d->arch.relmem == RELMEM_l4 )
- return;
-#endif
-
- for ( i = 0; i < L4_PAGETABLE_ENTRIES; i++ )
+ do {
if ( is_guest_l4_slot(d, i) )
- put_page_from_l4e(pl4e[i], pfn);
-}
+ rc = put_page_from_l4e(pl4e[i], pfn, partial, preemptible);
+ if ( rc < 0 )
+ break;
+ partial = 0;
+ } while ( i-- );
+ if ( rc == -EAGAIN )
+ {
+ page->nr_validated_ptes = i;
+ page->partial_pte = partial ?: -1;
+ }
+ else if ( rc == -EINTR && i < L4_PAGETABLE_ENTRIES - 1 )
+ {
+ page->nr_validated_ptes = i + 1;
+ page->partial_pte = 0;
+ rc = -EAGAIN;
+ }
+ return rc > 0 ? 0 : rc;
+}
+#else
+#define free_l4_table(page, preemptible) (-EINVAL)
#endif
@@ -1446,7 +1538,7 @@ static int mod_l2_entry(l2_pgentry_t *pl2e,
return UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, current, preserve_ad);
}
- if ( unlikely(!get_page_from_l2e(nl2e, pfn, d)) )
+ if ( unlikely(get_page_from_l2e(nl2e, pfn, d) < 0) )
return 0;
adjust_guest_l2e(nl2e, d);
@@ -1474,16 +1566,17 @@ static int mod_l2_entry(l2_pgentry_t *pl2e,
static int mod_l3_entry(l3_pgentry_t *pl3e,
l3_pgentry_t nl3e,
unsigned long pfn,
- int preserve_ad)
+ int preserve_ad,
+ int preemptible)
{
l3_pgentry_t ol3e;
struct domain *d = current->domain;
- int okay;
+ int rc = 0;
if ( unlikely(!is_guest_l3_slot(pgentry_ptr_to_slot(pl3e))) )
{
MEM_LOG("Illegal L3 update attempt in Xen-private area %p", pl3e);
- return 0;
+ return -EINVAL;
}
#if defined(CONFIG_X86_PAE) || defined(CONFIG_COMPAT)
@@ -1492,11 +1585,11 @@ static int mod_l3_entry(l3_pgentry_t *pl3e,
* would be a pain to ensure they remain continuously valid throughout.
*/
if ( is_pv_32bit_domain(d) && (pgentry_ptr_to_slot(pl3e) >= 3) )
- return 0;
+ return -EINVAL;
#endif
if ( unlikely(__copy_from_user(&ol3e, pl3e, sizeof(ol3e)) != 0) )
- return 0;
+ return -EFAULT;
if ( l3e_get_flags(nl3e) & _PAGE_PRESENT )
{
@@ -1504,41 +1597,46 @@ static int mod_l3_entry(l3_pgentry_t *pl3e,
{
MEM_LOG("Bad L3 flags %x",
l3e_get_flags(nl3e) & l3_disallow_mask(d));
- return 0;
+ return -EINVAL;
}
/* Fast path for identical mapping and presence. */
if (!l3e_has_changed(ol3e, nl3e, _PAGE_PRESENT))
{
adjust_guest_l3e(nl3e, d);
- return UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, current, preserve_ad);
+ rc = UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, current, preserve_ad);
+ return rc ? 0 : -EFAULT;
}
- if ( unlikely(!get_page_from_l3e(nl3e, pfn, d)) )
- return 0;
+ rc = get_page_from_l3e(nl3e, pfn, d, 0, preemptible);
+ if ( unlikely(rc < 0) )
+ return rc;
+ rc = 0;
adjust_guest_l3e(nl3e, d);
-
if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, current,
preserve_ad)) )
{
- put_page_from_l3e(nl3e, pfn);
- return 0;
+ ol3e = nl3e;
+ rc = -EFAULT;
}
}
else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, current,
preserve_ad)) )
{
- return 0;
+ return -EFAULT;
}
- okay = create_pae_xen_mappings(d, pl3e);
- BUG_ON(!okay);
+ if ( likely(rc == 0) )
+ {
+ if ( !create_pae_xen_mappings(d, pl3e) )
+ BUG();
- pae_flush_pgd(pfn, pgentry_ptr_to_slot(pl3e), nl3e);
+ pae_flush_pgd(pfn, pgentry_ptr_to_slot(pl3e), nl3e);
+ }
- put_page_from_l3e(ol3e, pfn);
- return 1;
+ put_page_from_l3e(ol3e, pfn, 0, 0);
+ return rc;
}
#endif
@@ -1550,18 +1648,20 @@ static int mod_l4_entry(struct domain *d,
l4_pgentry_t *pl4e,
l4_pgentry_t nl4e,
unsigned long pfn,
- int preserve_ad)
+ int preserve_ad,
+ int preemptible)
{
l4_pgentry_t ol4e;
+ int rc = 0;
if ( unlikely(!is_guest_l4_slot(d, pgentry_ptr_to_slot(pl4e))) )
{
MEM_LOG("Illegal L4 update attempt in Xen-private area %p", pl4e);
- return 0;
+ return -EINVAL;
}
if ( unlikely(__copy_from_user(&ol4e, pl4e, sizeof(ol4e)) != 0) )
- return 0;
+ return -EFAULT;
if ( l4e_get_flags(nl4e) & _PAGE_PRESENT )
{
@@ -1569,43 +1669,69 @@ static int mod_l4_entry(struct domain *d,
{
MEM_LOG("Bad L4 flags %x",
l4e_get_flags(nl4e) & L4_DISALLOW_MASK);
- return 0;
+ return -EINVAL;
}
/* Fast path for identical mapping and presence. */
if (!l4e_has_changed(ol4e, nl4e, _PAGE_PRESENT))
{
adjust_guest_l4e(nl4e, current->domain);
- return UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, current, preserve_ad);
+ rc = UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, current, preserve_ad);
+ return rc ? 0 : -EFAULT;
}
- if ( unlikely(!get_page_from_l4e(nl4e, pfn, current->domain)) )
- return 0;
+ rc = get_page_from_l4e(nl4e, pfn, current->domain, 0, preemptible);
+ if ( unlikely(rc < 0) )
+ return rc;
+ rc = 0;
adjust_guest_l4e(nl4e, current->domain);
-
if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, current,
preserve_ad)) )
{
- put_page_from_l4e(nl4e, pfn);
- return 0;
+ ol4e = nl4e;
+ rc = -EFAULT;
}
}
else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, current,
preserve_ad)) )
{
- return 0;
+ return -EFAULT;
}
- put_page_from_l4e(ol4e, pfn);
- return 1;
+ put_page_from_l4e(ol4e, pfn, 0, 0);
+ return rc;
}
#endif
-int alloc_page_type(struct page_info *page, unsigned long type)
+/*
+ * Special version of get_page() to be used exclusively when
+ * - a page is known to already have a non-zero reference count
+ * - the page does not need its owner to be checked
+ * - it will not be called more than once without dropping the thus
+ * acquired reference again.
+ * Due to get_page() reserving one reference, this call cannot fail.
+ */
+static void get_page_light(struct page_info *page)
+{
+ u32 x, nx, y = page->count_info;
+
+ do {
+ x = y;
+ nx = x + 1;
+ BUG_ON(!(x & PGC_count_mask)); /* Not allocated? */
+ BUG_ON(!(nx & PGC_count_mask)); /* Overflow? */
+ y = cmpxchg(&page->count_info, x, nx);
+ }
+ while ( unlikely(y != x) );
+}
+
+int alloc_page_type(struct page_info *page, unsigned long type,
+ int preemptible)
{
struct domain *owner = page_get_owner(page);
+ int rc;
/* A page table is dirtied when its type count becomes non-zero. */
if ( likely(owner != NULL) )
@@ -1614,31 +1740,66 @@ int alloc_page_type(struct page_info *page, unsigned long type)
switch ( type & PGT_type_mask )
{
case PGT_l1_page_table:
- return alloc_l1_table(page);
+ rc = alloc_l1_table(page);
+ break;
case PGT_l2_page_table:
- return alloc_l2_table(page, type);
+ rc = alloc_l2_table(page, type, preemptible);
+ break;
case PGT_l3_page_table:
- return alloc_l3_table(page);
+ rc = alloc_l3_table(page, preemptible);
+ break;
case PGT_l4_page_table:
- return alloc_l4_table(page);
+ rc = alloc_l4_table(page, preemptible);
+ break;
case PGT_gdt_page:
case PGT_ldt_page:
- return alloc_segdesc_page(page);
+ rc = alloc_segdesc_page(page);
+ break;
default:
printk("Bad type in alloc_page_type %lx t=%" PRtype_info " c=%x\n",
type, page->u.inuse.type_info,
page->count_info);
+ rc = -EINVAL;
BUG();
}
- return 0;
+ /* No need for atomic update of type_info here: noone else updates it. */
+ wmb();
+ if ( rc == -EAGAIN )
+ {
+ get_page_light(page);
+ page->u.inuse.type_info |= PGT_partial;
+ }
+ else if ( rc == -EINTR )
+ {
+ ASSERT((page->u.inuse.type_info &
+ (PGT_count_mask|PGT_validated|PGT_partial)) == 1);
+ page->u.inuse.type_info &= ~PGT_count_mask;
+ }
+ else if ( rc )
+ {
+ ASSERT(rc < 0);
+ MEM_LOG("Error while validating mfn %lx (pfn %lx) for type %"
+ PRtype_info ": caf=%08x taf=%" PRtype_info,
+ page_to_mfn(page), get_gpfn_from_mfn(page_to_mfn(page)),
+ type, page->count_info, page->u.inuse.type_info);
+ page->u.inuse.type_info = 0;
+ }
+ else
+ {
+ page->u.inuse.type_info |= PGT_validated;
+ }
+
+ return rc;
}
-void free_page_type(struct page_info *page, unsigned long type)
+int free_page_type(struct page_info *page, unsigned long type,
+ int preemptible)
{
struct domain *owner = page_get_owner(page);
unsigned long gmfn;
+ int rc;
if ( likely(owner != NULL) )
{
@@ -1658,7 +1819,7 @@ void free_page_type(struct page_info *page, unsigned long type)
paging_mark_dirty(owner, page_to_mfn(page));
if ( shadow_mode_refcounts(owner) )
- return;
+ return 0;
gmfn = mfn_to_gmfn(owner, page_to_mfn(page));
ASSERT(VALID_M2P(gmfn));
@@ -1666,42 +1827,97 @@ void free_page_type(struct page_info *page, unsigned long type)
}
}
+ if ( !(type & PGT_partial) )
+ {
+ page->nr_validated_ptes = 1U << PAGETABLE_ORDER;
+ page->partial_pte = 0;
+ }
+
switch ( type & PGT_type_mask )
{
case PGT_l1_page_table:
free_l1_table(page);
+ rc = 0;
break;
-
case PGT_l2_page_table:
- free_l2_table(page);
+ rc = free_l2_table(page, preemptible);
break;
-
#if CONFIG_PAGING_LEVELS >= 3
case PGT_l3_page_table:
- free_l3_table(page);
+#if CONFIG_PAGING_LEVELS == 3
+ if ( !(type & PGT_partial) )
+ page->nr_validated_ptes = L3_PAGETABLE_ENTRIES;
+#endif
+ rc = free_l3_table(page, preemptible);
break;
#endif
-
#if CONFIG_PAGING_LEVELS >= 4
case PGT_l4_page_table:
- free_l4_table(page);
+ rc = free_l4_table(page, preemptible);
break;
#endif
default:
- printk("%s: type %lx pfn %lx\n",__FUNCTION__,
- type, page_to_mfn(page));
+ MEM_LOG("type %lx pfn %lx\n", type, page_to_mfn(page));
+ rc = -EINVAL;
BUG();
}
+
+ return rc;
}
-void put_page_type(struct page_info *page)
+static int __put_final_page_type(
+ struct page_info *page, unsigned long type, int preemptible)
+{
+ int rc = free_page_type(page, type, preemptible);
+
+ /* No need for atomic update of type_info here: noone else updates it. */
+ if ( rc == 0 )
+ {
+ /*
+ * Record TLB information for flush later. We do not stamp page tables
+ * when running in shadow mode:
+ * 1. Pointless, since it's the shadow pt's which must be tracked.
+ * 2. Shadow mode reuses this field for shadowed page tables to
+ * store flags info -- we don't want to conflict with that.
+ */
+ if ( !(shadow_mode_enabled(page_get_owner(page)) &&
+ (page->count_info & PGC_page_table)) )
+ page->tlbflush_timestamp = tlbflush_current_time();
+ wmb();
+ page->u.inuse.type_info--;
+ }
+ else if ( rc == -EINTR )
+ {
+ ASSERT((page->u.inuse.type_info &
+ (PGT_count_mask|PGT_validated|PGT_partial)) == 1);
+ if ( !(shadow_mode_enabled(page_get_owner(page)) &&
+ (page->count_info & PGC_page_table)) )
+ page->tlbflush_timestamp = tlbflush_current_time();
+ wmb();
+ page->u.inuse.type_info |= PGT_validated;
+ }
+ else
+ {
+ BUG_ON(rc != -EAGAIN);
+ wmb();
+ get_page_light(page);
+ page->u.inuse.type_info |= PGT_partial;
+ }
+
+ return rc;
+}
+
+
+static int __put_page_type(struct page_info *page,
+ int preemptible)
{
unsigned long nx, x, y = page->u.inuse.type_info;
+ int rc = 0;
- again:
- do {
+ for ( ; ; )
+ {
x = y;
nx = x - 1;
@@ -1710,21 +1926,22 @@ void put_page_type(struct page_info *page)
if ( unlikely((nx & PGT_count_mask) == 0) )
{
if ( unlikely((nx & PGT_type_mask) <= PGT_l4_page_table) &&
- likely(nx & PGT_validated) )
+ likely(nx & (PGT_validated|PGT_partial)) )
{
/*
* Page-table pages must be unvalidated when count is zero. The
* 'free' is safe because the refcnt is non-zero and validated
* bit is clear => other ops will spin or fail.
*/
- if ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x,
- x & ~PGT_validated)) != x) )
- goto again;
+ nx = x & ~(PGT_validated|PGT_partial);
+ if ( unlikely((y = cmpxchg(&page->u.inuse.type_info,
+ x, nx)) != x) )
+ continue;
/* We cleared the 'valid bit' so we do the clean up. */
- free_page_type(page, x);
- /* Carry on, but with the 'valid bit' now clear. */
- x &= ~PGT_validated;
- nx &= ~PGT_validated;
+ rc = __put_final_page_type(page, x, preemptible);
+ if ( x & PGT_partial )
+ put_page(page);
+ break;
}
/*
@@ -1738,25 +1955,34 @@ void put_page_type(struct page_info *page)
(page->count_info & PGC_page_table)) )
page->tlbflush_timestamp = tlbflush_current_time();
}
+
+ if ( likely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) == x) )
+ break;
+
+ if ( preemptible && hypercall_preempt_check() )
+ return -EINTR;
}
- while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) );
+
+ return rc;
}
-int get_page_type(struct page_info *page, unsigned long type)
+static int __get_page_type(struct page_info *page, unsigned long type,
+ int preemptible)
{
unsigned long nx, x, y = page->u.inuse.type_info;
+ int rc = 0;
ASSERT(!(type & ~(PGT_type_mask | PGT_pae_xen_l2)));
- again:
- do {
+ for ( ; ; )
+ {
x = y;
nx = x + 1;
if ( unlikely((nx & PGT_count_mask) == 0) )
{
MEM_LOG("Type count overflow on pfn %lx", page_to_mfn(page));
- return 0;
+ return -EINVAL;
}
else if ( unlikely((x & PGT_count_mask) == 0) )
{
@@ -1803,50 +2029,85 @@ int get_page_type(struct page_info *page, unsigned long type)
/* Don't log failure if it could be a recursive-mapping attempt. */
if ( ((x & PGT_type_mask) == PGT_l2_page_table) &&
(type == PGT_l1_page_table) )
- return 0;
+ return -EINVAL;
if ( ((x & PGT_type_mask) == PGT_l3_page_table) &&
(type == PGT_l2_page_table) )
- return 0;
+ return -EINVAL;
if ( ((x & PGT_type_mask) == PGT_l4_page_table) &&
(type == PGT_l3_page_table) )
- return 0;
+ return -EINVAL;
MEM_LOG("Bad type (saw %" PRtype_info " != exp %" PRtype_info ") "
"for mfn %lx (pfn %lx)",
x, type, page_to_mfn(page),
get_gpfn_from_mfn(page_to_mfn(page)));
- return 0;
+ return -EINVAL;
}
else if ( unlikely(!(x & PGT_validated)) )
{
- /* Someone else is updating validation of this page. Wait... */
- while ( (y = page->u.inuse.type_info) == x )
- cpu_relax();
- goto again;
+ if ( !(x & PGT_partial) )
+ {
+ /* Someone else is updating validation of this page. Wait... */
+ while ( (y = page->u.inuse.type_info) == x )
+ {
+ if ( preemptible && hypercall_preempt_check() )
+ return -EINTR;
+ cpu_relax();
+ }
+ continue;
+ }
+ /* Type ref count was left at 1 when PGT_partial got set. */
+ ASSERT((x & PGT_count_mask) == 1);
+ nx = x & ~PGT_partial;
}
+
+ if ( likely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) == x) )
+ break;
+
+ if ( preemptible && hypercall_preempt_check() )
+ return -EINTR;
}
- while ( unlikely((y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x) );
if ( unlikely(!(nx & PGT_validated)) )
{
- /* Try to validate page type; drop the new reference on failure. */
- if ( unlikely(!alloc_page_type(page, type)) )
+ if ( !(x & PGT_partial) )
{
- MEM_LOG("Error while validating mfn %lx (pfn %lx) for type %"
- PRtype_info ": caf=%08x taf=%" PRtype_info,
- page_to_mfn(page), get_gpfn_from_mfn(page_to_mfn(page)),
- type, page->count_info, page->u.inuse.type_info);
- /* Noone else can get a reference. We hold the only ref. */
- page->u.inuse.type_info = 0;
- return 0;
+ page->nr_validated_ptes = 0;
+ page->partial_pte = 0;
}
-
- /* Noone else is updating simultaneously. */
- __set_bit(_PGT_validated, &page->u.inuse.type_info);
+ rc = alloc_page_type(page, type, preemptible);
}
- return 1;
+ if ( (x & PGT_partial) && !(nx & PGT_partial) )
+ put_page(page);
+
+ return rc;
+}
+
+void put_page_type(struct page_info *page)
+{
+ int rc = __put_page_type(page, 0);
+ ASSERT(rc == 0);
+ (void)rc;
+}
+
+int get_page_type(struct page_info *page, unsigned long type)
+{
+ int rc = __get_page_type(page, type, 0);
+ if ( likely(rc == 0) )
+ return 1;
+ ASSERT(rc == -EINVAL);
+ return 0;
+}
+
+int put_page_type_preemptible(struct page_info *page)
+{
+ return __put_page_type(page, 1);
}
+int get_page_type_preemptible(struct page_info *page, unsigned long type)
+{
+ return __get_page_type(page, type, 1);
+}
int new_guest_cr3(unsigned long mfn)
{
@@ -1866,7 +2127,7 @@ int new_guest_cr3(unsigned long mfn)
l4e_from_pfn(
mfn,
(_PAGE_PRESENT|_PAGE_RW|_PAGE_USER|_PAGE_ACCESSED)),
- pagetable_get_pfn(v->arch.guest_table), 0);
+ pagetable_get_pfn(v->arch.guest_table), 0, 0) == 0;
if ( unlikely(!okay) )
{
MEM_LOG("Error while installing new compat baseptr %lx", mfn);
@@ -1881,7 +2142,7 @@ int new_guest_cr3(unsigned long mfn)
#endif
okay = paging_mode_refcounts(d)
? get_page_from_pagenr(mfn, d)
- : get_page_and_type_from_pagenr(mfn, PGT_root_page_table, d);
+ : !get_page_and_type_from_pagenr(mfn, PGT_root_page_table, d, 0, 0);
if ( unlikely(!okay) )
{
MEM_LOG("Error while installing new baseptr %lx", mfn);
@@ -2061,9 +2322,7 @@ int do_mmuext_op(
{
if ( hypercall_preempt_check() )
{
- rc = hypercall_create_continuation(
- __HYPERVISOR_mmuext_op, "hihi",
- uops, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
+ rc = -EAGAIN;
break;
}
@@ -2106,10 +2365,14 @@ int do_mmuext_op(
if ( paging_mode_refcounts(FOREIGNDOM) )
break;
- okay = get_page_and_type_from_pagenr(mfn, type, FOREIGNDOM);
+ rc = get_page_and_type_from_pagenr(mfn, type, FOREIGNDOM, 0, 1);
+ okay = !rc;
if ( unlikely(!okay) )
{
- MEM_LOG("Error while pinning mfn %lx", mfn);
+ if ( rc == -EINTR )
+ rc = -EAGAIN;
+ else if ( rc != -EAGAIN )
+ MEM_LOG("Error while pinning mfn %lx", mfn);
break;
}
@@ -2154,8 +2417,11 @@ int do_mmuext_op(
{
put_page_and_type(page);
put_page(page);
- /* A page is dirtied when its pin status is cleared. */
- paging_mark_dirty(d, mfn);
+ if ( !rc )
+ {
+ /* A page is dirtied when its pin status is cleared. */
+ paging_mark_dirty(d, mfn);
+ }
}
else
{
@@ -2179,8 +2445,8 @@ int do_mmuext_op(
if ( paging_mode_refcounts(d) )
okay = get_page_from_pagenr(mfn, d);
else
- okay = get_page_and_type_from_pagenr(
- mfn, PGT_root_page_table, d);
+ okay = !get_page_and_type_from_pagenr(
+ mfn, PGT_root_page_table, d, 0, 0);
if ( unlikely(!okay) )
{
MEM_LOG("Error while installing new mfn %lx", mfn);
@@ -2299,6 +2565,11 @@ int do_mmuext_op(
guest_handle_add_offset(uops, 1);
}
+ if ( rc == -EAGAIN )
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_mmuext_op, "hihi",
+ uops, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
+
process_deferred_ops();
UNLOCK_BIGLOCK(d);
@@ -2362,9 +2633,7 @@ int do_mmu_update(
{
if ( hypercall_preempt_check() )
{
- rc = hypercall_create_continuation(
- __HYPERVISOR_mmu_update, "hihi",
- ureqs, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
+ rc = -EAGAIN;
break;
}
@@ -2439,8 +2708,9 @@ int do_mmu_update(
case PGT_l3_page_table:
{
l3_pgentry_t l3e = l3e_from_intpte(req.val);
- okay = mod_l3_entry(va, l3e, mfn,
- cmd == MMU_PT_UPDATE_PRESERVE_AD);
+ rc = mod_l3_entry(va, l3e, mfn,
+ cmd == MMU_PT_UPDATE_PRESERVE_AD, 1);
+ okay = !rc;
}
break;
#endif
@@ -2448,14 +2718,17 @@ int do_mmu_update(
case PGT_l4_page_table:
{
l4_pgentry_t l4e = l4e_from_intpte(req.val);
- okay = mod_l4_entry(d, va, l4e, mfn,
- cmd == MMU_PT_UPDATE_PRESERVE_AD);
+ rc = mod_l4_entry(d, va, l4e, mfn,
+ cmd == MMU_PT_UPDATE_PRESERVE_AD, 1);
+ okay = !rc;
}
break;
#endif
}
put_page_type(page);
+ if ( rc == -EINTR)
+ rc = -EAGAIN;
}
break;
@@ -2518,6 +2791,11 @@ int do_mmu_update(
guest_handle_add_offset(ureqs, 1);
}
+ if ( rc == -EAGAIN )
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_mmu_update, "hihi",
+ ureqs, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
+
process_deferred_ops();
UNLOCK_BIGLOCK(d);
diff --git a/include/asm-x86/config.h b/include/asm-x86/config.h
index cbdccf7..7fd368f 100644
--- a/include/asm-x86/config.h
+++ b/include/asm-x86/config.h
@@ -39,14 +39,6 @@
#define CONFIG_VGA 1
-/*
- * Avoid deep recursion when tearing down pagetables during domain destruction,
- * causing dom0 to become unresponsive and Xen to miss time-critical softirq
- * deadlines. This will ultimately be replaced by built-in preemptibility of
- * get_page_type().
- */
-#define DOMAIN_DESTRUCT_AVOID_RECURSION 1
-
#define HZ 100
#define OPT_CONSOLE_STR "com1,vga"
diff --git a/include/asm-x86/mm.h b/include/asm-x86/mm.h
index 8c02a76..65420bf 100644
--- a/include/asm-x86/mm.h
+++ b/include/asm-x86/mm.h
@@ -55,6 +55,41 @@ struct page_info
u32 tlbflush_timestamp;
/*
+ * When PGT_partial is true then this field is valid and indicates
+ * that PTEs in the range [0, @nr_validated_ptes) have been validated.
+ * An extra page reference must be acquired (or not dropped) whenever
+ * PGT_partial gets set, and it must be dropped when the flag gets
+ * cleared. This is so that a get() leaving a page in partially
+ * validated state (where the caller would drop the reference acquired
+ * due to the getting of the type [apparently] failing [-EAGAIN])
+ * would not accidentally result in a page left with zero general
+ * reference count, but non-zero type reference count (possible when
+ * the partial get() is followed immediately by domain destruction).
+ * Likewise, the ownership of the single type reference for partially
+ * (in-)validated pages is tied to this flag, i.e. the instance
+ * setting the flag must not drop that reference, whereas the instance
+ * clearing it will have to.
+ *
+ * If @partial_pte is positive then PTE at @nr_validated_ptes+1 has
+ * been partially validated. This implies that the general reference
+ * to the page (acquired from get_page_from_lNe()) would be dropped
+ * (again due to the apparent failure) and hence must be re-acquired
+ * when resuming the validation, but must not be dropped when picking
+ * up the page for invalidation.
+ *
+ * If @partial_pte is negative then PTE at @nr_validated_ptes+1 has
+ * been partially invalidated. This is basically the opposite case of
+ * above, i.e. the general reference to the page was not dropped in
+ * put_page_from_lNe() (due to the apparent failure), and hence it
+ * must be dropped when the put operation is resumed (and completes),
+ * but it must not be acquired if picking up the page for validation.
+ */
+ struct {
+ u16 nr_validated_ptes;
+ s8 partial_pte;
+ };
+
+ /*
* Guest pages with a shadow. This does not conflict with
* tlbflush_timestamp since page table pages are explicitly not
* tracked for TLB-flush avoidance when a guest runs in shadow mode.
@@ -83,9 +118,12 @@ struct page_info
/* PAE only: is this an L2 page directory containing Xen-private mappings? */
#define _PGT_pae_xen_l2 26
#define PGT_pae_xen_l2 (1U<<_PGT_pae_xen_l2)
+/* Has this page been *partially* validated for use as its current type? */
+#define _PGT_partial 25
+#define PGT_partial (1U<<_PGT_partial)
- /* 16-bit count of uses of this frame as its current type. */
-#define PGT_count_mask ((1U<<16)-1)
+ /* 25-bit count of uses of this frame as its current type. */
+#define PGT_count_mask ((1U<<25)-1)
/* Cleared when the owning guest 'frees' this page. */
#define _PGC_allocated 31
@@ -144,8 +182,9 @@ extern unsigned long max_page;
extern unsigned long total_pages;
void init_frametable(void);
-int alloc_page_type(struct page_info *page, unsigned long type);
-void free_page_type(struct page_info *page, unsigned long type);
+int alloc_page_type(struct page_info *page, unsigned long type, int preemptible);
+int free_page_type(struct page_info *page, unsigned long type,
+ int preemptible);
int _shadow_mode_refcounts(struct domain *d);
static inline void put_page(struct page_info *page)
@@ -199,6 +238,8 @@ static inline int get_page(struct page_info *page,
void put_page_type(struct page_info *page);
int get_page_type(struct page_info *page, unsigned long type);
+int put_page_type_preemptible(struct page_info *page);
+int get_page_type_preemptible(struct page_info *page, unsigned long type);
int get_page_from_l1e(l1_pgentry_t l1e, struct domain *d);
void put_page_from_l1e(l1_pgentry_t l1e, struct domain *d);
@@ -208,6 +249,19 @@ static inline void put_page_and_type(struct page_info *page)
put_page(page);
}
+static inline int put_page_and_type_preemptible(struct page_info *page,
+ int preemptible)
+{
+ int rc = 0;
+
+ if ( preemptible )
+ rc = put_page_type_preemptible(page);
+ else
+ put_page_type(page);
+ if ( likely(rc == 0) )
+ put_page(page);
+ return rc;
+}
static inline int get_page_and_type(struct page_info *page,
struct domain *domain,
distrib
>
Scientific%20Linux
>
5x
>
x86_64
>
by-pkgid
>
fc11cd6e1c513a17304da94a5390f3cd
>
files
>
4506
