From: Chris Lalancette <clalance@redhat.com>
Date: Fri, 14 Dec 2007 16:34:31 -0500
Subject: [xen] save/restore: pv oops when mmap prot_none
Message-id: 4762F6E7.1030009@redhat.com
O-Subject: [RHEL5.2 PATCH]: Fix Xen PV oops when mmaping PROT_NONE during save/restore
Bugzilla: 294811

All,
     Attached is a patch to fix an OOPS that can occur inside a Xen PV
guest when mmap()'ing a PROT_NONE region.  The oops is reproducible if you:

1.  mmap a PROT_NONE region, while
2.  saving the guest state

This patch is really a combination of 3 upstream changesets from
xen-unstable: c/s 12402, 13998, and 14006.  Before applying this patch,
I can trivially reproduce the bug by running the test case in the BZ and
doing save/restore in a loop on the domain.  After the patch, I have
been able to run the save/restore loop 100's of times without a crash on both
i386 and x86_64.

This resolves BZ 294811.  Please review and ACK.

Chris Lalancette

Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: "Stephen C. Tweedie" <sct@redhat.com>
Acked-by: "Stephen C. Tweedie" <sct@redhat.com>
Acked-by: Bill Burns <bburns@redhat.com>

diff --git a/include/asm-i386/mach-xen/asm/maddr.h b/include/asm-i386/mach-xen/asm/maddr.h
index b467320..b49131c 100644
--- a/include/asm-i386/mach-xen/asm/maddr.h
+++ b/include/asm-i386/mach-xen/asm/maddr.h
@@ -12,6 +12,7 @@
 #ifdef CONFIG_XEN
 
 extern unsigned long *phys_to_machine_mapping;
+extern unsigned long  max_mapnr;
 
 #undef machine_to_phys_mapping
 extern unsigned long *machine_to_phys_mapping;
@@ -21,20 +22,20 @@ static inline unsigned long pfn_to_mfn(unsigned long pfn)
 {
 	if (xen_feature(XENFEAT_auto_translated_physmap))
 		return pfn;
-	return phys_to_machine_mapping[(unsigned int)(pfn)] &
-		~FOREIGN_FRAME_BIT;
+	BUG_ON(max_mapnr && pfn >= max_mapnr);
+	return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
 }
 
 static inline int phys_to_machine_mapping_valid(unsigned long pfn)
 {
 	if (xen_feature(XENFEAT_auto_translated_physmap))
 		return 1;
+	BUG_ON(max_mapnr && pfn >= max_mapnr);
 	return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
 }
 
 static inline unsigned long mfn_to_pfn(unsigned long mfn)
 {
-	extern unsigned long max_mapnr;
 	unsigned long pfn;
 
 	if (xen_feature(XENFEAT_auto_translated_physmap))
@@ -83,7 +84,6 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn)
  */
 static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 {
-	extern unsigned long max_mapnr;
 	unsigned long pfn = mfn_to_pfn(mfn);
 	if ((pfn < max_mapnr)
 	    && !xen_feature(XENFEAT_auto_translated_physmap)
@@ -94,6 +94,7 @@ static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 
 static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 {
+	BUG_ON(max_mapnr && pfn >= max_mapnr);
 	if (xen_feature(XENFEAT_auto_translated_physmap)) {
 		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 		return;
@@ -127,6 +128,7 @@ static inline maddr_t phys_to_machine(paddr_t phys)
 	machine = (machine << PAGE_SHIFT) | (phys & ~PAGE_MASK);
 	return machine;
 }
+
 static inline paddr_t machine_to_phys(maddr_t machine)
 {
 	paddr_t phys = mfn_to_pfn(machine >> PAGE_SHIFT);
@@ -134,6 +136,34 @@ static inline paddr_t machine_to_phys(maddr_t machine)
 	return phys;
 }
 
+#ifdef CONFIG_X86_PAE
+static inline paddr_t pte_phys_to_machine(paddr_t phys)
+{
+	/*
+	 * In PAE mode, the NX bit needs to be dealt with in the value
+	 * passed to pfn_to_mfn(). On x86_64, we need to mask it off,
+	 * but for i386 the conversion to ulong for the argument will
+	 * clip it off.
+	 */
+	maddr_t machine = pfn_to_mfn(phys >> PAGE_SHIFT);
+	machine = (machine << PAGE_SHIFT) | (phys & ~PHYSICAL_PAGE_MASK);
+	return machine;
+}
+
+static inline paddr_t pte_machine_to_phys(maddr_t machine)
+{
+	/*
+	 * In PAE mode, the NX bit needs to be dealt with in the value
+	 * passed to mfn_to_pfn(). On x86_64, we need to mask it off,
+	 * but for i386 the conversion to ulong for the argument will
+	 * clip it off.
+	 */
+	paddr_t phys = mfn_to_pfn(machine >> PAGE_SHIFT);
+	phys = (phys << PAGE_SHIFT) | (machine & ~PHYSICAL_PAGE_MASK);
+	return phys;
+}
+#endif
+
 /* VIRT <-> MACHINE conversion */
 #define virt_to_machine(v)	(phys_to_machine(__pa(v)))
 #define virt_to_mfn(v)		(pfn_to_mfn(__pa(v) >> PAGE_SHIFT))
diff --git a/include/asm-i386/mach-xen/asm/page.h b/include/asm-i386/mach-xen/asm/page.h
index 5002bb1..57bb065 100644
--- a/include/asm-i386/mach-xen/asm/page.h
+++ b/include/asm-i386/mach-xen/asm/page.h
@@ -6,6 +6,16 @@
 #define PAGE_SIZE	(1UL << PAGE_SHIFT)
 #define PAGE_MASK	(~(PAGE_SIZE-1))
 
+#ifdef CONFIG_X86_PAE
+#define __PHYSICAL_MASK_SHIFT	36
+#define __PHYSICAL_MASK		((1ULL << __PHYSICAL_MASK_SHIFT) - 1)
+#define PHYSICAL_PAGE_MASK	(~((1ULL << PAGE_SHIFT) - 1) & __PHYSICAL_MASK)
+#else
+#define __PHYSICAL_MASK_SHIFT	32
+#define __PHYSICAL_MASK		(~0UL)
+#define PHYSICAL_PAGE_MASK	(PAGE_MASK & __PHYSICAL_MASK)
+#endif
+
 #define LARGE_PAGE_MASK (~(LARGE_PAGE_SIZE-1))
 #define LARGE_PAGE_SIZE (1UL << PMD_SHIFT)
 
@@ -20,6 +30,13 @@
 #include <xen/features.h>
 #include <xen/foreign_page.h>
 
+/*
+ * Need to repeat this here in order to not include pgtable.h (which in turn
+ * depends on definitions made here), but to be able to use the symbolic
+ * below. The preprocessor will warn if the two definitions aren't identical.
+ */
+#define _PAGE_PRESENT	0x001
+
 #define arch_free_page(_page,_order)			\
 ({	int foreign = PageForeign(_page);		\
 	if (foreign)					\
@@ -72,40 +89,38 @@ typedef struct { unsigned long long pgprot; } pgprot_t;
 #define pgprot_val(x)	((x).pgprot)
 #include <asm/maddr.h>
 #define __pte(x) ({ unsigned long long _x = (x);        \
-    if (_x & 1) _x = phys_to_machine(_x);               \
+    if (_x & _PAGE_PRESENT) _x = pte_phys_to_machine(_x);   \
     ((pte_t) {(unsigned long)(_x), (unsigned long)(_x>>32)}); })
 #define __pgd(x) ({ unsigned long long _x = (x); \
-    (((_x)&1) ? ((pgd_t) {phys_to_machine(_x)}) : ((pgd_t) {(_x)})); })
+    (pgd_t) {((_x) & _PAGE_PRESENT) ? pte_phys_to_machine(_x) : (_x)}; })
 #define __pmd(x) ({ unsigned long long _x = (x); \
-    (((_x)&1) ? ((pmd_t) {phys_to_machine(_x)}) : ((pmd_t) {(_x)})); })
+    (pmd_t) {((_x) & _PAGE_PRESENT) ? pte_phys_to_machine(_x) : (_x)}; })
+static inline unsigned long long pte_val_ma(pte_t x)
+{
+	return ((unsigned long long)x.pte_high << 32) | x.pte_low;
+}
 static inline unsigned long long pte_val(pte_t x)
 {
-	unsigned long long ret;
-
-	if (x.pte_low) {
-		ret = x.pte_low | (unsigned long long)x.pte_high << 32;
-		ret = machine_to_phys(ret) | 1;
-	} else {
-		ret = 0;
-	}
+	unsigned long long ret = pte_val_ma(x);
+	if (x.pte_low & _PAGE_PRESENT) ret = pte_machine_to_phys(ret);
 	return ret;
 }
 static inline unsigned long long pmd_val(pmd_t x)
 {
 	unsigned long long ret = x.pmd;
-	if (ret) ret = machine_to_phys(ret) | 1;
+#ifdef CONFIG_XEN_COMPAT_030002
+	if (ret) ret = pte_machine_to_phys(ret) | _PAGE_PRESENT;
+#else
+	if (ret & _PAGE_PRESENT) ret = pte_machine_to_phys(ret);
+#endif
 	return ret;
 }
 static inline unsigned long long pgd_val(pgd_t x)
 {
 	unsigned long long ret = x.pgd;
-	if (ret) ret = machine_to_phys(ret) | 1;
+	if (ret & _PAGE_PRESENT) ret = pte_machine_to_phys(ret);
 	return ret;
 }
-static inline unsigned long long pte_val_ma(pte_t x)
-{
-	return (unsigned long long)x.pte_high << 32 | x.pte_low;
-}
 #define HPAGE_SHIFT	21
 #else
 typedef struct { unsigned long pte_low; } pte_t;
@@ -114,22 +129,23 @@ typedef struct { unsigned long pgprot; } pgprot_t;
 #define pgprot_val(x)	((x).pgprot)
 #include <asm/maddr.h>
 #define boot_pte_t pte_t /* or would you rather have a typedef */
-#define pte_val(x)	(((x).pte_low & 1) ? machine_to_phys((x).pte_low) : \
+#define pte_val(x)	(((x).pte_low & _PAGE_PRESENT) ? \
+			 machine_to_phys((x).pte_low) : \
 			 (x).pte_low)
 #define pte_val_ma(x)	((x).pte_low)
 #define __pte(x) ({ unsigned long _x = (x); \
-    (((_x)&1) ? ((pte_t) {phys_to_machine(_x)}) : ((pte_t) {(_x)})); })
+    (pte_t) {((_x) & _PAGE_PRESENT) ? phys_to_machine(_x) : (_x)}; })
 #define __pgd(x) ({ unsigned long _x = (x); \
-    (((_x)&1) ? ((pgd_t) {phys_to_machine(_x)}) : ((pgd_t) {(_x)})); })
+    (pgd_t) {((_x) & _PAGE_PRESENT) ? phys_to_machine(_x) : (_x)}; })
 static inline unsigned long pgd_val(pgd_t x)
 {
 	unsigned long ret = x.pgd;
-	if (ret) ret = machine_to_phys(ret) | 1;
+	if (ret & _PAGE_PRESENT) ret = machine_to_phys(ret);
 	return ret;
 }
 #define HPAGE_SHIFT	22
 #endif
-#define PTE_MASK	PAGE_MASK
+#define PTE_MASK	PHYSICAL_PAGE_MASK
 
 #ifdef CONFIG_HUGETLB_PAGE
 #define HPAGE_SIZE	((1UL) << HPAGE_SHIFT)
diff --git a/include/asm-i386/mach-xen/asm/pgtable-2level.h b/include/asm-i386/mach-xen/asm/pgtable-2level.h
index 34cb67f..f7250a3 100644
--- a/include/asm-i386/mach-xen/asm/pgtable-2level.h
+++ b/include/asm-i386/mach-xen/asm/pgtable-2level.h
@@ -38,8 +38,11 @@
 
 #define ptep_get_and_clear(mm,addr,xp)	__pte_ma(xchg(&(xp)->pte_low, 0))
 #define pte_same(a, b)		((a).pte_low == (b).pte_low)
-#define pte_mfn(_pte) ((_pte).pte_low >> PAGE_SHIFT)
-#define pte_pfn(_pte) mfn_to_local_pfn(pte_mfn(_pte))
+#define __pte_mfn(_pte) ((_pte).pte_low >> PAGE_SHIFT)
+#define pte_mfn(_pte) ((_pte).pte_low & _PAGE_PRESENT ? \
+	__pte_mfn(_pte) : pfn_to_mfn(__pte_mfn(_pte)))
+#define pte_pfn(_pte) ((_pte).pte_low & _PAGE_PRESENT ? \
+	mfn_to_local_pfn(__pte_mfn(_pte)) : __pte_mfn(_pte))
 
 #define pte_page(_pte) pfn_to_page(pte_pfn(_pte))
 
diff --git a/include/asm-i386/mach-xen/asm/pgtable-3level.h b/include/asm-i386/mach-xen/asm/pgtable-3level.h
index 7c3c36b..8747c52 100644
--- a/include/asm-i386/mach-xen/asm/pgtable-3level.h
+++ b/include/asm-i386/mach-xen/asm/pgtable-3level.h
@@ -145,21 +145,24 @@ static inline int pte_none(pte_t pte)
 	return !pte.pte_low && !pte.pte_high;
 }
 
-#define pte_mfn(_pte) (((_pte).pte_low >> PAGE_SHIFT) |\
-		       (((_pte).pte_high & 0xfff) << (32-PAGE_SHIFT)))
-#define pte_pfn(_pte) mfn_to_local_pfn(pte_mfn(_pte))
+#define __pte_mfn(_pte) (((_pte).pte_low >> PAGE_SHIFT) | \
+			 ((_pte).pte_high << (32-PAGE_SHIFT)))
+#define pte_mfn(_pte) ((_pte).pte_low & _PAGE_PRESENT ? \
+	__pte_mfn(_pte) : pfn_to_mfn(__pte_mfn(_pte)))
+#define pte_pfn(_pte) ((_pte).pte_low & _PAGE_PRESENT ? \
+	mfn_to_local_pfn(__pte_mfn(_pte)) : __pte_mfn(_pte))
 
 extern unsigned long long __supported_pte_mask;
 
 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 {
-	return pfn_pte_ma(pfn_to_mfn(page_nr), pgprot);
+	return __pte((((unsigned long long)page_nr << PAGE_SHIFT) |
+			pgprot_val(pgprot)) & __supported_pte_mask);
 }
 
 static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
 {
-	BUG(); panic("needs review");
-	return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) | \
+	return __pmd((((unsigned long long)page_nr << PAGE_SHIFT) |
 			pgprot_val(pgprot)) & __supported_pte_mask);
 }
 
diff --git a/include/asm-i386/mach-xen/asm/pgtable.h b/include/asm-i386/mach-xen/asm/pgtable.h
index 388a5ba..3a404a7 100644
--- a/include/asm-i386/mach-xen/asm/pgtable.h
+++ b/include/asm-i386/mach-xen/asm/pgtable.h
@@ -312,18 +312,19 @@ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
 
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 {
-	pte.pte_low &= _PAGE_CHG_MASK;
-	pte.pte_low |= pgprot_val(newprot);
-#ifdef CONFIG_X86_PAE
 	/*
-	 * Chop off the NX bit (if present), and add the NX portion of
-	 * the newprot (if present):
+	 * Since this might change the present bit (which controls whether
+	 * a pte_t object has undergone p2m translation), we must use
+	 * pte_val() on the input pte and __pte() for the return value.
 	 */
-	pte.pte_high &= ~(1 << (_PAGE_BIT_NX - 32));
-	pte.pte_high |= (pgprot_val(newprot) >> 32) & \
-					(__supported_pte_mask >> 32);
+	paddr_t pteval = pte_val(pte);
+
+	pteval &= _PAGE_CHG_MASK;
+	pteval |= pgprot_val(newprot);
+#ifdef CONFIG_X86_PAE
+	pteval &= __supported_pte_mask;
 #endif
-	return pte;
+	return __pte(pteval);
 }
 
 #define pmd_large(pmd) \
diff --git a/include/asm-x86_64/mach-xen/asm/maddr.h b/include/asm-x86_64/mach-xen/asm/maddr.h
index 0104de8..baca328 100644
--- a/include/asm-x86_64/mach-xen/asm/maddr.h
+++ b/include/asm-x86_64/mach-xen/asm/maddr.h
@@ -21,14 +21,15 @@ static inline unsigned long pfn_to_mfn(unsigned long pfn)
 {
 	if (xen_feature(XENFEAT_auto_translated_physmap))
 		return pfn;
-	return phys_to_machine_mapping[(unsigned int)(pfn)] &
-		~FOREIGN_FRAME_BIT;
+	BUG_ON(end_pfn && pfn >= end_pfn);
+	return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
 }
 
 static inline int phys_to_machine_mapping_valid(unsigned long pfn)
 {
 	if (xen_feature(XENFEAT_auto_translated_physmap))
 		return 1;
+	BUG_ON(end_pfn && pfn >= end_pfn);
 	return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
 }
 
@@ -92,6 +93,7 @@ static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 
 static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 {
+	BUG_ON(end_pfn && pfn >= end_pfn);
 	if (xen_feature(XENFEAT_auto_translated_physmap)) {
 		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 		return;
@@ -127,6 +129,22 @@ static inline paddr_t machine_to_phys(maddr_t machine)
 	return phys;
 }
 
+static inline paddr_t pte_phys_to_machine(paddr_t phys)
+{
+	maddr_t machine;
+	machine = pfn_to_mfn((phys & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT);
+	machine = (machine << PAGE_SHIFT) | (phys & ~PHYSICAL_PAGE_MASK);
+	return machine;
+}
+
+static inline paddr_t pte_machine_to_phys(maddr_t machine)
+{
+	paddr_t phys;
+	phys = mfn_to_pfn((machine & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT);
+	phys = (phys << PAGE_SHIFT) | (machine & ~PHYSICAL_PAGE_MASK);
+	return phys;
+}
+
 /* VIRT <-> MACHINE conversion */
 #define virt_to_machine(v)	(phys_to_machine(__pa(v)))
 #define virt_to_mfn(v)		(pfn_to_mfn(__pa(v) >> PAGE_SHIFT))
diff --git a/include/asm-x86_64/mach-xen/asm/page.h b/include/asm-x86_64/mach-xen/asm/page.h
index 78c5dd0..50ad81d 100644
--- a/include/asm-x86_64/mach-xen/asm/page.h
+++ b/include/asm-x86_64/mach-xen/asm/page.h
@@ -12,6 +12,13 @@
 #include <xen/interface/xen.h> 
 #include <xen/foreign_page.h>
 
+/*
+ * Need to repeat this here in order to not include pgtable.h (which in turn
+ * depends on definitions made here), but to be able to use the symbolic
+ * below. The preprocessor will warn if the two definitions aren't identical.
+ */
+#define _PAGE_PRESENT	0x001
+
 #define arch_free_page(_page,_order)			\
 ({	int foreign = PageForeign(_page);		\
 	if (foreign)					\
@@ -30,6 +37,13 @@
 #define PAGE_SHIFT	12
 #define PAGE_SIZE	(_AC(1,UL) << PAGE_SHIFT)
 #define PAGE_MASK	(~(PAGE_SIZE-1))
+
+/* See Documentation/x86_64/mm.txt for a description of the memory map. */
+#define __PHYSICAL_MASK_SHIFT	46
+#define __PHYSICAL_MASK		((1UL << __PHYSICAL_MASK_SHIFT) - 1)
+#define __VIRTUAL_MASK_SHIFT	48
+#define __VIRTUAL_MASK		((1UL << __VIRTUAL_MASK_SHIFT) - 1)
+
 #define PHYSICAL_PAGE_MASK	(~(PAGE_SIZE-1) & __PHYSICAL_MASK)
 
 #define THREAD_ORDER 1 
@@ -87,28 +101,33 @@ typedef struct { unsigned long pgd; } pgd_t;
 
 typedef struct { unsigned long pgprot; } pgprot_t;
 
-#define pte_val(x)	(((x).pte & 1) ? machine_to_phys((x).pte) : \
+#define pte_val(x)	(((x).pte & _PAGE_PRESENT) ? \
+			 pte_machine_to_phys((x).pte) : \
 			 (x).pte)
 #define pte_val_ma(x)	((x).pte)
 
 static inline unsigned long pmd_val(pmd_t x)
 {
 	unsigned long ret = x.pmd;
-	if (ret) ret = machine_to_phys(ret);
+#ifdef CONFIG_XEN_COMPAT_030002
+	if (ret) ret = pte_machine_to_phys(ret) | _PAGE_PRESENT;
+#else
+	if (ret & _PAGE_PRESENT) ret = pte_machine_to_phys(ret);
+#endif
 	return ret;
 }
 
 static inline unsigned long pud_val(pud_t x)
 {
 	unsigned long ret = x.pud;
-	if (ret) ret = machine_to_phys(ret);
+	if (ret & _PAGE_PRESENT) ret = pte_machine_to_phys(ret);
 	return ret;
 }
 
 static inline unsigned long pgd_val(pgd_t x)
 {
 	unsigned long ret = x.pgd;
-	if (ret) ret = machine_to_phys(ret);
+	if (ret & _PAGE_PRESENT) ret = pte_machine_to_phys(ret);
 	return ret;
 }
 
@@ -116,25 +135,25 @@ static inline unsigned long pgd_val(pgd_t x)
 
 static inline pte_t __pte(unsigned long x)
 {
-	if (x & 1) x = phys_to_machine(x);
+	if (x & _PAGE_PRESENT) x = pte_phys_to_machine(x);
 	return ((pte_t) { (x) });
 }
 
 static inline pmd_t __pmd(unsigned long x)
 {
-	if ((x & 1)) x = phys_to_machine(x);
+	if (x & _PAGE_PRESENT) x = pte_phys_to_machine(x);
 	return ((pmd_t) { (x) });
 }
 
 static inline pud_t __pud(unsigned long x)
 {
-	if ((x & 1)) x = phys_to_machine(x);
+	if (x & _PAGE_PRESENT) x = pte_phys_to_machine(x);
 	return ((pud_t) { (x) });
 }
 
 static inline pgd_t __pgd(unsigned long x)
 {
-	if ((x & 1)) x = phys_to_machine(x);
+	if (x & _PAGE_PRESENT) x = pte_phys_to_machine(x);
 	return ((pgd_t) { (x) });
 }
 
@@ -153,12 +172,6 @@ static inline pgd_t __pgd(unsigned long x)
 /* to align the pointer to the (next) page boundary */
 #define PAGE_ALIGN(addr)	(((addr)+PAGE_SIZE-1)&PAGE_MASK)
 
-/* See Documentation/x86_64/mm.txt for a description of the memory map. */
-#define __PHYSICAL_MASK_SHIFT	46
-#define __PHYSICAL_MASK		((_AC(1,UL) << __PHYSICAL_MASK_SHIFT) - 1)
-#define __VIRTUAL_MASK_SHIFT	48
-#define __VIRTUAL_MASK		((_AC(1,UL) << __VIRTUAL_MASK_SHIFT) - 1)
-
 #define KERNEL_TEXT_SIZE  (_AC(40,UL)*1024*1024)
 #define KERNEL_TEXT_START _AC(0xffffffff80000000,UL)
 
diff --git a/include/asm-x86_64/mach-xen/asm/pgtable.h b/include/asm-x86_64/mach-xen/asm/pgtable.h
index e6eddf9..0bfa23d 100644
--- a/include/asm-x86_64/mach-xen/asm/pgtable.h
+++ b/include/asm-x86_64/mach-xen/asm/pgtable.h
@@ -322,19 +322,20 @@ static inline unsigned long pud_bad(pud_t pud)
 
 #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT))
 
-#define pte_mfn(_pte) (((_pte).pte & PTE_MASK) >> PAGE_SHIFT)
-#define pte_pfn(_pte) mfn_to_local_pfn(pte_mfn(_pte))
+#define __pte_mfn(_pte) (((_pte).pte & PTE_MASK) >> PAGE_SHIFT)
+#define pte_mfn(_pte) ((_pte).pte & _PAGE_PRESENT ? \
+	__pte_mfn(_pte) : pfn_to_mfn(__pte_mfn(_pte)))
+#define pte_pfn(_pte) ((_pte).pte & _PAGE_PRESENT ? \
+	mfn_to_local_pfn(__pte_mfn(_pte)) : __pte_mfn(_pte))
 
 #define pte_page(x)	pfn_to_page(pte_pfn(x))
 
 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 {
-	pte_t pte;
-        
-	(pte).pte = (pfn_to_mfn(page_nr) << PAGE_SHIFT);
-	(pte).pte |= pgprot_val(pgprot);
-	(pte).pte &= __supported_pte_mask;
-	return pte;
+	unsigned long pte = page_nr << PAGE_SHIFT;
+	pte |= pgprot_val(pgprot);
+	pte &= __supported_pte_mask;
+	return __pte(pte);
 }
 
 /*
@@ -466,18 +467,25 @@ static inline pud_t *pud_offset_k(pgd_t *pgd, unsigned long address)
 /* physical address -> PTE */
 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
 { 
-	pte_t pte;
-	(pte).pte = physpage | pgprot_val(pgprot); 
-	return pte; 
+	unsigned long pteval;
+	pteval = physpage | pgprot_val(pgprot);
+	return __pte(pteval);
 }
  
 /* Change flags of a PTE */
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 { 
-        (pte).pte &= _PAGE_CHG_MASK;
-	(pte).pte |= pgprot_val(newprot);
-	(pte).pte &= __supported_pte_mask;
-       return pte; 
+	/*
+	 * Since this might change the present bit (which controls whether
+	 * a pte_t object has undergone p2m translation), we must use
+	 * pte_val() on the input pte and __pte() for the return value.
+	 */
+	unsigned long pteval = pte_val(pte);
+
+	pteval &= _PAGE_CHG_MASK;
+	pteval |= pgprot_val(newprot);
+	pteval &= __supported_pte_mask;
+	return __pte(pteval);
 }
 
 #define pte_index(address) \