x86: Meltdown band-aid against malicious 64-bit PV guests

This is a very simplistic change limiting the amount of memory a running
64-bit PV guest has mapped (and hence available for attacking): Only the
mappings of stack, IDT, and TSS are being cloned from the direct map
into per-CPU page tables. Guest controlled parts of the page tables are
being copied into those per-CPU page tables upon entry into the guest.
Cross-vCPU synchronization of top level page table entry changes is
being effected by forcing other active vCPU-s of the guest into the
hypervisor.

The change to context_switch() isn't strictly necessary, but there's no
reason to keep switching page tables once a PV guest is being scheduled
out.

This isn't providing full isolation yet, but it should be covering all
pieces of information exposure of which would otherwise require an XSA.

There is certainly much room for improvement, especially of performance,
here - first and foremost suppressing all the negative effects on AMD
systems. But in the interest of backportability (including to really old
hypervisors, which may not even have alternative patching) any such is
being left out here.

WARNING: As to 32-bit Xen, the changes here are merely to allow it to
         continue to build and work, TOGETHER with the next patch, but
         not to implement a similar band-aid there.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v3: Add remark as to not doing full isolation here. Drop use of
    UNLIKELY_*() from CSTAR and INT80 entry paths.
v2: Use sensible numbers for the register classification constants also
    for the low 8 registers. Insert 'r' into their names to make their
    purpose more clear. Use "sub" instead of "add" when adding an
    immediate of 128. Defer sync IPI in do_mmu_update() until the end of
    the main loop. Use flush IPI there instead event check one. Make
    setup functions return a proper error code. Use better suited local
    variables in clone_mapping(). Add comment to new struct cpu_info
    fields.
---
Backporting notes:
- This needs f9eb74789a ("x86/entry: Remove support for partial
  cpu_user_regs frames") as a prereq, due to the uses of %r14 and %r15.
  But that's intended to be backported anyway (for Spectre/SP2).
- This further needs bd61fe94be ("x86/mm: Always set _PAGE_ACCESSED on
  L4e updates") as a prereq.
- The use of "root" instead of "l4" here is mainly to not make 5-level
  page table additions any harder. In backports "l4" should probably be
  preferred.

--- xen-4.0.3-testing.orig/xen/arch/x86/domain.c
+++ xen-4.0.3-testing/xen/arch/x86/domain.c
@@ -1362,6 +1362,12 @@ static void paravirt_ctxt_switch_to(stru
     set_int80_direct_trap(v);
     switch_kernel_stack(v);
 
+#ifdef __x86_64__
+    this_cpu(root_pgt)[root_table_offset(PERDOMAIN_VIRT_START)] =
+        root_from_paddr(__pa(v->domain->arch.mm_perdomain_l3),
+                        __PAGE_HYPERVISOR);
+#endif
+
     cr4 = pv_guest_cr4_to_real_cr4(v);
     if ( unlikely(cr4 != read_cr4()) )
         write_cr4(cr4);
@@ -1489,6 +1495,10 @@ void context_switch(struct vcpu *prev, s
 
     ASSERT(local_irq_is_enabled());
 
+#ifdef __x86_64__
+    get_cpu_info()->xen_cr3 = 0;
+#endif
+
     /* Allow at most one CPU at a time to be dirty. */
     ASSERT(cpus_weight(dirty_mask) <= 1);
     if ( unlikely(!cpu_isset(cpu, dirty_mask) && !cpus_empty(dirty_mask)) )
--- xen-4.0.3-testing.orig/xen/arch/x86/i8259.c
+++ xen-4.0.3-testing/xen/arch/x86/i8259.c
@@ -33,7 +33,9 @@
  * interrupt-controller happy.
  */
 
+#ifndef __x86_64__
 BUILD_COMMON_IRQ()
+#endif
 
 #define BI(x,y) \
     BUILD_IRQ(x##y)
--- xen-4.0.3-testing.orig/xen/arch/x86/mm.c
+++ xen-4.0.3-testing/xen/arch/x86/mm.c
@@ -3538,6 +3538,7 @@ int do_mmu_update(
     struct vcpu *curr = current, *v = curr;
     struct domain *d = current->domain, *pt_owner = d, *pg_owner;
     struct domain_mmap_cache mapcache;
+    bool_t sync_guest = 0;
     int rc = put_old_guest_table(curr);
 
     if ( unlikely(rc) )
@@ -3794,6 +3795,8 @@ int do_mmu_update(
 
                     rc = mod_l4_entry(va, l4e, mfn,
                                       cmd == MMU_PT_UPDATE_PRESERVE_AD, v);
+                    if ( !rc )
+                        sync_guest = 1;
                 }
                 break;
 #endif
@@ -3888,6 +3891,17 @@ int do_mmu_update(
 
     domain_mmap_cache_destroy(&mapcache);
 
+    if ( sync_guest )
+    {
+        /*
+         * Force other vCPU-s of the affected guest to pick up L4 entry
+         * changes (if any). Issue a flush IPI with empty operation mask to
+         * facilitate this (including ourselves waiting for the IPI to
+         * actually have arrived).
+         */
+        flush_mask(&pt_owner->domain_dirty_cpumask, 0);
+    }
+
     perfc_add(num_page_updates, i);
 
  out:
--- xen-4.0.3-testing.orig/xen/arch/x86/smpboot.c
+++ xen-4.0.3-testing/xen/arch/x86/smpboot.c
@@ -512,6 +512,11 @@ void __devinit start_secondary(void *unu
 	 */
 	spin_debug_disable();
 
+#ifdef __x86_64__
+	get_cpu_info()->xen_cr3 = 0;
+	get_cpu_info()->pv_cr3 = __pa(this_cpu(root_pgt));
+#endif
+
 	percpu_traps_init();
 
 	cpu_init();
@@ -842,6 +847,191 @@ static void *prepare_idle_stack(unsigned
 	return stack_base[cpu];
 }
 
+#ifdef __x86_64__
+static int clone_mapping(const void *ptr, root_pgentry_t *rpt)
+{
+    unsigned long linear = (unsigned long)ptr, pfn;
+    unsigned int flags;
+    l3_pgentry_t *pl3e = l4e_to_l3e(idle_pg_table[root_table_offset(linear)]) +
+                         l3_table_offset(linear);
+    l2_pgentry_t *pl2e;
+    l1_pgentry_t *pl1e;
+
+    if ( linear < DIRECTMAP_VIRT_START )
+        return 0;
+
+    flags = l3e_get_flags(*pl3e);
+    ASSERT(flags & _PAGE_PRESENT);
+    if ( flags & _PAGE_PSE )
+    {
+        pfn = (l3e_get_pfn(*pl3e) & ~((1UL << (2 * PAGETABLE_ORDER)) - 1)) |
+              (PFN_DOWN(linear) & ((1UL << (2 * PAGETABLE_ORDER)) - 1));
+        flags &= ~_PAGE_PSE;
+    }
+    else
+    {
+        pl2e = l3e_to_l2e(*pl3e) + l2_table_offset(linear);
+        flags = l2e_get_flags(*pl2e);
+        ASSERT(flags & _PAGE_PRESENT);
+        if ( flags & _PAGE_PSE )
+        {
+            pfn = (l2e_get_pfn(*pl2e) & ~((1UL << PAGETABLE_ORDER) - 1)) |
+                  (PFN_DOWN(linear) & ((1UL << PAGETABLE_ORDER) - 1));
+            flags &= ~_PAGE_PSE;
+        }
+        else
+        {
+            pl1e = l2e_to_l1e(*pl2e) + l1_table_offset(linear);
+            flags = l1e_get_flags(*pl1e);
+            if ( !(flags & _PAGE_PRESENT) )
+                return 0;
+            pfn = l1e_get_pfn(*pl1e);
+        }
+    }
+
+    if ( !(root_get_flags(rpt[root_table_offset(linear)]) & _PAGE_PRESENT) )
+    {
+        pl3e = alloc_xen_pagetable();
+        if ( !pl3e )
+            return -ENOMEM;
+        clear_page(pl3e);
+        l4e_write(&rpt[root_table_offset(linear)],
+                  l4e_from_paddr(__pa(pl3e), __PAGE_HYPERVISOR));
+    }
+    else
+        pl3e = l4e_to_l3e(rpt[root_table_offset(linear)]);
+
+    pl3e += l3_table_offset(linear);
+
+    if ( !(l3e_get_flags(*pl3e) & _PAGE_PRESENT) )
+    {
+        pl2e = alloc_xen_pagetable();
+        if ( !pl2e )
+            return -ENOMEM;
+        clear_page(pl2e);
+        l3e_write(pl3e, l3e_from_paddr(__pa(pl2e), __PAGE_HYPERVISOR));
+    }
+    else
+    {
+        ASSERT(!(l3e_get_flags(*pl3e) & _PAGE_PSE));
+        pl2e = l3e_to_l2e(*pl3e);
+    }
+
+    pl2e += l2_table_offset(linear);
+
+    if ( !(l2e_get_flags(*pl2e) & _PAGE_PRESENT) )
+    {
+        pl1e = alloc_xen_pagetable();
+        if ( !pl1e )
+            return -ENOMEM;
+        clear_page(pl1e);
+        l2e_write(pl2e, l2e_from_paddr(__pa(pl1e), __PAGE_HYPERVISOR));
+    }
+    else
+    {
+        ASSERT(!(l2e_get_flags(*pl2e) & _PAGE_PSE));
+        pl1e = l2e_to_l1e(*pl2e);
+    }
+
+    pl1e += l1_table_offset(linear);
+
+    if ( l1e_get_flags(*pl1e) & _PAGE_PRESENT )
+    {
+        ASSERT(l1e_get_pfn(*pl1e) == pfn);
+        ASSERT(l1e_get_flags(*pl1e) == flags);
+    }
+    else
+        l1e_write(pl1e, l1e_from_pfn(pfn, flags));
+
+    return 0;
+}
+
+DEFINE_PER_CPU(root_pgentry_t *, root_pgt);
+
+static int setup_cpu_root_pgt(unsigned int cpu)
+{
+    int rc = 0;
+    root_pgentry_t *rpt = alloc_xen_pagetable();
+    unsigned int off;
+
+    if ( !rpt )
+        return -ENOMEM;
+
+    clear_page(rpt);
+    per_cpu(root_pgt, cpu) = rpt;
+
+    rpt[root_table_offset(RO_MPT_VIRT_START)] =
+        idle_pg_table[root_table_offset(RO_MPT_VIRT_START)];
+    /* SH_LINEAR_PT inserted together with guest mappings. */
+    /* PERDOMAIN inserted during context switch. */
+    rpt[root_table_offset(XEN_VIRT_START)] =
+        idle_pg_table[root_table_offset(XEN_VIRT_START)];
+
+    /* Install direct map page table entries for stack, IDT, and TSS. */
+    for ( off = 0; !rc && off < STACK_SIZE; off += PAGE_SIZE )
+        rc = clone_mapping(__va(__pa(stack_base[cpu])) + off, rpt);
+
+    if ( !rc )
+        rc = clone_mapping(idt_tables[cpu], rpt);
+    if ( !rc )
+        rc = clone_mapping(&per_cpu(init_tss, cpu), rpt);
+
+    return rc;
+}
+#endif
+
+static void cleanup_cpu_root_pgt(unsigned int cpu)
+{
+#ifdef __x86_64__
+    root_pgentry_t *rpt = per_cpu(root_pgt, cpu);
+    unsigned int r;
+
+    if ( !rpt )
+        return;
+
+    per_cpu(root_pgt, cpu) = NULL;
+
+    for ( r = root_table_offset(DIRECTMAP_VIRT_START);
+          r < root_table_offset(HYPERVISOR_VIRT_END); ++r )
+    {
+        l3_pgentry_t *l3t;
+        unsigned int i3;
+
+        if ( !(root_get_flags(rpt[r]) & _PAGE_PRESENT) )
+            continue;
+
+        l3t = l4e_to_l3e(rpt[r]);
+
+        for ( i3 = 0; i3 < L3_PAGETABLE_ENTRIES; ++i3 )
+        {
+            l2_pgentry_t *l2t;
+            unsigned int i2;
+
+            if ( !(l3e_get_flags(l3t[i3]) & _PAGE_PRESENT) )
+                continue;
+
+            ASSERT(!(l3e_get_flags(l3t[i3]) & _PAGE_PSE));
+            l2t = l3e_to_l2e(l3t[i3]);
+
+            for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; ++i2 )
+            {
+                if ( !(l2e_get_flags(l2t[i2]) & _PAGE_PRESENT) )
+                    continue;
+
+                ASSERT(!(l2e_get_flags(l2t[i2]) & _PAGE_PSE));
+                free_xen_pagetable(l2e_to_l1e(l2t[i2]));
+            }
+
+            free_xen_pagetable(l2t);
+        }
+
+        free_xen_pagetable(l3t);
+    }
+
+    free_xen_pagetable(rpt);
+#endif
+}
+
 static int __devinit do_boot_cpu(int apicid, int cpu)
 /*
  * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
@@ -927,6 +1117,8 @@ static int __devinit do_boot_cpu(int api
 	set_ist(&idt_tables[cpu][TRAP_double_fault],  IST_NONE);
 	set_ist(&idt_tables[cpu][TRAP_nmi],           IST_NONE);
 	set_ist(&idt_tables[cpu][TRAP_machine_check], IST_NONE);
+
+	setup_cpu_root_pgt(cpu);
 #endif
 
 	/*
@@ -1234,8 +1426,20 @@ static void __init smp_boot_cpus(unsigne
    who understands all this stuff should rewrite it properly. --RR 15/Jul/02 */
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
+#ifdef __x86_64__
+	int rc;
+#endif
+
 	smp_commenced_mask = cpumask_of_cpu(0);
 	cpu_callin_map = cpumask_of_cpu(0);
+
+#ifdef __x86_64__
+	rc = setup_cpu_root_pgt(0);
+	if (rc)
+		panic("Error %d setting up PV root page table\n", rc);
+	get_cpu_info()->pv_cr3 = __pa(per_cpu(root_pgt, 0));
+#endif
+
 	mb();
 	smp_boot_cpus(max_cpus);
 	mtrr_aps_sync_begin();
@@ -1248,6 +1452,10 @@ void __devinit smp_prepare_boot_cpu(void
 	cpu_set(smp_processor_id(), cpu_present_map);
 	cpu_set(smp_processor_id(), cpu_possible_map);
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
+
+#ifdef __x86_64__
+	get_cpu_info()->xen_cr3 = 0;
+#endif
 }
 
 static void
@@ -1381,6 +1589,8 @@ int cpu_down(unsigned int cpu)
 
 	cpu_mcheck_distribute_cmci();
 
+	cleanup_cpu_root_pgt(cpu);
+
 out:
 	if (!err)
 		send_guest_global_virq(dom0, VIRQ_PCPU_STATE);
--- xen-4.0.3-testing.orig/xen/arch/x86/x86_64/asm-offsets.c
+++ xen-4.0.3-testing/xen/arch/x86/x86_64/asm-offsets.c
@@ -134,7 +134,10 @@ void __dummy__(void)
     OFFSET(COMPAT_VCPUINFO_upcall_mask, struct compat_vcpu_info, evtchn_upcall_mask);
     BLANK();
 
+    OFFSET(CPUINFO_guest_cpu_user_regs, struct cpu_info, guest_cpu_user_regs);
     OFFSET(CPUINFO_current_vcpu, struct cpu_info, current_vcpu);
+    OFFSET(CPUINFO_xen_cr3, struct cpu_info, xen_cr3);
+    OFFSET(CPUINFO_pv_cr3, struct cpu_info, pv_cr3);
     DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info));
     BLANK();
 
--- xen-4.0.3-testing.orig/xen/arch/x86/x86_64/compat/entry.S
+++ xen-4.0.3-testing/xen/arch/x86/x86_64/compat/entry.S
@@ -11,17 +11,6 @@
 #include <asm/desc.h>
 #include <public/xen.h>
 
-#define GET_GUEST_REGS(reg)                     \
-        movq $~(STACK_SIZE-1),reg;              \
-        andq %rsp,reg;                          \
-        orq  $(STACK_SIZE-CPUINFO_sizeof),reg;
-
-#define GET_CURRENT(reg)         \
-        movq $STACK_SIZE-8, reg; \
-        orq  %rsp, reg;          \
-        andq $~7,reg;            \
-        movq (reg),reg;
-
         ALIGN
 ENTRY(compat_hypercall)
         pushq $0
--- xen-4.0.3-testing.orig/xen/arch/x86/x86_64/entry.S
+++ xen-4.0.3-testing/xen/arch/x86/x86_64/entry.S
@@ -12,17 +12,6 @@
 #include <asm/page.h>
 #include <public/xen.h>
 
-#define GET_GUEST_REGS(reg)                     \
-        movq $~(STACK_SIZE-1),reg;              \
-        andq %rsp,reg;                          \
-        orq  $(STACK_SIZE-CPUINFO_sizeof),reg;
-
-#define GET_CURRENT(reg)         \
-        movq $STACK_SIZE-8, reg; \
-        orq  %rsp, reg;          \
-        andq $~7,reg;            \
-        movq (reg),reg;
-
         ALIGN
 /* %rbx: struct vcpu */
 switch_to_kernel:
@@ -47,6 +36,32 @@ switch_to_kernel:
 /* %rbx: struct vcpu, interrupts disabled */
 restore_all_guest:
         ASSERT_INTERRUPTS_DISABLED
+
+        /* Copy guest mappings and switch to per-CPU root page table. */
+        mov   %cr3, %r9
+        GET_STACK_BASE(%rdx)
+        mov   STACK_CPUINFO_FIELD(pv_cr3)(%rdx), %rdi
+        movabs $PADDR_MASK & PAGE_MASK, %rsi
+        movabs $DIRECTMAP_VIRT_START, %rcx
+        mov   %rdi, %rax
+        and   %rsi, %rdi
+        and   %r9, %rsi
+        add   %rcx, %rdi
+        add   %rcx, %rsi
+        mov   $ROOT_PAGETABLE_FIRST_XEN_SLOT, %ecx
+        mov   root_table_offset(SH_LINEAR_PT_VIRT_START)*8(%rsi), %r8
+        mov   %r8, root_table_offset(SH_LINEAR_PT_VIRT_START)*8(%rdi)
+        rep movsq
+        mov   $ROOT_PAGETABLE_ENTRIES - \
+               ROOT_PAGETABLE_LAST_XEN_SLOT - 1, %ecx
+        sub   $(ROOT_PAGETABLE_FIRST_XEN_SLOT - \
+                ROOT_PAGETABLE_LAST_XEN_SLOT - 1) * 8, %rsi
+        sub   $(ROOT_PAGETABLE_FIRST_XEN_SLOT - \
+                ROOT_PAGETABLE_LAST_XEN_SLOT - 1) * 8, %rdi
+        rep movsq
+        mov   %r9, STACK_CPUINFO_FIELD(xen_cr3)(%rdx)
+        write_cr3 rax, rdi, rsi
+
         RESTORE_ALL
         testw $TRAP_syscall,4(%rsp)
         jz    iret_exit_to_guest
@@ -115,6 +130,18 @@ failsafe_callback:
         ALIGN
 /* No special register assumptions. */
 restore_all_xen:
+        /*
+         * Check whether we need to switch to the per-CPU page tables, in
+         * case we return to late PV exit code (from an NMI or #MC).
+         */
+        GET_STACK_BASE(%rax)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rax), %rdx
+        mov   STACK_CPUINFO_FIELD(pv_cr3)(%rax), %rax
+        test  %rdx, %rdx
+UNLIKELY_START(g, exit_cr3)
+        write_cr3 rax, rdi, rsi
+UNLIKELY_END(exit_cr3)
+
         RESTORE_ALL
         addq  $8,%rsp
         iretq
@@ -144,7 +171,18 @@ ENTRY(syscall_enter)
         movl  $TRAP_syscall,4(%rsp)
         movq  24(%rsp),%r11 /* Re-load user RFLAGS into %r11 before SAVE_ALL */
         SAVE_ALL
-        GET_CURRENT(%rbx)
+
+        GET_STACK_BASE(%rbx)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx
+        neg   %rcx
+        jz    .Llstar_cr3_okay
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx)
+        neg   %rcx
+        write_cr3 rcx, r11, r12
+        movq  $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx)
+.Llstar_cr3_okay:
+
+        __GET_CURRENT(%rbx)
         movq  VCPU_domain(%rbx),%rcx
         testb $1,DOMAIN_is_32bit_pv(%rcx)
         jnz   compat_syscall
@@ -293,7 +331,18 @@ sysenter_eflags_saved:
         pushq $0
         movl  $TRAP_syscall,4(%rsp)
         SAVE_ALL
-        GET_CURRENT(%rbx)
+
+        GET_STACK_BASE(%rbx)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx
+        neg   %rcx
+        jz    .Lsyse_cr3_okay
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx)
+        neg   %rcx
+        write_cr3 rcx, rdi, rsi
+        movq  $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx)
+.Lsyse_cr3_okay:
+
+        __GET_CURRENT(%rbx)
         cmpb  $0,VCPU_sysenter_disables_events(%rbx)
         movq  $0,UREGS_rip(%rsp) /* null rip */
         movl  $3,UREGS_cs(%rsp)  /* ring 3 null cs */
@@ -327,15 +376,23 @@ ENTRY(int80_direct_trap)
         pushq $0
         SAVE_ALL
 
+        GET_STACK_BASE(%rbx)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx
+        neg   %rcx
+        jz    .Lint80_cr3_okay
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%rbx)
+        neg   %rcx
+        write_cr3 rcx, rdi, rsi
+        movq  $0, STACK_CPUINFO_FIELD(xen_cr3)(%rbx)
+.Lint80_cr3_okay:
+
         cmpb  $0,untrusted_msi(%rip)
         je    1f
         movl  $0x80,%edi
         call  check_for_unexpected_msi
-        RESTORE_ALL
-        SAVE_ALL
 1:
 
-        GET_CURRENT(%rbx)
+        __GET_CURRENT(%rbx)
 
         /* Check that the callback is non-null. */
         leaq  VCPU_int80_bounce(%rbx),%rdx
@@ -459,7 +516,7 @@ domain_crash_synchronous_string:
 
 ENTRY(domain_crash_synchronous)
         # Get out of the guest-save area of the stack.
-        GET_GUEST_REGS(%rax)
+        GET_CPUINFO_FIELD(guest_cpu_user_regs,%rax)
         movq  %rax,%rsp
         # create_bounce_frame() temporarily clobbers CS.RPL. Fix up.
         movq  CPUINFO_current_vcpu(%rax),%rax
@@ -475,6 +532,31 @@ ENTRY(domain_crash_synchronous)
         jmp  __domain_crash_synchronous
 
         ALIGN
+ENTRY(common_interrupt)
+        SAVE_ALL
+
+        GET_STACK_BASE(%r14)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
+        mov   %rcx, %r15
+        neg   %rcx
+        jz    .Lintr_cr3_okay
+        jns   .Lintr_cr3_load
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+        neg   %rcx
+.Lintr_cr3_load:
+        write_cr3 rcx, rdi, rsi
+        xor   %ecx, %ecx
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+        testb $3, UREGS_cs(%rsp)
+        cmovnz %rcx, %r15
+.Lintr_cr3_okay:
+
+        movq %rsp,%rdi
+        callq do_IRQ
+        mov   %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+        jmp ret_from_intr
+
+        ALIGN
 /* No special register assumptions. */
 ENTRY(ret_from_intr)
         GET_CURRENT(%rbx)
@@ -489,6 +571,23 @@ ENTRY(ret_from_intr)
 /* No special register assumptions. */
 ENTRY(handle_exception)
         SAVE_ALL
+
+        GET_STACK_BASE(%r14)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
+        mov   %rcx, %r15
+        neg   %rcx
+        jz    .Lxcpt_cr3_okay
+        jns   .Lxcpt_cr3_load
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+        neg   %rcx
+.Lxcpt_cr3_load:
+        write_cr3 rcx, rdi, rsi
+        xor   %ecx, %ecx
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+        testb $3, UREGS_cs(%rsp)
+        cmovnz %rcx, %r15
+.Lxcpt_cr3_okay:
+
 handle_exception_saved:
         testb $X86_EFLAGS_IF>>8,UREGS_eflags+1(%rsp)
         jz    exception_with_ints_disabled
@@ -499,6 +598,7 @@ handle_exception_saved:
         GET_CURRENT(%rbx)
         PERFC_INCR(exceptions, %rax, %rbx)
         callq *(%rdx,%rax,8)
+        mov   %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
         testb $3,UREGS_cs(%rsp)
         jz    restore_all_xen
         leaq  VCPU_trap_bounce(%rbx),%rdx
@@ -531,6 +631,7 @@ exception_with_ints_disabled:
         rep;  movsq                     # make room for ec/ev
 1:      movq  UREGS_error_code(%rsp),%rax # ec/ev
         movq  %rax,UREGS_kernel_sizeof(%rsp)
+        mov   %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
         jmp   restore_all_xen           # return to fixup code
 
 /* No special register assumptions. */
@@ -621,6 +722,17 @@ ENTRY(spurious_interrupt_bug)
 
 ENTRY(double_fault)
         SAVE_ALL
+
+        GET_STACK_BASE(%rbx)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rbx
+        test  %rbx, %rbx
+        jz    .Ldblf_cr3_okay
+        jns   .Ldblf_cr3_load
+        neg   %rbx
+.Ldblf_cr3_load:
+        write_cr3 rbx, rdi, rsi
+.Ldblf_cr3_okay:
+
         movq  %rsp,%rdi
         call  do_double_fault
         ud2
@@ -633,10 +745,28 @@ ENTRY(early_page_fault)
 
 handle_ist_exception:
         SAVE_ALL
+
+        GET_STACK_BASE(%r14)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
+        mov   %rcx, %r15
+        neg   %rcx
+        jz    .List_cr3_okay
+        jns   .List_cr3_load
+        mov   %rcx, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+        neg   %rcx
+.List_cr3_load:
+        write_cr3 rcx, rdi, rsi
+        movq  $0, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
+.List_cr3_okay:
+
         testb $3,UREGS_cs(%rsp)
         jz    1f
-        /* Interrupted guest context. Copy the context to stack bottom. */
-        GET_GUEST_REGS(%rdi)
+        /*
+         * Interrupted guest context. Clear the restore value for xen_cr3
+         * and copy the context to stack bottom.
+         */
+        xor   %r15, %r15
+        GET_CPUINFO_FIELD(guest_cpu_user_regs,%rdi)
         movq  %rsp,%rsi
         movl  $UREGS_kernel_sizeof/8,%ecx
         movq  %rdi,%rsp
@@ -645,6 +775,7 @@ handle_ist_exception:
         movl  UREGS_entry_vector(%rsp),%eax
         leaq  exception_table(%rip),%rdx
         callq *(%rdx,%rax,8)
+        mov   %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14)
         jmp   ret_from_intr
 
 ENTRY(nmi)
--- xen-4.0.3-testing.orig/xen/include/asm-x86/asm_defns.h
+++ xen-4.0.3-testing/xen/include/asm-x86/asm_defns.h
@@ -12,4 +12,27 @@
 #include <asm/x86_32/asm_defns.h>
 #endif
 
+#ifdef __ASSEMBLY__
+
+#define UNLIKELY_START(cond, tag) \
+        j##cond .Lunlikely.tag;   \
+        .subsection 1;            \
+        .Lunlikely.tag:
+
+#define UNLIKELY_END(tag)         \
+        jmp .Llikely.tag;         \
+        .subsection 0;            \
+        .Llikely.tag:
+
+.macro write_cr3 val:req, tmp1:req, tmp2:req
+        mov   %cr4, %\tmp1
+        mov   %\tmp1, %\tmp2
+        and   $~X86_CR4_PGE, %\tmp1
+        mov   %\tmp1, %cr4
+        mov   %\val, %cr3
+        mov   %\tmp2, %cr4
+.endm
+
+#endif /*  __ASSEMBLY__ */
+
 #endif /* __X86_ASM_DEFNS_H__ */
--- xen-4.0.3-testing.orig/xen/include/asm-x86/current.h
+++ xen-4.0.3-testing/xen/include/asm-x86/current.h
@@ -17,6 +17,21 @@ struct vcpu;
 struct cpu_info {
     struct cpu_user_regs guest_cpu_user_regs;
     unsigned int         processor_id;
+#ifdef __x86_64__
+    /*
+     * Of the two following fields the latter is being set to the CR3 value
+     * to be used on the given pCPU for loading whenever 64-bit PV guest
+     * context is being entered. The value never changes once set.
+     * The former is the value to restore when re-entering Xen, if any. IOW
+     * its value being zero means there's nothing to restore. However, its
+     * value can also be negative, indicating to the exit-to-Xen code that
+     * restoring is not necessary, but allowing any nested entry code paths
+     * to still know the value to put back into CR3.
+     */
+    unsigned long xen_cr3;
+    unsigned long pv_cr3;
+#endif
+    /* MUST BE LAST! */
     struct vcpu         *current_vcpu;
 };
 
--- xen-4.0.3-testing.orig/xen/include/asm-x86/processor.h
+++ xen-4.0.3-testing/xen/include/asm-x86/processor.h
@@ -478,6 +478,7 @@ extern idt_entry_t idt_table[];
 extern idt_entry_t *idt_tables[];
 
 DECLARE_PER_CPU(struct tss_struct, init_tss);
+DECLARE_PER_CPU(root_pgentry_t *, root_pgt);
 
 extern void init_int80_direct_trap(struct vcpu *v);
 
--- xen-4.0.3-testing.orig/xen/include/asm-x86/x86_64/asm_defns.h
+++ xen-4.0.3-testing/xen/include/asm-x86/x86_64/asm_defns.h
@@ -87,15 +87,6 @@
 #define REX64_PREFIX "rex64/"
 #endif
 
-#define BUILD_COMMON_IRQ()                      \
-__asm__(                                        \
-    "\n" __ALIGN_STR"\n"                        \
-    "common_interrupt:\n\t"                     \
-    STR(SAVE_ALL)                               \
-    "movq %rsp,%rdi\n\t"                        \
-    "callq " STR(do_IRQ) "\n\t"                 \
-    "jmp ret_from_intr\n");
-
 #define IRQ_NAME2(nr) nr##_interrupt(void)
 #define IRQ_NAME(nr) IRQ_NAME2(IRQ##nr)
 
@@ -108,4 +99,19 @@ STR(IRQ) #nr "_interrupt:\n\t"
     "movl $"#nr",4(%rsp)\n\t"                   \
     "jmp common_interrupt");
 
+#define STACK_CPUINFO_FIELD(field) (STACK_SIZE-CPUINFO_sizeof+CPUINFO_##field)
+#define GET_STACK_BASE(reg)                       \
+        movq $~(STACK_SIZE-1),reg;                \
+        andq %rsp,reg
+
+#define GET_CPUINFO_FIELD(field, reg)             \
+        GET_STACK_BASE(reg);                      \
+        addq $STACK_CPUINFO_FIELD(field),reg
+
+#define __GET_CURRENT(reg)                        \
+        movq STACK_CPUINFO_FIELD(current_vcpu)(reg),reg
+#define GET_CURRENT(reg)                          \
+        GET_STACK_BASE(reg);                      \
+        __GET_CURRENT(reg)
+
 #endif /* __X86_64_ASM_DEFNS_H__ */
--- xen-4.0.3-testing.orig/xen/include/asm-x86/x86_64/page.h
+++ xen-4.0.3-testing/xen/include/asm-x86/x86_64/page.h
@@ -22,8 +22,8 @@
 /* These are architectural limits. Current CPUs support only 40-bit phys. */
 #define PADDR_BITS              52
 #define VADDR_BITS              48
-#define PADDR_MASK              ((1UL << PADDR_BITS)-1)
-#define VADDR_MASK              ((1UL << VADDR_BITS)-1)
+#define PADDR_MASK              ((_AC(1,UL) << PADDR_BITS)-1)
+#define VADDR_MASK              ((_AC(1,UL) << VADDR_BITS)-1)
 
 #define is_canonical_address(x) (((long)(x) >> 47) == ((long)(x) >> 63))
 
@@ -123,6 +123,7 @@ typedef l4_pgentry_t root_pgentry_t;
       : (((_s) < ROOT_PAGETABLE_FIRST_XEN_SLOT) ||  \
          ((_s) > ROOT_PAGETABLE_LAST_XEN_SLOT)))
 
+#define root_table_offset         l4_table_offset
 #define root_get_pfn              l4e_get_pfn
 #define root_get_flags            l4e_get_flags
 #define root_get_intpte           l4e_get_intpte
--- xen-4.0.3-testing.orig/xen/include/xen/config.h
+++ xen-4.0.3-testing/xen/include/xen/config.h
@@ -98,4 +98,23 @@ int current_domain_id(void);
 #define __cpuinitdata
 #define __cpuinit
 
+/*
+ * Some constant macros are used in both assembler and
+ * C code.  Therefore we cannot annotate them always with
+ * 'UL' and other type specifiers unilaterally.  We
+ * use the following macros to deal with this.
+ *
+ * Similarly, _AT() will cast an expression with a type in C, but
+ * leave it unchanged in asm.
+ */
+
+#ifdef __ASSEMBLY__
+#define _AC(X,Y)	X
+#define _AT(T,X)	X
+#else
+#define __AC(X,Y)	(X##Y)
+#define _AC(X,Y)	__AC(X,Y)
+#define _AT(T,X)	((T)(X))
+#endif
+
 #endif /* __XEN_CONFIG_H__ */
