# Commit 5e7962901131186d3514528ed57c7a9901a15a3e
# Date 2018-01-26 14:10:21 +0000
# Author Andrew Cooper <andrew.cooper3@citrix.com>
# Committer Andrew Cooper <andrew.cooper3@citrix.com>
x86/entry: Organise the use of MSR_SPEC_CTRL at each entry/exit point

We need to be able to either set or clear IBRS in Xen context, as well as
restore appropriate guest values in guest context.  See the documentation in
asm-x86/spec_ctrl_asm.h for details.

With the contemporary microcode, writes to %cr3 are slower when SPEC_CTRL.IBRS
is set.  Therefore, the positioning of SPEC_CTRL_{ENTRY/EXIT}* is important.

Ideally, the IBRS_SET/IBRS_CLEAR hunks might be positioned either side of the
%cr3 change, but that is rather more complicated to arrange, and could still
result in a guest controlled value in SPEC_CTRL during the %cr3 change,
negating the saving if the guest chose to have IBRS set.

Therefore, we optimise for the pre-Skylake case (being far more common in the
field than Skylake and later, at the moment), where we have a Xen-preferred
value of IBRS clear when switching %cr3.

There is a semi-unrelated bugfix, where various asm_defn.h macros have a
hidden dependency on PAGE_SIZE, which results in an assembler error if used in
a .macro definition.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

--- a/xen/arch/x86/hvm/svm/entry.S
+++ b/xen/arch/x86/hvm/svm/entry.S
@@ -34,13 +34,11 @@
 #define get_current(reg) GET_CURRENT(r(reg))
         
 #if defined(__x86_64__)
-#define r(reg) %r##reg
 #define addr_of(lbl) lbl(%rip)
 #define call_with_regs(fn)                      \
         mov  %rsp,%rdi;                         \
         call fn;
 #else /* defined(__i386__) */
-#define r(reg) %e##reg
 #define addr_of(lbl) lbl
 #define UREGS_rax UREGS_eax
 #define UREGS_rip UREGS_eip
@@ -91,6 +89,11 @@ UNLIKELY_END(svm_trace)
         mov  UREGS_eflags(r(sp)),r(ax)
         mov  r(ax),VMCB_rflags(r(cx))
 
+        mov  VCPU_arch_spec_ctrl(r(bx)), %eax
+
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */
+
         mov  VCPU_svm_vmcb_pa(r(bx)),r(ax)
 
 #if defined(__x86_64__)
@@ -128,6 +131,10 @@ UNLIKELY_END(svm_trace)
 
         get_current(bx)
         movb $0,VCPU_svm_vmcb_in_sync(r(bx))
+
+        SPEC_CTRL_ENTRY_FROM_VMEXIT /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         mov  VCPU_svm_vmcb(r(bx)),r(cx)
         mov  VMCB_rax(r(cx)),r(ax)
         mov  r(ax),UREGS_rax(r(sp))
--- a/xen/arch/x86/hvm/vmx/entry.S
+++ b/xen/arch/x86/hvm/vmx/entry.S
@@ -39,13 +39,11 @@
 #define get_current(reg) GET_CURRENT(r(reg))
 
 #if defined(__x86_64__)
-#define r(reg) %r##reg
 #define addr_of(lbl) lbl(%rip)
 #define call_with_regs(fn)                      \
         mov  %rsp,%rdi;                         \
         call fn;
 #else /* defined(__i386__) */
-#define r(reg) %e##reg
 #define addr_of(lbl) lbl
 #define UREGS_rip UREGS_eip
 #define UREGS_rsp UREGS_esp
@@ -69,6 +67,9 @@ vmx_asm_vmexit_handler:
 
         movb $1,VCPU_vmx_launched(r(bx))
 
+        SPEC_CTRL_ENTRY_FROM_VMEXIT /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         lea  UREGS_rip(r(sp)),r(di)
         mov  $GUEST_RIP,%eax
         /*VMREAD(UREGS_rip)*/
@@ -132,6 +133,11 @@ UNLIKELY_END(realmode)
         mov  $GUEST_RFLAGS,%eax
         VMWRITE(UREGS_eflags)
 
+        mov VCPU_arch_spec_ctrl(r(bx)), %eax
+
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */
+
         cmpb $0,VCPU_vmx_launched(r(bx))
 #if defined(__x86_64__)
         pop  %r15
@@ -163,12 +169,36 @@ UNLIKELY_END(realmode)
 /*.Lvmx_resume:*/
         VMRESUME
         sti
+#ifdef __x86_64__
+        SAVE_ALL
+#else
+        SAVE_ALL_GPRS
+#endif
+
+        /*
+         * PV variant needed here as no guest code has executed (so
+         * MSR_SPEC_CTRL can't have changed value), and NMIs/MCEs are liable
+         * to hit (in which case the HVM variant might corrupt things).
+         */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         call vm_resume_fail
         ud2
 
 .Lvmx_launch:
         VMLAUNCH
         sti
+#ifdef __x86_64__
+        SAVE_ALL
+#else
+        SAVE_ALL_GPRS
+#endif
+
+        /* PV variant needed here again (see above). */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         call vm_launch_fail
         ud2
 
--- a/xen/arch/x86/i8259.c
+++ b/xen/arch/x86/i8259.c
@@ -35,10 +35,6 @@
 
 __asm__(".section .text");
 
-#ifndef __x86_64__
-BUILD_COMMON_IRQ()
-#endif
-
 #define BI(x,y) \
     BUILD_IRQ(x##y)
 
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -632,6 +632,7 @@ void __init __start_xen(unsigned long mb
     if ( cpu_has_efer )
         rdmsrl(MSR_EFER, this_cpu(efer));
     asm volatile ( "mov %%cr4,%0" : "=r" (this_cpu(cr4)) );
+    init_shadow_spec_ctrl_state();
 
     smp_prepare_boot_cpu();
 
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -41,6 +41,7 @@
 #include <asm/flushtlb.h>
 #include <asm/msr.h>
 #include <asm/mtrr.h>
+#include <asm/spec_ctrl.h>
 #include <asm/time.h>
 #include <asm/tboot.h>
 #include <mach_apic.h>
@@ -332,6 +333,7 @@ void start_secondary(void *unused)
     if ( cpu_has_efer )
         rdmsrl(MSR_EFER, this_cpu(efer));
     asm volatile ( "mov %%cr4,%0" : "=r" (this_cpu(cr4)) );
+    init_shadow_spec_ctrl_state();
 
     /*
      * Just as during early bootstrap, it is convenient here to disable
--- a/xen/arch/x86/x86_32/asm-offsets.c
+++ b/xen/arch/x86/x86_32/asm-offsets.c
@@ -64,6 +64,7 @@ void __dummy__(void)
     OFFSET(VCPU_kernel_ss, struct vcpu, arch.pv_vcpu.kernel_ss);
     OFFSET(VCPU_kernel_sp, struct vcpu, arch.pv_vcpu.kernel_sp);
     OFFSET(VCPU_guest_context_flags, struct vcpu, arch.vgc_flags);
+    OFFSET(VCPU_arch_spec_ctrl, struct vcpu, arch.spec_ctrl);
     OFFSET(VCPU_nmi_pending, struct vcpu, nmi_pending);
     OFFSET(VCPU_mce_pending, struct vcpu, mce_pending);
     OFFSET(VCPU_nmi_old_mask, struct vcpu, nmi_state.old_mask);
@@ -111,6 +112,9 @@ void __dummy__(void)
     OFFSET(CPUINFO_guest_cpu_user_regs, struct cpu_info, guest_cpu_user_regs);
     OFFSET(CPUINFO_processor_id, struct cpu_info, processor_id);
     OFFSET(CPUINFO_current_vcpu, struct cpu_info, current_vcpu);
+    OFFSET(CPUINFO_shadow_spec_ctrl, struct cpu_info, shadow_spec_ctrl);
+    OFFSET(CPUINFO_use_shadow_spec_ctrl, struct cpu_info, use_shadow_spec_ctrl);
+    OFFSET(CPUINFO_xen_ibrs, struct cpu_info, xen_ibrs);
     DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info));
     BLANK();
 
--- a/xen/arch/x86/x86_32/entry.S
+++ b/xen/arch/x86/x86_32/entry.S
@@ -65,6 +65,10 @@
         ALIGN
 restore_all_guest:
         ASSERT_INTERRUPTS_DISABLED
+
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %esp=regs/cpuinfo, Clob: cd */
+
         andl  $~X86_EFLAGS_IOPL,UREGS_eflags(%esp)
         orl   $X86_EFLAGS_IF,UREGS_eflags(%esp)
         testl $X86_EFLAGS_VM,UREGS_eflags(%esp)
@@ -131,6 +135,10 @@ failsafe_callback:
 
         ALIGN
 restore_all_xen:
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        GET_STACK_BASE(%ebx)
+        SPEC_CTRL_EXIT_TO_XEN /* Req: %ebx=end, Clob: acd */
+
         popl %ebx
         popl %ecx
         popl %edx
@@ -146,6 +154,10 @@ ENTRY(hypercall)
         FIXUP_RING0_GUEST_STACK
         SAVE_ALL(,1f)
 1:      sti
+
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %esp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         movl  UREGS_eax(%esp),%eax
         GET_CURRENT(%ebx)
         cmpl  $NR_hypercalls,%eax
@@ -422,6 +434,19 @@ domain_crash_synchronous:
         call  printk
         jmp   __domain_crash_synchronous
 
+ENTRY(common_interrupt)
+        FIXUP_RING0_GUEST_STACK
+        SAVE_ALL(1f,1f)
+1:
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %esp=regs, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
+        movl  %esp,%eax
+        pushl %eax
+        call  do_IRQ
+        addl  $4,%esp
+        /* jmp   ret_from_intr */
+
 ENTRY(ret_from_intr)
         GET_CURRENT(%ebx)
         movl  UREGS_eflags(%esp),%eax
@@ -441,7 +466,11 @@ handle_exception:
         mov   %ecx,%es
         jmp   2f
         .previous
-2:      testb $X86_EFLAGS_IF>>8,UREGS_eflags+1(%esp)
+2:
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %esp=regs, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
+        testb $X86_EFLAGS_IF>>8,UREGS_eflags+1(%esp)
         jz    exception_with_ints_disabled
         sti                             # re-enable interrupts
 1:      xorl  %eax,%eax
@@ -578,6 +607,9 @@ handle_nmi_mce:
         # Save state but do not trash the segment registers!
         SAVE_ALL(.Lnmi_mce_xen,.Lnmi_mce_common)
 .Lnmi_mce_common:
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %esp=regs, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         xorl  %eax,%eax
         movw  UREGS_entry_vector(%esp),%ax
         movl  %esp,%edx
--- a/xen/arch/x86/x86_64/asm-offsets.c
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -92,6 +92,7 @@ void __dummy__(void)
     OFFSET(VCPU_kernel_sp, struct vcpu, arch.pv_vcpu.kernel_sp);
     OFFSET(VCPU_kernel_ss, struct vcpu, arch.pv_vcpu.kernel_ss);
     OFFSET(VCPU_guest_context_flags, struct vcpu, arch.vgc_flags);
+    OFFSET(VCPU_arch_spec_ctrl, struct vcpu, arch.spec_ctrl);
     OFFSET(VCPU_nmi_pending, struct vcpu, nmi_pending);
     OFFSET(VCPU_mce_pending, struct vcpu, mce_pending);
     OFFSET(VCPU_nmi_old_mask, struct vcpu, nmi_state.old_mask);
@@ -142,6 +143,9 @@ void __dummy__(void)
     OFFSET(CPUINFO_current_vcpu, struct cpu_info, current_vcpu);
     OFFSET(CPUINFO_xen_cr3, struct cpu_info, xen_cr3);
     OFFSET(CPUINFO_pv_cr3, struct cpu_info, pv_cr3);
+    OFFSET(CPUINFO_shadow_spec_ctrl, struct cpu_info, shadow_spec_ctrl);
+    OFFSET(CPUINFO_use_shadow_spec_ctrl, struct cpu_info, use_shadow_spec_ctrl);
+    OFFSET(CPUINFO_xen_ibrs, struct cpu_info, xen_ibrs);
     DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info));
     BLANK();
 
--- a/xen/arch/x86/x86_64/compat/entry.S
+++ b/xen/arch/x86/x86_64/compat/entry.S
@@ -17,6 +17,9 @@ ENTRY(compat_hypercall)
         movl  $TRAP_syscall,4(%rsp)
         SAVE_ALL
 
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         cmpb  $0,untrusted_msi(%rip)
 UNLIKELY_START(ne, msi_check)
         movl  $HYPERCALL_VECTOR,%edi
@@ -176,6 +179,12 @@ ENTRY(compat_restore_all_guest)
         and   UREGS_eflags(%rsp),%r11d
         or    $X86_EFLAGS_IF,%r11
         mov   %r11d,UREGS_eflags(%rsp)
+
+        mov VCPU_arch_spec_ctrl(%rbx), %eax
+
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */
+
         RESTORE_ALL
         addq  $8,%rsp
 .Lft0:  iretq
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -38,6 +38,9 @@ switch_to_kernel:
 restore_all_guest:
         ASSERT_INTERRUPTS_DISABLED
 
+        /* Stash guest SPEC_CTRL value while we can read struct vcpu. */
+        mov   VCPU_arch_spec_ctrl(%rbx), %r15d
+
         /* Copy guest mappings and switch to per-CPU root page table. */
         mov   %cr3, %r9
         GET_STACK_BASE(%rdx)
@@ -65,6 +68,12 @@ restore_all_guest:
         write_cr3 rax, rdi, rsi
 .Lrag_keep_cr3:
 
+        /* Restore stashed SPEC_CTRL value. */
+        mov   %r15d, %eax
+
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        SPEC_CTRL_EXIT_TO_GUEST /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */
+
         RESTORE_ALL
         testw $TRAP_syscall,4(%rsp)
         jz    iret_exit_to_guest
@@ -133,14 +142,17 @@ restore_all_xen:
          * Check whether we need to switch to the per-CPU page tables, in
          * case we return to late PV exit code (from an NMI or #MC).
          */
-        GET_STACK_BASE(%rax)
-        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rax), %rdx
-        mov   STACK_CPUINFO_FIELD(pv_cr3)(%rax), %rax
+        GET_STACK_BASE(%rbx)
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rdx
+        mov   STACK_CPUINFO_FIELD(pv_cr3)(%rbx), %rax
         test  %rdx, %rdx
 UNLIKELY_START(g, exit_cr3)
         write_cr3 rax, rdi, rsi
 UNLIKELY_END(exit_cr3)
 
+        /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
+        SPEC_CTRL_EXIT_TO_XEN /* Req: %rbx=end, Clob: acd */
+
         RESTORE_ALL
         addq  $8,%rsp
         iretq
@@ -170,6 +182,9 @@ ENTRY(syscall_enter)
         movq  24(%rsp),%r11 /* Re-load user RFLAGS into %r11 before SAVE_ALL */
         SAVE_ALL
 
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         GET_STACK_BASE(%rbx)
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx
         neg   %rcx
@@ -333,6 +348,9 @@ sysenter_eflags_saved:
         movl  $TRAP_syscall,4(%rsp)
         SAVE_ALL
 
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         GET_STACK_BASE(%rbx)
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx
         neg   %rcx
@@ -377,6 +395,9 @@ ENTRY(int80_direct_trap)
         pushq $0
         SAVE_ALL
 
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         GET_STACK_BASE(%rbx)
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rcx
         neg   %rcx
@@ -544,6 +565,10 @@ ENTRY(common_interrupt)
         SAVE_ALL
 
         GET_STACK_BASE(%r14)
+
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
         mov   %rcx, %r15
         neg   %rcx
@@ -582,6 +607,10 @@ handle_exception:
         SAVE_ALL
 
         GET_STACK_BASE(%r14)
+
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
         mov   %rcx, %r15
         neg   %rcx
@@ -730,8 +759,12 @@ ENTRY(double_fault)
         movl  $TRAP_double_fault,4(%rsp)
         SAVE_ALL
 
-        GET_STACK_BASE(%rbx)
-        mov   STACK_CPUINFO_FIELD(xen_cr3)(%rbx), %rbx
+        GET_STACK_BASE(%r14)
+
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
+        mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rbx
         test  %rbx, %rbx
         jz    .Ldblf_cr3_okay
         jns   .Ldblf_cr3_load
@@ -759,6 +792,10 @@ handle_ist_exception:
         SAVE_ALL
 
         GET_STACK_BASE(%r14)
+
+        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
+
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
         mov   %rcx, %r15
         neg   %rcx
--- a/xen/include/asm-x86/asm_defns.h
+++ b/xen/include/asm-x86/asm_defns.h
@@ -6,6 +6,7 @@
 /* NB. Auto-generated from arch/.../asm-offsets.c */
 #include <asm/asm-offsets.h>
 #endif
+#include <asm/page.h>
 #include <asm/processor.h>
 
 #ifdef __ASSEMBLY__
@@ -78,4 +79,6 @@ void ret_from_intr(void);
 
 #endif
 
+#include <asm/spec_ctrl_asm.h>
+
 #endif /* __X86_ASM_DEFNS_H__ */
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -31,8 +31,13 @@ struct cpu_info {
      */
     unsigned long xen_cr3;
     unsigned long pv_cr3;
+
+    /* See asm-x86/spec_ctrl_asm.h for usage. */
+    unsigned int shadow_spec_ctrl;
+    bool_t       use_shadow_spec_ctrl;
+    int8_t       xen_ibrs;
+
 #ifdef __x86_64__ /* get_stack_bottom() must be 16-byte aligned */
-    unsigned long __pad_for_stack_bottom;
 #endif
 };
 
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -20,8 +20,18 @@
 #ifndef __X86_SPEC_CTRL_H__
 #define __X86_SPEC_CTRL_H__
 
+#include <asm/current.h>
+
 void init_speculation_mitigations(void);
 
+static inline void init_shadow_spec_ctrl_state(void)
+{
+    struct cpu_info *info = get_cpu_info();
+
+    info->shadow_spec_ctrl = info->use_shadow_spec_ctrl = 0;
+    info->xen_ibrs = -1;
+}
+
 #endif /* !__X86_SPEC_CTRL_H__ */
 
 /*
--- /dev/null
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -0,0 +1,232 @@
+/******************************************************************************
+ * include/asm-x86/spec_ctrl.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (c) 2017-2018 Citrix Systems Ltd.
+ */
+
+#ifndef __X86_SPEC_CTRL_ASM_H__
+#define __X86_SPEC_CTRL_ASM_H__
+
+#ifdef __ASSEMBLY__
+#include <asm/msr-index.h>
+
+/*
+ * Saving and restoring MSR_SPEC_CTRL state is a little tricky.
+ *
+ * We want the guests choice of SPEC_CTRL while in guest context, and Xen's
+ * choice (set or clear, depending on the hardware) while running in Xen
+ * context.  Therefore, a simplistic algorithm is:
+ *
+ *  - Set/clear IBRS on entry to Xen
+ *  - Set the guests' choice on exit to guest
+ *  - Leave SPEC_CTRL unchanged on exit to xen
+ *
+ * There are two complicating factors:
+ *  1) HVM guests can have direct access to the MSR, so it can change
+ *     behind Xen's back.
+ *  2) An NMI or MCE can interrupt at any point, including early in the entry
+ *     path, or late in the exit path after restoring the guest value.  This
+ *     will corrupt the guest value.
+ *
+ * Factor 1 is dealt with by relying on NMIs/MCEs being blocked immediately
+ * after VMEXIT.  The VMEXIT-specific code reads MSR_SPEC_CTRL and updates
+ * current before loading Xen's MSR_SPEC_CTRL setting.
+ *
+ * Factor 2 is harder.  We maintain a shadow_spec_ctrl value, and
+ * use_shadow_spec_ctrl boolean per cpu.  The synchronous use is:
+ *
+ *  1) Store guest value in shadow_spec_ctrl
+ *  2) Set use_shadow_spec_ctrl boolean
+ *  3) Load guest value into MSR_SPEC_CTRL
+ *  4) Exit to guest
+ *  5) Entry from guest
+ *  6) Clear use_shadow_spec_ctrl boolean
+ *  7) Load Xen's value into MSR_SPEC_CTRL
+ *
+ * The asynchronous use for interrupts/exceptions is:
+ *  -  Set/clear IBRS on entry to Xen
+ *  -  On exit to Xen, check use_shadow_spec_ctrl
+ *  -  If set, load shadow_spec_ctrl
+ *
+ * Therefore, an interrupt/exception which hits the synchronous path between
+ * steps 2 and 6 will restore the shadow value rather than leaving Xen's value
+ * loaded and corrupting the value used in guest context.
+ *
+ * The following ASM fragments implement this algorithm.  See their local
+ * comments for further details.
+ *  - SPEC_CTRL_ENTRY_FROM_VMEXIT
+ *  - SPEC_CTRL_ENTRY_FROM_PV
+ *  - SPEC_CTRL_ENTRY_FROM_INTR
+ *  - SPEC_CTRL_EXIT_TO_XEN
+ *  - SPEC_CTRL_EXIT_TO_GUEST
+ */
+
+.macro DO_SPEC_CTRL_ENTRY_FROM_VMEXIT
+/*
+ * Requires %rbx=current, %rsp=regs/cpuinfo
+ * Clobbers %rax, %rcx, %rdx
+ *
+ * The common case is that a guest has direct access to MSR_SPEC_CTRL, at
+ * which point we need to save the guest value before setting IBRS for Xen.
+ * Unilaterally saving the guest value is shorter and faster than checking.
+ */
+    cmpb $0, CPUINFO_xen_ibrs(r(sp))
+    jl .Lspec_entry_done\@
+
+    mov $MSR_SPEC_CTRL, %ecx
+    rdmsr
+
+    /* Stash the value from hardware. */
+    mov %eax, VCPU_arch_spec_ctrl(r(bx))
+    xor %edx, %edx
+
+    /* Clear SPEC_CTRL shadowing *before* loading Xen's value. */
+    movb %dl, CPUINFO_use_shadow_spec_ctrl(r(sp))
+
+    /* Load Xen's intended value. */
+    movzbl CPUINFO_xen_ibrs(r(sp)), %eax
+    wrmsr
+.Lspec_entry_done\@:
+.endm
+
+.macro DO_SPEC_CTRL_ENTRY maybexen:req
+/*
+ * Requires %rsp=regs (also cpuinfo if !maybexen)
+ * Requires %r14=stack_end (if 64-bit and maybexen)
+ * Clobbers %rax, %rcx, %rdx
+ *
+ * PV guests can't update MSR_SPEC_CTRL behind Xen's back, so no need to read
+ * it back.  Entries from guest context need to clear SPEC_CTRL shadowing,
+ * while entries from Xen must leave shadowing in its current state.
+ */
+#ifdef __i386__
+    .if \maybexen
+    GET_STACK_BASE(%ecx)
+    .endif
+#define r14 ecx
+#endif
+
+    xor %edx, %edx
+
+    .if \maybexen
+        cmp %dl, STACK_CPUINFO_FIELD(xen_ibrs)(%r14)
+    .else
+        cmp %dl, CPUINFO_xen_ibrs(r(sp))
+    .endif
+    jl .Lspec_entry_done\@
+
+    /*
+     * Clear SPEC_CTRL shadowing *before* loading Xen's value.  If entering
+     * from a possibly-xen context, %rsp doesn't necessarily alias the cpuinfo
+     * block so calculate the position directly.
+     */
+    .if \maybexen
+        /* Branchless `if ( !xen ) clear_shadowing` */
+        testb $3, UREGS_cs(r(sp))
+        setz %al
+        and %al, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(%r14)
+    .else
+        movb %dl, CPUINFO_use_shadow_spec_ctrl(r(sp))
+    .endif
+
+    /* Load Xen's intended value. */
+    .if \maybexen
+        movzbl STACK_CPUINFO_FIELD(xen_ibrs)(%r14), %eax
+    .else
+        movzbl CPUINFO_xen_ibrs(r(sp)), %eax
+    .endif
+#undef r14
+    mov $MSR_SPEC_CTRL, %ecx
+    wrmsr
+.Lspec_entry_done\@:
+.endm
+
+.macro DO_SPEC_CTRL_EXIT_TO_XEN
+/*
+ * Requires %rbx=stack_end
+ * Clobbers %rax, %rcx, %rdx
+ *
+ * When returning to Xen context, look to see whether SPEC_CTRL shadowing is
+ * in effect, and reload the shadow value.  This covers race conditions which
+ * exist with an NMI/MCE/etc hitting late in the return-to-guest path.
+ */
+    xor %edx, %edx
+
+    cmp %dl, STACK_CPUINFO_FIELD(use_shadow_spec_ctrl)(r(bx))
+    je .L\@_skip
+
+    mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(r(bx)), %eax
+    mov $MSR_SPEC_CTRL, %ecx
+    wrmsr
+
+.L\@_skip:
+.endm
+
+.macro DO_SPEC_CTRL_EXIT_TO_GUEST
+/*
+ * Requires %eax=spec_ctrl, %rsp=regs/cpuinfo
+ * Clobbers %rcx, %rdx
+ *
+ * When returning to guest context, set up SPEC_CTRL shadowing and load the
+ * guest value.
+ */
+    xor %edx, %edx
+    cmp %dl, CPUINFO_xen_ibrs(r(sp))
+    jl .Lspec_exit_done\@
+
+    /* Set up shadow value *before* enabling shadowing. */
+    mov %eax, CPUINFO_shadow_spec_ctrl(r(sp))
+
+    /* Set SPEC_CTRL shadowing *before* loading the guest value. */
+    movb $1, CPUINFO_use_shadow_spec_ctrl(r(sp))
+
+    mov $MSR_SPEC_CTRL, %ecx
+    wrmsr
+.Lspec_exit_done\@:
+.endm
+
+/* Use after a VMEXIT from an HVM guest. */
+#define SPEC_CTRL_ENTRY_FROM_VMEXIT                                     \
+        DO_SPEC_CTRL_ENTRY_FROM_VMEXIT
+
+/* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */
+#define SPEC_CTRL_ENTRY_FROM_PV                                         \
+        DO_SPEC_CTRL_ENTRY maybexen=0
+
+/* Use in interrupt/exception context.  May interrupt Xen or PV context. */
+#define SPEC_CTRL_ENTRY_FROM_INTR                                       \
+        DO_SPEC_CTRL_ENTRY maybexen=1
+
+/* Use when exiting to Xen context. */
+#define SPEC_CTRL_EXIT_TO_XEN                                           \
+        DO_SPEC_CTRL_EXIT_TO_XEN
+
+/* Use when exiting to guest context. */
+#define SPEC_CTRL_EXIT_TO_GUEST                                         \
+        DO_SPEC_CTRL_EXIT_TO_GUEST
+
+#endif /* __ASSEMBLY__ */
+#endif /* !__X86_SPEC_CTRL_ASM_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
--- a/xen/include/asm-x86/x86_32/asm_defns.h
+++ b/xen/include/asm-x86/x86_32/asm_defns.h
@@ -117,18 +117,6 @@
 #define FIXUP_RING0_GUEST_STACK
 #endif
 
-#define BUILD_COMMON_IRQ()                      \
-__asm__(                                        \
-    "\n" __ALIGN_STR"\n"                        \
-    "common_interrupt:\n\t"                     \
-    STR(FIXUP_RING0_GUEST_STACK)                \
-    STR(SAVE_ALL(1f,1f)) "\n\t"                 \
-    "1:movl %esp,%eax\n\t"                      \
-    "pushl %eax\n\t"                            \
-    "call " STR(do_IRQ) "\n\t"                  \
-    "addl $4,%esp\n\t"                          \
-    "jmp ret_from_intr\n");
-
 #define IRQ_NAME2(nr) nr##_interrupt(void)
 #define IRQ_NAME(nr) IRQ_NAME2(IRQ##nr)
 
@@ -140,15 +128,20 @@ STR(IRQ) #nr "_interrupt:\n\t"          
     "pushl $"#nr"<<16\n\t"                      \
     "jmp common_interrupt");
 
+#define STACK_CPUINFO_FIELD(field) (STACK_SIZE-CPUINFO_sizeof+CPUINFO_##field)
+#define GET_STACK_BASE(reg)                       \
+        movl $~(STACK_SIZE-1),reg;                \
+        andl %esp,reg
+
 #define GET_CPUINFO_FIELD(field,reg)                    \
-        movl $~(STACK_SIZE-1),reg;                      \
-        andl %esp,reg;                                  \
+        GET_STACK_BASE(reg);                            \
         orl  $(STACK_SIZE-CPUINFO_sizeof+field),reg;
 #define GET_CURRENT(reg)                                \
         GET_CPUINFO_FIELD(CPUINFO_current_vcpu,reg)     \
         movl (reg),reg;
 
 #ifdef __ASSEMBLY__
+# define r(reg) %e##reg
 # define _ASM_EX(p) p
 #else
 # define _ASM_EX(p) #p
--- a/xen/include/asm-x86/x86_64/asm_defns.h
+++ b/xen/include/asm-x86/x86_64/asm_defns.h
@@ -162,6 +162,7 @@ STR(IRQ) #nr "_interrupt:\n\t"          
         __GET_CURRENT(reg)
 
 #ifdef __ASSEMBLY__
+# define r(reg) %r##reg
 # define _ASM_EX(p) p-.
 #else
 # define _ASM_EX(p) #p "-."
