From: Andrew Cooper <andrew.cooper3@citrix.com>
Subject: x86/spec-ctrl: Support IBPB-on-entry

We are going to need this to mitigate Branch Type Confusion on AMD/Hygon CPUs,
but as we've talked about using it in other cases too, arrange to support it
generally.  However, this is also very expensive in some cases, so we're going
to want per-domain controls.

Introduce SCF_ist_ibpb and SCF_entry_ibpb controls, adding them to the IST and
DOM masks as appropriate.  Also introduce X86_FEATURE_IBPB_ENTRY_{PV,HVM} to
to patch the code blocks.

For SVM, the STGI is serialising enough to protect against Spectre-v1 attacks,
so no "else lfence" is necessary.  VT-x will use use the MSR host load list,
so doesn't need any code in the VMExit path.

For the IST path, we can't safely check CPL==0 to skip a flush, as we might
have hit an entry path before it's IBPB.  As IST hitting Xen is rare, flush
irrespective of CPL.  A later path, SCF_ist_sc_msr, provides Spectre-v1
safety.

For the PV paths, we know we're interrupting CPL>0, while for the INTR paths,
we can safely check CPL==0.  Only flush when interrupting guest context.

An "else lfence" is needed for safety, but we want to be able to skip it on
unaffected CPUs, so the block wants to be an alternative, which means the
lfence has to be inline rather than UNLIKELY() (the replacement block doesn't
have displacements fixed up for anything other than the first instruction).

As with SPEC_CTRL_ENTRY_FROM_INTR_IST, %rdx is 0 on entry so rely on this to
shrink the logic marginally.  Update the comments to specify this new
dependency.

This is part of XSA-407.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

--- a/xen/arch/x86/hvm/svm/entry.S
+++ b/xen/arch/x86/hvm/svm/entry.S
@@ -132,7 +132,13 @@ UNLIKELY_END(svm_trace)
         get_current(bx)
         movb $0,VCPU_svm_vmcb_in_sync(r(bx))
 
-        /* SPEC_CTRL_ENTRY_FROM_SVM    Req: b=curr %rsp=regs/cpuinfo, Clob: ac  */
+        /* SPEC_CTRL_ENTRY_FROM_SVM    Req: %rsp=regs/cpuinfo, %rdx=0 Clob: ac  */
+        testb  $XEN_IBPB_HVM, CPUINFO_xen_rsb(r(sp))
+        jz     .L_skip_ibpb
+        mov    $MSR_PRED_CMD, %ecx
+        mov    $PRED_CMD_IBPB, %eax
+        wrmsr
+.L_skip_ibpb:
         DO_OVERWRITE_RSB XEN_RSB_VMEXIT
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
@@ -159,6 +165,10 @@ UNLIKELY_END(svm_trace)
         mov  %ax,UREGS_ss(r(sp))
 #endif
 
+        /*
+         * STGI is executed unconditionally, and is sufficiently serialising
+         * to safely resolve any Spectre-v1 concerns in the above logic.
+         */
         STGI
 .globl svm_stgi_label
 svm_stgi_label:
--- a/xen/arch/x86/hvm/vmx/vmcs.c
+++ b/xen/arch/x86/hvm/vmx/vmcs.c
@@ -1058,6 +1058,10 @@ static int construct_vmcs(struct vcpu *v
         rc = vmx_add_msr(v, MSR_FLUSH_CMD, VMX_MSR_GUEST_LOADONLY)
              ?: vmx_write_guest_loadonly_msr(v, MSR_FLUSH_CMD, FLUSH_CMD_L1D);
 
+    if ( !rc && (d->arch.spec_ctrl_flags & XEN_IBPB_HVM) )
+        rc = vmx_add_msr(v, MSR_PRED_CMD, VMX_MSR_GUEST_LOADONLY)
+             ?: vmx_write_guest_loadonly_msr(v, MSR_PRED_CMD, PRED_CMD_IBPB);
+
  out:
     vmx_vmcs_exit(v);
 
--- a/xen/arch/x86/x86_32/entry.S
+++ b/xen/arch/x86/x86_32/entry.S
@@ -155,7 +155,7 @@ ENTRY(hypercall)
         SAVE_ALL(,1f)
 1:      sti
 
-        SPEC_CTRL_ENTRY_FROM_PV /* Req: %esp=regs/cpuinfo, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %esp=regs/cpuinfo, %edx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         movl  UREGS_eax(%esp),%eax
@@ -438,7 +438,7 @@ ENTRY(common_interrupt)
         FIXUP_RING0_GUEST_STACK
         SAVE_ALL(1f,1f)
 1:
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %esp=regs, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_INTR(0) /* Req: %esp=regs, %edx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         movl  %esp,%eax
@@ -467,7 +467,7 @@ handle_exception:
         jmp   2f
         .previous
 2:
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %esp=regs, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_INTR(0) /* Req: %esp=regs, %edx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         testb $X86_EFLAGS_IF>>8,UREGS_eflags+1(%esp)
@@ -607,7 +607,8 @@ handle_nmi_mce:
         # Save state but do not trash the segment registers!
         SAVE_ALL(.Lnmi_mce_xen,.Lnmi_mce_common)
 .Lnmi_mce_common:
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %esp=regs, Clob: acd */
+        xorl  %edx,%edx
+        SPEC_CTRL_ENTRY_FROM_INTR(1) /* Req: %esp=regs, %edx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         xorl  %eax,%eax
--- a/xen/arch/x86/x86_64/compat/entry.S
+++ b/xen/arch/x86/x86_64/compat/entry.S
@@ -17,7 +17,7 @@ ENTRY(compat_hypercall)
         movl  $TRAP_syscall,4(%rsp)
         SAVE_ALL
 
-        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         cmpb  $0,untrusted_msi(%rip)
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -182,7 +182,7 @@ ENTRY(syscall_enter)
         movq  24(%rsp),%r11 /* Re-load user RFLAGS into %r11 before SAVE_ALL */
         SAVE_ALL
 
-        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         GET_STACK_BASE(%rbx)
@@ -362,7 +362,7 @@ sysenter_eflags_saved:
         movl  $TRAP_syscall,4(%rsp)
         SAVE_ALL
 
-        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         GET_STACK_BASE(%rbx)
@@ -409,7 +409,7 @@ ENTRY(int80_direct_trap)
         pushq $0
         SAVE_ALL
 
-        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_PV /* Req: %rsp=regs/cpuinfo, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         GET_STACK_BASE(%rbx)
@@ -589,7 +589,7 @@ ENTRY(common_interrupt)
 
         GET_STACK_BASE(%r14)
 
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_INTR(0) /* Req: %rsp=regs, %r14=end, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
@@ -631,7 +631,7 @@ handle_exception:
 
         GET_STACK_BASE(%r14)
 
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_INTR(0) /* Req: %rsp=regs, %r14=end, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
@@ -778,7 +778,7 @@ ENTRY(double_fault)
 
         GET_STACK_BASE(%r14)
 
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_INTR(1) /* Req: %rsp=regs, %r14=end, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rbx
@@ -810,7 +810,7 @@ handle_ist_exception:
 
         GET_STACK_BASE(%r14)
 
-        SPEC_CTRL_ENTRY_FROM_INTR /* Req: %rsp=regs, %r14=end, Clob: acd */
+        SPEC_CTRL_ENTRY_FROM_INTR(1) /* Req: %rsp=regs, %r14=end, %rdx=0, Clob: acd */
         /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
 
         mov   STACK_CPUINFO_FIELD(xen_cr3)(%r14), %rcx
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -25,12 +25,14 @@
 #define XEN_RSB_VMEXIT 0x01
 #define XEN_RSB_NATIVE 0x02
 #define XEN_VERW       0x04
+#define XEN_IBPB_PV    0x08
+#define XEN_IBPB_HVM   0x10
 
 /*
  * Some speculative protections are per-domain.  These settings are merged
  * into the top-of-stack block in the context switch path.
  */
-#define SCF_DOM_MASK (XEN_VERW)
+#define SCF_DOM_MASK (XEN_VERW | XEN_IBPB_PV | XEN_IBPB_HVM)
 
 #ifndef __ASSEMBLY__
 
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -78,6 +78,40 @@
  *  - SPEC_CTRL_EXIT_TO_{SVM,VMX}
  */
 
+.macro DO_SPEC_CTRL_COND_IBPB maybexen:req
+/*
+ * Requires %rsp=regs (also cpuinfo if !maybexen)
+ * Requires %r14=stack_end (if maybexen), %rdx=0
+ * Clobbers %rax, %rcx, %rdx
+ *
+ * Conditionally issue IBPB if XEN_IBPB_PV is active.  In the maybexen
+ * case, we can safely look at UREGS_cs to skip taking the hit when
+ * interrupting Xen.
+ */
+    .if \maybexen
+#ifdef __i386__
+        GET_STACK_BASE(%ecx)
+        testb  $XEN_IBPB_PV, STACK_CPUINFO_FIELD(xen_rsb)(%ecx)
+#else
+        testb  $XEN_IBPB_PV, STACK_CPUINFO_FIELD(xen_rsb)(%r14)
+#endif
+        jz     .L\@_skip
+        testb  $3, UREGS_cs(r(sp))
+    .else
+        testb  $XEN_IBPB_PV, CPUINFO_xen_rsb(r(sp))
+    .endif
+    jz     .L\@_skip
+
+    mov     $MSR_PRED_CMD, %ecx
+    mov     $PRED_CMD_IBPB, %eax
+    wrmsr
+    jmp     .L\@_done
+
+.L\@_skip:
+    lfence
+.L\@_done:
+.endm
+
 .macro DO_OVERWRITE_RSB mask:req maybexen=0
 /*
  * Requires %rsp=cpuinfo (if !maybexen)
@@ -272,11 +306,13 @@
 
 /* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */
 #define SPEC_CTRL_ENTRY_FROM_PV                                         \
+        DO_SPEC_CTRL_COND_IBPB maybexen=0;                              \
         DO_OVERWRITE_RSB XEN_RSB_NATIVE;                                \
         DO_SPEC_CTRL_ENTRY maybexen=0
 
 /* Use in interrupt/exception context.  May interrupt Xen or PV context. */
-#define SPEC_CTRL_ENTRY_FROM_INTR                                       \
+#define SPEC_CTRL_ENTRY_FROM_INTR(ist)                                  \
+        DO_SPEC_CTRL_COND_IBPB maybexen=!(ist);                         \
         DO_OVERWRITE_RSB mask=XEN_RSB_NATIVE maybexen=1;                \
         DO_SPEC_CTRL_ENTRY maybexen=1
 
