Subject: x86/spec-ctrl: Mitigate IBPB not flushing the RSB/RAS
From: Andrew Cooper andrew.cooper3@citrix.com Tue Jun 14 16:18:36 2022 +0100
Date: Fri Nov 4 13:24:37 2022 +0000:
Git: 1151d260d7a0186978b80b708fcb712eb1470f49

Introduce spec_ctrl_new_guest_context() to encapsulate all logic pertaining to
using MSR_PRED_CMD for a new guest context, even if it only has one user
presently.

Introduce X86_BUG_IBPB_NO_RET, and use it extend spec_ctrl_new_guest_context()
with a manual fixup for hardware which mis-implements IBPB.

This is part of XSA-422 / CVE-2022-23824.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Jan Beulich <jbeulich@suse.com>
(cherry picked from commit 2b27967fb89d7904a1571a2fb963b1c9cac548db)

--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1834,7 +1834,7 @@ void context_switch(struct vcpu *prev, s
              */
             if ( *last_id != next_id )
             {
-                wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
+                spec_ctrl_new_guest_context();
                 *last_id = next_id;
             }
         }
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -554,6 +554,14 @@ static void __init ibpb_calculations(voi
     }
 
     /*
+     * AMD/Hygon CPUs to date (June 2022) don't flush the the RAS.  Future
+     * CPUs are expected to enumerate IBPB_RET when this has been fixed.
+     * Until then, cover the difference with the software sequence.
+     */
+    if ( boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_IBPB_RET) )
+        __set_bit(X86_BUG_IBPB_NO_RET, boot_cpu_data.x86_capability);
+
+    /*
      * IBPB-on-entry mitigations for Branch Type Confusion.
      *
      * IBPB && !BTC_NO selects all AMD/Hygon hardware, not known to be safe,
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -82,6 +82,7 @@
 #define X86_FEATURE_CPUID_FAULTING (3*32+14) /* cpuid faulting */
 #define X86_FEATURE_CLFLUSH_MONITOR (3*32+15) /* clflush reqd with monitor */
 #define X86_FEATURE_XEN_SELFSNOOP   (3*32+16) /* SELFSNOOP gets used by Xen itself */
+#define X86_BUG_IBPB_NO_RET         (3*32+30) /* IBPB doesn't flush the RSB/RAS */
 #define X86_BUG_CLFLUSH_MFENCE      (3*32+31) /* MFENCE needed to serialise CLFLUSH */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -26,6 +26,27 @@
 void init_speculation_mitigations(void);
 void spec_ctrl_init_domain(struct domain *d);
 
+/*
+ * Switch to a new guest prediction context.
+ *
+ * This flushes all indirect branch predictors (BTB, RSB/RAS), so guest code
+ * which has previously run on this CPU can't attack subsequent guest code.
+ *
+ * As this flushes the RSB/RAS, it destroys the predictions of the calling
+ * context.  For best performace, arrange for this to be used when we're going
+ * to jump out of the current context, e.g. with reset_stack_and_jump().
+ *
+ * For hardware which mis-implements IBPB, fix up by flushing the RSB/RAS
+ * manually.
+ */
+static always_inline void spec_ctrl_new_guest_context(void)
+{
+    wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
+
+    if ( boot_cpu_has(X86_BUG_IBPB_NO_RET) )
+        asm volatile ( "DO_OVERWRITE_RSB" ::: "rax", "rcx" );
+}
+
 extern int8_t opt_ibpb_ctxt_switch;
 extern bool_t opt_ssbd;
 extern bool_t opt_msr_sc_pv, opt_msr_sc_hvm;
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -112,7 +112,31 @@
 .L\@_done:
 .endm
 
-.macro DO_OVERWRITE_RSB mask:req maybexen=0
+#define ARGS(args...) args
+#define LBL(name) .L\@_##name
+#define LBLn(name) .L\@_##name##_\n
+#define asm(insn...) insn
+
+#ifdef __i386__
+#define r14 ecx
+#endif
+
+#else /* !__ASSEMBLY__ */
+
+#define ARGS(args...)
+#define LBL(name) .L\\@_##name
+#define LBLn(name) .L\\@_##name##_\\n
+#define asm(insn...) asm ( STR(insn) )
+
+#ifdef __i386__
+#define r(reg) %e##reg
+#else
+#define r(reg) %r##reg
+#endif
+
+#endif /* __ASSEMBLY__ */
+
+asm(.macro DO_OVERWRITE_RSB ARGS(mask:req maybexen=0);
 /*
  * Requires %rsp=cpuinfo (if !maybexen)
  * Requires %r14=stack_end (if 64-bit and maybexen)
@@ -130,42 +154,52 @@
  * b) the two movs are shorter to encode than `add $32*8, %rsp`, and c) can be
  * optimised with mov-elimination in modern cores.
  */
+#ifdef __ASSEMBLY__
 #ifdef __i386__
-    .if \maybexen
-    GET_STACK_BASE(%ecx)
-    .endif
-#define r14 ecx
+    .if \maybexen;
+    GET_STACK_BASE(%ecx);
+    .endif;
 #endif
 
-    .if \maybexen
-        testb $\mask, STACK_CPUINFO_FIELD(xen_rsb)(%r14)
-    .else
-        testb $\mask, CPUINFO_xen_rsb(r(sp))
-    .endif
-    jz .Lrsb_done\@
-#undef r14
+    .if \maybexen;
+        testb $\mask, STACK_CPUINFO_FIELD(xen_rsb)(%r14);
+    .else;
+        testb $\mask, CPUINFO_xen_rsb(r(sp));
+    .endif;
+    jz LBL(rsb_done);
+#endif
 
-    mov $16, %ecx                   /* 16 iterations, two calls per loop */
-    mov r(sp), r(ax)                /* Store the current %rsp */
+    mov $16, %ecx;                  /* 16 iterations, two calls per loop */
+    mov r(sp), r(ax);               /* Store the current r(sp) */
 
-.L\@_fill_rsb_loop:
+LBL(fill_rsb_loop):;
 
-    .irp n, 1, 2                    /* Unrolled twice. */
-    call .L\@_insert_rsb_entry_\n   /* Create an RSB entry. */
+    .irp n, 1, 2;                   /* Unrolled twice. */
+    call LBLn(insert_rsb_entry);    /* Create an RSB entry. */
 
-.L\@_capture_speculation_\n:
-    pause
-    lfence
-    jmp .L\@_capture_speculation_\n /* Capture rogue speculation. */
-
-.L\@_insert_rsb_entry_\n:
-    .endr
-
-    sub $1, %ecx
-    jnz .L\@_fill_rsb_loop
-    mov r(ax), r(sp)                /* Restore old %rsp */
-.Lrsb_done\@:
-.endm
+LBLn(capture_speculation):
+    pause;
+    lfence;
+    jmp LBLn(capture_speculation);  /* Capture rogue speculation. */
+
+LBLn(insert_rsb_entry):;
+    .endr;
+
+    sub $1, %ecx;
+    jnz LBL(fill_rsb_loop);
+    mov r(ax), r(sp);               /* Restore old r(sp) */
+LBL(rsb_done):;
+.endm);
+
+#undef asm
+#undef LBLn
+#undef LBL
+#undef ARGS
+
+#ifndef __ASSEMBLY__
+#undef r
+#else
+#undef r14
 
 .macro DO_SPEC_CTRL_ENTRY_FROM_HVM
 /*
