From: Andrew Cooper <andrew.cooper3@citrix.com>
Subject: x86/spec-ctrl: Make VERW flushing runtime conditional

Currently, VERW flushing to mitigate MDS is boot time conditional per domain
type.  However, to provide mitigations for DRPW (CVE-2022-21166), we need to
conditionally use VERW based on the trustworthiness of the guest, and the
devices passed through.

Remove the PV/HVM alternatives and instead issue a VERW on the return-to-guest
path depending on the SCF_verw bit in cpuinfo spec_ctrl_flags.

Introduce spec_ctrl_init_domain() and d->arch.verw to calculate the VERW
disposition at domain creation time, and context switch the SCF_verw bit.

For now, VERW flushing is used and controlled exactly as before, but later
patches will add per-domain cases too.

No change in behaviour.

This is part of XSA-404.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Roger Pau Monné <roger.pau@citrix.com>

--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -944,9 +944,8 @@ in place for guests to use.
 Use of a positive boolean value for either of these options is invalid.
 
 The booleans `pv=`, `hvm=`, `msr-sc=`, `rsb=` and `md-clear=` offer fine
-grained control over the alternative blocks used by Xen.  These impact Xen's
-ability to protect itself, and Xen's ability to virtualise support for guests
-to use.
+grained control over the primitives by Xen.  These impact Xen's ability to
+protect itself, and Xen's ability to virtualise support for guests to use.
 
 * `pv=` and `hvm=` offer control over all suboptions for PV and HVM guests
   respectively.
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -673,6 +673,8 @@ int arch_domain_create(struct domain *d,
     tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0);
     spin_lock_init(&d->arch.vtsc_lock);
 
+    spec_ctrl_init_domain(d);
+
     return 0;
 
  fail:
@@ -1749,11 +1751,12 @@ static void __context_switch(void)
 void context_switch(struct vcpu *prev, struct vcpu *next)
 {
     unsigned int cpu = smp_processor_id();
+    struct cpu_info *info = get_cpu_info();
     cpumask_t dirty_mask;
 
     ASSERT(local_irq_is_enabled());
 
-    get_cpu_info()->xen_cr3 = 0;
+    info->xen_cr3 = 0;
 
     cpumask_copy(&dirty_mask, next->vcpu_dirty_cpumask);
     /* Allow at most one CPU at a time to be dirty. */
@@ -1835,6 +1838,11 @@ void context_switch(struct vcpu *prev, s
                 *last_id = next_id;
             }
         }
+
+        /* Update the top-of-stack block with the VERW disposition. */
+        info->xen_rsb &= ~XEN_VERW;
+        if ( next->domain->arch.verw )
+            info->xen_rsb |= XEN_VERW;
     }
 
     context_saved(prev);
--- a/xen/arch/x86/hvm/vmx/entry.S
+++ b/xen/arch/x86/hvm/vmx/entry.S
@@ -137,6 +137,7 @@ UNLIKELY_END(realmode)
 
         /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */
         SPEC_CTRL_EXIT_TO_HVM   /* Req: a=spec_ctrl %rsp=regs/cpuinfo, Clob: cd */
+        DO_SPEC_CTRL_COND_VERW
 
         cmpb $0,VCPU_vmx_launched(r(bx))
 #if defined(__x86_64__)
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -19,6 +19,7 @@
 #include <xen/init.h>
 #include <xen/errno.h>
 #include <xen/lib.h>
+#include <xen/sched.h>
 
 #include <asm/processor.h>
 #include <asm/setup.h>
@@ -29,8 +30,8 @@ bool_t __read_mostly opt_msr_sc_pv = 1;
 bool_t __read_mostly opt_msr_sc_hvm = 1;
 static bool_t __initdata opt_rsb_pv = 1;
 static bool_t __initdata opt_rsb_hvm = 1;
-static int8_t __initdata opt_md_clear_pv = -1;
-static int8_t __initdata opt_md_clear_hvm = -1;
+int8_t __read_mostly opt_md_clear_pv = -1;
+int8_t __read_mostly opt_md_clear_hvm = -1;
 
 /* Cmdline controls for Xen's speculative settings. */
 enum ind_thunk {
@@ -664,6 +665,13 @@ static __init void mds_calculations(void
     }
 }
 
+void spec_ctrl_init_domain(struct domain *d)
+{
+    bool_t pv = !is_hvm_domain(d);
+
+    d->arch.verw = pv ? opt_md_clear_pv : opt_md_clear_hvm;
+}
+
 void __init init_speculation_mitigations(void)
 {
     enum ind_thunk thunk = THUNK_DEFAULT;
@@ -786,14 +794,10 @@ void __init init_speculation_mitigations
     if ( opt_md_clear_pv == -1 )
         opt_md_clear_pv = ((cpu_has_bug_mds || cpu_has_bug_msbds_only) &&
                            boot_cpu_has(X86_FEATURE_MD_CLEAR));
-    if ( opt_md_clear_pv )
-        default_xen_rsb |= XEN_VERW_PV;
     if ( opt_md_clear_hvm == -1 )
         opt_md_clear_hvm = ((cpu_has_bug_mds || cpu_has_bug_msbds_only) &&
                             !opt_l1d_flush &&
                             boot_cpu_has(X86_FEATURE_MD_CLEAR));
-    if ( opt_md_clear_hvm )
-        default_xen_rsb |= XEN_VERW_HVM;
 
     /*
      * Warn the user if they are on MLPDS/MFBDS-vulnerable hardware with HT
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -14,8 +14,7 @@
 
 #define XEN_RSB_VMEXIT 0x01
 #define XEN_RSB_NATIVE 0x02
-#define XEN_VERW_PV    0x10
-#define XEN_VERW_HVM   0x20
+#define XEN_VERW       0x04
 
 #ifndef __ASSEMBLY__
 
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -279,13 +279,16 @@ struct arch_domain
     struct mapcache_domain mapcache;
 #endif
 
-    bool_t s3_integrity;
-
     /* I/O-port admin-specified access capabilities. */
     struct rangeset *ioport_caps;
     uint32_t pci_cf8;
     uint8_t cmos_idx;
 
+    /* Use VERW on return-to-guest for its flushing side effect. */
+    bool_t verw;
+
+    bool_t s3_integrity;
+
     struct list_head pdev_list;
 
     union {
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -24,10 +24,12 @@
 #include <asm/msr.h>
 
 void init_speculation_mitigations(void);
+void spec_ctrl_init_domain(struct domain *d);
 
 extern bool_t opt_ibpb;
 extern bool_t opt_ssbd;
 extern bool_t opt_msr_sc_pv, opt_msr_sc_hvm;
+extern int8_t opt_md_clear_pv, opt_md_clear_hvm;
 extern int8_t opt_eager_fpu;
 extern int8_t opt_l1d_flush;
 
@@ -99,7 +101,7 @@ static always_inline void spec_ctrl_ente
      * Note: VERW must be encoded with a memory operand, as it is only that
      * form which causes a flush.
      */
-    if ( info->xen_rsb & (XEN_VERW_PV | XEN_VERW_HVM) )
+    if ( opt_md_clear_pv || opt_md_clear_hvm )
         asm volatile ( "verw %[sel]" :: [sel] "m" (info->verw_sel) );
 }
 
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -159,6 +159,19 @@
 .Lspec_entry_done\@:
 .endm
 
+.macro DO_SPEC_CTRL_COND_VERW
+/*
+ * Requires %rsp=cpuinfo
+ *
+ * Issue a VERW for its flushing side effect, if indicated.  This is a Spectre
+ * v1 gadget, but the IRET/VMEntry is serialising.
+ */
+    testb $XEN_VERW, CPUINFO_xen_rsb(r(sp))
+    jz .L\@_verw_skip
+    verw CPUINFO_verw_sel(r(sp))
+.L\@_verw_skip:
+.endm
+
 .macro DO_SPEC_CTRL_ENTRY maybexen:req
 /*
  * Requires %rsp=regs (also cpuinfo if !maybexen)
@@ -232,7 +245,7 @@
 .L\@_skip:
 .endm
 
-.macro DO_SPEC_CTRL_EXIT_TO_GUEST mask:req
+.macro DO_SPEC_CTRL_EXIT_TO_GUEST
 /*
  * Requires %eax=spec_ctrl, %rsp=regs/cpuinfo
  * Clobbers %rcx, %rdx
@@ -253,10 +266,6 @@
     mov $MSR_SPEC_CTRL, %ecx
     wrmsr
 .Lspec_exit_done\@:
-    testb $\mask, CPUINFO_xen_rsb(r(sp))
-    jz .Lverw_done\@
-    verw CPUINFO_verw_sel(r(sp))
-.Lverw_done\@:
 .endm
 
 /* Use after a VMEXIT from an HVM guest. */
@@ -280,11 +289,12 @@
 
 /* Use when exiting to PV guest context. */
 #define SPEC_CTRL_EXIT_TO_PV                                            \
-        DO_SPEC_CTRL_EXIT_TO_GUEST XEN_VERW_PV
+        DO_SPEC_CTRL_EXIT_TO_GUEST;                                     \
+        DO_SPEC_CTRL_COND_VERW
 
 /* Use when exiting to HVM guest context. */
 #define SPEC_CTRL_EXIT_TO_HVM                                           \
-        DO_SPEC_CTRL_EXIT_TO_GUEST XEN_VERW_HVM
+        DO_SPEC_CTRL_EXIT_TO_GUEST
 
 #endif /* __ASSEMBLY__ */
 #endif /* !__X86_SPEC_CTRL_ASM_H__ */
