x86/spec-ctrl: Introduce options to control VERW flushing

The Microarchitectural Data Sampling vulnerability is split into categories
with subtly different properties:

 MLPDS - Microarchitectural Load Port Data Sampling
 MSBDS - Microarchitectural Store Buffer Data Sampling
 MFBDS - Microarchitectural Fill Buffer Data Sampling
 MDSUM - Microarchitectural Data Sampling Uncacheable Memory

MDSUM is a special case of the other three, and isn't distinguished further.

These issues pertain to three microarchitectural buffers.  The Load Ports, the
Store Buffers and the Fill Buffers.  Each of these structures are flushed by
the new enhanced VERW functionality, but the conditions under which flushing
is necessary vary.

For this concise overview of the issues and default logic, the abbreviations
SP (Store Port), FB (Fill Buffer), LP (Load Port) and HT (Hyperthreading) are
used for brevity:

 * Vulnerable hardware is divided into two categories - parts which suffer
   from SP only, and parts with any other combination of vulnerabilities.

 * SP only has an HT interaction when the thread goes idle, due to the static
   partitioning of resources.  LP and FB have HT interactions at all points,
   due to the competitive sharing of resources.  All issues potentially leak
   data across the return-to-guest transition.

 * The microcode which implements VERW flushing also extends MSR_FLUSH_CMD, so
   we don't need to do both on the HVM return-to-guest path.  However, the
   Knights range of processors are immune to L1TF (therefore have no
   MSR_FLUSH_CMD), but are vulnerable to MDS, so do require VERW on the HVM
   path.

Note that we deliberately support mds=1 even without MD_CLEAR in case the
microcode has been updated but the feature bit not exposed.

This is part of XSA-297, CVE-2018-12126, CVE-2018-12127, CVE-2018-12130, CVE-2019-11091.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -29,6 +29,8 @@ bool_t __read_mostly opt_msr_sc_pv = 1;
 bool_t __read_mostly opt_msr_sc_hvm = 1;
 static bool_t __initdata opt_rsb_pv = 1;
 static bool_t __initdata opt_rsb_hvm = 1;
+static int8_t __initdata opt_md_clear_pv = -1;
+static int8_t __initdata opt_md_clear_hvm = -1;
 
 /* Cmdline controls for Xen's speculative settings. */
 enum ind_thunk {
@@ -51,6 +53,9 @@ paddr_t __read_mostly l1tf_addr_mask, __
 static bool_t __initdata cpu_has_bug_l1tf;
 static unsigned int __initdata l1d_maxphysaddr;
 
+static bool_t __initdata cpu_has_bug_msbds_only; /* => minimal HT impact. */
+static bool_t __initdata cpu_has_bug_mds; /* Any other M{LP,SB,FB}DS combination. */
+
 static int __init parse_bti(const char *s)
 {
     const char *ss;
@@ -112,6 +117,8 @@ static int __init parse_spec_ctrl(char *
         disable_common:
             opt_rsb_pv = 0;
             opt_rsb_hvm = 0;
+            opt_md_clear_pv = 0;
+            opt_md_clear_hvm = 0;
 
             opt_ibrs = 0;
             opt_ibpb = 0;
@@ -133,11 +140,13 @@ static int __init parse_spec_ctrl(char *
         {
             opt_msr_sc_pv = val;
             opt_rsb_pv = val;
+            opt_md_clear_pv = val;
         }
         else if ( (val = parse_boolean("hvm", s, ss)) >= 0 )
         {
             opt_msr_sc_hvm = val;
             opt_rsb_hvm = val;
+            opt_md_clear_hvm = val;
         }
         else if ( (val = parse_boolean("msr-sc", s, ss)) >= 0 )
         {
@@ -149,6 +158,12 @@ static int __init parse_spec_ctrl(char *
             opt_rsb_pv = val;
             opt_rsb_hvm = val;
         }
+        else if ( (val = parse_boolean("md-clear", s, ss)) >= 0 ||
+                  (val = parse_boolean("mds", s, ss)) >= 0 )
+        {
+            opt_md_clear_pv = val;
+            opt_md_clear_hvm = val;
+        }
 
         /* Xen's speculative sidechannel mitigation settings. */
         else if ( (val = parse_boolean("ibrs", s, ss)) >= 0 )
@@ -247,7 +262,7 @@ static void __init print_details(enum in
 #endif
 
     /* Settings for Xen's protection, irrespective of guests. */
-    printk("  Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s\n",
+    printk("  Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s%s\n",
            thunk == THUNK_NONE      ? "N/A" :
            thunk == THUNK_RETPOLINE ? "RETPOLINE" : "?",
            default_xen_spec_ctrl < 0 ? "" :
@@ -256,7 +271,8 @@ static void __init print_details(enum in
                                                      ? "" :
            (default_xen_spec_ctrl & SPEC_CTRL_SSBD)  ? " SSBD+" : " SSBD-",
            opt_ibpb                 ? " IBPB"  : "",
-           opt_l1d_flush            ? " L1D_FLUSH" : "");
+           opt_l1d_flush            ? " L1D_FLUSH" : "",
+           opt_md_clear_pv || opt_md_clear_hvm ? " VERW"  : "");
 
     /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */
     if ( cpu_has_bug_l1tf || opt_pv_l1tf )
@@ -265,17 +281,19 @@ static void __init print_details(enum in
                cpu_has_bug_l1tf ? "" : " not",
                l1d_maxphysaddr, paddr_bits, l1tf_safe_maddr);
 
-    printk("  Support for VMs: PV:%s%s%s%s, HVM:%s%s%s%s\n",
-           (opt_msr_sc_pv || opt_rsb_pv ||
-            opt_eager_fpu)               ? ""               : " None",
+    printk("  Support for VMs: PV:%s%s%s%s%s, HVM:%s%s%s%s%s\n",
+           (opt_msr_sc_pv || opt_rsb_pv || opt_eager_fpu ||
+            opt_md_clear_pv)             ? ""               : " None",
            opt_msr_sc_pv                 ? " MSR_SPEC_CTRL" : "",
            opt_rsb_pv                    ? " RSB"           : "",
            opt_eager_fpu                 ? " EAGER_FPU"     : "",
-           (opt_msr_sc_hvm || opt_rsb_hvm ||
-            opt_eager_fpu)               ? ""               : " None",
+           opt_md_clear_pv               ? " MD_CLEAR"      : "",
+           (opt_msr_sc_hvm || opt_rsb_hvm || opt_eager_fpu ||
+            opt_md_clear_hvm)            ? ""               : " None",
            opt_msr_sc_hvm                ? " MSR_SPEC_CTRL" : "",
            opt_rsb_hvm                   ? " RSB"           : "",
-           opt_eager_fpu                 ? " EAGER_FPU"     : "");
+           opt_eager_fpu                 ? " EAGER_FPU"     : "",
+           opt_md_clear_hvm              ? " MD_CLEAR"      : "");
 
     printk("  PV L1TF shadowing: Dom0 %s, DomU %s\n",
            opt_pv_l1tf & OPT_PV_L1TF_DOM0  ? "enabled"  : "disabled",
@@ -540,6 +558,103 @@ static __init void l1tf_calculations(voi
                                             : ((paddr_t)3 << (paddr_bits - 2))));
 }
 
+/* Calculate whether this CPU is vulnerable to MDS. */
+static __init void mds_calculations(void)
+{
+    /* MDS is only known to affect Intel Family 6 processors at this time. */
+    if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
+         boot_cpu_data.x86 != 6 )
+        return;
+
+    switch ( boot_cpu_data.x86_model )
+    {
+        /*
+         * Core processors since at least Nehalem are vulnerable.
+         */
+    case 0x1f: /* Auburndale / Havendale */
+    case 0x1e: /* Nehalem */
+    case 0x1a: /* Nehalem EP */
+    case 0x2e: /* Nehalem EX */
+    case 0x25: /* Westmere */
+    case 0x2c: /* Westmere EP */
+    case 0x2f: /* Westmere EX */
+    case 0x2a: /* SandyBridge */
+    case 0x2d: /* SandyBridge EP/EX */
+    case 0x3a: /* IvyBridge */
+    case 0x3e: /* IvyBridge EP/EX */
+    case 0x3c: /* Haswell */
+    case 0x3f: /* Haswell EX/EP */
+    case 0x45: /* Haswell D */
+    case 0x46: /* Haswell H */
+    case 0x3d: /* Broadwell */
+    case 0x47: /* Broadwell H */
+    case 0x4f: /* Broadwell EP/EX */
+    case 0x56: /* Broadwell D */
+    case 0x4e: /* Skylake M */
+    case 0x5e: /* Skylake D */
+        cpu_has_bug_mds = 1;
+        break;
+
+        /*
+         * Some Core processors have per-stepping vulnerability.
+         */
+    case 0x55: /* Skylake-X / Cascade Lake */
+        if ( boot_cpu_data.x86_mask <= 5 )
+            cpu_has_bug_mds = 1;
+        break;
+
+    case 0x8e: /* Kaby / Coffee / Whiskey Lake M */
+        if ( boot_cpu_data.x86_mask <= 0xb )
+            cpu_has_bug_mds = 1;
+        break;
+
+    case 0x9e: /* Kaby / Coffee / Whiskey Lake D */
+        if ( boot_cpu_data.x86_mask <= 0xc )
+            cpu_has_bug_mds = 1;
+        break;
+
+        /*
+         * Very old and very new Atom processors are not vulnerable.
+         */
+    case 0x1c: /* Pineview */
+    case 0x26: /* Lincroft */
+    case 0x27: /* Penwell */
+    case 0x35: /* Cloverview */
+    case 0x36: /* Cedarview */
+    case 0x7a: /* Goldmont */
+        break;
+
+        /*
+         * Middling Atom processors are vulnerable to just the Store Buffer
+         * aspect.
+         */
+    case 0x37: /* Baytrail / Valleyview (Silvermont) */
+    case 0x4a: /* Merrifield */
+    case 0x4c: /* Cherrytrail / Brasswell */
+    case 0x4d: /* Avaton / Rangely (Silvermont) */
+    case 0x5a: /* Moorefield */
+    case 0x5d:
+    case 0x65:
+    case 0x6e:
+    case 0x75:
+        /*
+         * Knights processors (which are based on the Silvermont/Airmont
+         * microarchitecture) are similarly only affected by the Store Buffer
+         * aspect.
+         */
+    case 0x57: /* Knights Landing */
+    case 0x85: /* Knights Mill */
+        cpu_has_bug_msbds_only = 1;
+        break;
+
+    default:
+        printk("Unrecognised CPU model %#x - assuming vulnerable to MDS\n",
+               boot_cpu_data.x86_model);
+        cpu_has_bug_mds = 1;
+        break;
+    }
+}
+
 void __init init_speculation_mitigations(void)
 {
     enum ind_thunk thunk = THUNK_DEFAULT;
@@ -652,6 +767,37 @@ void __init init_speculation_mitigations
                "* explicit 'smt=<bool>' setting.  See XSA-273.               *\n"
                "**************************************************************\n");
 
+    mds_calculations();
+
+    /*
+     * By default, enable PV and HVM mitigations on MDS-vulnerable hardware.
+     * This will only be a token effort for MLPDS/MFBDS when HT is enabled,
+     * but it is somewhat better than nothing.
+     */
+    if ( opt_md_clear_pv == -1 )
+        opt_md_clear_pv = ((cpu_has_bug_mds || cpu_has_bug_msbds_only) &&
+                           boot_cpu_has(X86_FEATURE_MD_CLEAR));
+    if ( opt_md_clear_pv )
+        default_xen_rsb |= XEN_VERW_PV;
+    if ( opt_md_clear_hvm == -1 )
+        opt_md_clear_hvm = ((cpu_has_bug_mds || cpu_has_bug_msbds_only) &&
+                            !opt_l1d_flush &&
+                            boot_cpu_has(X86_FEATURE_MD_CLEAR));
+    if ( opt_md_clear_hvm )
+        default_xen_rsb |= XEN_VERW_HVM;
+
+    /*
+     * Warn the user if they are on MLPDS/MFBDS-vulnerable hardware with HT
+     * active and no explicit SMT choice.
+     */
+    if ( opt_smt == -1 && cpu_has_bug_mds && hw_smt_enabled )
+        printk("**************************************************************\n"
+               "* Booted on MLPDS/MFBDS-vulnerable hardware with             *\n"
+               "* SMT/Hyperthreading enabled.  Mitigations will not be fully *\n"
+               "* effective.  Please choose an explicit smt=<bool> setting.  *\n"
+               "* See XSA-297.                                               *\n"
+               "**************************************************************\n");
+
     print_details(thunk);
 
     /*
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -14,6 +14,8 @@
 
 #define XEN_RSB_VMEXIT 0x01
 #define XEN_RSB_NATIVE 0x02
+#define XEN_VERW_PV    0x10
+#define XEN_VERW_HVM   0x20
 
 #ifndef __ASSEMBLY__
 
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -97,7 +97,8 @@ static always_inline void spec_ctrl_ente
      * Note: VERW must be encoded with a memory operand, as it is only that
      * form which causes a flush.
      */
-    asm volatile ( "verw %[sel]" :: [sel] "m" (info->verw_sel) );
+    if ( info->xen_rsb & (XEN_VERW_PV | XEN_VERW_HVM) )
+        asm volatile ( "verw %[sel]" :: [sel] "m" (info->verw_sel) );
 }
 
 /* WARNING! `ret`, `call *`, `jmp *` not safe before this call. */
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -232,7 +232,7 @@
 .L\@_skip:
 .endm
 
-.macro DO_SPEC_CTRL_EXIT_TO_GUEST
+.macro DO_SPEC_CTRL_EXIT_TO_GUEST mask:req
 /*
  * Requires %eax=spec_ctrl, %rsp=regs/cpuinfo
  * Clobbers %rcx, %rdx
@@ -253,7 +253,10 @@
     mov $MSR_SPEC_CTRL, %ecx
     wrmsr
 .Lspec_exit_done\@:
+    testb $\mask, CPUINFO_xen_rsb(r(sp))
+    jz .Lverw_done\@
     verw CPUINFO_verw_sel(r(sp))
+.Lverw_done\@:
 .endm
 
 /* Use after a VMEXIT from an HVM guest. */
@@ -277,11 +280,11 @@
 
 /* Use when exiting to PV guest context. */
 #define SPEC_CTRL_EXIT_TO_PV                                            \
-        DO_SPEC_CTRL_EXIT_TO_GUEST
+        DO_SPEC_CTRL_EXIT_TO_GUEST XEN_VERW_PV
 
 /* Use when exiting to HVM guest context. */
 #define SPEC_CTRL_EXIT_TO_HVM                                           \
-        DO_SPEC_CTRL_EXIT_TO_GUEST
+        DO_SPEC_CTRL_EXIT_TO_GUEST XEN_VERW_HVM
 
 #endif /* __ASSEMBLY__ */
 #endif /* !__X86_SPEC_CTRL_ASM_H__ */
