# Commit fb5b6f6744713410c74cfc12b7176c108e3c9a31
# Date 2024-03-12 15:50:04 +0000
# Author Andrew Cooper <andrew.cooper3@citrix.com>
# Committer Andrew Cooper <andrew.cooper3@citrix.com>
x86/spec-ctrl: Mitigation Register File Data Sampling

RFDS affects Atom cores, also branded E-cores, between the Goldmont and
Gracemont microarchitectures.  This includes Alder Lake and Raptor Lake hybrid
clien systems which have a mix of Gracemont and other types of cores.

Two new bits have been defined; RFDS_CLEAR to indicate VERW has more side
effets, and RFDS_NO to incidate that the system is unaffected.  Plenty of
unaffected CPUs won't be getting RFDS_NO retrofitted in microcode, so we
synthesise it.  Alder Lake and Raptor Lake Xeon-E's are unaffected due to
their platform configuration, and we must use the Hybrid CPUID bit to
distinguish them from their non-Xeon counterparts.

Like MD_CLEAR and FB_CLEAR, RFDS_CLEAR needs OR-ing across a resource pool, so
set it in the max policies and reflect the host setting in default.

This is part of XSA-452 / CVE-2023-28746.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

--- a/tools/misc/xen-cpuid.c
+++ b/tools/misc/xen-cpuid.c
@@ -172,7 +172,7 @@ static const char *const str_7d0[32] =
     [ 8] = "avx512-vp2intersect", [ 9] = "srbds-ctrl",
     [10] = "md-clear",            [11] = "rtm-always-abort",
     /* 12 */                [13] = "tsx-force-abort",
-    [14] = "serialize",
+    [14] = "serialize",     [15] = "hybrid",
     [16] = "tsxldtrk",
     [18] = "pconfig",
     [20] = "cet-ibt",
@@ -237,7 +237,8 @@ static const char *const str_m10Al[32] =
     [20] = "bhi-no",              [21] = "xapic-status",
     /* 22 */                      [23] = "ovrclk-status",
     [24] = "pbrsb-no",            [25] = "gds-ctrl",
-    [26] = "gds-no",
+    [26] = "gds-no",              [27] = "rfds-no",
+    [28] = "rfds-clear",
 };
 
 static const char *const str_m10Ah[32] =
--- a/xen/arch/x86/cpu-policy.c
+++ b/xen/arch/x86/cpu-policy.c
@@ -444,6 +444,7 @@ static void __init guest_common_max_feat
          */
         __set_bit(X86_FEATURE_MD_CLEAR, fs);
         __set_bit(X86_FEATURE_FB_CLEAR, fs);
+        __set_bit(X86_FEATURE_RFDS_CLEAR, fs);
 
         /*
          * The Gather Data Sampling microcode mitigation (August 2023) has an
@@ -493,6 +494,10 @@ static void __init guest_common_default_
         if ( cpu_has_fb_clear )
             __set_bit(X86_FEATURE_FB_CLEAR, fs);
 
+        __clear_bit(X86_FEATURE_RFDS_CLEAR, fs);
+        if ( cpu_has_rfds_clear )
+            __set_bit(X86_FEATURE_RFDS_CLEAR, fs);
+
         /*
          * The Gather Data Sampling microcode mitigation (August 2023) has an
          * adverse performance impact on the CLWB instruction on SKX/CLX/CPX.
--- a/xen/arch/x86/include/asm/cpufeature.h
+++ b/xen/arch/x86/include/asm/cpufeature.h
@@ -140,6 +140,7 @@
 #define cpu_has_rtm_always_abort boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)
 #define cpu_has_tsx_force_abort boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)
 #define cpu_has_serialize       boot_cpu_has(X86_FEATURE_SERIALIZE)
+#define cpu_has_hybrid          boot_cpu_has(X86_FEATURE_HYBRID)
 #define cpu_has_avx512_fp16     boot_cpu_has(X86_FEATURE_AVX512_FP16)
 #define cpu_has_arch_caps       boot_cpu_has(X86_FEATURE_ARCH_CAPS)
 
@@ -161,6 +162,8 @@
 #define cpu_has_rrsba           boot_cpu_has(X86_FEATURE_RRSBA)
 #define cpu_has_gds_ctrl        boot_cpu_has(X86_FEATURE_GDS_CTRL)
 #define cpu_has_gds_no          boot_cpu_has(X86_FEATURE_GDS_NO)
+#define cpu_has_rfds_no         boot_cpu_has(X86_FEATURE_RFDS_NO)
+#define cpu_has_rfds_clear      boot_cpu_has(X86_FEATURE_RFDS_CLEAR)
 
 /* Synthesized. */
 #define cpu_has_arch_perfmon    boot_cpu_has(X86_FEATURE_ARCH_PERFMON)
--- a/xen/arch/x86/include/asm/msr-index.h
+++ b/xen/arch/x86/include/asm/msr-index.h
@@ -88,6 +88,8 @@
 #define  ARCH_CAPS_PBRSB_NO                 (_AC(1, ULL) << 24)
 #define  ARCH_CAPS_GDS_CTRL                 (_AC(1, ULL) << 25)
 #define  ARCH_CAPS_GDS_NO                   (_AC(1, ULL) << 26)
+#define  ARCH_CAPS_RFDS_NO                  (_AC(1, ULL) << 27)
+#define  ARCH_CAPS_RFDS_CLEAR               (_AC(1, ULL) << 28)
 
 #define MSR_FLUSH_CMD                       0x0000010b
 #define  FLUSH_CMD_L1D                      (_AC(1, ULL) <<  0)
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -24,6 +24,7 @@
 
 #include <asm/amd.h>
 #include <asm/hvm/svm/svm.h>
+#include <asm/intel-family.h>
 #include <asm/microcode.h>
 #include <asm/msr.h>
 #include <asm/pv/domain.h>
@@ -447,7 +448,7 @@ static void __init print_details(enum in
      * Hardware read-only information, stating immunity to certain issues, or
      * suggestions of which mitigation to use.
      */
-    printk("  Hardware hints:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+    printk("  Hardware hints:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
            (caps & ARCH_CAPS_RDCL_NO)                        ? " RDCL_NO"        : "",
            (caps & ARCH_CAPS_EIBRS)                          ? " EIBRS"          : "",
            (caps & ARCH_CAPS_RSBA)                           ? " RSBA"           : "",
@@ -463,6 +464,7 @@ static void __init print_details(enum in
            (caps & ARCH_CAPS_FB_CLEAR)                       ? " FB_CLEAR"       : "",
            (caps & ARCH_CAPS_PBRSB_NO)                       ? " PBRSB_NO"       : "",
            (caps & ARCH_CAPS_GDS_NO)                         ? " GDS_NO"         : "",
+           (caps & ARCH_CAPS_RFDS_NO)                        ? " RFDS_NO"        : "",
            (e8b  & cpufeat_mask(X86_FEATURE_IBRS_ALWAYS))    ? " IBRS_ALWAYS"    : "",
            (e8b  & cpufeat_mask(X86_FEATURE_STIBP_ALWAYS))   ? " STIBP_ALWAYS"   : "",
            (e8b  & cpufeat_mask(X86_FEATURE_IBRS_FAST))      ? " IBRS_FAST"      : "",
@@ -473,7 +475,7 @@ static void __init print_details(enum in
            (e21a & cpufeat_mask(X86_FEATURE_SRSO_NO))        ? " SRSO_NO"        : "");
 
     /* Hardware features which need driving to mitigate issues. */
-    printk("  Hardware features:%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+    printk("  Hardware features:%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
            (e8b  & cpufeat_mask(X86_FEATURE_IBPB)) ||
            (_7d0 & cpufeat_mask(X86_FEATURE_IBRSB))          ? " IBPB"           : "",
            (e8b  & cpufeat_mask(X86_FEATURE_IBRS)) ||
@@ -491,6 +493,7 @@ static void __init print_details(enum in
            (caps & ARCH_CAPS_TSX_CTRL)                       ? " TSX_CTRL"       : "",
            (caps & ARCH_CAPS_FB_CLEAR_CTRL)                  ? " FB_CLEAR_CTRL"  : "",
            (caps & ARCH_CAPS_GDS_CTRL)                       ? " GDS_CTRL"       : "",
+           (caps & ARCH_CAPS_RFDS_CLEAR)                     ? " RFDS_CLEAR"     : "",
            (e21a & cpufeat_mask(X86_FEATURE_SBPB))           ? " SBPB"           : "");
 
     /* Compiled-in support which pertains to mitigations. */
@@ -1359,6 +1362,83 @@ static __init void mds_calculations(void
     }
 }
 
+/*
+ * Register File Data Sampling affects Atom cores from the Goldmont to
+ * Gracemont microarchitectures.  The March 2024 microcode adds RFDS_NO to
+ * some but not all unaffected parts, and RFDS_CLEAR to affected parts still
+ * in support.
+ *
+ * Alder Lake and Raptor Lake client CPUs have a mix of P cores
+ * (Golden/Raptor Cove, not vulnerable) and E cores (Gracemont,
+ * vulnerable), and both enumerate RFDS_CLEAR.
+ *
+ * Both exist in a Xeon SKU, which has the E cores (Gracemont) disabled by
+ * platform configuration, and enumerate RFDS_NO.
+ *
+ * With older parts, or with out-of-date microcode, synthesise RFDS_NO when
+ * safe to do so.
+ *
+ * https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/advisory-guidance/register-file-data-sampling.html
+ */
+static void __init rfds_calculations(void)
+{
+    /* RFDS is only known to affect Intel Family 6 processors at this time. */
+    if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
+         boot_cpu_data.x86 != 6 )
+        return;
+
+    /*
+     * If RFDS_NO or RFDS_CLEAR are visible, we've either got suitable
+     * microcode, or an RFDS-aware hypervisor is levelling us in a pool.
+     */
+    if ( cpu_has_rfds_no || cpu_has_rfds_clear )
+        return;
+
+    /* If we're virtualised, don't attempt to synthesise RFDS_NO. */
+    if ( cpu_has_hypervisor )
+        return;
+
+    /*
+     * Not all CPUs are expected to get a microcode update enumerating one of
+     * RFDS_{NO,CLEAR}, or we might have out-of-date microcode.
+     */
+    switch ( boot_cpu_data.x86_model )
+    {
+    case INTEL_FAM6_ALDERLAKE:
+    case INTEL_FAM6_RAPTORLAKE:
+        /*
+         * Alder Lake and Raptor Lake might be a client SKU (with the
+         * Gracemont cores active, and therefore vulnerable) or might be a
+         * server SKU (with the Gracemont cores disabled, and therefore not
+         * vulnerable).
+         *
+         * See if the CPU identifies as hybrid to distinguish the two cases.
+         */
+        if ( !cpu_has_hybrid )
+            break;
+        fallthrough;
+    case INTEL_FAM6_ALDERLAKE_L:
+    case INTEL_FAM6_RAPTORLAKE_P:
+    case INTEL_FAM6_RAPTORLAKE_S:
+
+    case INTEL_FAM6_ATOM_GOLDMONT:      /* Apollo Lake */
+    case INTEL_FAM6_ATOM_GOLDMONT_D:    /* Denverton */
+    case INTEL_FAM6_ATOM_GOLDMONT_PLUS: /* Gemini Lake */
+    case INTEL_FAM6_ATOM_TREMONT_D:     /* Snow Ridge / Parker Ridge */
+    case INTEL_FAM6_ATOM_TREMONT:       /* Elkhart Lake */
+    case INTEL_FAM6_ATOM_TREMONT_L:     /* Jasper Lake */
+    case INTEL_FAM6_ATOM_GRACEMONT:     /* Alder Lake N */
+        return;
+    }
+
+    /*
+     * We appear to be on an unaffected CPU which didn't enumerate RFDS_NO,
+     * perhaps because of it's age or because of out-of-date microcode.
+     * Synthesise it.
+     */
+    setup_force_cpu_cap(X86_FEATURE_RFDS_NO);
+}
+
 static bool __init cpu_has_gds(void)
 {
     /*
@@ -1872,6 +1952,7 @@ void __init init_speculation_mitigations
      *
      * https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/intel-analysis-microarchitectural-data-sampling.html
      * https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/processor-mmio-stale-data-vulnerabilities.html
+     * https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/advisory-guidance/register-file-data-sampling.html
      *
      * Relevant ucodes:
      *
@@ -1901,8 +1982,12 @@ void __init init_speculation_mitigations
      *
      *   If FB_CLEAR is enumerated, L1D_FLUSH does not have the same scrubbing
      *   side effects as VERW and cannot be used in its place.
+     *
+     * - March 2023, for RFDS.  Enumerate RFDS_CLEAR to mean that VERW now
+     *   scrubs non-architectural entries from certain register files.
      */
     mds_calculations();
+    rfds_calculations();
 
     /*
      * Parts which enumerate FB_CLEAR are those with now-updated microcode
@@ -1934,15 +2019,19 @@ void __init init_speculation_mitigations
      * MLPDS/MFBDS when SMT is enabled.
      */
     if ( opt_verw_pv == -1 )
-        opt_verw_pv = cpu_has_useful_md_clear;
+        opt_verw_pv = cpu_has_useful_md_clear || cpu_has_rfds_clear;
 
     if ( opt_verw_hvm == -1 )
-        opt_verw_hvm = cpu_has_useful_md_clear;
+        opt_verw_hvm = cpu_has_useful_md_clear || cpu_has_rfds_clear;
 
     /*
      * If SMT is active, and we're protecting against MDS or MMIO stale data,
      * we need to scrub before going idle as well as on return to guest.
      * Various pipeline resources are repartitioned amongst non-idle threads.
+     *
+     * We don't need to scrub on idle for RFDS.  There are no affected cores
+     * which support SMT, despite there being affected cores in hybrid systems
+     * which have SMT elsewhere in the platform.
      */
     if ( ((cpu_has_useful_md_clear && (opt_verw_pv || opt_verw_hvm)) ||
           opt_verw_mmio) && hw_smt_enabled )
@@ -1956,7 +2045,8 @@ void __init init_speculation_mitigations
      * It is only safe to use L1D_FLUSH in place of VERW when MD_CLEAR is the
      * only *_CLEAR we can see.
      */
-    if ( opt_l1d_flush && cpu_has_md_clear && !cpu_has_fb_clear )
+    if ( opt_l1d_flush && cpu_has_md_clear && !cpu_has_fb_clear &&
+         !cpu_has_rfds_clear )
         opt_verw_hvm = false;
 
     /*
--- a/xen/include/public/arch-x86/cpufeatureset.h
+++ b/xen/include/public/arch-x86/cpufeatureset.h
@@ -264,6 +264,7 @@ XEN_CPUFEATURE(MD_CLEAR,      9*32+10) /
 XEN_CPUFEATURE(RTM_ALWAYS_ABORT, 9*32+11) /*! June 2021 TSX defeaturing in microcode. */
 XEN_CPUFEATURE(TSX_FORCE_ABORT, 9*32+13) /* MSR_TSX_FORCE_ABORT.RTM_ABORT */
 XEN_CPUFEATURE(SERIALIZE,     9*32+14) /*A  SERIALIZE insn */
+XEN_CPUFEATURE(HYBRID,        9*32+15) /*   Heterogeneous platform */
 XEN_CPUFEATURE(TSXLDTRK,      9*32+16) /*a  TSX load tracking suspend/resume insns */
 XEN_CPUFEATURE(CET_IBT,       9*32+20) /*   CET - Indirect Branch Tracking */
 XEN_CPUFEATURE(AVX512_FP16,   9*32+23) /*   AVX512 FP16 instructions */
@@ -330,6 +331,8 @@ XEN_CPUFEATURE(OVRCLK_STATUS,      16*32
 XEN_CPUFEATURE(PBRSB_NO,           16*32+24) /*A  No Post-Barrier RSB predictions */
 XEN_CPUFEATURE(GDS_CTRL,           16*32+25) /*   MCU_OPT_CTRL.GDS_MIT_{DIS,LOCK} */
 XEN_CPUFEATURE(GDS_NO,             16*32+26) /*A  No Gather Data Sampling */
+XEN_CPUFEATURE(RFDS_NO,            16*32+27) /*A  No Register File Data Sampling */
+XEN_CPUFEATURE(RFDS_CLEAR,         16*32+28) /*!A Register File(s) cleared by VERW */
 
 /* Intel-defined CPU features, MSR_ARCH_CAPS 0x10a.edx, word 17 */
 
