From: Andrew Cooper <andrew.cooper3@citrix.com>
Subject: x86: Split cache_flush() out of cache_writeback()

Subsequent changes will want a fully flushing version.

Use the new helper rather than opencoding it in flush_area_local().  This
resolves an outstanding issue where the conditional sfence is on the wrong
side of the clflushopt loop.  clflushopt is ordered with respect to older
stores, not to younger stores.

Rename gnttab_cache_flush()'s helper to avoid colliding in name.
grant_table.c can see the prototype from cache.h so the build fails
otherwise.

This is part of XSA-402.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

Xen 4.16 and earlier:
 * Also backport half of c/s 3330013e67396 "VT-d / x86: re-arrange cache
   syncing" to split cache_writeback() out of the IOMMU logic, but without the
   associated hooks changes.

--- a/xen/arch/x86/flushtlb.c
+++ b/xen/arch/x86/flushtlb.c
@@ -147,7 +147,7 @@ void flush_area_local(const void *va, un
 
     if ( flags & FLUSH_CACHE )
     {
-        unsigned long i, sz = 0;
+        unsigned long sz = 0;
 
         if ( order < (BITS_PER_LONG - PAGE_SHIFT) )
             sz = 1UL << (order + PAGE_SHIFT);
@@ -158,9 +158,7 @@ void flush_area_local(const void *va, un
              ((sz >> 10) < c->x86_cache_size) )
         {
             va = (const void *)((unsigned long)va & ~(sz - 1));
-            for ( i = 0; i < sz; i += c->x86_clflush_size )
-                 asm volatile ( "clflush %0"
-                                : : "m" (((const char *)va)[i]) );
+            cache_flush(va, sz);
         }
         else
         {
@@ -170,3 +168,49 @@ void flush_area_local(const void *va, un
 
     local_irq_restore(irqfl);
 }
+
+void cache_flush(const void *addr, unsigned int size)
+{
+    unsigned int clflush_size = current_cpu_data.x86_clflush_size;
+    const void *end = addr + size;
+
+    if ( !clflush_size )
+    {
+        wbinvd();
+        return;
+    }
+
+    addr -= (unsigned long)addr & (clflush_size - 1);
+    if ( boot_cpu_has(X86_FEATURE_CLFLUSHOPT) )
+    {
+        for ( ; addr < end; addr += clflush_size )
+            asm volatile ( "data16 clflush %[p]" /* clflushopt */
+                           :: [p] "m" (*(const char *)(addr)) );
+        asm volatile ( "sfence" );
+    }
+    else
+        for ( ; addr < end; addr += clflush_size )
+            clflush(addr);
+}
+
+void cache_writeback(const void *addr, unsigned int size)
+{
+    unsigned int clflush_size;
+    const void *end = addr + size;
+
+    /* Fall back to CLFLUSH{,OPT} when CLWB isn't available. */
+    if ( !boot_cpu_has(X86_FEATURE_CLWB) )
+        return cache_flush(addr, size);
+
+    /*
+     * This function may be called before current_cpu_data is established.
+     * Hence a fallback is needed to prevent the loop below becoming infinite.
+     */
+    clflush_size = current_cpu_data.x86_clflush_size ?: 16;
+    addr -= (unsigned long)addr & (clflush_size - 1);
+    for ( ; addr < end; addr += clflush_size )
+        asm volatile ( ".byte 0x66, 0x0f, 0xae, 0x30" /* clwb (%%rax) */
+                       :: "a" (addr), "m" (*(const char *)(addr)) );
+
+    asm volatile ( "sfence" ::: "memory" );
+}
--- a/xen/drivers/passthrough/vtd/extern.h
+++ b/xen/drivers/passthrough/vtd/extern.h
@@ -84,8 +84,6 @@ static inline int dev_invalidate_iotlb(s
 int qinval_device_iotlb(struct iommu *iommu,
                         u32 max_invs_pend, u16 sid, u16 size, u64 addr);
 
-unsigned int get_cache_line_size(void);
-void cacheline_flush(char *);
 void flush_all_cache(void);
 
 u64 alloc_pgtable_maddr(struct acpi_drhd_unit *drhd, unsigned long npages);
--- a/xen/drivers/passthrough/vtd/iommu.c
+++ b/xen/drivers/passthrough/vtd/iommu.c
@@ -32,6 +32,7 @@
 #include <xen/pci.h>
 #include <xen/pci_regs.h>
 #include <xen/keyhandler.h>
+#include <asm/cache.h>
 #include <asm/msi.h>
 #include <asm/irq.h>
 #if defined(__i386__) || defined(__x86_64__)
@@ -180,18 +181,10 @@ static int iommus_incoherent;
 
 static void sync_cache(const void *addr, unsigned int size)
 {
-    static unsigned long clflush_size = 0;
-    const void *end = addr + size;
-
     if ( !iommus_incoherent )
         return;
 
-    if ( clflush_size == 0 )
-        clflush_size = get_cache_line_size();
-
-    addr -= (unsigned long)addr & (clflush_size - 1);
-    for ( ; addr < end; addr += clflush_size )
-        cacheline_flush((char *)addr);
+    cache_writeback(addr, size);
 }
 
 /* Allocate page table, return its machine address */
--- a/xen/drivers/passthrough/vtd/x86/vtd.c
+++ b/xen/drivers/passthrough/vtd/x86/vtd.c
@@ -49,16 +49,6 @@ void unmap_vtd_domain_page(void *va)
     unmap_domain_page(va);
 }
 
-unsigned int get_cache_line_size(void)
-{
-    return ((cpuid_ebx(1) >> 8) & 0xff) * 8;
-}
-
-void cacheline_flush(char * addr)
-{
-    clflush(addr);
-}
-
 void flush_all_cache()
 {
     wbinvd();
--- a/xen/include/asm-x86/cache.h
+++ b/xen/include/asm-x86/cache.h
@@ -12,4 +12,11 @@
 
 #define __read_mostly __section(".data.read_mostly")
 
+#ifndef __ASSEMBLY__
+
+void cache_flush(const void *addr, unsigned int size);
+void cache_writeback(const void *addr, unsigned int size);
+
+#endif
+
 #endif
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -161,6 +161,8 @@
 #define X86_FEATURE_RTM 	(7*32+11) /* Restricted Transactional Memory */
 #define X86_FEATURE_NO_FPU_SEL 	(7*32+13) /* FPU CS/DS stored as zero */
 #define X86_FEATURE_SMAP	(7*32+20) /* Supervisor Mode Access Prevention */
+#define X86_FEATURE_CLFLUSHOPT  (7*32+23) /* CLFLUSHOPT instruction */
+#define X86_FEATURE_CLWB        (7*32+24) /* CLWB instruction */
 
 /* AMD-defined CPU features, CPUID level 0x80000008.ebx, word 8 */
 #define X86_FEATURE_IBPB	(8*32+12) /* IBPB support only (no IBRS, used by AMD) */
