Subject: atomic: Define {read,write}_atomic() for reading/writing memory atomically.
From: Keir Fraser keir@xen.org Fri Nov 25 13:31:58 2011 +0000
Date: Fri Nov 25 13:31:58 2011 +0000:
Git: 5e0f79980e4ace10de74923f604ba523ede0ef90

Signed-off-by: Keir Fraser <keir@xen.org>

--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -4638,7 +4638,7 @@ long do_update_descriptor(u64 pa, u64 de
 
     /* All is good so make the update. */
     gdt_pent = map_domain_page(mfn);
-    atomic_write64((uint64_t *)&gdt_pent[offset], *(uint64_t *)&d);
+    write_atomic((uint64_t *)&gdt_pent[offset], *(uint64_t *)&d);
     unmap_domain_page(gdt_pent);
 
     put_page_type(page);
--- a/xen/arch/x86/x86_32/seg_fixup.c
+++ b/xen/arch/x86/x86_32/seg_fixup.c
@@ -314,7 +314,7 @@ static int fixup_seg(u16 seg, unsigned l
     b &= ~0xf0000; b |= limit & 0xf0000;
     b ^= _SEGMENT_EC; /* grows-up <-> grows-down */
     /* NB. This can't fault. Checked readable above; must also be writable. */
-    atomic_write64((uint64_t *)&table[2*idx], ((uint64_t)b<<32) | a);
+    write_atomic((uint64_t *)&table[2*idx], ((uint64_t)b<<32) | a);
     return 1;
 }
 
--- a/xen/include/asm-x86/atomic.h
+++ b/xen/include/asm-x86/atomic.h
@@ -10,36 +10,34 @@
 #define LOCK ""
 #endif
 
-#define build_atomic_read(name, size, type, reg, barrier) \
+#define build_read_atomic(name, size, type, reg, barrier) \
 static inline type name(const volatile type *addr) \
 { type ret; asm volatile("mov" size " %1,%0":reg (ret) \
 :"m" (*(volatile type *)addr) barrier); return ret; }
 
-#define build_atomic_write(name, size, type, reg, barrier) \
+#define build_write_atomic(name, size, type, reg, barrier) \
 static inline void name(volatile type *addr, type val) \
 { asm volatile("mov" size " %1,%0": "=m" (*(volatile type *)addr) \
 :reg (val) barrier); }
 
-build_atomic_read(atomic_read8, "b", uint8_t, "=q", )
-build_atomic_read(atomic_read16, "w", uint16_t, "=r", )
-build_atomic_read(atomic_read32, "l", uint32_t, "=r", )
-build_atomic_read(atomic_read_int, "l", int, "=r", )
-
-build_atomic_write(atomic_write8, "b", uint8_t, "q", )
-build_atomic_write(atomic_write16, "w", uint16_t, "r", )
-build_atomic_write(atomic_write32, "l", uint32_t, "r", )
-build_atomic_write(atomic_write_int, "l", int, "r", )
+build_read_atomic(read_u8_atomic, "b", uint8_t, "=q", )
+build_read_atomic(read_u16_atomic, "w", uint16_t, "=r", )
+build_read_atomic(read_u32_atomic, "l", uint32_t, "=r", )
+
+build_write_atomic(write_u8_atomic, "b", uint8_t, "q", )
+build_write_atomic(write_u16_atomic, "w", uint16_t, "r", )
+build_write_atomic(write_u32_atomic, "l", uint32_t, "r", )
 
 #ifdef __x86_64__
-build_atomic_read(atomic_read64, "q", uint64_t, "=r", )
-build_atomic_write(atomic_write64, "q", uint64_t, "r", )
+build_read_atomic(read_u64_atomic, "q", uint64_t, "=r", )
+build_write_atomic(write_u64_atomic, "q", uint64_t, "r", )
 #else
-static inline uint64_t atomic_read64(const volatile uint64_t *addr)
+static inline uint64_t read_u64_atomic(const volatile uint64_t *addr)
 {
     uint64_t *__addr = (uint64_t *)addr;
     return __cmpxchg8b(__addr, 0, 0);
 }
-static inline void atomic_write64(volatile uint64_t *addr, uint64_t val)
+static inline void write_u64_atomic(volatile uint64_t *addr, uint64_t val)
 {
     uint64_t old = *addr, new, *__addr = (uint64_t *)addr;
     while ( (new = __cmpxchg8b(__addr, old, val)) != old )
@@ -47,8 +45,34 @@ static inline void atomic_write64(volati
 }
 #endif
 
-#undef build_atomic_read
-#undef build_atomic_write
+#undef build_read_atomic
+#undef build_write_atomic
+
+void __bad_atomic_size(void);
+
+#define read_atomic(p) ({                                               \
+    typeof(*p) __x;                                                     \
+    switch ( sizeof(*p) ) {                                             \
+    case 1: __x = (typeof(*p))read_u8_atomic((uint8_t *)p); break;      \
+    case 2: __x = (typeof(*p))read_u16_atomic((uint16_t *)p); break;    \
+    case 4: __x = (typeof(*p))read_u32_atomic((uint32_t *)p); break;    \
+    case 8: __x = (typeof(*p))read_u64_atomic((uint64_t *)p); break;    \
+    default: __x = 0; __bad_atomic_size(); break;                       \
+    }                                                                   \
+    __x;                                                                \
+})
+
+#define write_atomic(p, x) ({                                           \
+    typeof(*p) __x = (x);                                               \
+    switch ( sizeof(*p) ) {                                             \
+    case 1: write_u8_atomic((uint8_t *)p, (uint8_t)__x); break;         \
+    case 2: write_u16_atomic((uint16_t *)p, (uint16_t)__x); break;      \
+    case 4: write_u32_atomic((uint32_t *)p, (uint32_t)__x); break;      \
+    case 8: write_u64_atomic((uint64_t *)p, (uint64_t)__x); break;      \
+    default: __bad_atomic_size(); break;                                \
+    }                                                                   \
+    __x;                                                                \
+})
 
 /*
  * NB. I've pushed the volatile qualifier into the operations. This allows
@@ -66,7 +90,7 @@ typedef struct { int counter; } atomic_t
  * Atomically reads the value of @v.
  */
 #define _atomic_read(v)		((v).counter)
-#define atomic_read(v)		atomic_read_int(&((v)->counter))
+#define atomic_read(v)		read_atomic(&((v)->counter))
 
 /**
  * atomic_set - set atomic variable
@@ -76,7 +100,7 @@ typedef struct { int counter; } atomic_t
  * Atomically sets the value of @v to @i.
  */ 
 #define _atomic_set(v,i)	(((v).counter) = (i))
-#define atomic_set(v,i)		atomic_write_int(&((v)->counter), (i))
+#define atomic_set(v,i)		write_atomic(&((v)->counter), (i))
 
 /**
  * atomic_add - add integer to atomic variable
--- a/xen/include/asm-x86/x86_32/page.h
+++ b/xen/include/asm-x86/x86_32/page.h
@@ -79,15 +79,15 @@ extern unsigned int PAGE_HYPERVISOR_NOCA
 
 #endif
 
-#define pte_read_atomic(ptep)       atomic_read64(ptep)
-#define pte_write_atomic(ptep, pte) atomic_write64(ptep, pte)
-#define pte_write(ptep, pte) do {                             \
-    u32 *__ptep_words = (u32 *)(ptep);                        \
-    atomic_write32(&__ptep_words[0], 0);                      \
-    wmb();                                                    \
-    atomic_write32(&__ptep_words[1], (pte) >> 32);            \
-    wmb();                                                    \
-    atomic_write32(&__ptep_words[0], (pte) >>  0);            \
+#define pte_read_atomic(ptep)       read_atomic(ptep)
+#define pte_write_atomic(ptep, pte) write_atomic(ptep, pte)
+#define pte_write(ptep, pte) do {                   \
+    u32 *__ptep_words = (u32 *)(ptep);              \
+    write_atomic(&__ptep_words[0], 0);              \
+    wmb();                                          \
+    write_atomic(&__ptep_words[1], (pte) >> 32);    \
+    wmb();                                          \
+    write_atomic(&__ptep_words[0], (pte) >>  0);    \
 } while ( 0 )
 
 /* root table */
--- a/xen/include/asm-x86/x86_64/page.h
+++ b/xen/include/asm-x86/x86_64/page.h
@@ -101,9 +101,9 @@ typedef l4_pgentry_t root_pgentry_t;
 
 #endif /* !__ASSEMBLY__ */
 
-#define pte_read_atomic(ptep)       atomic_read64(ptep)
-#define pte_write_atomic(ptep, pte) atomic_write64(ptep, pte)
-#define pte_write(ptep, pte)        atomic_write64(ptep, pte)
+#define pte_read_atomic(ptep)       read_atomic(ptep)
+#define pte_write_atomic(ptep, pte) write_atomic(ptep, pte)
+#define pte_write(ptep, pte)        write_atomic(ptep, pte)
 
 /* Given a virtual address, get an entry offset into a linear page table. */
 #define l1_linear_offset(_a) (((_a) & VADDR_MASK) >> L1_PAGETABLE_SHIFT)
