@@ -313,6 +313,35 @@ do { \
#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif
+#ifndef __SIZEOF_INT128__
+#define raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) \
+({ \
+ typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
+ typeof(pcp) __val = *__p, __old = *(ovalp); \
+ bool __ret; \
+ if (!__builtin_memcmp(&__val, &__old, sizeof(pcp))) { \
+ *__p = nval; \
+ __ret = true; \
+ } else { \
+ *(ovalp) = __val; \
+ __ret = false; \
+ } \
+ __ret; \
+})
+
+#define raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) \
+({ \
+ typeof(pcp) __old = (oval); \
+ raw_cpu_generic_try_cmpxchg_memcpy(pcp, &__old, nval); \
+ __old; \
+})
+
+#define raw_cpu_cmpxchg128(pcp, oval, nval) \
+ raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval)
+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval)
+#endif
+
#ifndef raw_cpu_try_cmpxchg_1
#ifdef raw_cpu_cmpxchg_1
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
@@ -503,6 +532,33 @@ do { \
#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif
+#ifndef __SIZEOF_INT128__
+#define this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) \
+({ \
+ bool __ret; \
+ unsigned long __flags; \
+ raw_local_irq_save(__flags); \
+ __ret = raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval); \
+ raw_local_irq_restore(__flags); \
+ __ret; \
+})
+
+#define this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) \
+({ \
+ typeof(pcp) __ret; \
+ unsigned long __flags; \
+ raw_local_irq_save(__flags); \
+ __ret = raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval); \
+ raw_local_irq_restore(__flags); \
+ __ret; \
+})
+
+#define this_cpu_cmpxchg128(pcp, oval, nval) \
+ this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval)
+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval)
+#endif
+
#ifndef this_cpu_try_cmpxchg_1
#ifdef this_cpu_cmpxchg_1
#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
@@ -13,6 +13,13 @@
#ifdef __SIZEOF_INT128__
typedef __s128 s128;
typedef __u128 u128;
+#else
+#ifdef CONFIG_64BIT
+/* hack for this_cpu_cmpxchg128 */
+typedef struct {
+ u64 a, b;
+} u128 __attribute__((aligned(16)));
+#endif
#endif
typedef u32 __kernel_dev_t;
Some 64bit architectures do not advertise __SIZEOF_INT128__ on all supported compiler versions. Notably the HPPA64 only started doing with GCC-11. Since the per-cpu ops are universally availably, and this_cpu_{,try_}cmpxchg128() is expected to be available on all 64bit architectures a wee bodge is in order. Sadly, while C reverts to memcpy() for assignment of POD types, it does not revert to memcmp() for for equality. Therefore frob that manually. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- include/asm-generic/percpu.h | 56 +++++++++++++++++++++++++++++++++++++++++++ include/linux/types.h | 7 +++++ 2 files changed, 63 insertions(+)