diff options
| author | George Guo <guodongtai@kylinos.cn> | 2026-02-10 19:31:12 +0800 |
|---|---|---|
| committer | Huacai Chen <chenhuacai@loongson.cn> | 2026-02-10 19:31:12 +0800 |
| commit | f0e4b1b6e295299f5c9c79ad546dedbdf7132f45 (patch) | |
| tree | bf156aa60e2867ce7025bdeb17768fb6dbaea0c7 /arch/loongarch/include | |
| parent | 48543c4283e76d561d11b9955222b1a3054abdb9 (diff) | |
LoongArch: Add 128-bit atomic cmpxchg support
Implement 128-bit atomic compare-and-exchange using LoongArch's
LL.D/SC.Q instructions.
At the same time, this fix the BPF scheduler test failures (scx_central
and scx_qmap) caused by kmalloc_nolock_noprof() returning NULL, due to
missing 128-bit atomics. The NULL returns lead to -ENOMEM errors during
scheduler initialization, causing test cases to fail.
Verified by testing with the scx_qmap scheduler
(located in tools/sched_ext/).
Building with `make` and running ./tools/sched_ext/build/bin/scx_qmap.
Link: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/commit/?id=5fb750e8a9ae
Acked-by: Hengqi Chen <hengqi.chen@gmail.com>
Tested-by: Hengqi Chen <hengqi.chen@gmail.com>
Co-developed-by:: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
Diffstat (limited to 'arch/loongarch/include')
| -rw-r--r-- | arch/loongarch/include/asm/cmpxchg.h | 54 |
1 files changed, 54 insertions, 0 deletions
diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h index 0494c2ab553e..58cabab6d90d 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -8,6 +8,7 @@ #include <linux/bits.h> #include <linux/build_bug.h> #include <asm/barrier.h> +#include <asm/cpu-features.h> #define __xchg_amo_asm(amswap_db, m, val) \ ({ \ @@ -236,6 +237,59 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg((ptr), (o), (n)); \ }) + +union __u128_halves { + u128 full; + struct { + u64 low; + u64 high; + }; +}; + +#define system_has_cmpxchg128() cpu_opt(LOONGARCH_CPU_SCQ) + +#define __arch_cmpxchg128(ptr, old, new, llsc_mb) \ +({ \ + union __u128_halves __old, __new, __ret; \ + volatile u64 *__ptr = (volatile u64 *)(ptr); \ + \ + __old.full = (old); \ + __new.full = (new); \ + \ + __asm__ __volatile__( \ + "1: ll.d %0, %3 # 128-bit cmpxchg low \n" \ + llsc_mb \ + " ld.d %1, %4 # 128-bit cmpxchg high \n" \ + " move $t0, %0 \n" \ + " move $t1, %1 \n" \ + " bne %0, %z5, 2f \n" \ + " bne %1, %z6, 2f \n" \ + " move $t0, %z7 \n" \ + " move $t1, %z8 \n" \ + "2: sc.q $t0, $t1, %2 \n" \ + " beqz $t0, 1b \n" \ + llsc_mb \ + : "=&r" (__ret.low), "=&r" (__ret.high) \ + : "r" (__ptr), \ + "ZC" (__ptr[0]), "m" (__ptr[1]), \ + "Jr" (__old.low), "Jr" (__old.high), \ + "Jr" (__new.low), "Jr" (__new.high) \ + : "t0", "t1", "memory"); \ + \ + __ret.full; \ +}) + +#define arch_cmpxchg128(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ + __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB); \ +}) + +#define arch_cmpxchg128_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ + __arch_cmpxchg128(ptr, o, n, ""); \ +}) #else #include <asm-generic/cmpxchg-local.h> #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) |