Skip to content

Add support arena atomics for RV64 #9329

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: bpf-next_base
Choose a base branch
from
+214 −389
Open

Add support arena atomics for RV64 #9329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cc9aeb9
riscv, bpf: Extract emit_stx() helper
Jul 19, 2025
4fa4d58
riscv, bpf: Extract emit_st() helper
Jul 19, 2025
39fb93d
riscv, bpf: Extract emit_ldx() helper
Jul 19, 2025
da61af8
riscv: Separate toolchain support dependency from RISCV_ISA_ZACAS
Jul 19, 2025
7e91c2c
riscv, bpf: Add rv_ext_enabled macro for runtime detection extentsion
Jul 19, 2025
298a7f9
riscv, bpf: Add Zacas instructions
Jul 19, 2025
147e460
riscv, bpf: Optimize cmpxchg insn with Zacas support
Jul 19, 2025
9fca13c
riscv, bpf: Add ex_insn_off and ex_jmp_off for exception table handling
Jul 19, 2025
b999ed9
riscv, bpf: Add support arena atomics for RV64
Jul 19, 2025
bc2eece
selftests/bpf: Enable arena atomics tests for RV64
Jul 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion arch/riscv/Kconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,7 +717,6 @@ config TOOLCHAIN_HAS_ZACAS

config RISCV_ISA_ZACAS
bool "Zacas extension support for atomic CAS"
depends on TOOLCHAIN_HAS_ZACAS
depends on RISCV_ALTERNATIVE
default y
help
Expand Down
6 changes: 4 additions & 2 deletions arch/riscv/include/asm/cmpxchg.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
({ \
if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \
riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \
riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
r = o; \
Expand Down Expand Up @@ -180,6 +181,7 @@
r, p, co, o, n) \
({ \
if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \
riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
r = o; \
\
Expand Down Expand Up @@ -315,7 +317,7 @@
arch_cmpxchg_release((ptr), (o), (n)); \
})

#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS)
#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS) && defined(CONFIG_TOOLCHAIN_HAS_ZACAS)

#define system_has_cmpxchg128() riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)

Expand Down Expand Up @@ -351,7 +353,7 @@ union __u128_halves {
#define arch_cmpxchg128_local(ptr, o, n) \
__arch_cmpxchg128((ptr), (o), (n), "")

#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS */
#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS && CONFIG_TOOLCHAIN_HAS_ZACAS */

#ifdef CONFIG_RISCV_ISA_ZAWRS
/*
Expand Down
1 change: 1 addition & 0 deletions arch/riscv/kernel/setup.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ static void __init riscv_spinlock_init(void)

if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) &&
IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&
IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) &&
riscv_isa_extension_available(NULL, ZABHA) &&
riscv_isa_extension_available(NULL, ZACAS)) {
using_ext = "using Zabha";
Expand Down
70 changes: 53 additions & 17 deletions arch/riscv/net/bpf_jit.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,15 @@
#include <linux/filter.h>
#include <asm/cacheflush.h>

/* verify runtime detection extension status */
#define rv_ext_enabled(ext) \
(IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext))

static inline bool rvc_enabled(void)
{
return IS_ENABLED(CONFIG_RISCV_ISA_C);
}

static inline bool rvzba_enabled(void)
{
return IS_ENABLED(CONFIG_RISCV_ISA_ZBA) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBA);
}

static inline bool rvzbb_enabled(void)
{
return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB);
}

enum {
RV_REG_ZERO = 0, /* The constant value 0 */
RV_REG_RA = 1, /* Return address */
Expand Down Expand Up @@ -84,6 +78,8 @@ struct rv_jit_context {
int epilogue_offset;
int *offset; /* BPF to RV */
int nexentries;
int ex_insn_off;
int ex_jmp_off;
unsigned long flags;
int stack_size;
u64 arena_vm_start;
Expand Down Expand Up @@ -757,6 +753,17 @@ static inline u16 rvc_swsp(u32 imm8, u8 rs2)
return rv_css_insn(0x6, imm, rs2, 0x2);
}

/* RVZACAS instructions. */
static inline u32 rvzacas_amocas_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
{
return rv_amo_insn(0x5, aq, rl, rs2, rs1, 2, rd, 0x2f);
}

static inline u32 rvzacas_amocas_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
{
return rv_amo_insn(0x5, aq, rl, rs2, rs1, 3, rd, 0x2f);
}

/* RVZBA instructions. */
static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2)
{
Expand Down Expand Up @@ -1123,7 +1130,7 @@ static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)

static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
{
if (rvzba_enabled()) {
if (rv_ext_enabled(ZBA)) {
emit(rvzba_sh2add(rd, rs1, rs2), ctx);
return;
}
Expand All @@ -1134,7 +1141,7 @@ static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx

static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
{
if (rvzba_enabled()) {
if (rv_ext_enabled(ZBA)) {
emit(rvzba_sh3add(rd, rs1, rs2), ctx);
return;
}
Expand Down Expand Up @@ -1184,7 +1191,7 @@ static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)

static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx)
{
if (rvzbb_enabled()) {
if (rv_ext_enabled(ZBB)) {
emit(rvzbb_sextb(rd, rs), ctx);
return;
}
Expand All @@ -1195,7 +1202,7 @@ static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx)

static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
{
if (rvzbb_enabled()) {
if (rv_ext_enabled(ZBB)) {
emit(rvzbb_sexth(rd, rs), ctx);
return;
}
Expand All @@ -1211,7 +1218,7 @@ static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx)

static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
{
if (rvzbb_enabled()) {
if (rv_ext_enabled(ZBB)) {
emit(rvzbb_zexth(rd, rs), ctx);
return;
}
Expand All @@ -1222,7 +1229,7 @@ static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx)

static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
{
if (rvzba_enabled()) {
if (rv_ext_enabled(ZBA)) {
emit(rvzba_zextw(rd, rs), ctx);
return;
}
Expand All @@ -1233,7 +1240,7 @@ static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx)

static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
{
if (rvzbb_enabled()) {
if (rv_ext_enabled(ZBB)) {
int bits = 64 - imm;

emit(rvzbb_rev8(rd, rd), ctx);
Expand Down Expand Up @@ -1289,6 +1296,35 @@ static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
emit_mv(rd, RV_REG_T2, ctx);
}

static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx)
{
int jmp_offset;

if (rv_ext_enabled(ZACAS)) {
ctx->ex_insn_off = ctx->ninsns;
emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) :
rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx);
ctx->ex_jmp_off = ctx->ninsns;
if (!is64)
emit_zextw(r0, r0, ctx);
return;
}

if (is64)
emit_mv(RV_REG_T2, r0, ctx);
else
emit_addiw(RV_REG_T2, r0, 0, ctx);
emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
rv_lr_w(r0, 0, rd, 0, 0), ctx);
jmp_offset = ninsns_rvoff(8);
emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
jmp_offset = ninsns_rvoff(-6);
emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
emit_fence_rw_rw(ctx);
}

#endif /* __riscv_xlen == 64 */

void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);
Expand Down
Loading
Loading