Skip to content

Commit cc88f54

Browse files
Xu Kuohaiborkmann
Xu Kuohai
authored and
borkmann
committed
bpf, arm64: Support sign-extension load instructions
Add JIT support for sign-extension load instructions. Signed-off-by: Xu Kuohai <xukuohai@huawei.com> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Tested-by: Florent Revest <revest@chromium.org> Acked-by: Florent Revest <revest@chromium.org> Link: https://lore.kernel.org/bpf/20230815154158.717901-3-xukuohai@huaweicloud.com
1 parent 6c9f86d commit cc88f54

File tree

2 files changed

+43
-8
lines changed

2 files changed

+43
-8
lines changed

‎arch/arm64/net/bpf_jit.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,13 @@
5959
AARCH64_INSN_LDST_##type##_REG_OFFSET)
6060
#define A64_STRB(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 8, STORE)
6161
#define A64_LDRB(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 8, LOAD)
62+
#define A64_LDRSB(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 8, SIGNED_LOAD)
6263
#define A64_STRH(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 16, STORE)
6364
#define A64_LDRH(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 16, LOAD)
65+
#define A64_LDRSH(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 16, SIGNED_LOAD)
6466
#define A64_STR32(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 32, STORE)
6567
#define A64_LDR32(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 32, LOAD)
68+
#define A64_LDRSW(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 32, SIGNED_LOAD)
6669
#define A64_STR64(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 64, STORE)
6770
#define A64_LDR64(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 64, LOAD)
6871

@@ -73,10 +76,13 @@
7376
AARCH64_INSN_LDST_##type##_IMM_OFFSET)
7477
#define A64_STRBI(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 8, STORE)
7578
#define A64_LDRBI(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 8, LOAD)
79+
#define A64_LDRSBI(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 8, SIGNED_LOAD)
7680
#define A64_STRHI(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 16, STORE)
7781
#define A64_LDRHI(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 16, LOAD)
82+
#define A64_LDRSHI(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 16, SIGNED_LOAD)
7883
#define A64_STR32I(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 32, STORE)
7984
#define A64_LDR32I(Wt, Xn, imm) A64_LS_IMM(Wt, Xn, imm, 32, LOAD)
85+
#define A64_LDRSWI(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 32, SIGNED_LOAD)
8086
#define A64_STR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, STORE)
8187
#define A64_LDR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, LOAD)
8288

‎arch/arm64/net/bpf_jit_comp.c‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
715715
/* First pass */
716716
return 0;
717717

718-
if (BPF_MODE(insn->code) != BPF_PROBE_MEM)
718+
if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
719+
BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
719720
return 0;
720721

721722
if (!ctx->prog->aux->extable ||
@@ -779,6 +780,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
779780
u8 dst_adj;
780781
int off_adj;
781782
int ret;
783+
bool sign_extend;
782784

783785
switch (code) {
784786
/* dst = src */
@@ -1122,7 +1124,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
11221124
return 1;
11231125
}
11241126

1125-
/* LDX: dst = *(size *)(src + off) */
1127+
/* LDX: dst = (u64)*(unsigned size *)(src + off) */
11261128
case BPF_LDX | BPF_MEM | BPF_W:
11271129
case BPF_LDX | BPF_MEM | BPF_H:
11281130
case BPF_LDX | BPF_MEM | BPF_B:
@@ -1131,36 +1133,63 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
11311133
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
11321134
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
11331135
case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1136+
/* LDXS: dst_reg = (s64)*(signed size *)(src_reg + off) */
1137+
case BPF_LDX | BPF_MEMSX | BPF_B:
1138+
case BPF_LDX | BPF_MEMSX | BPF_H:
1139+
case BPF_LDX | BPF_MEMSX | BPF_W:
1140+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1141+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1142+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
11341143
if (ctx->fpb_offset > 0 && src == fp) {
11351144
src_adj = fpb;
11361145
off_adj = off + ctx->fpb_offset;
11371146
} else {
11381147
src_adj = src;
11391148
off_adj = off;
11401149
}
1150+
sign_extend = (BPF_MODE(insn->code) == BPF_MEMSX ||
1151+
BPF_MODE(insn->code) == BPF_PROBE_MEMSX);
11411152
switch (BPF_SIZE(code)) {
11421153
case BPF_W:
11431154
if (is_lsi_offset(off_adj, 2)) {
1144-
emit(A64_LDR32I(dst, src_adj, off_adj), ctx);
1155+
if (sign_extend)
1156+
emit(A64_LDRSWI(dst, src_adj, off_adj), ctx);
1157+
else
1158+
emit(A64_LDR32I(dst, src_adj, off_adj), ctx);
11451159
} else {
11461160
emit_a64_mov_i(1, tmp, off, ctx);
1147-
emit(A64_LDR32(dst, src, tmp), ctx);
1161+
if (sign_extend)
1162+
emit(A64_LDRSW(dst, src_adj, off_adj), ctx);
1163+
else
1164+
emit(A64_LDR32(dst, src, tmp), ctx);
11481165
}
11491166
break;
11501167
case BPF_H:
11511168
if (is_lsi_offset(off_adj, 1)) {
1152-
emit(A64_LDRHI(dst, src_adj, off_adj), ctx);
1169+
if (sign_extend)
1170+
emit(A64_LDRSHI(dst, src_adj, off_adj), ctx);
1171+
else
1172+
emit(A64_LDRHI(dst, src_adj, off_adj), ctx);
11531173
} else {
11541174
emit_a64_mov_i(1, tmp, off, ctx);
1155-
emit(A64_LDRH(dst, src, tmp), ctx);
1175+
if (sign_extend)
1176+
emit(A64_LDRSH(dst, src, tmp), ctx);
1177+
else
1178+
emit(A64_LDRH(dst, src, tmp), ctx);
11561179
}
11571180
break;
11581181
case BPF_B:
11591182
if (is_lsi_offset(off_adj, 0)) {
1160-
emit(A64_LDRBI(dst, src_adj, off_adj), ctx);
1183+
if (sign_extend)
1184+
emit(A64_LDRSBI(dst, src_adj, off_adj), ctx);
1185+
else
1186+
emit(A64_LDRBI(dst, src_adj, off_adj), ctx);
11611187
} else {
11621188
emit_a64_mov_i(1, tmp, off, ctx);
1163-
emit(A64_LDRB(dst, src, tmp), ctx);
1189+
if (sign_extend)
1190+
emit(A64_LDRSB(dst, src, tmp), ctx);
1191+
else
1192+
emit(A64_LDRB(dst, src, tmp), ctx);
11641193
}
11651194
break;
11661195
case BPF_DW:

0 commit comments

Comments
 (0)