Skip to content

Commit a9ce3a5

Browse files
liu-song-6kernel-patches-bot
liu-song-6
authored and
kernel-patches-bot
committed
bpf: enable BPF_PROG_TEST_RUN for raw_tracepoint
Add .test_run for raw_tracepoint. Also, introduce a new feature that runs the target program on a specific CPU. This is achieved by a new flag in bpf_attr.test, BPF_F_TEST_RUN_ON_CPU. When this flag is set, the program is triggered on cpu with id bpf_attr.test.cpu. This feature is needed for BPF programs that handle perf_event and other percpu resources, as the program can access these resource locally. Acked-by: John Fastabend <john.fastabend@gmail.com> Acked-by: Andrii Nakryiko <andriin@fb.com> Signed-off-by: Song Liu <songliubraving@fb.com>
1 parent a79ecc1 commit a9ce3a5

File tree

6 files changed

+110
-1
lines changed

6 files changed

+110
-1
lines changed

‎include/linux/bpf.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1396,6 +1396,9 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
13961396
int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
13971397
const union bpf_attr *kattr,
13981398
union bpf_attr __user *uattr);
1399+
int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
1400+
const union bpf_attr *kattr,
1401+
union bpf_attr __user *uattr);
13991402
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
14001403
const struct bpf_prog *prog,
14011404
struct bpf_insn_access_aux *info);

‎include/uapi/linux/bpf.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,11 @@ enum {
424424
*/
425425
#define BPF_F_QUERY_EFFECTIVE (1U << 0)
426426

427+
/* Flags for BPF_PROG_TEST_RUN */
428+
429+
/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
430+
#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
431+
427432
/* type for BPF_ENABLE_STATS */
428433
enum bpf_stats_type {
429434
/* enabled run_time_ns and run_cnt */
@@ -566,6 +571,8 @@ union bpf_attr {
566571
*/
567572
__aligned_u64 ctx_in;
568573
__aligned_u64 ctx_out;
574+
__u32 flags;
575+
__u32 cpu;
569576
} test;
570577

571578
struct { /* anonymous struct used by BPF_*_GET_*_ID */

‎kernel/bpf/syscall.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2979,7 +2979,7 @@ static int bpf_prog_query(const union bpf_attr *attr,
29792979
}
29802980
}
29812981

2982-
#define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out
2982+
#define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
29832983

29842984
static int bpf_prog_test_run(const union bpf_attr *attr,
29852985
union bpf_attr __user *uattr)

‎kernel/trace/bpf_trace.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1678,6 +1678,7 @@ const struct bpf_verifier_ops raw_tracepoint_verifier_ops = {
16781678
};
16791679

16801680
const struct bpf_prog_ops raw_tracepoint_prog_ops = {
1681+
.test_run = bpf_prog_test_run_raw_tp,
16811682
};
16821683

16831684
const struct bpf_verifier_ops tracing_verifier_ops = {

‎net/bpf/test_run.c‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <net/sock.h>
1212
#include <net/tcp.h>
1313
#include <linux/error-injection.h>
14+
#include <linux/smp.h>
1415

1516
#define CREATE_TRACE_POINTS
1617
#include <trace/events/bpf_test_run.h>
@@ -204,6 +205,9 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
204205
int b = 2, err = -EFAULT;
205206
u32 retval = 0;
206207

208+
if (kattr->test.flags || kattr->test.cpu)
209+
return -EINVAL;
210+
207211
switch (prog->expected_attach_type) {
208212
case BPF_TRACE_FENTRY:
209213
case BPF_TRACE_FEXIT:
@@ -236,6 +240,87 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
236240
return err;
237241
}
238242

243+
struct bpf_raw_tp_test_run_info {
244+
struct bpf_prog *prog;
245+
void *ctx;
246+
u32 retval;
247+
};
248+
249+
static void
250+
__bpf_prog_test_run_raw_tp(void *data)
251+
{
252+
struct bpf_raw_tp_test_run_info *info = data;
253+
254+
rcu_read_lock();
255+
migrate_disable();
256+
info->retval = BPF_PROG_RUN(info->prog, info->ctx);
257+
migrate_enable();
258+
rcu_read_unlock();
259+
}
260+
261+
int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
262+
const union bpf_attr *kattr,
263+
union bpf_attr __user *uattr)
264+
{
265+
void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
266+
__u32 ctx_size_in = kattr->test.ctx_size_in;
267+
struct bpf_raw_tp_test_run_info info;
268+
int cpu = kattr->test.cpu, err = 0;
269+
270+
/* doesn't support data_in/out, ctx_out, duration, or repeat */
271+
if (kattr->test.data_in || kattr->test.data_out ||
272+
kattr->test.ctx_out || kattr->test.duration ||
273+
kattr->test.repeat)
274+
return -EINVAL;
275+
276+
if (ctx_size_in < prog->aux->max_ctx_offset)
277+
return -EINVAL;
278+
279+
if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
280+
return -EINVAL;
281+
282+
if (ctx_size_in) {
283+
info.ctx = kzalloc(ctx_size_in, GFP_USER);
284+
if (!info.ctx)
285+
return -ENOMEM;
286+
if (copy_from_user(info.ctx, ctx_in, ctx_size_in)) {
287+
err = -EFAULT;
288+
goto out;
289+
}
290+
} else {
291+
info.ctx = NULL;
292+
}
293+
294+
info.prog = prog;
295+
296+
if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
297+
cpu == smp_processor_id()) {
298+
__bpf_prog_test_run_raw_tp(&info);
299+
} else {
300+
/* smp_call_function_single() also checks cpu_online()
301+
* after csd_lock(). However, since cpu is from user
302+
* space, let's do an extra quick check to filter out
303+
* invalid value before smp_call_function_single().
304+
*/
305+
if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
306+
err = -ENXIO;
307+
goto out;
308+
}
309+
310+
err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
311+
&info, 1);
312+
if (err)
313+
goto out;
314+
}
315+
316+
if (copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
317+
err = -EFAULT;
318+
319+
out:
320+
kfree(info.ctx);
321+
return err;
322+
}
323+
239324
static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
240325
{
241326
void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
@@ -410,6 +495,9 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
410495
void *data;
411496
int ret;
412497

498+
if (kattr->test.flags || kattr->test.cpu)
499+
return -EINVAL;
500+
413501
data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
414502
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
415503
if (IS_ERR(data))
@@ -607,6 +695,9 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
607695
if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
608696
return -EINVAL;
609697

698+
if (kattr->test.flags || kattr->test.cpu)
699+
return -EINVAL;
700+
610701
if (size < ETH_HLEN)
611702
return -EINVAL;
612703

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,11 @@ enum {
424424
*/
425425
#define BPF_F_QUERY_EFFECTIVE (1U << 0)
426426

427+
/* Flags for BPF_PROG_TEST_RUN */
428+
429+
/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
430+
#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
431+
427432
/* type for BPF_ENABLE_STATS */
428433
enum bpf_stats_type {
429434
/* enabled run_time_ns and run_cnt */
@@ -566,6 +571,8 @@ union bpf_attr {
566571
*/
567572
__aligned_u64 ctx_in;
568573
__aligned_u64 ctx_out;
574+
__u32 flags;
575+
__u32 cpu;
569576
} test;
570577

571578
struct { /* anonymous struct used by BPF_*_GET_*_ID */

0 commit comments

Comments
 (0)