From 389f282a8f1940d2a22cd5a6501254869af70e3e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Jun 2024 15:09:05 -1000 Subject: [PATCH] tools/sched_ext: Sync from scx repo (d3b34d1df7aabb66244607f1473ecc5774b5c9b3) --- tools/sched_ext/include/scx/common.bpf.h | 142 ++++++++++++++----- tools/sched_ext/include/scx/common.h | 17 +-- tools/sched_ext/include/scx/user_exit_info.h | 8 +- tools/sched_ext/scx_central.c | 4 +- tools/sched_ext/scx_qmap.bpf.c | 6 +- 5 files changed, 123 insertions(+), 54 deletions(-) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index d0b708e959c1f..818c0febcb156 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -235,50 +235,15 @@ BPF_PROG(name, ##args) __addr; \ }) + /* - * BPF core and other generic helpers + * BPF declarations and helpers */ /* list and rbtree */ #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) -/* - * bpf_log2 - Compute the base 2 logarithm of a 32-bit exponential value. - * @v: The value for which we're computing the base 2 logarithm. - */ -static inline u32 bpf_log2(u32 v) -{ - u32 r; - u32 shift; - - r = (v > 0xFFFF) << 4; v >>= r; - shift = (v > 0xFF) << 3; v >>= shift; r |= shift; - shift = (v > 0xF) << 2; v >>= shift; r |= shift; - shift = (v > 0x3) << 1; v >>= shift; r |= shift; - r |= (v >> 1); - return r; -} - -/* - * bpf_log2l - Compute the base 2 logarithm of a 64-bit exponential value. - * @v: The value for which we're computing the base 2 logarithm. - */ -static inline u32 bpf_log2l(u64 v) -{ - u32 hi = v >> 32; - if (hi) - return bpf_log2(hi) + 32 + 1; - else - return bpf_log2(v) + 1; -} - -/* useful compiler attributes */ -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#define __maybe_unused __attribute__((__unused__)) - - void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; @@ -311,6 +276,16 @@ struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym; void bpf_cgroup_release(struct cgroup *cgrp) __ksym; struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym; +/* css iteration */ +struct bpf_iter_css; +struct cgroup_subsys_state; +extern int bpf_iter_css_new(struct bpf_iter_css *it, + struct cgroup_subsys_state *start, + unsigned int flags) __weak __ksym; +extern struct cgroup_subsys_state * +bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym; +extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym; + /* cpumask */ struct bpf_cpumask *bpf_cpumask_create(void) __ksym; struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym; @@ -344,6 +319,99 @@ u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; + +/* + * Other helpers + */ + +/* useful compiler attributes */ +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#define __maybe_unused __attribute__((__unused__)) + +/* + * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They + * prevent compiler from caching, redoing or reordering reads or writes. + */ +typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; +typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; +typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; +typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; + +static __always_inline void __read_once_size(const volatile void *p, void *res, int size) +{ + switch (size) { + case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break; + case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break; + case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break; + case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break; + default: + barrier(); + __builtin_memcpy((void *)res, (const void *)p, size); + barrier(); + } +} + +static __always_inline void __write_once_size(volatile void *p, void *res, int size) +{ + switch (size) { + case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break; + case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break; + case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break; + case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break; + default: + barrier(); + __builtin_memcpy((void *)p, (const void *)res, size); + barrier(); + } +} + +#define READ_ONCE(x) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u = \ + { .__c = { 0 } }; \ + __read_once_size(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) + +#define WRITE_ONCE(x, val) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u = \ + { .__val = (val) }; \ + __write_once_size(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) + +/* + * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value. + * @v: The value for which we're computing the base 2 logarithm. + */ +static inline u32 log2_u32(u32 v) +{ + u32 r; + u32 shift; + + r = (v > 0xFFFF) << 4; v >>= r; + shift = (v > 0xFF) << 3; v >>= shift; r |= shift; + shift = (v > 0xF) << 2; v >>= shift; r |= shift; + shift = (v > 0x3) << 1; v >>= shift; r |= shift; + r |= (v >> 1); + return r; +} + +/* + * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value. + * @v: The value for which we're computing the base 2 logarithm. + */ +static inline u32 log2_u64(u64 v) +{ + u32 hi = v >> 32; + if (hi) + return log2_u32(hi) + 32 + 1; + else + return log2_u32(v) + 1; +} + #include "compat.bpf.h" #endif /* __SCX_COMMON_BPF_H */ diff --git a/tools/sched_ext/include/scx/common.h b/tools/sched_ext/include/scx/common.h index 8d5a6775f64d9..49234dad00a15 100644 --- a/tools/sched_ext/include/scx/common.h +++ b/tools/sched_ext/include/scx/common.h @@ -44,7 +44,8 @@ typedef int64_t s64; /** * RESIZE_ARRAY - Convenience macro for resizing a BPF array - * @elfsec: the data section of the BPF program in which to the array exists + * @__skel: the skeleton containing the array + * @elfsec: the data section of the BPF program in which the array exists * @arr: the name of the array * @n: the desired array element count * @@ -56,13 +57,13 @@ typedef int64_t s64; * for that custom data section so that it points to the newly memory mapped * region. */ -#define RESIZE_ARRAY(elfsec, arr, n) \ - do { \ - size_t __sz; \ - bpf_map__set_value_size(skel->maps.elfsec##_##arr, \ - sizeof(skel->elfsec##_##arr->arr[0]) * (n)); \ - skel->elfsec##_##arr = \ - bpf_map__initial_value(skel->maps.elfsec##_##arr, &__sz); \ +#define RESIZE_ARRAY(__skel, elfsec, arr, n) \ + do { \ + size_t __sz; \ + bpf_map__set_value_size((__skel)->maps.elfsec##_##arr, \ + sizeof((__skel)->elfsec##_##arr->arr[0]) * (n)); \ + (__skel)->elfsec##_##arr = \ + bpf_map__initial_value((__skel)->maps.elfsec##_##arr, &__sz); \ } while (0) #include "user_exit_info.h" diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h index 2d86d01a95752..891693ee604ec 100644 --- a/tools/sched_ext/include/scx/user_exit_info.h +++ b/tools/sched_ext/include/scx/user_exit_info.h @@ -53,10 +53,10 @@ struct user_exit_info { #include /* no need to call the following explicitly if SCX_OPS_LOAD() is used */ -#define UEI_SET_SIZE(__skel, __ops_name, __uei_name) ({ \ - u32 __len = (__skel)->struct_ops.__ops_name->exit_dump_len ?: UEI_DUMP_DFL_LEN; \ - (__skel)->rodata->__uei_name##_dump_len = __len; \ - RESIZE_ARRAY(data, __uei_name##_dump, __len); \ +#define UEI_SET_SIZE(__skel, __ops_name, __uei_name) ({ \ + u32 __len = (__skel)->struct_ops.__ops_name->exit_dump_len ?: UEI_DUMP_DFL_LEN; \ + (__skel)->rodata->__uei_name##_dump_len = __len; \ + RESIZE_ARRAY((__skel), data, __uei_name##_dump, __len); \ }) #define UEI_EXITED(__skel, __uei_name) ({ \ diff --git a/tools/sched_ext/scx_central.c b/tools/sched_ext/scx_central.c index 1e0568624ccc2..21deea320bd78 100644 --- a/tools/sched_ext/scx_central.c +++ b/tools/sched_ext/scx_central.c @@ -77,8 +77,8 @@ int main(int argc, char **argv) } /* Resize arrays so their element count is equal to cpu count. */ - RESIZE_ARRAY(data, cpu_gimme_task, skel->rodata->nr_cpu_ids); - RESIZE_ARRAY(data, cpu_started_at, skel->rodata->nr_cpu_ids); + RESIZE_ARRAY(skel, data, cpu_gimme_task, skel->rodata->nr_cpu_ids); + RESIZE_ARRAY(skel, data, cpu_started_at, skel->rodata->nr_cpu_ids); SCX_OPS_LOAD(skel, central_ops, scx_central, uei); diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index d74c5cf2a2517..e22f190f52249 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -579,7 +579,7 @@ struct { __uint(max_entries, 1); __type(key, u32); __type(value, struct monitor_timer); -} central_timer SEC(".maps"); +} monitor_timer SEC(".maps"); /* * Print out the min, avg and max performance levels of CPUs every second to @@ -692,11 +692,11 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) if (ret) return ret; - timer = bpf_map_lookup_elem(¢ral_timer, &key); + timer = bpf_map_lookup_elem(&monitor_timer, &key); if (!timer) return -ESRCH; - bpf_timer_init(timer, ¢ral_timer, CLOCK_MONOTONIC); + bpf_timer_init(timer, &monitor_timer, CLOCK_MONOTONIC); bpf_timer_set_callback(timer, monitor_timerfn); return bpf_timer_start(timer, ONE_SEC_IN_NS, 0);