From 8621495f785a0cb525fdfa6bc7717900a286ee2b Mon Sep 17 00:00:00 2001 From: Nicholas Sun Date: Sat, 20 Jul 2024 03:31:07 +0800 Subject: [PATCH] v23.05.4 build test Fresh build workflows Backport bpf-loop to kernel k5.15 Remove PPPoE offload Signed-off-by: Nicholas Sun --- .github/workflows/R2S-OpenWrt.yml | 37 +- .github/workflows/R4S-OpenWrt.yml | 37 +- .github/workflows/X86-OpenWrt.yml | 37 +- ...s-of-BPF_CAST_CALL-with-BPF_CALL_IMM.patch | 160 +++ ...ST_CALL-with-proper-function-typedef.patch | 150 ++ .../070-v5.17-01-Add-bpf_loop-helper.patch | 258 ++++ ...7-02-selftests-bpf-Add-bpf_loop-test.patch | 277 ++++ ...easure-bpf_loop-verifier-performance.patch | 302 ++++ ...elftests-bpf-Extract-syscall-wrapper.patch | 44 + ...rchitectures-without-syscall-wrapper.patch | 26 + ...-arch-dependent-syscall-entry-points.patch | 177 +++ ...s-to-bpf_loop-when-callback-is-known.patch | 1245 +++++++++++++++++ .../net/01-maximize_nic_rx_tx_buffers | 41 +- SCRIPTS/02_prepare_package.sh | 7 +- SCRIPTS/08_fix_permissions.sh | 32 + SCRIPTS/R2S/02_R2S.sh | 1 - SCRIPTS/R4S/02_R4S.sh | 1 - SCRIPTS/X86/02_X86.sh | 1 - SEED/R2S/config.seed | 1 + SEED/R4S/config.seed | 1 + SEED/X86/config.seed | 1 + 21 files changed, 2770 insertions(+), 66 deletions(-) create mode 100644 PATCH/bpf_loop/069-v5.16-01-bpf-Replace-want-address-users-of-BPF_CAST_CALL-with-BPF_CALL_IMM.patch create mode 100644 PATCH/bpf_loop/069-v5.16-02-bpf-Replace-callers-of-BPF_CAST_CALL-with-proper-function-typedef.patch create mode 100644 PATCH/bpf_loop/070-v5.17-01-Add-bpf_loop-helper.patch create mode 100644 PATCH/bpf_loop/070-v5.17-02-selftests-bpf-Add-bpf_loop-test.patch create mode 100644 PATCH/bpf_loop/070-v5.17-03-selftests-bpf-measure-bpf_loop-verifier-performance.patch create mode 100644 PATCH/bpf_loop/070-v5.17-04-selftests-bpf-Extract-syscall-wrapper.patch create mode 100644 PATCH/bpf_loop/070-v5.17-05-selftests-bpf-Use-_se-prefix-on-architectures-without-syscall-wrapper.patch create mode 100644 PATCH/bpf_loop/070-v5.17-06-selftests-bpf-Fix-tests-to-use-arch-dependent-syscall-entry-points.patch create mode 100644 PATCH/bpf_loop/071-v5.17-bpf-Inline-calls-to-bpf_loop-when-callback-is-known.patch mode change 100755 => 100644 PATCH/duplicate/files/etc/hotplug.d/net/01-maximize_nic_rx_tx_buffers create mode 100644 SCRIPTS/08_fix_permissions.sh diff --git a/.github/workflows/R2S-OpenWrt.yml b/.github/workflows/R2S-OpenWrt.yml index cd9734f5..a48fb5ff 100644 --- a/.github/workflows/R2S-OpenWrt.yml +++ b/.github/workflows/R2S-OpenWrt.yml @@ -14,7 +14,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.event.repository.owner.id == github.event.sender.id steps: @@ -39,14 +39,11 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: | - sudo swapoff -a - sudo rm -rf /etc/apt/sources.list.d/* /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo -E apt-get -qq update - sudo -E apt-get -qq install aria2 - sudo -E wget -P /usr/local/sbin/ https://github.com/HiGarfield/lede-17.01.4-Mod/raw/master/.github/backup/apt-fast - sudo -E chmod -R 755 /usr/local/sbin/apt-fast - sudo -E apt-fast -y -qq install dwarves llvm clang lldb lld build-essential rsync asciidoc binutils bzip2 gawk gettext git libncurses5-dev patch python2.7 unzip zlib1g-dev lib32gcc-s1 libc6-dev-i386 subversion flex uglifyjs gcc-multilib p7zip-full msmtp libssl-dev texinfo libreadline-dev libglib2.0-dev xmlto qemu-utils upx-ucl libelf-dev autoconf automake libtool autopoint device-tree-compiler g++-multilib antlr3 gperf wget ccache curl swig coreutils vim nano python3 python3-dev python3-pip python3-ply python3-pyelftools lrzsz scons - pip3 install --user -U pylibfdt + sudo /bin/bash -c "$(curl -sL https://git.io/vokNn)" + sudo -E apt-fast -y -qq install asciidoc bash bcc bin86 binutils bison bzip2 clang file flex g++ g++-multilib gawk gcc gcc-multilib gettext git gzip help2man intltool libboost-dev libelf-dev libncurses-dev libncurses5-dev libssl-dev libthread-queue-any-perl libusb-dev libxml-parser-perl make patch perl-modules python3-dev python3-pip python3-pyelftools python3-setuptools rsync sharutils swig time unzip util-linux wget xsltproc zlib1g-dev zip + sudo -E apt-fast -y -qq install dos2unix dwarves quilt + pip3 install --user -U pylibfdt --break-system-packages sudo -E apt-get -qq autoremove --purge sudo -E apt-get -qq clean sudo -E git config --global user.name 'GitHub Actions' && git config --global user.email 'noreply@github.com' @@ -59,26 +56,31 @@ jobs: cp -r ./SCRIPTS/. ./ /bin/bash 01_get_ready.sh - name: Prepare Package + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt cp -r ../SCRIPTS/. ./ /bin/bash 02_prepare_package.sh /bin/bash 02_R2S.sh - name: Convert Translation + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 03_convert_translation.sh - name: Remove Upx + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 04_remove_upx.sh - name: Add ACL + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 05_create_acl_for_luci.sh -a + - name: Fix Permissions + working-directory: ${{ github.workspace }}/openwrt + run: | + sudo -E chmod -R 755 ./08_fix_permissions.sh + /bin/bash 08_fix_permissions.sh - name: Make Config + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt mv ../SEED/R2S/config.seed .config make defconfig - name: Smart chmod @@ -91,20 +93,21 @@ jobs: rm -f ${MY_Filter} unset MY_Filter - name: Make Download + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt make download -j48 - name: Make Toolchain + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt IGNORE_ERRORS=1 make toolchain/install -j$(($(nproc) + 1)) V=s - name: Compile Openwrt + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt IGNORE_ERRORS=1 make -j$(($(nproc) + 1)) V=s - name: Cleaning + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt/bin/targets/rockchip/armv8 + cd bin/targets/rockchip/armv8 /bin/bash ../../../../../SCRIPTS/06_cleaning.sh - name: Print Disk Space After run: df -h diff --git a/.github/workflows/R4S-OpenWrt.yml b/.github/workflows/R4S-OpenWrt.yml index b673acc9..3fe3b7a6 100644 --- a/.github/workflows/R4S-OpenWrt.yml +++ b/.github/workflows/R4S-OpenWrt.yml @@ -14,7 +14,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.event.repository.owner.id == github.event.sender.id steps: @@ -39,14 +39,11 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: | - sudo swapoff -a - sudo rm -rf /etc/apt/sources.list.d/* /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo -E apt-get -qq update - sudo -E apt-get -qq install aria2 - sudo -E wget -P /usr/local/sbin/ https://github.com/HiGarfield/lede-17.01.4-Mod/raw/master/.github/backup/apt-fast - sudo -E chmod -R 755 /usr/local/sbin/apt-fast - sudo -E apt-fast -y -qq install dwarves llvm clang lldb lld build-essential rsync asciidoc binutils bzip2 gawk gettext git libncurses5-dev patch python2.7 unzip zlib1g-dev lib32gcc-s1 libc6-dev-i386 subversion flex uglifyjs gcc-multilib p7zip-full msmtp libssl-dev texinfo libreadline-dev libglib2.0-dev xmlto qemu-utils upx-ucl libelf-dev autoconf automake libtool autopoint device-tree-compiler g++-multilib antlr3 gperf wget ccache curl swig coreutils vim nano python3 python3-dev python3-pip python3-ply python3-pyelftools lrzsz scons - pip3 install --user -U pylibfdt + sudo /bin/bash -c "$(curl -sL https://git.io/vokNn)" + sudo -E apt-fast -y -qq install asciidoc bash bcc bin86 binutils bison bzip2 clang file flex g++ g++-multilib gawk gcc gcc-multilib gettext git gzip help2man intltool libboost-dev libelf-dev libncurses-dev libncurses5-dev libssl-dev libthread-queue-any-perl libusb-dev libxml-parser-perl make patch perl-modules python3-dev python3-pip python3-pyelftools python3-setuptools rsync sharutils swig time unzip util-linux wget xsltproc zlib1g-dev zip + sudo -E apt-fast -y -qq install dos2unix dwarves quilt + pip3 install --user -U pylibfdt --break-system-packages sudo -E apt-get -qq autoremove --purge sudo -E apt-get -qq clean sudo -E git config --global user.name 'GitHub Actions' && git config --global user.email 'noreply@github.com' @@ -59,26 +56,31 @@ jobs: cp -r ./SCRIPTS/. ./ /bin/bash 01_get_ready.sh - name: Prepare Package + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt cp -r ../SCRIPTS/. ./ /bin/bash 02_prepare_package.sh /bin/bash 02_R4S.sh - name: Convert Translation + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 03_convert_translation.sh - name: Remove Upx + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 04_remove_upx.sh - name: Add ACL + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 05_create_acl_for_luci.sh -a + - name: Fix Permissions + working-directory: ${{ github.workspace }}/openwrt + run: | + sudo -E chmod -R 755 ./08_fix_permissions.sh + /bin/bash 08_fix_permissions.sh - name: Make Config + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt mv ../SEED/R4S/config.seed .config make defconfig - name: Smart chmod @@ -91,20 +93,21 @@ jobs: rm -f ${MY_Filter} unset MY_Filter - name: Make Download + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt make download -j48 - name: Make Toolchain + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt IGNORE_ERRORS=1 make toolchain/install -j$(($(nproc) + 1)) V=s - name: Compile Openwrt + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt IGNORE_ERRORS=1 make -j$(($(nproc) + 1)) V=s - name: Cleaning + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt/bin/targets/rockchip/armv8 + cd bin/targets/rockchip/armv8 /bin/bash ../../../../../SCRIPTS/06_cleaning.sh - name: Print Disk Space After run: df -h diff --git a/.github/workflows/X86-OpenWrt.yml b/.github/workflows/X86-OpenWrt.yml index 6ca8bc72..efb9cd88 100644 --- a/.github/workflows/X86-OpenWrt.yml +++ b/.github/workflows/X86-OpenWrt.yml @@ -14,7 +14,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.event.repository.owner.id == github.event.sender.id steps: @@ -39,14 +39,11 @@ jobs: env: DEBIAN_FRONTEND: noninteractive run: | - sudo swapoff -a - sudo rm -rf /etc/apt/sources.list.d/* /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo -E apt-get -qq update - sudo -E apt-get -qq install aria2 - sudo -E wget -P /usr/local/sbin/ https://github.com/HiGarfield/lede-17.01.4-Mod/raw/master/.github/backup/apt-fast - sudo -E chmod -R 755 /usr/local/sbin/apt-fast - sudo -E apt-fast -y -qq install dwarves llvm clang lldb lld build-essential rsync asciidoc binutils bzip2 gawk gettext git libncurses5-dev patch python2.7 unzip zlib1g-dev lib32gcc-s1 libc6-dev-i386 subversion flex uglifyjs gcc-multilib p7zip-full msmtp libssl-dev texinfo libreadline-dev libglib2.0-dev xmlto qemu-utils upx-ucl libelf-dev autoconf automake libtool autopoint device-tree-compiler g++-multilib antlr3 gperf wget ccache curl swig coreutils vim nano python3 python3-dev python3-pip python3-ply python3-pyelftools lrzsz scons - pip3 install --user -U pylibfdt + sudo /bin/bash -c "$(curl -sL https://git.io/vokNn)" + sudo -E apt-fast -y -qq install asciidoc bash bcc bin86 binutils bison bzip2 clang file flex g++ g++-multilib gawk gcc gcc-multilib gettext git gzip help2man intltool libboost-dev libelf-dev libncurses-dev libncurses5-dev libssl-dev libthread-queue-any-perl libusb-dev libxml-parser-perl make patch perl-modules python3-dev python3-pip python3-pyelftools python3-setuptools rsync sharutils swig time unzip util-linux wget xsltproc zlib1g-dev zip + sudo -E apt-fast -y -qq install dos2unix dwarves quilt + pip3 install --user -U pylibfdt --break-system-packages sudo -E apt-get -qq autoremove --purge sudo -E apt-get -qq clean sudo -E git config --global user.name 'GitHub Actions' && git config --global user.email 'noreply@github.com' @@ -59,26 +56,31 @@ jobs: cp -r ./SCRIPTS/. ./ /bin/bash 01_get_ready.sh - name: Prepare Package + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt cp -r ../SCRIPTS/. ./ /bin/bash 02_prepare_package.sh /bin/bash 02_X86.sh - name: Convert Translation + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 03_convert_translation.sh - name: Remove Upx + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 04_remove_upx.sh - name: Add ACL + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt /bin/bash 05_create_acl_for_luci.sh -a + - name: Fix Permissions + working-directory: ${{ github.workspace }}/openwrt + run: | + sudo -E chmod -R 755 ./08_fix_permissions.sh + /bin/bash 08_fix_permissions.sh - name: Make Config + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt mv ../SEED/X86/config.seed .config make defconfig - name: Smart chmod @@ -91,20 +93,21 @@ jobs: rm -f ${MY_Filter} unset MY_Filter - name: Make Download + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt make download -j48 - name: Make Toolchain + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt IGNORE_ERRORS=1 make toolchain/install -j$(($(nproc) + 1)) V=s - name: Compile Openwrt + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt IGNORE_ERRORS=1 make -j$(($(nproc) + 1)) V=s - name: Cleaning + working-directory: ${{ github.workspace }}/openwrt run: | - cd openwrt/bin/targets/x86/64 + cd bin/targets/x86/64 /bin/bash ../../../../../SCRIPTS/06_cleaning.sh - name: Print Disk Space After run: df -h diff --git a/PATCH/bpf_loop/069-v5.16-01-bpf-Replace-want-address-users-of-BPF_CAST_CALL-with-BPF_CALL_IMM.patch b/PATCH/bpf_loop/069-v5.16-01-bpf-Replace-want-address-users-of-BPF_CAST_CALL-with-BPF_CALL_IMM.patch new file mode 100644 index 00000000..974c88c5 --- /dev/null +++ b/PATCH/bpf_loop/069-v5.16-01-bpf-Replace-want-address-users-of-BPF_CAST_CALL-with-BPF_CALL_IMM.patch @@ -0,0 +1,160 @@ +From: Kees Cook +To: Alexei Starovoitov +Cc: Kees Cook , + Daniel Borkmann , + Andrii Nakryiko , + Martin KaFai Lau , Song Liu , + Yonghong Song , + John Fastabend , + KP Singh , + netdev@vger.kernel.org, bpf@vger.kernel.org, + "Gustavo A . R . Silva" , + linux-kernel@vger.kernel.org, linux-hardening@vger.kernel.org +Subject: [PATCH 1/2] bpf: Replace "want address" users of BPF_CAST_CALL with BPF_CALL_IMM +Date: Mon, 27 Sep 2021 11:26:59 -0700 [thread overview] +Message-ID: <20210927182700.2980499-2-keescook@chromium.org> (raw) +In-Reply-To: <20210927182700.2980499-1-keescook@chromium.org> + +In order to keep ahead of cases in the kernel where Control Flow +Integrity (CFI) may trip over function call casts, enabling +-Wcast-function-type is helpful. To that end, BPF_CAST_CALL causes +various warnings and is one of the last places in the kernel triggering +this warning. + +Most places using BPF_CAST_CALL actually just want a void * to perform +math on. It's not actually performing a call, so just use a different +helper to get the void *, by way of the new BPF_CALL_IMM() helper, which +can clean up a common copy/paste idiom as well. + +This change results in no object code difference. + +Cc: Alexei Starovoitov +Cc: Daniel Borkmann +Cc: Andrii Nakryiko +Cc: Martin KaFai Lau +Cc: Song Liu +Cc: Yonghong Song +Cc: John Fastabend +Cc: KP Singh +Cc: netdev@vger.kernel.org +Cc: bpf@vger.kernel.org +Cc: Gustavo A. R. Silva +Link: https://github.com/KSPP/linux/issues/20 +Signed-off-by: Kees Cook +--- + include/linux/filter.h | 6 +++++- + kernel/bpf/hashtab.c | 6 +++--- + kernel/bpf/verifier.c | 26 +++++++++----------------- + 3 files changed, 17 insertions(+), 21 deletions(-) + +--- a/include/linux/filter.h ++++ b/include/linux/filter.h +@@ -365,13 +365,17 @@ static inline bool insn_is_zext(const st + #define BPF_CAST_CALL(x) \ + ((u64 (*)(u64, u64, u64, u64, u64))(x)) + ++/* Convert function address to BPF immediate */ ++ ++#define BPF_CALL_IMM(x) ((void *)(x) - (void *)__bpf_call_base) ++ + #define BPF_EMIT_CALL(FUNC) \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_CALL, \ + .dst_reg = 0, \ + .src_reg = 0, \ + .off = 0, \ +- .imm = ((FUNC) - __bpf_call_base) }) ++ .imm = BPF_CALL_IMM(FUNC) }) + + /* Raw code statement block */ + +--- a/kernel/bpf/hashtab.c ++++ b/kernel/bpf/hashtab.c +@@ -681,7 +681,7 @@ static int htab_map_gen_lookup(struct bp + + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, + (void *(*)(struct bpf_map *map, void *key))NULL)); +- *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); ++ *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); + *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, + offsetof(struct htab_elem, key) + +@@ -722,7 +722,7 @@ static int htab_lru_map_gen_lookup(struc + + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, + (void *(*)(struct bpf_map *map, void *key))NULL)); +- *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); ++ *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); + *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, + offsetof(struct htab_elem, lru_node) + +@@ -2417,7 +2417,7 @@ static int htab_of_map_gen_lookup(struct + + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, + (void *(*)(struct bpf_map *map, void *key))NULL)); +- *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); ++ *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2); + *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, + offsetof(struct htab_elem, key) + +--- a/kernel/bpf/verifier.c ++++ b/kernel/bpf/verifier.c +@@ -1736,7 +1736,7 @@ static int add_kfunc_call(struct bpf_ver + + desc = &tab->descs[tab->nr_descs++]; + desc->func_id = func_id; +- desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base; ++ desc->imm = BPF_CALL_IMM(addr); + err = btf_distill_func_proto(&env->log, btf_vmlinux, + func_proto, func_name, + &desc->func_model); +@@ -12778,8 +12778,7 @@ static int jit_subprogs(struct bpf_verif + if (!bpf_pseudo_call(insn)) + continue; + subprog = insn->off; +- insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); + } + + /* we use the aux data to keep a list of the start addresses +@@ -13267,32 +13266,25 @@ static int do_misc_fixups(struct bpf_ver + patch_map_ops_generic: + switch (insn->imm) { + case BPF_FUNC_map_lookup_elem: +- insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); + continue; + case BPF_FUNC_map_update_elem: +- insn->imm = BPF_CAST_CALL(ops->map_update_elem) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_update_elem); + continue; + case BPF_FUNC_map_delete_elem: +- insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_delete_elem); + continue; + case BPF_FUNC_map_push_elem: +- insn->imm = BPF_CAST_CALL(ops->map_push_elem) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_push_elem); + continue; + case BPF_FUNC_map_pop_elem: +- insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_pop_elem); + continue; + case BPF_FUNC_map_peek_elem: +- insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_peek_elem); + continue; + case BPF_FUNC_redirect_map: +- insn->imm = BPF_CAST_CALL(ops->map_redirect) - +- __bpf_call_base; ++ insn->imm = BPF_CALL_IMM(ops->map_redirect); + continue; + } + diff --git a/PATCH/bpf_loop/069-v5.16-02-bpf-Replace-callers-of-BPF_CAST_CALL-with-proper-function-typedef.patch b/PATCH/bpf_loop/069-v5.16-02-bpf-Replace-callers-of-BPF_CAST_CALL-with-proper-function-typedef.patch new file mode 100644 index 00000000..2888a124 --- /dev/null +++ b/PATCH/bpf_loop/069-v5.16-02-bpf-Replace-callers-of-BPF_CAST_CALL-with-proper-function-typedef.patch @@ -0,0 +1,150 @@ +From: Kees Cook +To: Alexei Starovoitov +Cc: Kees Cook , + Daniel Borkmann , + Andrii Nakryiko , + Martin KaFai Lau , Song Liu , + Yonghong Song , + John Fastabend , + KP Singh , + netdev@vger.kernel.org, bpf@vger.kernel.org, + "Gustavo A . R . Silva" , + linux-kernel@vger.kernel.org, linux-hardening@vger.kernel.org +Subject: [PATCH 2/2] bpf: Replace callers of BPF_CAST_CALL with proper function typedef +Date: Mon, 27 Sep 2021 11:27:00 -0700 [thread overview] +Message-ID: <20210927182700.2980499-3-keescook@chromium.org> (raw) +In-Reply-To: <20210927182700.2980499-1-keescook@chromium.org> + +In order to keep ahead of cases in the kernel where Control Flow +Integrity (CFI) may trip over function call casts, enabling +-Wcast-function-type is helpful. To that end, BPF_CAST_CALL causes +various warnings and is one of the last places in the kernel +triggering this warning. + +For actual function calls, replace BPF_CAST_CALL() with a typedef, which +captures the same details about the given function pointers. + +This change results in no object code difference. + +Cc: Alexei Starovoitov +Cc: Daniel Borkmann +Cc: Andrii Nakryiko +Cc: Martin KaFai Lau +Cc: Song Liu +Cc: Yonghong Song +Cc: John Fastabend +Cc: KP Singh +Cc: netdev@vger.kernel.org +Cc: bpf@vger.kernel.org +Cc: Gustavo A. R. Silva +Link: https://github.com/KSPP/linux/issues/20 +Signed-off-by: Kees Cook +--- + include/linux/bpf.h | 4 +++- + include/linux/filter.h | 5 ----- + kernel/bpf/arraymap.c | 7 +++---- + kernel/bpf/hashtab.c | 7 +++---- + kernel/bpf/helpers.c | 5 ++--- + 5 files changed, 11 insertions(+), 17 deletions(-) + +--- a/include/linux/bpf.h ++++ b/include/linux/bpf.h +@@ -48,6 +48,7 @@ extern struct idr btf_idr; + extern spinlock_t btf_idr_lock; + extern struct kobject *btf_kobj; + ++typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); + typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, + struct bpf_iter_aux_info *aux); + typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); +@@ -146,7 +147,8 @@ struct bpf_map_ops { + int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee); +- int (*map_for_each_callback)(struct bpf_map *map, void *callback_fn, ++ int (*map_for_each_callback)(struct bpf_map *map, ++ bpf_callback_t callback_fn, + void *callback_ctx, u64 flags); + + /* BTF name and id of struct allocated by map_alloc */ +--- a/include/linux/filter.h ++++ b/include/linux/filter.h +@@ -360,11 +360,6 @@ static inline bool insn_is_zext(const st + .off = 0, \ + .imm = TGT }) + +-/* Function call */ +- +-#define BPF_CAST_CALL(x) \ +- ((u64 (*)(u64, u64, u64, u64, u64))(x)) +- + /* Convert function address to BPF immediate */ + + #define BPF_CALL_IMM(x) ((void *)(x) - (void *)__bpf_call_base) +--- a/kernel/bpf/arraymap.c ++++ b/kernel/bpf/arraymap.c +@@ -651,7 +651,7 @@ static const struct bpf_iter_seq_info it + .seq_priv_size = sizeof(struct bpf_iter_seq_array_map_info), + }; + +-static int bpf_for_each_array_elem(struct bpf_map *map, void *callback_fn, ++static int bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn, + void *callback_ctx, u64 flags) + { + u32 i, key, num_elems = 0; +@@ -674,9 +674,8 @@ static int bpf_for_each_array_elem(struc + val = array->value + array->elem_size * i; + num_elems++; + key = i; +- ret = BPF_CAST_CALL(callback_fn)((u64)(long)map, +- (u64)(long)&key, (u64)(long)val, +- (u64)(long)callback_ctx, 0); ++ ret = callback_fn((u64)(long)map, (u64)(long)&key, ++ (u64)(long)val, (u64)(long)callback_ctx, 0); + /* return value: 0 - continue, 1 - stop and return */ + if (ret) + break; +--- a/kernel/bpf/hashtab.c ++++ b/kernel/bpf/hashtab.c +@@ -2069,7 +2069,7 @@ static const struct bpf_iter_seq_info it + .seq_priv_size = sizeof(struct bpf_iter_seq_hash_map_info), + }; + +-static int bpf_for_each_hash_elem(struct bpf_map *map, void *callback_fn, ++static int bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_fn, + void *callback_ctx, u64 flags) + { + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); +@@ -2109,9 +2109,8 @@ static int bpf_for_each_hash_elem(struct + val = elem->key + roundup_key_size; + } + num_elems++; +- ret = BPF_CAST_CALL(callback_fn)((u64)(long)map, +- (u64)(long)key, (u64)(long)val, +- (u64)(long)callback_ctx, 0); ++ ret = callback_fn((u64)(long)map, (u64)(long)key, ++ (u64)(long)val, (u64)(long)callback_ctx, 0); + /* return value: 0 - continue, 1 - stop and return */ + if (ret) { + rcu_read_unlock(); +--- a/kernel/bpf/helpers.c ++++ b/kernel/bpf/helpers.c +@@ -1068,7 +1068,7 @@ static enum hrtimer_restart bpf_timer_cb + struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer); + struct bpf_map *map = t->map; + void *value = t->value; +- void *callback_fn; ++ bpf_callback_t callback_fn; + void *key; + u32 idx; + +@@ -1093,8 +1093,7 @@ static enum hrtimer_restart bpf_timer_cb + key = value - round_up(map->key_size, 8); + } + +- BPF_CAST_CALL(callback_fn)((u64)(long)map, (u64)(long)key, +- (u64)(long)value, 0, 0); ++ callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + /* The verifier checked that return value is zero. */ + + this_cpu_write(hrtimer_running, NULL); diff --git a/PATCH/bpf_loop/070-v5.17-01-Add-bpf_loop-helper.patch b/PATCH/bpf_loop/070-v5.17-01-Add-bpf_loop-helper.patch new file mode 100644 index 00000000..a4fc4268 --- /dev/null +++ b/PATCH/bpf_loop/070-v5.17-01-Add-bpf_loop-helper.patch @@ -0,0 +1,258 @@ +From: Joanne Koong @ 2021-11-29 22:37 UTC (permalink / raw) + To: bpf; +Cc: andrii, ast, daniel, Kernel-team, Joanne Koong + +This patch adds the kernel-side and API changes for a new helper +function, bpf_loop: + +long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, +u64 flags); + +where long (*callback_fn)(u32 index, void *ctx); + +bpf_loop invokes the "callback_fn" **nr_loops** times or until the +callback_fn returns 1. The callback_fn can only return 0 or 1, and +this is enforced by the verifier. The callback_fn index is zero-indexed. + +A few things to please note: +~ The "u64 flags" parameter is currently unused but is included in +case a future use case for it arises. +~ In the kernel-side implementation of bpf_loop (kernel/bpf/bpf_iter.c), +bpf_callback_t is used as the callback function cast. +~ A program can have nested bpf_loop calls but the program must +still adhere to the verifier constraint of its stack depth (the stack depth +cannot exceed MAX_BPF_STACK)) +~ Recursive callback_fns do not pass the verifier, due to the call stack +for these being too deep. +~ The next patch will include the tests and benchmark + +Signed-off-by: Joanne Koong +--- + include/linux/bpf.h | 1 + + include/uapi/linux/bpf.h | 25 ++++++++++ + kernel/bpf/bpf_iter.c | 35 ++++++++++++++ + kernel/bpf/helpers.c | 2 + + kernel/bpf/verifier.c | 88 +++++++++++++++++++++------------- + tools/include/uapi/linux/bpf.h | 25 ++++++++++ + 6 files changed, 142 insertions(+), 34 deletions(-) + +--- a/include/linux/bpf.h ++++ b/include/linux/bpf.h +@@ -2184,6 +2184,7 @@ extern const struct bpf_func_proto bpf_f + extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; + extern const struct bpf_func_proto bpf_sk_setsockopt_proto; + extern const struct bpf_func_proto bpf_sk_getsockopt_proto; ++extern const struct bpf_func_proto bpf_loop_proto; + + const struct bpf_func_proto *tracing_prog_func_proto( + enum bpf_func_id func_id, const struct bpf_prog *prog); +--- a/include/uapi/linux/bpf.h ++++ b/include/uapi/linux/bpf.h +@@ -5074,6 +5074,12 @@ union bpf_attr { + FN(get_func_ip), \ + FN(get_attach_cookie), \ + FN(task_pt_regs), \ ++ FN(get_branch_snapshot), \ ++ FN(trace_vprintk), \ ++ FN(skc_to_unix_sock), \ ++ FN(kallsyms_lookup_name), \ ++ FN(find_vma), \ ++ FN(loop), \ + /* */ + + /* integer value in 'imm' field of BPF_CALL instruction selects which helper +--- a/kernel/bpf/bpf_iter.c ++++ b/kernel/bpf/bpf_iter.c +@@ -714,3 +714,38 @@ const struct bpf_func_proto bpf_for_each + .arg3_type = ARG_PTR_TO_STACK_OR_NULL, + .arg4_type = ARG_ANYTHING, + }; ++ ++/* maximum number of loops */ ++#define MAX_LOOPS BIT(23) ++ ++BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, ++ u64, flags) ++{ ++ bpf_callback_t callback = (bpf_callback_t)callback_fn; ++ u64 ret; ++ u32 i; ++ ++ if (flags) ++ return -EINVAL; ++ if (nr_loops > MAX_LOOPS) ++ return -E2BIG; ++ ++ for (i = 0; i < nr_loops; i++) { ++ ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0); ++ /* return value: 0 - continue, 1 - stop and return */ ++ if (ret) ++ return i + 1; ++ } ++ ++ return i; ++} ++ ++const struct bpf_func_proto bpf_loop_proto = { ++ .func = bpf_loop, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_ANYTHING, ++ .arg2_type = ARG_PTR_TO_FUNC, ++ .arg3_type = ARG_PTR_TO_STACK_OR_NULL, ++ .arg4_type = ARG_ANYTHING, ++}; +--- a/kernel/bpf/helpers.c ++++ b/kernel/bpf/helpers.c +@@ -1397,6 +1397,8 @@ bpf_base_func_proto(enum bpf_func_id fun + return &bpf_ringbuf_query_proto; + case BPF_FUNC_for_each_map_elem: + return &bpf_for_each_map_elem_proto; ++ case BPF_FUNC_loop: ++ return &bpf_loop_proto; + default: + break; + } +--- a/kernel/bpf/verifier.c ++++ b/kernel/bpf/verifier.c +@@ -6246,6 +6246,27 @@ static int set_timer_callback_state(stru + return 0; + } + ++static int set_loop_callback_state(struct bpf_verifier_env *env, ++ struct bpf_func_state *caller, ++ struct bpf_func_state *callee, ++ int insn_idx) ++{ ++ /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, ++ * u64 flags); ++ * callback_fn(u32 index, void *callback_ctx); ++ */ ++ callee->regs[BPF_REG_1].type = SCALAR_VALUE; ++ callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; ++ ++ /* unused */ ++ __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); ++ __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); ++ __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); ++ ++ callee->in_callback_fn = true; ++ return 0; ++} ++ + static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) + { + struct bpf_verifier_state *state = env->cur_state; +@@ -6582,13 +6603,7 @@ static int check_helper_call(struct bpf_ + return err; + } + +- if (func_id == BPF_FUNC_tail_call) { +- err = check_reference_leak(env); +- if (err) { +- verbose(env, "tail_call would lead to reference leak\n"); +- return err; +- } +- } else if (is_release_function(func_id)) { ++ if (is_release_function(func_id)) { + err = release_reference(env, meta.ref_obj_id); + if (err) { + verbose(env, "func %s#%d reference has not been acquired before\n", +@@ -6599,35 +6614,43 @@ static int check_helper_call(struct bpf_ + + regs = cur_regs(env); + +- /* check that flags argument in get_local_storage(map, flags) is 0, +- * this is required because get_local_storage() can't return an error. +- */ +- if (func_id == BPF_FUNC_get_local_storage && +- !register_is_null(®s[BPF_REG_2])) { +- verbose(env, "get_local_storage() doesn't support non-zero flags\n"); +- return -EINVAL; +- } +- +- if (func_id == BPF_FUNC_for_each_map_elem) { ++ switch (func_id) { ++ case BPF_FUNC_tail_call: ++ err = check_reference_leak(env); ++ if (err) { ++ verbose(env, "tail_call would lead to reference leak\n"); ++ return err; ++ } ++ break; ++ case BPF_FUNC_get_local_storage: ++ /* check that flags argument in get_local_storage(map, flags) is 0, ++ * this is required because get_local_storage() can't return an error. ++ */ ++ if (!register_is_null(®s[BPF_REG_2])) { ++ verbose(env, "get_local_storage() doesn't support non-zero flags\n"); ++ return -EINVAL; ++ } ++ break; ++ case BPF_FUNC_for_each_map_elem: + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_map_elem_callback_state); +- if (err < 0) +- return -EINVAL; +- } +- +- if (func_id == BPF_FUNC_timer_set_callback) { ++ break; ++ case BPF_FUNC_timer_set_callback: + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_timer_callback_state); +- if (err < 0) +- return -EINVAL; +- } +- +- if (func_id == BPF_FUNC_snprintf) { ++ break; ++ case BPF_FUNC_snprintf: + err = check_bpf_snprintf_call(env, regs); +- if (err < 0) +- return err; ++ break; ++ case BPF_FUNC_loop: ++ err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, ++ set_loop_callback_state); ++ break; + } + ++ if (err) ++ return err; ++ + /* reset caller saved regs */ + for (i = 0; i < CALLER_SAVED_REGS; i++) { + mark_reg_not_init(env, regs, caller_saved[i]); +--- a/scripts/bpf_doc.py ++++ b/scripts/bpf_doc.py +@@ -537,6 +537,7 @@ class PrinterHelpers(Printer): + 'struct tcp_timewait_sock', + 'struct tcp_request_sock', + 'struct udp6_sock', ++ 'struct unix_sock', + 'struct task_struct', + + 'struct __sk_buff', +@@ -589,6 +590,7 @@ class PrinterHelpers(Printer): + 'struct tcp_timewait_sock', + 'struct tcp_request_sock', + 'struct udp6_sock', ++ 'struct unix_sock', + 'struct task_struct', + 'struct path', + 'struct btf_ptr', +--- a/tools/include/uapi/linux/bpf.h ++++ b/tools/include/uapi/linux/bpf.h +@@ -5074,6 +5074,12 @@ union bpf_attr { + FN(get_func_ip), \ + FN(get_attach_cookie), \ + FN(task_pt_regs), \ ++ FN(get_branch_snapshot), \ ++ FN(trace_vprintk), \ ++ FN(skc_to_unix_sock), \ ++ FN(kallsyms_lookup_name), \ ++ FN(find_vma), \ ++ FN(loop), \ + /* */ + + /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/PATCH/bpf_loop/070-v5.17-02-selftests-bpf-Add-bpf_loop-test.patch b/PATCH/bpf_loop/070-v5.17-02-selftests-bpf-Add-bpf_loop-test.patch new file mode 100644 index 00000000..0bcdde5a --- /dev/null +++ b/PATCH/bpf_loop/070-v5.17-02-selftests-bpf-Add-bpf_loop-test.patch @@ -0,0 +1,277 @@ +From: Joanne Koong @ 2021-11-29 22:37 UTC (permalink / raw) + To: bpf; +Cc: andrii, ast, daniel, Kernel-team, Joanne Koong + +Add test for bpf_loop testing a variety of cases: +various nr_loops, null callback ctx, invalid flags, nested callbacks. + +Signed-off-by: Joanne Koong +--- + .../selftests/bpf/prog_tests/bpf_loop.c | 138 ++++++++++++++++++ + tools/testing/selftests/bpf/progs/bpf_loop.c | 99 +++++++++++++ + 2 files changed, 237 insertions(+) + create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_loop.c + create mode 100644 tools/testing/selftests/bpf/progs/bpf_loop.c + +--- /dev/null ++++ b/tools/testing/selftests/bpf/prog_tests/bpf_loop.c +@@ -0,0 +1,145 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* Copyright (c) 2021 Facebook */ ++ ++#include ++#include ++#include "bpf_loop.skel.h" ++ ++static void check_nr_loops(struct bpf_loop *skel) ++{ ++ struct bpf_link *link; ++ ++ link = bpf_program__attach(skel->progs.test_prog); ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ /* test 0 loops */ ++ skel->bss->nr_loops = 0; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->nr_loops_returned, skel->bss->nr_loops, ++ "0 loops"); ++ ++ /* test 500 loops */ ++ skel->bss->nr_loops = 500; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->nr_loops_returned, skel->bss->nr_loops, ++ "500 loops"); ++ ASSERT_EQ(skel->bss->g_output, (500 * 499) / 2, "g_output"); ++ ++ /* test exceeding the max limit */ ++ skel->bss->nr_loops = -1; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->err, -E2BIG, "over max limit"); ++ ++ bpf_link__destroy(link); ++} ++ ++static void check_callback_fn_stop(struct bpf_loop *skel) ++{ ++ struct bpf_link *link; ++ ++ link = bpf_program__attach(skel->progs.test_prog); ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ /* testing that loop is stopped when callback_fn returns 1 */ ++ skel->bss->nr_loops = 400; ++ skel->data->stop_index = 50; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->nr_loops_returned, skel->data->stop_index + 1, ++ "nr_loops_returned"); ++ ASSERT_EQ(skel->bss->g_output, (50 * 49) / 2, ++ "g_output"); ++ ++ bpf_link__destroy(link); ++} ++ ++static void check_null_callback_ctx(struct bpf_loop *skel) ++{ ++ struct bpf_link *link; ++ ++ /* check that user is able to pass in a null callback_ctx */ ++ link = bpf_program__attach(skel->progs.prog_null_ctx); ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ skel->bss->nr_loops = 10; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->nr_loops_returned, skel->bss->nr_loops, ++ "nr_loops_returned"); ++ ++ bpf_link__destroy(link); ++} ++ ++static void check_invalid_flags(struct bpf_loop *skel) ++{ ++ struct bpf_link *link; ++ ++ /* check that passing in non-zero flags returns -EINVAL */ ++ link = bpf_program__attach(skel->progs.prog_invalid_flags); ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->err, -EINVAL, "err"); ++ ++ bpf_link__destroy(link); ++} ++ ++static void check_nested_calls(struct bpf_loop *skel) ++{ ++ __u32 nr_loops = 100, nested_callback_nr_loops = 4; ++ struct bpf_link *link; ++ ++ /* check that nested calls are supported */ ++ link = bpf_program__attach(skel->progs.prog_nested_calls); ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ skel->bss->nr_loops = nr_loops; ++ skel->bss->nested_callback_nr_loops = nested_callback_nr_loops; ++ ++ usleep(1); ++ ++ ASSERT_EQ(skel->bss->nr_loops_returned, nr_loops * nested_callback_nr_loops ++ * nested_callback_nr_loops, "nr_loops_returned"); ++ ASSERT_EQ(skel->bss->g_output, (4 * 3) / 2 * nested_callback_nr_loops ++ * nr_loops, "g_output"); ++ ++ bpf_link__destroy(link); ++} ++ ++void test_bpf_loop(void) ++{ ++ struct bpf_loop *skel; ++ ++ skel = bpf_loop__open_and_load(); ++ if (!ASSERT_OK_PTR(skel, "bpf_loop__open_and_load")) ++ return; ++ ++ skel->bss->pid = getpid(); ++ ++ if (test__start_subtest("check_nr_loops")) ++ check_nr_loops(skel); ++ if (test__start_subtest("check_callback_fn_stop")) ++ check_callback_fn_stop(skel); ++ if (test__start_subtest("check_null_callback_ctx")) ++ check_null_callback_ctx(skel); ++ if (test__start_subtest("check_invalid_flags")) ++ check_invalid_flags(skel); ++ if (test__start_subtest("check_nested_calls")) ++ check_nested_calls(skel); ++ ++ bpf_loop__destroy(skel); ++} +--- /dev/null ++++ b/tools/testing/selftests/bpf/progs/bpf_loop.c +@@ -0,0 +1,112 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* Copyright (c) 2021 Facebook */ ++ ++#include "vmlinux.h" ++#include ++ ++char _license[] SEC("license") = "GPL"; ++ ++struct callback_ctx { ++ int output; ++}; ++ ++/* These should be set by the user program */ ++u32 nested_callback_nr_loops; ++u32 stop_index = -1; ++u32 nr_loops; ++int pid; ++ ++/* Making these global variables so that the userspace program ++ * can verify the output through the skeleton ++ */ ++int nr_loops_returned; ++int g_output; ++int err; ++ ++static int callback(__u32 index, void *data) ++{ ++ struct callback_ctx *ctx = data; ++ ++ if (index >= stop_index) ++ return 1; ++ ++ ctx->output += index; ++ ++ return 0; ++} ++ ++static int empty_callback(__u32 index, void *data) ++{ ++ return 0; ++} ++ ++static int nested_callback2(__u32 index, void *data) ++{ ++ nr_loops_returned += bpf_loop(nested_callback_nr_loops, callback, data, 0); ++ ++ return 0; ++} ++ ++static int nested_callback1(__u32 index, void *data) ++{ ++ bpf_loop(nested_callback_nr_loops, nested_callback2, data, 0); ++ return 0; ++} ++ ++SEC("fentry/__x64_sys_nanosleep") ++int test_prog(void *ctx) ++{ ++ struct callback_ctx data = {}; ++ ++ if (bpf_get_current_pid_tgid() >> 32 != pid) ++ return 0; ++ ++ nr_loops_returned = bpf_loop(nr_loops, callback, &data, 0); ++ ++ if (nr_loops_returned < 0) ++ err = nr_loops_returned; ++ else ++ g_output = data.output; ++ ++ return 0; ++} ++ ++SEC("fentry/__x64_sys_nanosleep") ++int prog_null_ctx(void *ctx) ++{ ++ if (bpf_get_current_pid_tgid() >> 32 != pid) ++ return 0; ++ ++ nr_loops_returned = bpf_loop(nr_loops, empty_callback, NULL, 0); ++ ++ return 0; ++} ++ ++SEC("fentry/__x64_sys_nanosleep") ++int prog_invalid_flags(void *ctx) ++{ ++ struct callback_ctx data = {}; ++ ++ if (bpf_get_current_pid_tgid() >> 32 != pid) ++ return 0; ++ ++ err = bpf_loop(nr_loops, callback, &data, 1); ++ ++ return 0; ++} ++ ++SEC("fentry/__x64_sys_nanosleep") ++int prog_nested_calls(void *ctx) ++{ ++ struct callback_ctx data = {}; ++ ++ if (bpf_get_current_pid_tgid() >> 32 != pid) ++ return 0; ++ ++ nr_loops_returned = 0; ++ bpf_loop(nr_loops, nested_callback1, &data, 0); ++ ++ g_output = data.output; ++ ++ return 0; ++} diff --git a/PATCH/bpf_loop/070-v5.17-03-selftests-bpf-measure-bpf_loop-verifier-performance.patch b/PATCH/bpf_loop/070-v5.17-03-selftests-bpf-measure-bpf_loop-verifier-performance.patch new file mode 100644 index 00000000..64482881 --- /dev/null +++ b/PATCH/bpf_loop/070-v5.17-03-selftests-bpf-measure-bpf_loop-verifier-performance.patch @@ -0,0 +1,302 @@ +From: Joanne Koong @ 2021-11-29 22:37 UTC (permalink / raw) + To: bpf; +Cc: andrii, ast, daniel, Kernel-team, Joanne Koong + +This patch tests bpf_loop in pyperf and strobemeta, and measures the +verifier performance of replacing the traditional for loop +with bpf_loop. + +The results are as follows: + +~strobemeta~ + +Baseline + verification time 6808200 usec + stack depth 496 + processed 554252 insns (limit 1000000) max_states_per_insn 16 + total_states 15878 peak_states 13489 mark_read 3110 + #192 verif_scale_strobemeta:OK (unrolled loop) + +Using bpf_loop + verification time 31589 usec + stack depth 96+400 + processed 1513 insns (limit 1000000) max_states_per_insn 2 + total_states 106 peak_states 106 mark_read 60 + #193 verif_scale_strobemeta_bpf_loop:OK + +~pyperf600~ + +Baseline + verification time 29702486 usec + stack depth 368 + processed 626838 insns (limit 1000000) max_states_per_insn 7 + total_states 30368 peak_states 30279 mark_read 748 + #182 verif_scale_pyperf600:OK (unrolled loop) + +Using bpf_loop + verification time 148488 usec + stack depth 320+40 + processed 10518 insns (limit 1000000) max_states_per_insn 10 + total_states 705 peak_states 517 mark_read 38 + #183 verif_scale_pyperf600_bpf_loop:OK + +Using the bpf_loop helper led to approximately a 99% decrease +in the verification time and in the number of instructions. + +Signed-off-by: Joanne Koong +--- + .../bpf/prog_tests/bpf_verif_scale.c | 12 +++ + tools/testing/selftests/bpf/progs/pyperf.h | 71 +++++++++++++++++- + .../selftests/bpf/progs/pyperf600_bpf_loop.c | 6 ++ + .../testing/selftests/bpf/progs/strobemeta.h | 75 ++++++++++++++++++- + .../selftests/bpf/progs/strobemeta_bpf_loop.c | 9 +++ + 5 files changed, 169 insertions(+), 4 deletions(-) + create mode 100644 tools/testing/selftests/bpf/progs/pyperf600_bpf_loop.c + create mode 100644 tools/testing/selftests/bpf/progs/strobemeta_bpf_loop.c + +--- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c ++++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +@@ -64,6 +64,9 @@ void test_bpf_verif_scale(void) + */ + { "pyperf600.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, + ++ /* use the bpf_loop helper*/ ++ { "pyperf600_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, ++ + /* no unroll at all. + * C loop count -> 600. + * ASM loop count -> 600. +@@ -84,6 +87,9 @@ void test_bpf_verif_scale(void) + */ + { "strobemeta.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, + ++ /* use the bpf_loop helper*/ ++ { "strobemeta_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, ++ + /* no unroll, tiny loops */ + { "strobemeta_nounroll1.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, + { "strobemeta_nounroll2.o", BPF_PROG_TYPE_RAW_TRACEPOINT }, +--- a/tools/testing/selftests/bpf/progs/pyperf.h ++++ b/tools/testing/selftests/bpf/progs/pyperf.h +@@ -159,6 +159,59 @@ struct { + __uint(value_size, sizeof(long long) * 127); + } stackmap SEC(".maps"); + ++#ifdef USE_BPF_LOOP ++struct process_frame_ctx { ++ int cur_cpu; ++ int32_t *symbol_counter; ++ void *frame_ptr; ++ FrameData *frame; ++ PidData *pidData; ++ Symbol *sym; ++ Event *event; ++ bool done; ++}; ++ ++#define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var)) ++ ++static int process_frame_callback(__u32 i, struct process_frame_ctx *ctx) ++{ ++ int zero = 0; ++ void *frame_ptr = ctx->frame_ptr; ++ PidData *pidData = ctx->pidData; ++ FrameData *frame = ctx->frame; ++ int32_t *symbol_counter = ctx->symbol_counter; ++ int cur_cpu = ctx->cur_cpu; ++ Event *event = ctx->event; ++ Symbol *sym = ctx->sym; ++ ++ if (frame_ptr && get_frame_data(frame_ptr, pidData, frame, sym)) { ++ int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu; ++ int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, sym); ++ ++ if (!symbol_id) { ++ bpf_map_update_elem(&symbolmap, sym, &zero, 0); ++ symbol_id = bpf_map_lookup_elem(&symbolmap, sym); ++ if (!symbol_id) { ++ ctx->done = true; ++ return 1; ++ } ++ } ++ if (*symbol_id == new_symbol_id) ++ (*symbol_counter)++; ++ ++ barrier_var(i); ++ if (i >= STACK_MAX_LEN) ++ return 1; ++ ++ event->stack[i] = *symbol_id; ++ ++ event->stack_len = i + 1; ++ frame_ptr = frame->f_back; ++ } ++ return 0; ++} ++#endif /* USE_BPF_LOOP */ ++ + #ifdef GLOBAL_FUNC + __noinline + #elif defined(SUBPROGS) +@@ -228,6 +281,21 @@ int __on_event(struct bpf_raw_tracepoint + int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym); + if (symbol_counter == NULL) + return 0; ++#ifdef USE_BPF_LOOP ++ struct process_frame_ctx ctx = { ++ .cur_cpu = cur_cpu, ++ .symbol_counter = symbol_counter, ++ .frame_ptr = frame_ptr, ++ .frame = &frame, ++ .pidData = pidData, ++ .sym = &sym, ++ .event = event, ++ }; ++ ++ bpf_loop(STACK_MAX_LEN, process_frame_callback, &ctx, 0); ++ if (ctx.done) ++ return 0; ++#else + #ifdef NO_UNROLL + #pragma clang loop unroll(disable) + #else +@@ -251,6 +319,7 @@ int __on_event(struct bpf_raw_tracepoint + frame_ptr = frame.f_back; + } + } ++#endif /* USE_BPF_LOOP */ + event->stack_complete = frame_ptr == NULL; + } else { + event->stack_complete = 1; +--- /dev/null ++++ b/tools/testing/selftests/bpf/progs/pyperf600_bpf_loop.c +@@ -0,0 +1,6 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* Copyright (c) 2021 Facebook */ ++ ++#define STACK_MAX_LEN 600 ++#define USE_BPF_LOOP ++#include "pyperf.h" +--- a/tools/testing/selftests/bpf/progs/strobemeta.h ++++ b/tools/testing/selftests/bpf/progs/strobemeta.h +@@ -445,6 +445,48 @@ static __always_inline void *read_map_va + return payload; + } + ++#ifdef USE_BPF_LOOP ++enum read_type { ++ READ_INT_VAR, ++ READ_MAP_VAR, ++ READ_STR_VAR, ++}; ++ ++struct read_var_ctx { ++ struct strobemeta_payload *data; ++ void *tls_base; ++ struct strobemeta_cfg *cfg; ++ void *payload; ++ /* value gets mutated */ ++ struct strobe_value_generic *value; ++ enum read_type type; ++}; ++ ++static int read_var_callback(__u32 index, struct read_var_ctx *ctx) ++{ ++ switch (ctx->type) { ++ case READ_INT_VAR: ++ if (index >= STROBE_MAX_INTS) ++ return 1; ++ read_int_var(ctx->cfg, index, ctx->tls_base, ctx->value, ctx->data); ++ break; ++ case READ_MAP_VAR: ++ if (index >= STROBE_MAX_MAPS) ++ return 1; ++ ctx->payload = read_map_var(ctx->cfg, index, ctx->tls_base, ++ ctx->value, ctx->data, ctx->payload); ++ break; ++ case READ_STR_VAR: ++ if (index >= STROBE_MAX_STRS) ++ return 1; ++ ctx->payload += read_str_var(ctx->cfg, index, ctx->tls_base, ++ ctx->value, ctx->data, ctx->payload); ++ break; ++ } ++ return 0; ++} ++#endif /* USE_BPF_LOOP */ ++ + /* + * read_strobe_meta returns NULL, if no metadata was read; otherwise returns + * pointer to *right after* payload ends +@@ -475,11 +517,36 @@ static void *read_strobe_meta(struct tas + */ + tls_base = (void *)task; + ++#ifdef USE_BPF_LOOP ++ struct read_var_ctx ctx = { ++ .cfg = cfg, ++ .tls_base = tls_base, ++ .value = &value, ++ .data = data, ++ .payload = payload, ++ }; ++ int err; ++ ++ ctx.type = READ_INT_VAR; ++ err = bpf_loop(STROBE_MAX_INTS, read_var_callback, &ctx, 0); ++ if (err != STROBE_MAX_INTS) ++ return NULL; ++ ++ ctx.type = READ_STR_VAR; ++ err = bpf_loop(STROBE_MAX_STRS, read_var_callback, &ctx, 0); ++ if (err != STROBE_MAX_STRS) ++ return NULL; ++ ++ ctx.type = READ_MAP_VAR; ++ err = bpf_loop(STROBE_MAX_MAPS, read_var_callback, &ctx, 0); ++ if (err != STROBE_MAX_MAPS) ++ return NULL; ++#else + #ifdef NO_UNROLL + #pragma clang loop unroll(disable) + #else + #pragma unroll +-#endif ++#endif /* NO_UNROLL */ + for (int i = 0; i < STROBE_MAX_INTS; ++i) { + read_int_var(cfg, i, tls_base, &value, data); + } +@@ -487,7 +554,7 @@ static void *read_strobe_meta(struct tas + #pragma clang loop unroll(disable) + #else + #pragma unroll +-#endif ++#endif /* NO_UNROLL */ + for (int i = 0; i < STROBE_MAX_STRS; ++i) { + payload += read_str_var(cfg, i, tls_base, &value, data, payload); + } +@@ -495,10 +562,12 @@ static void *read_strobe_meta(struct tas + #pragma clang loop unroll(disable) + #else + #pragma unroll +-#endif ++#endif /* NO_UNROLL */ + for (int i = 0; i < STROBE_MAX_MAPS; ++i) { + payload = read_map_var(cfg, i, tls_base, &value, data, payload); + } ++#endif /* USE_BPF_LOOP */ ++ + /* + * return pointer right after end of payload, so it's possible to + * calculate exact amount of useful data that needs to be sent +--- /dev/null ++++ b/tools/testing/selftests/bpf/progs/strobemeta_bpf_loop.c +@@ -0,0 +1,9 @@ ++// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) ++/* Copyright (c) 2021 Facebook */ ++ ++#define STROBE_MAX_INTS 2 ++#define STROBE_MAX_STRS 25 ++#define STROBE_MAX_MAPS 100 ++#define STROBE_MAX_MAP_ENTRIES 20 ++#define USE_BPF_LOOP ++#include "strobemeta.h" diff --git a/PATCH/bpf_loop/070-v5.17-04-selftests-bpf-Extract-syscall-wrapper.patch b/PATCH/bpf_loop/070-v5.17-04-selftests-bpf-Extract-syscall-wrapper.patch new file mode 100644 index 00000000..f6dc1b28 --- /dev/null +++ b/PATCH/bpf_loop/070-v5.17-04-selftests-bpf-Extract-syscall-wrapper.patch @@ -0,0 +1,44 @@ +commit 78a2054156dd6265619b230cc5372b74f9ba5233 upstream. + +Extract the helper to set up SYS_PREFIX for fentry and kprobe selftests +that use __x86_sys_* attach functions. + +Suggested-by: Andrii Nakryiko +Signed-off-by: Kenta Tada +Signed-off-by: Andrii Nakryiko +Link: https://lore.kernel.org/bpf/20220124141622.4378-2-Kenta.Tada@sony.com +Signed-off-by: Tianchen Ding +--- + +--- /dev/null ++++ b/tools/testing/selftests/bpf/progs/bpf_misc.h +@@ -0,0 +1,19 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef __BPF_MISC_H__ ++#define __BPF_MISC_H__ ++ ++#if defined(__TARGET_ARCH_x86) ++#define SYSCALL_WRAPPER 1 ++#define SYS_PREFIX "__x64_" ++#elif defined(__TARGET_ARCH_s390) ++#define SYSCALL_WRAPPER 1 ++#define SYS_PREFIX "__s390x_" ++#elif defined(__TARGET_ARCH_arm64) ++#define SYSCALL_WRAPPER 1 ++#define SYS_PREFIX "__arm64_" ++#else ++#define SYSCALL_WRAPPER 0 ++#define SYS_PREFIX "" ++#endif ++ ++#endif +--- a/tools/testing/selftests/bpf/progs/test_probe_user.c ++++ b/tools/testing/selftests/bpf/progs/test_probe_user.c +@@ -7,6 +7,7 @@ + + #include + #include ++#include "bpf_misc.h" + + static struct sockaddr_in old; + diff --git a/PATCH/bpf_loop/070-v5.17-05-selftests-bpf-Use-_se-prefix-on-architectures-without-syscall-wrapper.patch b/PATCH/bpf_loop/070-v5.17-05-selftests-bpf-Use-_se-prefix-on-architectures-without-syscall-wrapper.patch new file mode 100644 index 00000000..049d33b7 --- /dev/null +++ b/PATCH/bpf_loop/070-v5.17-05-selftests-bpf-Use-_se-prefix-on-architectures-without-syscall-wrapper.patch @@ -0,0 +1,26 @@ +commit 046b841ea7c528931e7d2e74d5e668aa6c94c1fc upstream. + +On architectures that don't use a syscall wrapper, sys_* function names +are set as an alias of __se_sys_* functions. Due to this, there is no +BTF associated with sys_* function names. This results in some of the +test progs failing to load. Set the SYS_PREFIX to "__se_" to fix this +issue. + +Fixes: 38261f369fb905 ("selftests/bpf: Fix probe_user test failure with clang build kernel") +Signed-off-by: Naveen N. Rao +Signed-off-by: Andrii Nakryiko +Link: https://lore.kernel.org/bpf/013d632aacd3e41290445c0025db6a7055ec6e18.1643973917.git.naveen.n.rao@linux.vnet.ibm.com +Signed-off-by: Tianchen Ding +--- + +--- a/tools/testing/selftests/bpf/progs/bpf_misc.h ++++ b/tools/testing/selftests/bpf/progs/bpf_misc.h +@@ -13,7 +13,7 @@ + #define SYS_PREFIX "__arm64_" + #else + #define SYSCALL_WRAPPER 0 +-#define SYS_PREFIX "" ++#define SYS_PREFIX "__se_" + #endif + + #endif diff --git a/PATCH/bpf_loop/070-v5.17-06-selftests-bpf-Fix-tests-to-use-arch-dependent-syscall-entry-points.patch b/PATCH/bpf_loop/070-v5.17-06-selftests-bpf-Fix-tests-to-use-arch-dependent-syscall-entry-points.patch new file mode 100644 index 00000000..fc8039b7 --- /dev/null +++ b/PATCH/bpf_loop/070-v5.17-06-selftests-bpf-Fix-tests-to-use-arch-dependent-syscall-entry-points.patch @@ -0,0 +1,177 @@ +commit e91d280c840f133560072f246321f9a4d1f4eb14 upstream. + +Some of the tests are using x86_64 ABI-specific syscall entry points +(such as __x64_sys_nanosleep and __x64_sys_getpgid). Update them to use +architecture-dependent syscall entry names. + +Also update fexit_sleep test to not use BPF_PROG() so that it is clear +that the syscall parameters aren't being accessed in the bpf prog. + +Note that none of the bpf progs in these tests are actually accessing +any of the syscall parameters. The only exception is perfbuf_bench, which +passes on the bpf prog context into bpf_perf_event_output() as a pointer +to pt_regs, but that looks to be mostly ignored. + +Signed-off-by: Naveen N. Rao +Signed-off-by: Andrii Nakryiko +Link: https://lore.kernel.org/bpf/e35f7051f03e269b623a68b139d8ed131325f7b7.1643973917.git.naveen.n.rao@linux.vnet.ibm.com +[dtcccc: NOTE: this patch is mainly for bpf_loop selftests on aarch64, +but now it will still fail because we have not supported fentry on +aarch64 yet. However, I do not plan to "fix" it by replacing fentry with +kprobe or other things because: +1. Other test cases like bloom_filter and ringbuf also use fentry and +they have already broken on aarch64. +2. We may support fentry on aarch64 in future, I don't want to "fix" it +now and then revert the fix at that time. +If we really want to test bpf_loop on aarch64 now, just set the section +as SEC("kprobe/hrtimer_nanosleep").] +Signed-off-by: Tianchen Ding +--- + +--- a/tools/testing/selftests/bpf/progs/bpf_loop.c ++++ b/tools/testing/selftests/bpf/progs/bpf_loop.c +@@ -3,6 +3,7 @@ + + #include "vmlinux.h" + #include ++#include "bpf_misc.h" + + char _license[] SEC("license") = "GPL"; + +@@ -53,7 +54,7 @@ static int nested_callback1(__u32 index, + return 0; + } + +-SEC("fentry/__x64_sys_nanosleep") ++SEC("fentry/" SYS_PREFIX "sys_nanosleep") + int test_prog(void *ctx) + { + struct callback_ctx data = {}; +@@ -71,7 +72,7 @@ int test_prog(void *ctx) + return 0; + } + +-SEC("fentry/__x64_sys_nanosleep") ++SEC("fentry/" SYS_PREFIX "sys_nanosleep") + int prog_null_ctx(void *ctx) + { + if (bpf_get_current_pid_tgid() >> 32 != pid) +@@ -82,7 +83,7 @@ int prog_null_ctx(void *ctx) + return 0; + } + +-SEC("fentry/__x64_sys_nanosleep") ++SEC("fentry/" SYS_PREFIX "sys_nanosleep") + int prog_invalid_flags(void *ctx) + { + struct callback_ctx data = {}; +@@ -95,7 +96,7 @@ int prog_invalid_flags(void *ctx) + return 0; + } + +-SEC("fentry/__x64_sys_nanosleep") ++SEC("fentry/" SYS_PREFIX "sys_nanosleep") + int prog_nested_calls(void *ctx) + { + struct callback_ctx data = {}; +--- a/tools/testing/selftests/bpf/progs/perfbuf_bench.c ++++ b/tools/testing/selftests/bpf/progs/perfbuf_bench.c +@@ -4,6 +4,7 @@ + #include + #include + #include ++#include "bpf_misc.h" + + char _license[] SEC("license") = "GPL"; + +@@ -18,7 +19,7 @@ const volatile int batch_cnt = 0; + long sample_val = 42; + long dropped __attribute__((aligned(128))) = 0; + +-SEC("fentry/__x64_sys_getpgid") ++SEC("fentry/" SYS_PREFIX "sys_getpgid") + int bench_perfbuf(void *ctx) + { + __u64 *sample; +--- a/tools/testing/selftests/bpf/progs/ringbuf_bench.c ++++ b/tools/testing/selftests/bpf/progs/ringbuf_bench.c +@@ -4,6 +4,7 @@ + #include + #include + #include ++#include "bpf_misc.h" + + char _license[] SEC("license") = "GPL"; + +@@ -30,7 +31,7 @@ static __always_inline long get_flags() + return sz >= wakeup_data_size ? BPF_RB_FORCE_WAKEUP : BPF_RB_NO_WAKEUP; + } + +-SEC("fentry/__x64_sys_getpgid") ++SEC("fentry/" SYS_PREFIX "sys_getpgid") + int bench_ringbuf(void *ctx) + { + long *sample, flags; +--- a/tools/testing/selftests/bpf/progs/test_ringbuf.c ++++ b/tools/testing/selftests/bpf/progs/test_ringbuf.c +@@ -3,6 +3,7 @@ + + #include + #include ++#include "bpf_misc.h" + + char _license[] SEC("license") = "GPL"; + +--- a/tools/testing/selftests/bpf/progs/trace_printk.c ++++ b/tools/testing/selftests/bpf/progs/trace_printk.c +@@ -4,6 +4,7 @@ + #include "vmlinux.h" + #include + #include ++#include "bpf_misc.h" + + char _license[] SEC("license") = "GPL"; + +--- a/tools/testing/selftests/bpf/progs/trigger_bench.c ++++ b/tools/testing/selftests/bpf/progs/trigger_bench.c +@@ -5,6 +5,7 @@ + #include + #include + #include ++#include "bpf_misc.h" + + char _license[] SEC("license") = "GPL"; + +@@ -25,28 +26,28 @@ int BPF_PROG(bench_trigger_raw_tp, struc + return 0; + } + +-SEC("kprobe/__x64_sys_getpgid") ++SEC("kprobe/" SYS_PREFIX "sys_getpgid") + int bench_trigger_kprobe(void *ctx) + { + __sync_add_and_fetch(&hits, 1); + return 0; + } + +-SEC("fentry/__x64_sys_getpgid") ++SEC("fentry/" SYS_PREFIX "sys_getpgid") + int bench_trigger_fentry(void *ctx) + { + __sync_add_and_fetch(&hits, 1); + return 0; + } + +-SEC("fentry.s/__x64_sys_getpgid") ++SEC("fentry.s/" SYS_PREFIX "sys_getpgid") + int bench_trigger_fentry_sleep(void *ctx) + { + __sync_add_and_fetch(&hits, 1); + return 0; + } + +-SEC("fmod_ret/__x64_sys_getpgid") ++SEC("fmod_ret/" SYS_PREFIX "sys_getpgid") + int bench_trigger_fmodret(void *ctx) + { + __sync_add_and_fetch(&hits, 1); diff --git a/PATCH/bpf_loop/071-v5.17-bpf-Inline-calls-to-bpf_loop-when-callback-is-known.patch b/PATCH/bpf_loop/071-v5.17-bpf-Inline-calls-to-bpf_loop-when-callback-is-known.patch new file mode 100644 index 00000000..b8b830cc --- /dev/null +++ b/PATCH/bpf_loop/071-v5.17-bpf-Inline-calls-to-bpf_loop-when-callback-is-known.patch @@ -0,0 +1,1245 @@ +--- a/include/linux/bpf.h ++++ b/include/linux/bpf.h +@@ -1127,6 +1127,9 @@ struct bpf_array { + #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ + #define MAX_TAIL_CALL_CNT 32 + ++/* Maximum number of loops for bpf_loop */ ++#define BPF_MAX_LOOPS BIT(23) ++ + #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ + BPF_F_RDONLY_PROG | \ + BPF_F_WRONLY | \ +--- a/include/linux/bpf_verifier.h ++++ b/include/linux/bpf_verifier.h +@@ -356,6 +356,14 @@ struct bpf_verifier_state_list { + int miss_cnt, hit_cnt; + }; + ++struct bpf_loop_inline_state { ++ unsigned int initialized:1; /* set to true upon first entry */ ++ unsigned int fit_for_inline:1; /* true if callback function is the same ++ * at each call and flags are always zero ++ */ ++ u32 callback_subprogno; /* valid when fit_for_inline is true */ ++}; ++ + /* Possible states for alu_state member. */ + #define BPF_ALU_SANITIZE_SRC (1U << 0) + #define BPF_ALU_SANITIZE_DST (1U << 1) +@@ -385,6 +393,10 @@ struct bpf_insn_aux_data { + u32 mem_size; /* mem_size for non-struct typed var */ + }; + } btf_var; ++ /* if instruction is a call to bpf_loop this field tracks ++ * the state of the relevant registers to make decision about inlining ++ */ ++ struct bpf_loop_inline_state loop_inline_state; + }; + u64 map_key_state; /* constant (32 bit) key tracking for maps */ + int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ +--- a/kernel/bpf/bpf_iter.c ++++ b/kernel/bpf/bpf_iter.c +@@ -715,9 +715,6 @@ const struct bpf_func_proto bpf_for_each + .arg4_type = ARG_ANYTHING, + }; + +-/* maximum number of loops */ +-#define MAX_LOOPS BIT(23) +- + BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, + u64, flags) + { +@@ -725,9 +722,13 @@ BPF_CALL_4(bpf_loop, u32, nr_loops, void + u64 ret; + u32 i; + ++ /* Note: these safety checks are also verified when bpf_loop ++ * is inlined, be careful to modify this code in sync. See ++ * function verifier.c:inline_bpf_loop. ++ */ + if (flags) + return -EINVAL; +- if (nr_loops > MAX_LOOPS) ++ if (nr_loops > BPF_MAX_LOOPS) + return -E2BIG; + + for (i = 0; i < nr_loops; i++) { +--- a/kernel/bpf/verifier.c ++++ b/kernel/bpf/verifier.c +@@ -6520,6 +6520,41 @@ static int check_get_func_ip(struct bpf_ + return -ENOTSUPP; + } + ++static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) ++{ ++ return &env->insn_aux_data[env->insn_idx]; ++} ++ ++static bool loop_flag_is_zero(struct bpf_verifier_env *env) ++{ ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_reg_state *reg = ®s[BPF_REG_4]; ++ bool reg_is_null = register_is_null(reg); ++ ++ if (reg_is_null) ++ mark_chain_precision(env, BPF_REG_4); ++ ++ return reg_is_null; ++} ++ ++static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) ++{ ++ struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; ++ ++ if (!state->initialized) { ++ state->initialized = 1; ++ state->fit_for_inline = loop_flag_is_zero(env); ++ state->callback_subprogno = subprogno; ++ return; ++ } ++ ++ if (!state->fit_for_inline) ++ return; ++ ++ state->fit_for_inline = (loop_flag_is_zero(env) && ++ state->callback_subprogno == subprogno); ++} ++ + static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx_p) + { +@@ -6643,6 +6678,7 @@ static int check_helper_call(struct bpf_ + err = check_bpf_snprintf_call(env, regs); + break; + case BPF_FUNC_loop: ++ update_loop_inline_state(env, meta.subprogno); + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_loop_callback_state); + break; +@@ -6979,11 +7015,6 @@ static bool check_reg_sane_offset(struct + return true; + } + +-static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) +-{ +- return &env->insn_aux_data[env->insn_idx]; +-} +- + enum { + REASON_BOUNDS = -1, + REASON_TYPE = -2, +@@ -13390,6 +13421,142 @@ patch_call_imm: + return 0; + } + ++static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, ++ int position, ++ s32 stack_base, ++ u32 callback_subprogno, ++ u32 *cnt) ++{ ++ s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; ++ s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; ++ s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; ++ int reg_loop_max = BPF_REG_6; ++ int reg_loop_cnt = BPF_REG_7; ++ int reg_loop_ctx = BPF_REG_8; ++ ++ struct bpf_prog *new_prog; ++ u32 callback_start; ++ u32 call_insn_offset; ++ s32 callback_offset; ++ ++ /* This represents an inlined version of bpf_iter.c:bpf_loop, ++ * be careful to modify this code in sync. ++ */ ++ struct bpf_insn insn_buf[] = { ++ /* Return error and jump to the end of the patch if ++ * expected number of iterations is too big. ++ */ ++ BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), ++ BPF_MOV32_IMM(BPF_REG_0, -E2BIG), ++ BPF_JMP_IMM(BPF_JA, 0, 0, 16), ++ /* spill R6, R7, R8 to use these as loop vars */ ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), ++ /* initialize loop vars */ ++ BPF_MOV64_REG(reg_loop_max, BPF_REG_1), ++ BPF_MOV32_IMM(reg_loop_cnt, 0), ++ BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), ++ /* loop header, ++ * if reg_loop_cnt >= reg_loop_max skip the loop body ++ */ ++ BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), ++ /* callback call, ++ * correct callback offset would be set after patching ++ */ ++ BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), ++ BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), ++ BPF_CALL_REL(0), ++ /* increment loop counter */ ++ BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), ++ /* jump to loop header if callback returned 0 */ ++ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), ++ /* return value of bpf_loop, ++ * set R0 to the number of iterations ++ */ ++ BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), ++ /* restore original values of R6, R7, R8 */ ++ BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), ++ BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), ++ BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), ++ }; ++ ++ *cnt = ARRAY_SIZE(insn_buf); ++ new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); ++ if (!new_prog) ++ return new_prog; ++ ++ /* callback start is known only after patching */ ++ callback_start = env->subprog_info[callback_subprogno].start; ++ /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ ++ call_insn_offset = position + 12; ++ callback_offset = callback_start - call_insn_offset - 1; ++ new_prog->insnsi[call_insn_offset].imm = callback_offset; ++ ++ return new_prog; ++} ++ ++static bool is_bpf_loop_call(struct bpf_insn *insn) ++{ ++ return insn->code == (BPF_JMP | BPF_CALL) && ++ insn->src_reg == 0 && ++ insn->imm == BPF_FUNC_loop; ++} ++ ++/* For all sub-programs in the program (including main) check ++ * insn_aux_data to see if there are bpf_loop calls that require ++ * inlining. If such calls are found the calls are replaced with a ++ * sequence of instructions produced by `inline_bpf_loop` function and ++ * subprog stack_depth is increased by the size of 3 registers. ++ * This stack space is used to spill values of the R6, R7, R8. These ++ * registers are used to store the loop bound, counter and context ++ * variables. ++ */ ++static int optimize_bpf_loop(struct bpf_verifier_env *env) ++{ ++ struct bpf_subprog_info *subprogs = env->subprog_info; ++ int i, cur_subprog = 0, cnt, delta = 0; ++ struct bpf_insn *insn = env->prog->insnsi; ++ int insn_cnt = env->prog->len; ++ u16 stack_depth = subprogs[cur_subprog].stack_depth; ++ u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; ++ u16 stack_depth_extra = 0; ++ ++ for (i = 0; i < insn_cnt; i++, insn++) { ++ struct bpf_loop_inline_state *inline_state = ++ &env->insn_aux_data[i + delta].loop_inline_state; ++ ++ if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { ++ struct bpf_prog *new_prog; ++ ++ stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; ++ new_prog = inline_bpf_loop(env, ++ i + delta, ++ -(stack_depth + stack_depth_extra), ++ inline_state->callback_subprogno, ++ &cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ } ++ ++ if (subprogs[cur_subprog + 1].start == i + delta + 1) { ++ subprogs[cur_subprog].stack_depth += stack_depth_extra; ++ cur_subprog++; ++ stack_depth = subprogs[cur_subprog].stack_depth; ++ stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; ++ stack_depth_extra = 0; ++ } ++ } ++ ++ env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; ++ ++ return 0; ++} ++ + static void free_states(struct bpf_verifier_env *env) + { + struct bpf_verifier_state_list *sl, *sln; +@@ -14131,6 +14298,9 @@ skip_full_check: + ret = check_max_stack_depth(env); + + /* instruction rewrites happen after this point */ ++ if (ret == 0) ++ ret = optimize_bpf_loop(env); ++ + if (is_priv) { + if (ret == 0) + opt_hard_wire_dead_code_branches(env); +--- a/tools/testing/selftests/bpf/prog_tests/bpf_loop.c ++++ b/tools/testing/selftests/bpf/prog_tests/bpf_loop.c +@@ -120,6 +120,64 @@ static void check_nested_calls(struct bp + bpf_link__destroy(link); + } + ++static void check_non_constant_callback(struct bpf_loop *skel) ++{ ++ struct bpf_link *link = ++ bpf_program__attach(skel->progs.prog_non_constant_callback); ++ ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ skel->bss->callback_selector = 0x0F; ++ usleep(1); ++ ASSERT_EQ(skel->bss->g_output, 0x0F, "g_output #1"); ++ ++ skel->bss->callback_selector = 0xF0; ++ usleep(1); ++ ASSERT_EQ(skel->bss->g_output, 0xF0, "g_output #2"); ++ ++ bpf_link__destroy(link); ++} ++ ++static void check_stack(struct bpf_loop *skel) ++{ ++ struct bpf_link *link = bpf_program__attach(skel->progs.stack_check); ++ const int max_key = 12; ++ int key; ++ int map_fd; ++ ++ if (!ASSERT_OK_PTR(link, "link")) ++ return; ++ ++ map_fd = bpf_map__fd(skel->maps.map1); ++ ++ if (!ASSERT_GE(map_fd, 0, "bpf_map__fd")) ++ goto out; ++ ++ for (key = 1; key <= max_key; ++key) { ++ int val = key; ++ int err = bpf_map_update_elem(map_fd, &key, &val, BPF_NOEXIST); ++ ++ if (!ASSERT_OK(err, "bpf_map_update_elem")) ++ goto out; ++ } ++ ++ usleep(1); ++ ++ for (key = 1; key <= max_key; ++key) { ++ int val; ++ int err = bpf_map_lookup_elem(map_fd, &key, &val); ++ ++ if (!ASSERT_OK(err, "bpf_map_lookup_elem")) ++ goto out; ++ if (!ASSERT_EQ(val, key + 1, "bad value in the map")) ++ goto out; ++ } ++ ++out: ++ bpf_link__destroy(link); ++} ++ + void test_bpf_loop(void) + { + struct bpf_loop *skel; +@@ -140,6 +198,10 @@ void test_bpf_loop(void) + check_invalid_flags(skel); + if (test__start_subtest("check_nested_calls")) + check_nested_calls(skel); ++ if (test__start_subtest("check_non_constant_callback")) ++ check_non_constant_callback(skel); ++ if (test__start_subtest("check_stack")) ++ check_stack(skel); + + bpf_loop__destroy(skel); + } +--- a/tools/testing/selftests/bpf/prog_tests/btf.c ++++ b/tools/testing/selftests/bpf/prog_tests/btf.c +@@ -36,7 +36,6 @@ static bool always_log; + #undef CHECK + #define CHECK(condition, format...) _CHECK(condition, "check", duration, format) + +-#define BTF_END_RAW 0xdeadbeef + #define NAME_TBD 0xdeadb33f + + #define NAME_NTH(N) (0xffff0000 | N) +--- a/tools/testing/selftests/bpf/progs/bpf_loop.c ++++ b/tools/testing/selftests/bpf/progs/bpf_loop.c +@@ -11,11 +11,19 @@ struct callback_ctx { + int output; + }; + ++struct { ++ __uint(type, BPF_MAP_TYPE_HASH); ++ __uint(max_entries, 32); ++ __type(key, int); ++ __type(value, int); ++} map1 SEC(".maps"); ++ + /* These should be set by the user program */ + u32 nested_callback_nr_loops; + u32 stop_index = -1; + u32 nr_loops; + int pid; ++int callback_selector; + + /* Making these global variables so that the userspace program + * can verify the output through the skeleton +@@ -111,3 +119,109 @@ int prog_nested_calls(void *ctx) + + return 0; + } ++ ++static int callback_set_f0(int i, void *ctx) ++{ ++ g_output = 0xF0; ++ return 0; ++} ++ ++static int callback_set_0f(int i, void *ctx) ++{ ++ g_output = 0x0F; ++ return 0; ++} ++ ++/* ++ * non-constant callback is a corner case for bpf_loop inline logic ++ */ ++SEC("fentry/" SYS_PREFIX "sys_nanosleep") ++int prog_non_constant_callback(void *ctx) ++{ ++ struct callback_ctx data = {}; ++ ++ if (bpf_get_current_pid_tgid() >> 32 != pid) ++ return 0; ++ ++ int (*callback)(int i, void *ctx); ++ ++ g_output = 0; ++ ++ if (callback_selector == 0x0F) ++ callback = callback_set_0f; ++ else ++ callback = callback_set_f0; ++ ++ bpf_loop(1, callback, NULL, 0); ++ ++ return 0; ++} ++ ++static int stack_check_inner_callback(void *ctx) ++{ ++ return 0; ++} ++ ++static int map1_lookup_elem(int key) ++{ ++ int *val = bpf_map_lookup_elem(&map1, &key); ++ ++ return val ? *val : -1; ++} ++ ++static void map1_update_elem(int key, int val) ++{ ++ bpf_map_update_elem(&map1, &key, &val, BPF_ANY); ++} ++ ++static int stack_check_outer_callback(void *ctx) ++{ ++ int a = map1_lookup_elem(1); ++ int b = map1_lookup_elem(2); ++ int c = map1_lookup_elem(3); ++ int d = map1_lookup_elem(4); ++ int e = map1_lookup_elem(5); ++ int f = map1_lookup_elem(6); ++ ++ bpf_loop(1, stack_check_inner_callback, NULL, 0); ++ ++ map1_update_elem(1, a + 1); ++ map1_update_elem(2, b + 1); ++ map1_update_elem(3, c + 1); ++ map1_update_elem(4, d + 1); ++ map1_update_elem(5, e + 1); ++ map1_update_elem(6, f + 1); ++ ++ return 0; ++} ++ ++/* Some of the local variables in stack_check and ++ * stack_check_outer_callback would be allocated on stack by ++ * compiler. This test should verify that stack content for these ++ * variables is preserved between calls to bpf_loop (might be an issue ++ * if loop inlining allocates stack slots incorrectly). ++ */ ++SEC("fentry/" SYS_PREFIX "sys_nanosleep") ++int stack_check(void *ctx) ++{ ++ if (bpf_get_current_pid_tgid() >> 32 != pid) ++ return 0; ++ ++ int a = map1_lookup_elem(7); ++ int b = map1_lookup_elem(8); ++ int c = map1_lookup_elem(9); ++ int d = map1_lookup_elem(10); ++ int e = map1_lookup_elem(11); ++ int f = map1_lookup_elem(12); ++ ++ bpf_loop(1, stack_check_outer_callback, NULL, 0); ++ ++ map1_update_elem(7, a + 1); ++ map1_update_elem(8, b + 1); ++ map1_update_elem(9, c + 1); ++ map1_update_elem(10, d + 1); ++ map1_update_elem(11, e + 1); ++ map1_update_elem(12, f + 1); ++ ++ return 0; ++} +--- a/tools/testing/selftests/bpf/test_btf.h ++++ b/tools/testing/selftests/bpf/test_btf.h +@@ -4,6 +4,8 @@ + #ifndef _TEST_BTF_H + #define _TEST_BTF_H + ++#define BTF_END_RAW 0xdeadbeef ++ + #define BTF_INFO_ENC(kind, kind_flag, vlen) \ + ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + +--- a/tools/testing/selftests/bpf/test_verifier.c ++++ b/tools/testing/selftests/bpf/test_verifier.c +@@ -48,12 +48,24 @@ + #include "../../../include/linux/filter.h" + + #define MAX_INSNS BPF_MAXINSNS ++#define MAX_EXPECTED_INSNS 32 ++#define MAX_UNEXPECTED_INSNS 32 + #define MAX_TEST_INSNS 1000000 + #define MAX_FIXUPS 8 + #define MAX_NR_MAPS 21 + #define MAX_TEST_RUNS 8 + #define POINTER_VALUE 0xcafe4all + #define TEST_DATA_LEN 64 ++#define MAX_FUNC_INFOS 8 ++#define MAX_BTF_STRINGS 256 ++#define MAX_BTF_TYPES 256 ++ ++#define INSN_OFF_MASK ((__s16)0xFFFF) ++#define INSN_IMM_MASK ((__s32)0xFFFFFFFF) ++#define SKIP_INSNS() BPF_RAW_INSN(0xde, 0xa, 0xd, 0xbeef, 0xdeadbeef) ++ ++#define DEFAULT_LIBBPF_LOG_LEVEL 4 ++#define VERBOSE_LIBBPF_LOG_LEVEL 1 + + #define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS (1 << 0) + #define F_LOAD_WITH_STRICT_ALIGNMENT (1 << 1) +@@ -76,6 +88,23 @@ struct bpf_test { + const char *descr; + struct bpf_insn insns[MAX_INSNS]; + struct bpf_insn *fill_insns; ++ /* If specified, test engine looks for this sequence of ++ * instructions in the BPF program after loading. Allows to ++ * test rewrites applied by verifier. Use values ++ * INSN_OFF_MASK and INSN_IMM_MASK to mask `off` and `imm` ++ * fields if content does not matter. The test case fails if ++ * specified instructions are not found. ++ * ++ * The sequence could be split into sub-sequences by adding ++ * SKIP_INSNS instruction at the end of each sub-sequence. In ++ * such case sub-sequences are searched for one after another. ++ */ ++ struct bpf_insn expected_insns[MAX_EXPECTED_INSNS]; ++ /* If specified, test engine applies same pattern matching ++ * logic as for `expected_insns`. If the specified pattern is ++ * matched test case is marked as failed. ++ */ ++ struct bpf_insn unexpected_insns[MAX_UNEXPECTED_INSNS]; + int fixup_map_hash_8b[MAX_FIXUPS]; + int fixup_map_hash_48b[MAX_FIXUPS]; + int fixup_map_hash_16b[MAX_FIXUPS]; +@@ -130,6 +159,14 @@ struct bpf_test { + }; + enum bpf_attach_type expected_attach_type; + const char *kfunc; ++ struct bpf_func_info func_info[MAX_FUNC_INFOS]; ++ int func_info_cnt; ++ char btf_strings[MAX_BTF_STRINGS]; ++ /* A set of BTF types to load when specified, ++ * use macro definitions from test_btf.h, ++ * must end with BTF_END_RAW ++ */ ++ __u32 btf_types[MAX_BTF_TYPES]; + }; + + /* Note we want this to be 64 bit aligned so that the end of our array is +@@ -383,6 +420,45 @@ static void bpf_fill_torturous_jumps(str + } + } + ++static void bpf_fill_big_prog_with_loop_1(struct bpf_test *self) ++{ ++ struct bpf_insn *insn = self->fill_insns; ++ /* This test was added to catch a specific use after free ++ * error, which happened upon BPF program reallocation. ++ * Reallocation is handled by core.c:bpf_prog_realloc, which ++ * reuses old memory if page boundary is not crossed. The ++ * value of `len` is chosen to cross this boundary on bpf_loop ++ * patching. ++ */ ++ const int len = getpagesize() - 25; ++ int callback_load_idx; ++ int callback_idx; ++ int i = 0; ++ ++ insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1); ++ callback_load_idx = i; ++ insn[i++] = BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, ++ BPF_REG_2, BPF_PSEUDO_FUNC, 0, ++ 777 /* filled below */); ++ insn[i++] = BPF_RAW_INSN(0, 0, 0, 0, 0); ++ insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); ++ insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0); ++ insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop); ++ ++ while (i < len - 3) ++ insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); ++ insn[i++] = BPF_EXIT_INSN(); ++ ++ callback_idx = i; ++ insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); ++ insn[i++] = BPF_EXIT_INSN(); ++ ++ insn[callback_load_idx].imm = callback_idx - callback_load_idx - 1; ++ self->func_info[1].insn_off = callback_idx; ++ self->prog_len = i; ++ assert(i == len); ++} ++ + /* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */ + #define BPF_SK_LOOKUP(func) \ + /* struct bpf_sock_tuple tuple = {} */ \ +@@ -624,34 +700,59 @@ static __u32 btf_raw_types[] = { + BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */ + }; + +-static int load_btf(void) ++static char bpf_vlog[UINT_MAX >> 8]; ++ ++static int load_btf_spec(__u32 *types, int types_len, ++ const char *strings, int strings_len) + { + struct btf_header hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), +- .type_len = sizeof(btf_raw_types), +- .str_off = sizeof(btf_raw_types), +- .str_len = sizeof(btf_str_sec), ++ .type_len = types_len, ++ .str_off = types_len, ++ .str_len = strings_len, + }; + void *ptr, *raw_btf; + int btf_fd; + +- ptr = raw_btf = malloc(sizeof(hdr) + sizeof(btf_raw_types) + +- sizeof(btf_str_sec)); ++ raw_btf = malloc(sizeof(hdr) + types_len + strings_len); + ++ ptr = raw_btf; + memcpy(ptr, &hdr, sizeof(hdr)); + ptr += sizeof(hdr); +- memcpy(ptr, btf_raw_types, hdr.type_len); ++ memcpy(ptr, types, hdr.type_len); + ptr += hdr.type_len; +- memcpy(ptr, btf_str_sec, hdr.str_len); ++ memcpy(ptr, strings, hdr.str_len); + ptr += hdr.str_len; + +- btf_fd = bpf_load_btf(raw_btf, ptr - raw_btf, 0, 0, 0); +- free(raw_btf); ++ btf_fd = bpf_load_btf(raw_btf, ptr - raw_btf, bpf_vlog, sizeof(bpf_vlog), 1); + if (btf_fd < 0) +- return -1; +- return btf_fd; ++ printf("Failed to load BTF spec: '%s'\n", strerror(errno)); ++ ++ free(raw_btf); ++ ++ return btf_fd < 0 ? -1 : btf_fd; ++} ++ ++static int load_btf(void) ++{ ++ return load_btf_spec(btf_raw_types, sizeof(btf_raw_types), ++ btf_str_sec, sizeof(btf_str_sec)); ++} ++ ++static int load_btf_for_test(struct bpf_test *test) ++{ ++ int types_num = 0; ++ ++ while (types_num < MAX_BTF_TYPES && ++ test->btf_types[types_num] != BTF_END_RAW) ++ ++types_num; ++ ++ int types_len = types_num * sizeof(test->btf_types[0]); ++ ++ return load_btf_spec(test->btf_types, types_len, ++ test->btf_strings, sizeof(test->btf_strings)); + } + + static int create_map_spin_lock(void) +@@ -702,8 +803,6 @@ static int create_sk_storage_map(void) + return fd; + } + +-static char bpf_vlog[UINT_MAX >> 8]; +- + static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type, + struct bpf_insn *prog, int *map_fds) + { +@@ -1038,10 +1137,220 @@ static bool cmp_str_seq(const char *log, + return true; + } + ++static struct bpf_insn *get_xlated_program(int fd_prog, int *cnt) ++{ ++ __u32 buf_element_size = sizeof(struct bpf_insn); ++ struct bpf_prog_info info = {}; ++ __u32 info_len = sizeof(info); ++ __u32 xlated_prog_len; ++ struct bpf_insn *buf; ++ ++ if (bpf_obj_get_info_by_fd(fd_prog, &info, &info_len)) { ++ perror("bpf_obj_get_info_by_fd failed"); ++ return NULL; ++ } ++ ++ xlated_prog_len = info.xlated_prog_len; ++ if (xlated_prog_len % buf_element_size) { ++ printf("Program length %d is not multiple of %d\n", ++ xlated_prog_len, buf_element_size); ++ return NULL; ++ } ++ ++ *cnt = xlated_prog_len / buf_element_size; ++ buf = calloc(*cnt, buf_element_size); ++ if (!buf) { ++ perror("can't allocate xlated program buffer"); ++ return NULL; ++ } ++ ++ bzero(&info, sizeof(info)); ++ info.xlated_prog_len = xlated_prog_len; ++ info.xlated_prog_insns = (__u64)(unsigned long)buf; ++ if (bpf_obj_get_info_by_fd(fd_prog, &info, &info_len)) { ++ perror("second bpf_obj_get_info_by_fd failed"); ++ goto out_free_buf; ++ } ++ ++ return buf; ++ ++out_free_buf: ++ free(buf); ++ return NULL; ++} ++ ++static bool is_null_insn(struct bpf_insn *insn) ++{ ++ struct bpf_insn null_insn = {}; ++ ++ return memcmp(insn, &null_insn, sizeof(null_insn)) == 0; ++} ++ ++static bool is_skip_insn(struct bpf_insn *insn) ++{ ++ struct bpf_insn skip_insn = SKIP_INSNS(); ++ ++ return memcmp(insn, &skip_insn, sizeof(skip_insn)) == 0; ++} ++ ++static int null_terminated_insn_len(struct bpf_insn *seq, int max_len) ++{ ++ int i; ++ ++ for (i = 0; i < max_len; ++i) { ++ if (is_null_insn(&seq[i])) ++ return i; ++ } ++ return max_len; ++} ++ ++static bool compare_masked_insn(struct bpf_insn *orig, struct bpf_insn *masked) ++{ ++ struct bpf_insn orig_masked; ++ ++ memcpy(&orig_masked, orig, sizeof(orig_masked)); ++ if (masked->imm == INSN_IMM_MASK) ++ orig_masked.imm = INSN_IMM_MASK; ++ if (masked->off == INSN_OFF_MASK) ++ orig_masked.off = INSN_OFF_MASK; ++ ++ return memcmp(&orig_masked, masked, sizeof(orig_masked)) == 0; ++} ++ ++static int find_insn_subseq(struct bpf_insn *seq, struct bpf_insn *subseq, ++ int seq_len, int subseq_len) ++{ ++ int i, j; ++ ++ if (subseq_len > seq_len) ++ return -1; ++ ++ for (i = 0; i < seq_len - subseq_len + 1; ++i) { ++ bool found = true; ++ ++ for (j = 0; j < subseq_len; ++j) { ++ if (!compare_masked_insn(&seq[i + j], &subseq[j])) { ++ found = false; ++ break; ++ } ++ } ++ if (found) ++ return i; ++ } ++ ++ return -1; ++} ++ ++static int find_skip_insn_marker(struct bpf_insn *seq, int len) ++{ ++ int i; ++ ++ for (i = 0; i < len; ++i) ++ if (is_skip_insn(&seq[i])) ++ return i; ++ ++ return -1; ++} ++ ++/* Return true if all sub-sequences in `subseqs` could be found in ++ * `seq` one after another. Sub-sequences are separated by a single ++ * nil instruction. ++ */ ++static bool find_all_insn_subseqs(struct bpf_insn *seq, struct bpf_insn *subseqs, ++ int seq_len, int max_subseqs_len) ++{ ++ int subseqs_len = null_terminated_insn_len(subseqs, max_subseqs_len); ++ ++ while (subseqs_len > 0) { ++ int skip_idx = find_skip_insn_marker(subseqs, subseqs_len); ++ int cur_subseq_len = skip_idx < 0 ? subseqs_len : skip_idx; ++ int subseq_idx = find_insn_subseq(seq, subseqs, ++ seq_len, cur_subseq_len); ++ ++ if (subseq_idx < 0) ++ return false; ++ seq += subseq_idx + cur_subseq_len; ++ seq_len -= subseq_idx + cur_subseq_len; ++ subseqs += cur_subseq_len + 1; ++ subseqs_len -= cur_subseq_len + 1; ++ } ++ ++ return true; ++} ++ ++static void print_insn(struct bpf_insn *buf, int cnt) ++{ ++ int i; ++ ++ printf(" addr op d s off imm\n"); ++ for (i = 0; i < cnt; ++i) { ++ struct bpf_insn *insn = &buf[i]; ++ ++ if (is_null_insn(insn)) ++ break; ++ ++ if (is_skip_insn(insn)) ++ printf(" ...\n"); ++ else ++ printf(" %04x: %02x %1x %x %04hx %08x\n", ++ i, insn->code, insn->dst_reg, ++ insn->src_reg, insn->off, insn->imm); ++ } ++} ++ ++static bool check_xlated_program(struct bpf_test *test, int fd_prog) ++{ ++ struct bpf_insn *buf; ++ int cnt; ++ bool result = true; ++ bool check_expected = !is_null_insn(test->expected_insns); ++ bool check_unexpected = !is_null_insn(test->unexpected_insns); ++ ++ if (!check_expected && !check_unexpected) ++ goto out; ++ ++ buf = get_xlated_program(fd_prog, &cnt); ++ if (!buf) { ++ printf("FAIL: can't get xlated program\n"); ++ result = false; ++ goto out; ++ } ++ ++ if (check_expected && ++ !find_all_insn_subseqs(buf, test->expected_insns, ++ cnt, MAX_EXPECTED_INSNS)) { ++ printf("FAIL: can't find expected subsequence of instructions\n"); ++ result = false; ++ if (verbose) { ++ printf("Program:\n"); ++ print_insn(buf, cnt); ++ printf("Expected subsequence:\n"); ++ print_insn(test->expected_insns, MAX_EXPECTED_INSNS); ++ } ++ } ++ ++ if (check_unexpected && ++ find_all_insn_subseqs(buf, test->unexpected_insns, ++ cnt, MAX_UNEXPECTED_INSNS)) { ++ printf("FAIL: found unexpected subsequence of instructions\n"); ++ result = false; ++ if (verbose) { ++ printf("Program:\n"); ++ print_insn(buf, cnt); ++ printf("Un-expected subsequence:\n"); ++ print_insn(test->unexpected_insns, MAX_UNEXPECTED_INSNS); ++ } ++ } ++ ++ free(buf); ++ out: ++ return result; ++} ++ + static void do_test_single(struct bpf_test *test, bool unpriv, + int *passes, int *errors) + { +- int fd_prog, expected_ret, alignment_prevented_execution; ++ int fd_prog, btf_fd, expected_ret, alignment_prevented_execution; + int prog_len, prog_type = test->prog_type; + struct bpf_insn *prog = test->insns; + struct bpf_load_program_attr attr; +@@ -1053,8 +1362,10 @@ static void do_test_single(struct bpf_te + __u32 pflags; + int i, err; + ++ fd_prog = -1; + for (i = 0; i < MAX_NR_MAPS; i++) + map_fds[i] = -1; ++ btf_fd = -1; + + if (!prog_type) + prog_type = BPF_PROG_TYPE_SOCKET_FILTER; +@@ -1109,6 +1420,19 @@ static void do_test_single(struct bpf_te + } + } + ++ if (test->btf_types[0] != 0) { ++ btf_fd = load_btf_for_test(test); ++ if (btf_fd < 0) ++ goto fail_log; ++ attr.prog_btf_fd = btf_fd; ++ } ++ ++ if (test->func_info_cnt != 0) { ++ attr.func_info = test->func_info; ++ attr.func_info_cnt = test->func_info_cnt; ++ attr.func_info_rec_size = sizeof(test->func_info[0]); ++ } ++ + fd_prog = bpf_load_program_xattr(&attr, bpf_vlog, sizeof(bpf_vlog)); + saved_errno = errno; + +@@ -1166,6 +1490,9 @@ static void do_test_single(struct bpf_te + if (verbose) + printf(", verifier log:\n%s", bpf_vlog); + ++ if (!check_xlated_program(test, fd_prog)) ++ goto fail_log; ++ + run_errs = 0; + run_successes = 0; + if (!alignment_prevented_execution && fd_prog >= 0 && test->runs >= 0) { +@@ -1209,6 +1536,7 @@ close_fds: + if (test->fill_insns) + free(test->fill_insns); + close(fd_prog); ++ close(btf_fd); + for (i = 0; i < MAX_NR_MAPS; i++) + close(map_fds[i]); + sched_yield(); +--- /dev/null ++++ b/tools/testing/selftests/bpf/verifier/bpf_loop_inline.c +@@ -0,0 +1,263 @@ ++#define BTF_TYPES \ ++ .btf_strings = "\0int\0i\0ctx\0callback\0main\0", \ ++ .btf_types = { \ ++ /* 1: int */ BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), \ ++ /* 2: int* */ BTF_PTR_ENC(1), \ ++ /* 3: void* */ BTF_PTR_ENC(0), \ ++ /* 4: int __(void*) */ BTF_FUNC_PROTO_ENC(1, 1), \ ++ BTF_FUNC_PROTO_ARG_ENC(7, 3), \ ++ /* 5: int __(int, int*) */ BTF_FUNC_PROTO_ENC(1, 2), \ ++ BTF_FUNC_PROTO_ARG_ENC(5, 1), \ ++ BTF_FUNC_PROTO_ARG_ENC(7, 2), \ ++ /* 6: main */ BTF_FUNC_ENC(20, 4), \ ++ /* 7: callback */ BTF_FUNC_ENC(11, 5), \ ++ BTF_END_RAW \ ++ } ++ ++#define MAIN_TYPE 6 ++#define CALLBACK_TYPE 7 ++ ++/* can't use BPF_CALL_REL, jit_subprogs adjusts IMM & OFF ++ * fields for pseudo calls ++ */ ++#define PSEUDO_CALL_INSN() \ ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, \ ++ INSN_OFF_MASK, INSN_IMM_MASK) ++ ++/* can't use BPF_FUNC_loop constant, ++ * do_mix_fixups adjusts the IMM field ++ */ ++#define HELPER_CALL_INSN() \ ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, INSN_OFF_MASK, INSN_IMM_MASK) ++ ++{ ++ "inline simple bpf_loop call", ++ .insns = { ++ /* main */ ++ /* force verifier state branching to verify logic on first and ++ * subsequent bpf_loop insn processing steps ++ */ ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), ++ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 777, 2), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1), ++ BPF_JMP_IMM(BPF_JA, 0, 0, 1), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 2), ++ ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 6), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), ++ BPF_EXIT_INSN(), ++ /* callback */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ }, ++ .expected_insns = { PSEUDO_CALL_INSN() }, ++ .unexpected_insns = { HELPER_CALL_INSN() }, ++ .prog_type = BPF_PROG_TYPE_TRACEPOINT, ++ .result = ACCEPT, ++ .runs = 0, ++ .func_info = { { 0, MAIN_TYPE }, { 12, CALLBACK_TYPE } }, ++ .func_info_cnt = 2, ++ BTF_TYPES ++}, ++{ ++ "don't inline bpf_loop call, flags non-zero", ++ .insns = { ++ /* main */ ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), ++ BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), ++ BPF_ALU64_REG(BPF_MOV, BPF_REG_7, BPF_REG_0), ++ BPF_JMP_IMM(BPF_JNE, BPF_REG_6, 0, 9), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_JMP_IMM(BPF_JNE, BPF_REG_7, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 7), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), ++ BPF_EXIT_INSN(), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 1), ++ BPF_JMP_IMM(BPF_JA, 0, 0, -10), ++ /* callback */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ }, ++ .expected_insns = { HELPER_CALL_INSN() }, ++ .unexpected_insns = { PSEUDO_CALL_INSN() }, ++ .prog_type = BPF_PROG_TYPE_TRACEPOINT, ++ .result = ACCEPT, ++ .runs = 0, ++ .func_info = { { 0, MAIN_TYPE }, { 16, CALLBACK_TYPE } }, ++ .func_info_cnt = 2, ++ BTF_TYPES ++}, ++{ ++ "don't inline bpf_loop call, callback non-constant", ++ .insns = { ++ /* main */ ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), ++ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 777, 4), /* pick a random callback */ ++ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 10), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_JMP_IMM(BPF_JA, 0, 0, 3), ++ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 8), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), ++ BPF_EXIT_INSN(), ++ /* callback */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ /* callback #2 */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ }, ++ .expected_insns = { HELPER_CALL_INSN() }, ++ .unexpected_insns = { PSEUDO_CALL_INSN() }, ++ .prog_type = BPF_PROG_TYPE_TRACEPOINT, ++ .result = ACCEPT, ++ .runs = 0, ++ .func_info = { ++ { 0, MAIN_TYPE }, ++ { 14, CALLBACK_TYPE }, ++ { 16, CALLBACK_TYPE } ++ }, ++ .func_info_cnt = 3, ++ BTF_TYPES ++}, ++{ ++ "bpf_loop_inline and a dead func", ++ .insns = { ++ /* main */ ++ ++ /* A reference to callback #1 to make verifier count it as a func. ++ * This reference is overwritten below and callback #1 is dead. ++ */ ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 9), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 8), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), ++ BPF_EXIT_INSN(), ++ /* callback */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ /* callback #2 */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ }, ++ .expected_insns = { PSEUDO_CALL_INSN() }, ++ .unexpected_insns = { HELPER_CALL_INSN() }, ++ .prog_type = BPF_PROG_TYPE_TRACEPOINT, ++ .result = ACCEPT, ++ .runs = 0, ++ .func_info = { ++ { 0, MAIN_TYPE }, ++ { 10, CALLBACK_TYPE }, ++ { 12, CALLBACK_TYPE } ++ }, ++ .func_info_cnt = 3, ++ BTF_TYPES ++}, ++{ ++ "bpf_loop_inline stack locations for loop vars", ++ .insns = { ++ /* main */ ++ BPF_ST_MEM(BPF_W, BPF_REG_10, -12, 0x77), ++ /* bpf_loop call #1 */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 1), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 22), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ /* bpf_loop call #2 */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 2), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 16), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ /* call func and exit */ ++ BPF_CALL_REL(2), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), ++ BPF_EXIT_INSN(), ++ /* func */ ++ BPF_ST_MEM(BPF_DW, BPF_REG_10, -32, 0x55), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_1, 2), ++ BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, 6), ++ BPF_RAW_INSN(0, 0, 0, 0, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_4, 0), ++ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_loop), ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), ++ BPF_EXIT_INSN(), ++ /* callback */ ++ BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 1), ++ BPF_EXIT_INSN(), ++ }, ++ .expected_insns = { ++ BPF_ST_MEM(BPF_W, BPF_REG_10, -12, 0x77), ++ SKIP_INSNS(), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, -40), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, -32), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, -24), ++ SKIP_INSNS(), ++ /* offsets are the same as in the first call */ ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, -40), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, -32), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, -24), ++ SKIP_INSNS(), ++ BPF_ST_MEM(BPF_DW, BPF_REG_10, -32, 0x55), ++ SKIP_INSNS(), ++ /* offsets differ from main because of different offset ++ * in BPF_ST_MEM instruction ++ */ ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, -56), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, -48), ++ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, -40), ++ }, ++ .unexpected_insns = { HELPER_CALL_INSN() }, ++ .prog_type = BPF_PROG_TYPE_TRACEPOINT, ++ .result = ACCEPT, ++ .func_info = { ++ { 0, MAIN_TYPE }, ++ { 16, MAIN_TYPE }, ++ { 25, CALLBACK_TYPE }, ++ }, ++ .func_info_cnt = 3, ++ BTF_TYPES ++}, ++{ ++ "inline bpf_loop call in a big program", ++ .insns = {}, ++ .fill_helper = bpf_fill_big_prog_with_loop_1, ++ .expected_insns = { PSEUDO_CALL_INSN() }, ++ .unexpected_insns = { HELPER_CALL_INSN() }, ++ .result = ACCEPT, ++ .func_info = { { 0, MAIN_TYPE }, { 16, CALLBACK_TYPE } }, ++ .func_info_cnt = 2, ++ BTF_TYPES ++}, ++ ++#undef HELPER_CALL_INSN ++#undef PSEUDO_CALL_INSN ++#undef CALLBACK_TYPE ++#undef MAIN_TYPE ++#undef BTF_TYPES diff --git a/PATCH/duplicate/files/etc/hotplug.d/net/01-maximize_nic_rx_tx_buffers b/PATCH/duplicate/files/etc/hotplug.d/net/01-maximize_nic_rx_tx_buffers old mode 100755 new mode 100644 index 00c75505..6e3e3553 --- a/PATCH/duplicate/files/etc/hotplug.d/net/01-maximize_nic_rx_tx_buffers +++ b/PATCH/duplicate/files/etc/hotplug.d/net/01-maximize_nic_rx_tx_buffers @@ -1,22 +1,51 @@ #!/bin/sh -[ "$ACTION" = add ] || exit -# 遍历所有网口 +# Exit if the script is not triggered by adding a device +#[ "$ACTION" = add ] || exit + +# Iterate over all network interfaces for NIC in $(ls /sys/class/net/) do - # 检查网口是否支持ethtool命令 + # Check if ethtool command is supported for the interface if command -v ethtool &> /dev/null && ethtool $NIC &> /dev/null; then - # 获取rx和tx缓存最大值 + # Get the maximum RX and TX buffer sizes RX_MAX=$(ethtool -g $NIC 2>/dev/null | awk '/^RX:/ {print $2}' | awk 'NR==1') TX_MAX=$(ethtool -g $NIC 2>/dev/null | awk '/^TX:/ {print $2}' | awk 'NR==1') - # 如果无法获取rx和tx缓存最大值，则跳过该网卡 + # Skip the network card if unable to retrieve max RX and TX buffer sizes if [ -z "$RX_MAX" ] || [ -z "$TX_MAX" ]; then continue fi - # 调整rx和tx缓存为驱动支持的最大值 + # Set RX and TX buffers to maximum supported by the driver + echo "Setting RX buffer size to $RX_MAX on $NIC" ethtool -G $NIC rx $RX_MAX + echo "Setting TX buffer size to $TX_MAX on $NIC" ethtool -G $NIC tx $TX_MAX + + # Initialize flags to check if adaptive coalescing was enabled for RX and TX + adaptive_rx_supported=false + adaptive_tx_supported=false + + # Check and enable adaptive RX coalescing if supported + if ethtool -c $NIC | grep "Adaptive RX:" > /dev/null; then + adaptive_rx_supported=true + fi + + # Check and enable adaptive TX coalescing if supported + if ethtool -c $NIC | grep "TX: off" > /dev/null; then + adaptive_tx_supported=true + fi + + # If both adaptive RX and TX coalescing are supported, enable them + if [ "$adaptive_rx_supported" = true ] && [ "$adaptive_tx_supported" = true ]; then + if ethtool -C $NIC adaptive-rx on adaptive-tx on &> /dev/null; then + echo "Adaptive RX/TX coalescing enabled on $NIC" + else + echo "Failed to set adaptive RX/TX coalescing on $NIC." + adaptive_rx_supported=false + adaptive_tx_supported=false + fi + fi fi done diff --git a/SCRIPTS/02_prepare_package.sh b/SCRIPTS/02_prepare_package.sh index 87670947..5788eca5 100755 --- a/SCRIPTS/02_prepare_package.sh +++ b/SCRIPTS/02_prepare_package.sh @@ -17,12 +17,6 @@ rm -rf ./package/network/config/firewall4 cp -rf ../openwrt_ma/package/network/config/firewall4 ./package/network/config/firewall4 ## Important Patches -# Temp fix libpfring -wget -qO - https://github.com/SergeyFilippov/openwrt/commit/e66ca39.patch | patch -p1 -# PPPoE offload -wget https://github.com/openwrt/openwrt/raw/98834a4c3f81c6e4f20329ff266f9bd85731d114/target/linux/generic/backport-5.15/741-v6.9-01-netfilter-flowtable-validate-pppoe-header.patch -O target/linux/generic/backport-5.15/741-v6.9-01-netfilter-flowtable-validate-pppoe-header.patch -wget https://github.com/openwrt/openwrt/raw/98834a4c3f81c6e4f20329ff266f9bd85731d114/target/linux/generic/backport-5.15/741-v6.9-02-netfilter-flowtable-incorrect-pppoe-tuple.patch -O target/linux/generic/backport-5.15/741-v6.9-02-netfilter-flowtable-incorrect-pppoe-tuple.patch -wget https://github.com/openwrt/openwrt/raw/98834a4c3f81c6e4f20329ff266f9bd85731d114/target/linux/generic/hack-5.15/650-netfilter-add-xt_FLOWOFFLOAD-target.patch -O target/linux/generic/hack-5.15/650-netfilter-add-xt_FLOWOFFLOAD-target.patch # ARM64: Add CPU model name in proc cpuinfo cp -rf ../immortalwrt/target/linux/generic/hack-5.15/312-arm64-cpuinfo-Add-model-name-in-proc-cpuinfo-for-64bit-ta.patch ./target/linux/generic/hack-5.15/312-arm64-cpuinfo-Add-model-name-in-proc-cpuinfo-for-64bit-ta.patch # Patches for SSL @@ -121,6 +115,7 @@ mkdir -p feeds/packages/utils/cgroupfs-mount/patches cp -rf ../PATCH/cgroupfs-mount/900-mount-cgroup-v2-hierarchy-to-sys-fs-cgroup-cgroup2.patch ./feeds/packages/utils/cgroupfs-mount/patches/ cp -rf ../PATCH/cgroupfs-mount/901-fix-cgroupfs-umount.patch ./feeds/packages/utils/cgroupfs-mount/patches/ cp -rf ../PATCH/script/updategeo.sh ./package/base-files/files/bin/updategeo +cp -f ../PATCH/bpf_loop/*.patch ./target/linux/generic/backport-5.15/ # Dae update sed -i "s,0.6.0rc2,$(curl -s "https://api.github.com/repos/daeuniverse/dae/releases" | grep -m 1 -oP '"tag_name": "\K(.*?)(?=")' | cut -d '"' -f 4)-$(date +'%Y%m%d'),g" feeds/packages/net/dae/Makefile sed -i "s,b5ebd4f8cb82c5a0b44a49b53f3e9df4f01419c8,$(curl -s https://api.github.com/repos/daeuniverse/dae/commits | grep '"sha"' | head -1 | cut -d '"' -f 4),g" feeds/packages/net/dae/Makefile diff --git a/SCRIPTS/08_fix_permissions.sh b/SCRIPTS/08_fix_permissions.sh new file mode 100644 index 00000000..ed37a829 --- /dev/null +++ b/SCRIPTS/08_fix_permissions.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Set base directory to the current directory +BASE_DIR=$(pwd) + + +# Setting permissions for directories, excluding staging_dir +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type d -exec chmod 755 "{}" + + +# Setting permissions for files, excluding staging_dir +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec chmod 644 "{}" + +#find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f ! -name "*.c" ! -name "*.h" -exec dos2unix "{}" + + +# Setting executable permissions for specific scripts, excluding staging_dir +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f \( -name "*.sh" -o -name "*.pl" -o -name "*.py" -o -name "*.awk" -o -name "*bin*" -o -name "Makefile" -o -name "configure" \) -exec chmod 755 "{}" + + +# Setting executable permissions for scripts in scripts directory, excluding staging_dir +find "$BASE_DIR/scripts" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec chmod 755 "{}" + + +# Setting special permissions for feeds, excluding staging_dir +find "$BASE_DIR/feeds" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec chmod 755 "{}" + + +# Setting executable permissions for init scripts in package, excluding staging_dir +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec grep -l 'config' "{}" + -exec chmod 644 "{}" + +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec grep -l 'bin' "{}" + -exec chmod 755 "{}" + +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec grep -l 'PKG' "{}" + -exec chmod 755 "{}" + +find "$BASE_DIR" -path "$BASE_DIR/staging_dir" -prune -o -type f -exec grep -l 'uci' "{}" + -exec chmod 755 "{}" + + +# Setting special permissions for the build system scripts +chmod 755 "$BASE_DIR/scripts/feeds" + +echo "Permissions have been fixed." diff --git a/SCRIPTS/R2S/02_R2S.sh b/SCRIPTS/R2S/02_R2S.sh index 23a7a888..b3cdc384 100755 --- a/SCRIPTS/R2S/02_R2S.sh +++ b/SCRIPTS/R2S/02_R2S.sh @@ -28,7 +28,6 @@ zgrep -m 1 "Depends: kernel (=.*)$" Packages.gz | sed -e 's/.*-\(.*\))/\1/' >.ve sed -i -e 's/^\(.\).*vermagic$/\1cp $(TOPDIR)\/.vermagic $(LINUX_DIR)\/.vermagic/' include/kernel-defaults.mk # Final Cleanup -chmod -R 755 ./ find ./ -name *.orig | xargs rm -f find ./ -name *.rej | xargs rm -f diff --git a/SCRIPTS/R4S/02_R4S.sh b/SCRIPTS/R4S/02_R4S.sh index a2e06230..a8c09ec4 100755 --- a/SCRIPTS/R4S/02_R4S.sh +++ b/SCRIPTS/R4S/02_R4S.sh @@ -22,7 +22,6 @@ zgrep -m 1 "Depends: kernel (=.*)$" Packages.gz | sed -e 's/.*-\(.*\))/\1/' >.ve sed -i -e 's/^\(.\).*vermagic$/\1cp $(TOPDIR)\/.vermagic $(LINUX_DIR)\/.vermagic/' include/kernel-defaults.mk # Final Cleanup -chmod -R 755 ./ find ./ -name *.orig | xargs rm -f find ./ -name *.rej | xargs rm -f diff --git a/SCRIPTS/X86/02_X86.sh b/SCRIPTS/X86/02_X86.sh index feaa077b..2b239b17 100755 --- a/SCRIPTS/X86/02_X86.sh +++ b/SCRIPTS/X86/02_X86.sh @@ -33,7 +33,6 @@ zgrep -m 1 "Depends: kernel (=.*)$" Packages.gz | sed -e 's/.*-\(.*\))/\1/' >.ve sed -i -e 's/^\(.\).*vermagic$/\1cp $(TOPDIR)\/.vermagic $(LINUX_DIR)\/.vermagic/' include/kernel-defaults.mk # Final Cleanup -chmod -R 755 ./ find ./ -name *.orig | xargs rm -f find ./ -name *.rej | xargs rm -f diff --git a/SEED/R2S/config.seed b/SEED/R2S/config.seed index be3ce22b..6078bb33 100644 --- a/SEED/R2S/config.seed +++ b/SEED/R2S/config.seed @@ -8,6 +8,7 @@ CONFIG_KERNEL_BUILD_USER="nicholas-sun" CONFIG_ALL_KMODS=y CONFIG_DEVEL=y # CONFIG_KERNEL_KALLSYMS is not set +# CONFIG_KERNEL_WERROR is not set CONFIG_PACKAGE_r8152-firmware=y CONFIG_PACKAGE_trusted-firmware-a-rk3328=y CONFIG_TOOLCHAINOPTS=y diff --git a/SEED/R4S/config.seed b/SEED/R4S/config.seed index e1fef702..4fcee753 100644 --- a/SEED/R4S/config.seed +++ b/SEED/R4S/config.seed @@ -8,6 +8,7 @@ CONFIG_KERNEL_BUILD_USER="nicholas-sun" CONFIG_ALL_KMODS=y CONFIG_DEVEL=y # CONFIG_KERNEL_KALLSYMS is not set +# CONFIG_KERNEL_WERROR is not set CONFIG_PACKAGE_r8169-firmware=y CONFIG_PACKAGE_trusted-firmware-a-rk3399=y CONFIG_TOOLCHAINOPTS=y diff --git a/SEED/X86/config.seed b/SEED/X86/config.seed index 20767533..a471d870 100644 --- a/SEED/X86/config.seed +++ b/SEED/X86/config.seed @@ -20,6 +20,7 @@ CONFIG_PACKAGE_kmod-xdp-sockets-diag=y CONFIG_ALL_KMODS=y CONFIG_DEVEL=y # CONFIG_KERNEL_KALLSYMS is not set +# CONFIG_KERNEL_WERROR is not set CONFIG_TARGET_ROOTFS_SQUASHFS=y # CONFIG_TARGET_ROOTFS_TARGZ is not set # CONFIG_TARGET_ROOTFS_EXT4FS is not set