Merge branch 'fluent:master' into enable-aws-http-debug

Author: runderwo

.github/workflows/call-build-linux-packages.yaml

@@ -163,7 +163,9 @@ jobs:
         # For ubuntu map to codename using the disto-info list (CSV)
         run: |
           sudo apt-get update
-          sudo apt-get install -y distro-info awscli
+          sudo apt-get install -y distro-info
+          sudo apt-get install -y awscli || sudo snap install aws-cli --classic
+
           TARGET=${DISTRO%*.arm64v8}
           if [[ "$TARGET" == "ubuntu/"* ]]; then
               UBUNTU_CODENAME=$(cut -d ',' -f 1,3 < "/usr/share/distro-info/ubuntu.csv"|grep "${TARGET##*/}"|cut -d ',' -f 2)
@@ -224,7 +226,9 @@ jobs:
         timeout-minutes: 10
         run: |
           sudo apt-get update
-          sudo apt-get install -y createrepo-c aptly awscli
+          sudo apt-get install -y createrepo-c aptly
+          sudo apt-get install -y awscli || sudo snap install aws-cli --classic
+        shell: bash
         env:
           DEBIAN_FRONTEND: noninteractive
 

.github/workflows/call-windows-unit-tests.yaml

@@ -28,7 +28,7 @@ on:
 
 jobs:
   call-build-windows-unit-test:
-    runs-on: windows-latest
+    runs-on: ${{ matrix.config.os }}
     environment: ${{ inputs.environment }}
     strategy:
       fail-fast: false
@@ -39,11 +39,19 @@ jobs:
             cmake_additional_opt: ""
             vcpkg_triplet: x86-windows-static
             cmake_version: "3.31.6"
+            os: windows-latest
           - name: "Windows 64bit"
             arch: x64
             cmake_additional_opt: ""
             vcpkg_triplet: x64-windows-static
             cmake_version: "3.31.6"
+            os: windows-latest
+          - name: "Windows 64bit (Arm64)"
+            arch: amd64_arm64
+            cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64"
+            vcpkg_triplet: arm64-windows-static
+            cmake_version: "3.31.6"
+            os: windows-11-arm
     permissions:
       contents: read
     # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures.
@@ -62,7 +70,7 @@ jobs:
           Copy-Item -Path C:\WinFlexBison/win_bison.exe C:\WinFlexBison/bison.exe
           Copy-Item -Path C:\WinFlexBison/win_flex.exe C:\WinFlexBison/flex.exe
           echo "C:\WinFlexBison" | Out-File -FilePath $env:GITHUB_PATH -Append
-          choco install cmake --version "${{ matrix.config.cmake_version }}" --force
+          choco install cmake.portable --version "${{ matrix.config.cmake_version }}" --force
         env:
           WINFLEXBISON: https://github.com/lexxmark/winflexbison/releases/download/v2.5.22/win_flex_bison-2.5.22.zip
         shell: pwsh
@@ -119,7 +127,7 @@ jobs:
           key: ${{ steps.cache-unit-test-vcpkg-sources.outputs.cache-primary-key }}
           enableCrossOsArchive: false
 
-      - name: Build unit-test for Fluent Bit packages (only for x86 and x64)
+      - name: Build unit-test for Fluent Bit packages (x86, x64, and ARM64)
         run: |
           cmake -G "NMake Makefiles" `
             -D FLB_TESTS_INTERNAL=On `
@@ -162,7 +170,7 @@ jobs:
           dumpbin /dependents .\bin\fluent-bit.exe
         working-directory: build
 
-      - name: Build unit-test for Fluent Bit packages (only for x86 and x64)
+      - name: Run unit tests for Fluent Bit packages (x86, x64, and ARM64)
         run: |
             ctest --build-run-dir "$PWD" --output-on-failure
         shell: pwsh

.github/workflows/cron-scorecards-analysis.yaml

@@ -31,7 +31,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a
         with:
           results_file: results.sarif
           results_format: sarif
@@ -46,7 +46,7 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v4
         with:
           sarif_file: results.sarif
           category: ossf-scorecard

.github/workflows/cron-trivy.yaml

@@ -72,7 +72,7 @@ jobs:
             format: table
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v4
         with:
             sarif_file: trivy-results-${{ matrix.local_tag }}.sarif
             category: ${{ matrix.arch }} container

.github/workflows/pr-compile-check.yaml

@@ -120,3 +120,35 @@ jobs:
           echo "$ldd_result" | grep libmsgpack
           echo "$ldd_result" | grep librdkafka
         working-directory: build
+
+  # Sanity check for compilation w/ CXX support
+  pr-compile-without-cxx:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        cmake_version:
+          - "3.31.6"
+    steps:
+      - name: Setup environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y bison cmake flex gcc libssl-dev libyaml-dev
+          sudo apt-get install -y libzstd-dev librdkafka-dev
+
+      - name: Install cmake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: "${{ matrix.cmake_version }}"
+
+      - name: Checkout Fluent Bit code
+        uses: actions/checkout@v5
+
+      - name: Compile w/ CXX support
+        run: |
+          export CXX=/bin/false
+          export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+          cmake -DFLB_PREFER_SYSTEM_LIB_ZSTD=ON -DFLB_PREFER_SYSTEM_LIB_KAFKA=ON ../
+          make -j $nparallel
+        working-directory: build

.github/workflows/pr-fuzz.yaml

@@ -33,7 +33,7 @@ jobs:
         path: ./out/artifacts
     - name: Upload Sarif
       if: always() && steps.build.outcome == 'success'
-      uses: github/codeql-action/upload-sarif@v3
+      uses: github/codeql-action/upload-sarif@v4
       with:
         # Path to SARIF file relative to the root of the repository
         sarif_file: cifuzz-sarif/results.sarif

.github/workflows/staging-release.yaml

@@ -831,9 +831,20 @@ jobs:
           target_commitish: '3.2'
           make_latest: false
 
-      - name: Release 4.0 and latest
+      - name: Release 4.0 - not latest
         uses: softprops/action-gh-release@v2
         if: startsWith(inputs.version, '4.0')
+        with:
+          body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
+          draft: false
+          generate_release_notes: true
+          name: "Fluent Bit ${{ inputs.version }}"
+          tag_name: v${{ inputs.version }}
+          make_latest: false
+
+      - name: Release 4.1 and latest
+        uses: softprops/action-gh-release@v2
+        if: startsWith(inputs.version, '4.1')
         with:
           body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
           draft: false
@@ -950,12 +961,20 @@ jobs:
           token: ${{ secrets.GH_PA_TOKEN }}
           ref: 3.2
 
-      - name: Release 4.0 and latest
+      - name: Release 4.0 - not latest
         if: startsWith(inputs.version, '4.0')
         uses: actions/checkout@v5
         with:
           repository: fluent/fluent-bit-docs
           token: ${{ secrets.GH_PA_TOKEN }}
+          ref: 4.0
+
+      - name: Release 4.1 and latest
+        if: startsWith(inputs.version, '4.1')
+        uses: actions/checkout@v5
+        with:
+          repository: fluent/fluent-bit-docs
+          token: ${{ secrets.GH_PA_TOKEN }}
 
       - name: Ensure we have the script we need
         run: |
@@ -1056,7 +1075,7 @@ jobs:
 
       - name: Release 4.1
         if: startsWith(inputs.version, '4.1')
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           ref: master
 

.github/workflows/unit-tests.yaml

@@ -156,7 +156,7 @@ jobs:
           FLB_OPT: ${{ matrix.flb_option }}
 
   run-aarch64-unit-tests:
-    runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'ubuntu-22.04-arm' || 'ubuntu-latest' }}
+    runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     permissions:
       contents: read
     needs:
@@ -183,8 +183,8 @@ jobs:
       - name: Setup environment
         run: |
           sudo apt-get update
-          sudo apt-get install -y gcc-9 g++-9 clang-12 flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common curl tar gzip
-          sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
+          sudo apt-get install -y gcc-14 g++-14 clang-14 flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common curl tar gzip
+          sudo ln -s /usr/bin/llvm-symbolizer-14 /usr/bin/llvm-symbolizer || true
           sudo mkdir -p "${CMAKE_HOME}"
           cmake_url="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz"
           cmake_dist="$(mktemp --suffix ".tar.gz")"
@@ -199,9 +199,9 @@ jobs:
 
       - name: Build and test with arm runners
         run: |
-            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
-            sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
-            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90
+            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 90
+            sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 90
+            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 90
 
             export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
             export FLB_OPTION="${{ matrix.config.flb_option }}"
@@ -255,13 +255,32 @@ jobs:
           install: |
             apt-get update
             apt-get install -y gcc-12 g++-12 libyaml-dev flex bison libssl-dev libbpf-dev linux-tools-common
+            arch="$(dpkg --print-architecture)"
+            case "$arch" in
+              riscv64)
+                apt-get install -y gcc-12 g++-12 libyaml-dev flex bison libssl-dev \
+                                   libbpf-dev linux-tools-common lld-15
+                update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-15 50
+                ;;
+              *)
+                ;;
+            esac
             apt-get satisfy -y cmake "cmake (<< 4.0)"
 
             update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90
             update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90
           run: |
             cd build
             export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) ))
+            export CMAKE_LINKER_OPTION=""
+            arch="$(dpkg --print-architecture)"
+            case "$arch" in
+              riscv64)
+                export CMAKE_LINKER_OPTION='-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"'
+                ;;
+              *)
+                ;;
+            esac
             export FLB_OPTION="-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1"
             export FLB_OMIT_OPTION=""
             export GLOBAL_OPTION="-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
@@ -272,7 +291,7 @@ jobs:
 
             echo "CC = $CC, CXX = $CXX, FLB_OPT = $FLB_OPT"
 
-            cmake ${FLB_OPT}  ../
+            cmake ${CMAKE_LINKER_OPTION} ${FLB_OPT}  ../
             make -j $nparallel
             ctest -j $nparallel --build-run-dir . --output-on-failure
 

.github/workflows/update-dockerhub.yaml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Docker Hub Description
-        uses: peter-evans/dockerhub-description@v4
+        uses: peter-evans/dockerhub-description@v5
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

CMakeLists.txt

@@ -7,7 +7,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  4)
-set(FLB_VERSION_MINOR  1)
+set(FLB_VERSION_MINOR  2)
 set(FLB_VERSION_PATCH  0)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
@@ -214,6 +214,9 @@ option(FLB_CUSTOM_CALYPTIA     "Enable Calyptia Support"      Yes)
 # Config formats
 option(FLB_CONFIG_YAML         "Enable YAML config format"    Yes)
 
+option(FLB_BENCHMARKS          "Enable benchmarks"             No)
+
+
 # List of plugins available and defaults for each option
 include(cmake/plugins_options.cmake)
 
@@ -318,6 +321,7 @@ if(FLB_DEV)
   set(FLB_HTTP_SERVER       On)
   set(FLB_HTTP_CLIENT_DEBUG On)
   set(FLB_TESTS_INTERNAL    On)
+  set(FLB_BENCHMARKS        On)
 endif()
 
 if(FLB_TRACE)
@@ -588,6 +592,9 @@ endif()
 # ring buffer library
 add_subdirectory(${FLB_PATH_LIB_RING_BUFFER} EXCLUDE_FROM_ALL)
 
+# yyson
+add_subdirectory(${FLB_PATH_LIB_YYJSON} EXCLUDE_FROM_ALL)
+
 # Avro
 if(FLB_AVRO_ENCODER)
   # jansson
@@ -1067,6 +1074,10 @@ else()
   FLB_DEFINITION_VAL(FLB_MSGPACK_TO_JSON_REALLOC_BUFFER_SIZE ${FLB_MSGPACK_TO_JSON_REALLOC_BUFFER_SIZE})
 endif()
 
+if (FLB_BENCHMARKS)
+  add_subdirectory(benchmarks)
+endif()
+
 # Build tools/xxd-c
 add_subdirectory(tools/xxd-c)