From 6c27e61ad5c0dbae23a080c23d289ae0146b663a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Furkan=20T=C3=BCrkal?= Date: Fri, 7 Jul 2023 21:34:45 +0300 Subject: [PATCH 1/4] Add aws-for-fluent-bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Furkan Türkal --- aws-for-fluent-bit.yaml | 110 ++++++++++++++++++++++++++++++++++++++++ packages.txt | 1 + 2 files changed, 111 insertions(+) create mode 100644 aws-for-fluent-bit.yaml diff --git a/aws-for-fluent-bit.yaml b/aws-for-fluent-bit.yaml new file mode 100644 index 00000000000..12b3a2d3463 --- /dev/null +++ b/aws-for-fluent-bit.yaml @@ -0,0 +1,110 @@ +package: + name: aws-for-fluent-bit + version: 2.31.12.20230629 + epoch: 0 + description: AWS provides a Fluent Bit image with plugins for both CloudWatch Logs and Kinesis Data Firehose. + copyright: + - license: Apache-2.0 + +environment: + contents: + packages: + - build-base + - busybox + - ca-certificates-bundle + - glibc + - glibc-dev + - yaml-dev + - gcc + - make + - wget + - gnutar + - git + - openssl-dev + - cyrus-sasl-dev + - pkgconf + - dpkg + - systemd-dev + - zlib-dev + - flex + - bison + - autoconf + - automake + - go + - aws-flb-kinesis-compat + - aws-flb-firehose-compat + - aws-flb-cloudwatch-compat + environment: + FLB_VERSION: "1.9.10" + FLB_DOCKER_BRANCH: "1.8" + +pipeline: + - uses: git-checkout + with: + repository: https://github.com/aws/aws-for-fluent-bit + expected-commit: dfb7a7f63eadbe46a846cfbff05e57948154d39c + tag: v${{package.version}} + + - runs: | + # GHSA-p782-xgp4-8hr8 + go get -u golang.org/x/sys@v0.0.0-20220412211240-33da011f77ad + + go mod tidy + + - runs: | + mkdir -p "${{targets.destdir}}"/fluent-bit/etc + mkdir -p "${{targets.destdir}}"/fluent-bit/parsers + cp fluent-bit.conf "${{targets.destdir}}"/fluent-bit/etc/ + cp AWS_FLB_CHERRY_PICKS "${{targets.destdir}}"/AWS_FLB_CHERRY_PICKS + cp ./entrypoint.sh "${{targets.destdir}}"/fluent-bit/entrypoint.sh + chmod +x "${{targets.destdir}}"/fluent-bit/entrypoint.sh + + - runs: | + git clone --depth=1 https://github.com/fluent/fluent-bit-docker-image.git + pushd fluent-bit-docker-image + git fetch && git checkout ${FLB_DOCKER_BRANCH} + cp conf/parsers*.conf "${{targets.destdir}}"/fluent-bit/etc + cp conf/parsers*.conf "${{targets.destdir}}"/fluent-bit/parsers/ + popd + + - runs: | + git clone --depth=1 https://github.com/fluent/fluent-bit.git /tmp/fluent-bit-$FLB_VERSION/ + pushd fluent-bit-$FLB_VERSION/build + git fetch --all --tags && git checkout tags/v${FLB_VERSION} -b v${FLB_VERSION} && git describe --tags + + cd build + cmake -DFLB_RELEASE=On \ + -DFLB_TRACE=Off \ + -DFLB_JEMALLOC=On \ + -DFLB_TLS=On \ + -DFLB_SHARED_LIB=Off \ + -DFLB_EXAMPLES=Off \ + -DFLB_HTTP_SERVER=On \ + -DFLB_IN_SYSTEMD=On \ + -DFLB_OUT_KAFKA=On \ + -DFLB_ARROW=On \ + .. + make + + - runs: | + make -C build DESTDIR="${{targets.destdir}}" install + + - runs: | + mkdir -p "${{targets.destdir}}"/fluent-bit/configs + mv configs/parse-json.conf \ + configs/minimize-log-loss.conf \ + configs/output-metrics-healthcheck.conf \ + configs/plugin-metrics-to-cloudwatch.conf \ + configs/plugin-and-storage-metrics-to-cloudwatch.conf \ + configs/plugin-metrics-parser.conf.conf \ + "${{targets.destdir}}"/fluent-bit/configs/ + popd + + - uses: strip + +update: + enabled: true + github: + identifier: aws/aws-for-fluent-bit + strip-prefix: v + use-tag: true diff --git a/packages.txt b/packages.txt index de5b488885e..db96e1289e4 100644 --- a/packages.txt +++ b/packages.txt @@ -678,6 +678,7 @@ aws-c-http aws-c-auth aws-c-s3 aws-c-mqtt +aws-for-fluent-bit nri-prometheus kube-bench hugo From a79c2cdc553a581394f62ad026fd1ca3a7f62f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Furkan=20T=C3=BCrkal?= Date: Sat, 8 Jul 2023 14:34:49 +0300 Subject: [PATCH 2/4] Fix build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Furkan Türkal --- aws-for-fluent-bit.yaml | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/aws-for-fluent-bit.yaml b/aws-for-fluent-bit.yaml index 12b3a2d3463..a2f5cefb0d0 100644 --- a/aws-for-fluent-bit.yaml +++ b/aws-for-fluent-bit.yaml @@ -52,24 +52,24 @@ pipeline: go mod tidy - runs: | - mkdir -p "${{targets.destdir}}"/fluent-bit/etc - mkdir -p "${{targets.destdir}}"/fluent-bit/parsers - cp fluent-bit.conf "${{targets.destdir}}"/fluent-bit/etc/ - cp AWS_FLB_CHERRY_PICKS "${{targets.destdir}}"/AWS_FLB_CHERRY_PICKS - cp ./entrypoint.sh "${{targets.destdir}}"/fluent-bit/entrypoint.sh - chmod +x "${{targets.destdir}}"/fluent-bit/entrypoint.sh + mkdir -p "${{targets.destdir}}"/etc/fluent-bit/etc + mkdir -p "${{targets.destdir}}"/etc/fluent-bit/parsers + cp fluent-bit.conf "${{targets.destdir}}"/etc/fluent-bit/etc/ + cp AWS_FLB_CHERRY_PICKS "${{targets.destdir}}"/etc/fluent-bit/AWS_FLB_CHERRY_PICKS + cp ./entrypoint.sh "${{targets.destdir}}"/etc/fluent-bit/entrypoint.sh + chmod +x "${{targets.destdir}}"/etc/fluent-bit/entrypoint.sh - runs: | git clone --depth=1 https://github.com/fluent/fluent-bit-docker-image.git - pushd fluent-bit-docker-image + cd fluent-bit-docker-image git fetch && git checkout ${FLB_DOCKER_BRANCH} - cp conf/parsers*.conf "${{targets.destdir}}"/fluent-bit/etc - cp conf/parsers*.conf "${{targets.destdir}}"/fluent-bit/parsers/ - popd + cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/etc + cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/parsers/ + cd .. - runs: | - git clone --depth=1 https://github.com/fluent/fluent-bit.git /tmp/fluent-bit-$FLB_VERSION/ - pushd fluent-bit-$FLB_VERSION/build + git clone --depth=1 https://github.com/fluent/fluent-bit.git fluent-bit-$FLB_VERSION/ + cd fluent-bit-$FLB_VERSION/build git fetch --all --tags && git checkout tags/v${FLB_VERSION} -b v${FLB_VERSION} && git describe --tags cd build @@ -90,18 +90,29 @@ pipeline: make -C build DESTDIR="${{targets.destdir}}" install - runs: | - mkdir -p "${{targets.destdir}}"/fluent-bit/configs + mkdir -p "${{targets.destdir}}"/etc/fluent-bit/configs mv configs/parse-json.conf \ configs/minimize-log-loss.conf \ configs/output-metrics-healthcheck.conf \ configs/plugin-metrics-to-cloudwatch.conf \ configs/plugin-and-storage-metrics-to-cloudwatch.conf \ configs/plugin-metrics-parser.conf.conf \ - "${{targets.destdir}}"/fluent-bit/configs/ - popd + "${{targets.destdir}}"/etc/fluent-bit/configs/ - uses: strip +subpackages: + - name: "aws-for-fluent-bit-compat" + description: "Compatibility package to place binaries in the location expected by AWS for Fluent Bit" + pipeline: + - runs: | + # The AWS for Fluent Bit expects the .so libraries to be in /fluent-bit + mkdir -p "${{targets.subpkgdir}}"/fluent-bit/bin + mkdir -p "${{targets.subpkgdir}}"/fluent-bit/etc + ln -sf /usr/bin/fluent-bit/bin/fluent-bit ${{targets.subpkgdir}}/fluent-bit/bin/fluent-bit + ln -sf /usr/bin/fluent-bit/etc/fluent-bit.conf ${{targets.subpkgdir}}/fluent-bit/etc/fluent-bit.conf + - uses: strip + update: enabled: true github: From fd334cfa51944f6bd7307f6ef7f6dbbac680c455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Furkan=20T=C3=BCrkal?= Date: Sat, 8 Jul 2023 15:10:33 +0300 Subject: [PATCH 3/4] Use pipelines for clarify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Furkan Türkal --- aws-for-fluent-bit.yaml | 83 ++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/aws-for-fluent-bit.yaml b/aws-for-fluent-bit.yaml index a2f5cefb0d0..5e3f9f0a7f0 100644 --- a/aws-for-fluent-bit.yaml +++ b/aws-for-fluent-bit.yaml @@ -16,6 +16,7 @@ environment: - glibc-dev - yaml-dev - gcc + - cmake - make - wget - gnutar @@ -34,9 +35,10 @@ environment: - aws-flb-kinesis-compat - aws-flb-firehose-compat - aws-flb-cloudwatch-compat - environment: - FLB_VERSION: "1.9.10" - FLB_DOCKER_BRANCH: "1.8" + +vars: + FLB_VERSION: "1.9.10" + FLB_DOCKER_BRANCH: "1.8" pipeline: - uses: git-checkout @@ -59,36 +61,6 @@ pipeline: cp ./entrypoint.sh "${{targets.destdir}}"/etc/fluent-bit/entrypoint.sh chmod +x "${{targets.destdir}}"/etc/fluent-bit/entrypoint.sh - - runs: | - git clone --depth=1 https://github.com/fluent/fluent-bit-docker-image.git - cd fluent-bit-docker-image - git fetch && git checkout ${FLB_DOCKER_BRANCH} - cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/etc - cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/parsers/ - cd .. - - - runs: | - git clone --depth=1 https://github.com/fluent/fluent-bit.git fluent-bit-$FLB_VERSION/ - cd fluent-bit-$FLB_VERSION/build - git fetch --all --tags && git checkout tags/v${FLB_VERSION} -b v${FLB_VERSION} && git describe --tags - - cd build - cmake -DFLB_RELEASE=On \ - -DFLB_TRACE=Off \ - -DFLB_JEMALLOC=On \ - -DFLB_TLS=On \ - -DFLB_SHARED_LIB=Off \ - -DFLB_EXAMPLES=Off \ - -DFLB_HTTP_SERVER=On \ - -DFLB_IN_SYSTEMD=On \ - -DFLB_OUT_KAFKA=On \ - -DFLB_ARROW=On \ - .. - make - - - runs: | - make -C build DESTDIR="${{targets.destdir}}" install - - runs: | mkdir -p "${{targets.destdir}}"/etc/fluent-bit/configs mv configs/parse-json.conf \ @@ -96,9 +68,52 @@ pipeline: configs/output-metrics-healthcheck.conf \ configs/plugin-metrics-to-cloudwatch.conf \ configs/plugin-and-storage-metrics-to-cloudwatch.conf \ - configs/plugin-metrics-parser.conf.conf \ + configs/plugin-metrics-parser.conf \ "${{targets.destdir}}"/etc/fluent-bit/configs/ + - working-directory: fluent-bit-docker-image + pipeline: + - uses: git-checkout + with: + repository: https://github.com/fluent/fluent-bit-docker-image + branch: ${{vars.FLB_DOCKER_BRANCH}} + expected-commit: 98ce3316ea93751ddd33bc319a8f9d177493155b + - runs: | + cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/etc + cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/parsers/ + + - working-directory: fluent-bit + pipeline: + - uses: git-checkout + with: + repository: https://github.com/fluent/fluent-bit.git + tag: v${{vars.FLB_VERSION}} + expected-commit: 760956f50cdc7eef3047df20d2299202a8c68594 + - runs: | + cd build + + # We remove fortify when building the libraries + export CFLAGS=${CFLAGS/-Wp,-D_FORTIFY_SOURCE=3/} + export CPPFLAGS=${CPPFLAGS/-Wp,-D_FORTIFY_SOURCE=3/} + export CXXFLAGS=${CXXFLAGS/-Wp,-D_FORTIFY_SOURCE=3/} + + cmake -DFLB_RELEASE=On \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DFLB_TRACE=Off \ + -DFLB_JEMALLOC=On \ + -DFLB_TLS=On \ + -DFLB_SHARED_LIB=Off \ + -DFLB_EXAMPLES=Off \ + -DFLB_HTTP_SERVER=On \ + -DFLB_IN_SYSTEMD=On \ + -DFLB_OUT_KAFKA=On \ + -DFLB_ARROW=On \ + .. + make + - runs: | + make -C build DESTDIR="${{targets.destdir}}" install + - uses: strip subpackages: From 8357e55b5f323c645f5ed7dd84e6328ba47ee150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Furkan=20T=C3=BCrkal?= Date: Tue, 11 Jul 2023 15:45:10 +0300 Subject: [PATCH 4/4] Simplfy the package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Furkan Türkal --- aws-for-fluent-bit.yaml | 81 ++++++++--------------------------------- 1 file changed, 15 insertions(+), 66 deletions(-) diff --git a/aws-for-fluent-bit.yaml b/aws-for-fluent-bit.yaml index 5e3f9f0a7f0..9a1ec754ce3 100644 --- a/aws-for-fluent-bit.yaml +++ b/aws-for-fluent-bit.yaml @@ -5,6 +5,13 @@ package: description: AWS provides a Fluent Bit image with plugins for both CloudWatch Logs and Kinesis Data Firehose. copyright: - license: Apache-2.0 + dependencies: + runtime: + - ca-certificates-bundle + - fluent-bit + - aws-flb-kinesis-compat + - aws-flb-firehose-compat + - aws-flb-cloudwatch-compat environment: contents: @@ -12,32 +19,8 @@ environment: - build-base - busybox - ca-certificates-bundle - - glibc - - glibc-dev - - yaml-dev - - gcc - - cmake - - make - - wget - - gnutar - - git - - openssl-dev - - cyrus-sasl-dev - - pkgconf - - dpkg - - systemd-dev - - zlib-dev - - flex - - bison - - autoconf - - automake - - go - - aws-flb-kinesis-compat - - aws-flb-firehose-compat - - aws-flb-cloudwatch-compat vars: - FLB_VERSION: "1.9.10" FLB_DOCKER_BRANCH: "1.8" pipeline: @@ -48,17 +31,13 @@ pipeline: tag: v${{package.version}} - runs: | - # GHSA-p782-xgp4-8hr8 - go get -u golang.org/x/sys@v0.0.0-20220412211240-33da011f77ad - - go mod tidy - - - runs: | + mkdir -p "${{targets.destdir}}"/etc/ecs mkdir -p "${{targets.destdir}}"/etc/fluent-bit/etc mkdir -p "${{targets.destdir}}"/etc/fluent-bit/parsers + cp -r ./ecs/* ${{targets.destdir}}/etc/ecs cp fluent-bit.conf "${{targets.destdir}}"/etc/fluent-bit/etc/ - cp AWS_FLB_CHERRY_PICKS "${{targets.destdir}}"/etc/fluent-bit/AWS_FLB_CHERRY_PICKS cp ./entrypoint.sh "${{targets.destdir}}"/etc/fluent-bit/entrypoint.sh + cp AWS_FOR_FLUENT_BIT_VERSION "${{targets.destdir}}"/etc/fluent-bit/AWS_FOR_FLUENT_BIT_VERSION chmod +x "${{targets.destdir}}"/etc/fluent-bit/entrypoint.sh - runs: | @@ -82,38 +61,6 @@ pipeline: cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/etc cp conf/parsers*.conf "${{targets.destdir}}"/etc/fluent-bit/parsers/ - - working-directory: fluent-bit - pipeline: - - uses: git-checkout - with: - repository: https://github.com/fluent/fluent-bit.git - tag: v${{vars.FLB_VERSION}} - expected-commit: 760956f50cdc7eef3047df20d2299202a8c68594 - - runs: | - cd build - - # We remove fortify when building the libraries - export CFLAGS=${CFLAGS/-Wp,-D_FORTIFY_SOURCE=3/} - export CPPFLAGS=${CPPFLAGS/-Wp,-D_FORTIFY_SOURCE=3/} - export CXXFLAGS=${CXXFLAGS/-Wp,-D_FORTIFY_SOURCE=3/} - - cmake -DFLB_RELEASE=On \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DFLB_TRACE=Off \ - -DFLB_JEMALLOC=On \ - -DFLB_TLS=On \ - -DFLB_SHARED_LIB=Off \ - -DFLB_EXAMPLES=Off \ - -DFLB_HTTP_SERVER=On \ - -DFLB_IN_SYSTEMD=On \ - -DFLB_OUT_KAFKA=On \ - -DFLB_ARROW=On \ - .. - make - - runs: | - make -C build DESTDIR="${{targets.destdir}}" install - - uses: strip subpackages: @@ -122,10 +69,12 @@ subpackages: pipeline: - runs: | # The AWS for Fluent Bit expects the .so libraries to be in /fluent-bit + mkdir -p "${{targets.subpkgdir}}"/fluent-bit mkdir -p "${{targets.subpkgdir}}"/fluent-bit/bin - mkdir -p "${{targets.subpkgdir}}"/fluent-bit/etc - ln -sf /usr/bin/fluent-bit/bin/fluent-bit ${{targets.subpkgdir}}/fluent-bit/bin/fluent-bit - ln -sf /usr/bin/fluent-bit/etc/fluent-bit.conf ${{targets.subpkgdir}}/fluent-bit/etc/fluent-bit.conf + mkdir -p "${{targets.subpkgdir}}"/ecs + ln -sf /etc/ecs ${{targets.subpkgdir}}/ecs + ln -sf /etc/fluent-bit/ ${{targets.subpkgdir}}/fluent-bit/ + ln -sf /usr/bin/fluent-bit ${{targets.subpkgdir}}/fluent-bit/bin/fluent-bit - uses: strip update: