Misc code cleanup (#18540) #320
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # Handles building of binary packages for the agent. | |
| name: Packages | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - labeled | |
| - synchronize | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| type: | |
| description: Package build type | |
| default: devel | |
| required: true | |
| version: | |
| description: Package version | |
| required: false | |
| env: | |
| DISABLE_TELEMETRY: 1 | |
| REPO_PREFIX: netdata/netdata | |
| concurrency: | |
| group: packages-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| file-check: # Check what files changed if we’re being run in a PR or on a push. | |
| name: Check Modified Files | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run: ${{ steps.check-run.outputs.run }} | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Check files | |
| id: check-files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| since_last_remote_commit: ${{ github.event_name != 'pull_request' }} | |
| files: | | |
| **/*.c | |
| **/*.cc | |
| **/*.h | |
| **/*.hh | |
| **/*.in | |
| **/*.patch | |
| **/*.cmake | |
| netdata.spec.in | |
| CMakeLists.txt | |
| .github/data/distros.yml | |
| .github/workflows/packaging.yml | |
| .github/scripts/gen-matrix-packaging.py | |
| .github/scripts/pkg-test.sh | |
| packaging/cmake/ | |
| packaging/*.sh | |
| packaging/*.version | |
| packaging/*.checksums | |
| src/aclk/aclk-schemas/ | |
| src/ml/dlib/ | |
| src/fluent-bit/ | |
| src/web/server/h2o/libh2o/ | |
| files_ignore: | | |
| **/*.md | |
| packaging/repoconfig/ | |
| - name: List all changed files in pattern | |
| continue-on-error: true | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} | |
| run: | | |
| for file in ${ALL_CHANGED_FILES}; do | |
| echo "$file was changed" | |
| done | |
| - name: Check Run | |
| id: check-run | |
| run: | | |
| if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo 'run=true' >> "${GITHUB_OUTPUT}" | |
| else | |
| echo 'run=false' >> "${GITHUB_OUTPUT}" | |
| fi | |
| matrix: | |
| name: Prepare Build Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@v4 | |
| - name: Prepare tools | |
| id: prepare | |
| run: | | |
| sudo apt-get update || true | |
| sudo apt-get install -y python3-ruamel.yaml | |
| - name: Read build matrix | |
| id: set-matrix | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ] && \ | |
| [ "${{ !contains(github.event.pull_request.labels.*.name, 'run-ci/packaging') }}" = "true" ]; then | |
| matrix="$(.github/scripts/gen-matrix-packaging.py 1)" | |
| else | |
| matrix="$(.github/scripts/gen-matrix-packaging.py 0)" | |
| fi | |
| echo "Generated matrix: ${matrix}" | |
| echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Package Build matrix generation failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to generate build matrix for package build. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Prepare Tools: ${{ steps.prepare.outcome }} | |
| Read Build Matrix: ${{ steps.set-matrix.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && github.event_name != 'pull_request' | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.repository == 'netdata/netdata' | |
| }} | |
| version-check: | |
| name: Version check | |
| runs-on: ubuntu-latest | |
| outputs: | |
| repo: ${{ steps.check-version.outputs.repo }} | |
| version: ${{ steps.check-version.outputs.version }} | |
| retention: ${{ steps.check-version.outputs.retention }} | |
| steps: | |
| - name: Checkout | |
| id: checkout | |
| uses: actions/checkout@v4 | |
| - name: Check Version | |
| id: check-version | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| case "${{ github.event.inputs.type }}" in | |
| "release") | |
| echo "repo=${REPO_PREFIX}" >> "${GITHUB_OUTPUT}" | |
| echo "version=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}" | |
| echo "retention=365" >> "${GITHUB_OUTPUT}" | |
| ;; | |
| "nightly") | |
| echo "repo=${REPO_PREFIX}-edge" >> "${GITHUB_OUTPUT}" | |
| echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}" | |
| echo "retention=30" >> "${GITHUB_OUTPUT}" | |
| ;; | |
| *) | |
| echo "repo=${REPO_PREFIX}-devel" >> "${GITHUB_OUTPUT}" | |
| echo "version=0.${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" | |
| echo "retention=30" >> "${GITHUB_OUTPUT}" | |
| ;; | |
| esac | |
| else | |
| echo "version=$(cut -d'-' -f 1 packaging/version | tr -d 'v')" >> "${GITHUB_OUTPUT}" | |
| echo "retention=0" >> "${GITHUB_OUTPUT}" | |
| fi | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Package Build version check failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: Failed to generate version information for package build. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Check Version: ${{ steps.check-version.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && github.event_name != 'pull_request' | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.repository == 'netdata/netdata' | |
| }} | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKER_CLI_EXPERIMENTAL: enabled | |
| needs: | |
| - matrix | |
| - version-check | |
| - file-check | |
| strategy: | |
| matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} | |
| # We intentiaonally disable the fail-fast behavior so that a | |
| # build failure for one version doesn't prevent us from publishing | |
| # successfully built and tested packages for another version. | |
| fail-fast: false | |
| max-parallel: 8 | |
| steps: | |
| - name: Skip Check | |
| id: skip | |
| if: needs.file-check.outputs.run != 'true' | |
| run: echo "SKIPPED" | |
| - name: Checkout | |
| id: checkout | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # We need full history for versioning | |
| submodules: recursive | |
| - name: Set Sentry telemetry env vars | |
| id: set-telemetry-env-vars | |
| run: | | |
| if [ "${{ github.repository }}" = 'netdata/netdata' ] && \ | |
| [ "${{ matrix.bundle_sentry }}" = 'true' ] && \ | |
| [ "${{ github.event_name }}" = 'workflow_dispatch' ]; then | |
| echo "RELEASE_PIPELINE=Production" >> "${GITHUB_ENV}" | |
| echo "UPLOAD_SENTRY=true" >> "${GITHUB_ENV}" | |
| else | |
| echo "RELEASE_PIPELINE=Unknown" >> "${GITHUB_ENV}" | |
| echo "UPLOAD_SENTRY=false" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Setup QEMU | |
| id: qemu | |
| if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' && needs.file-check.outputs.run == 'true' | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Prepare Docker Environment | |
| id: docker-config | |
| if: needs.file-check.outputs.run == 'true' | |
| shell: bash | |
| run: | | |
| echo '{"cgroup-parent": "actions-job.slice", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null | |
| sudo service docker restart | |
| - name: Fetch images | |
| id: fetch-images | |
| if: needs.file-check.outputs.run == 'true' | |
| uses: nick-invision/retry@v3 | |
| with: | |
| max_attempts: 3 | |
| retry_wait_seconds: 30 | |
| timeout_seconds: 900 | |
| command: | | |
| docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }} | |
| docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }} | |
| - name: Build Packages | |
| id: build | |
| if: needs.file-check.outputs.run == 'true' | |
| shell: bash | |
| run: | | |
| docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e VERSION=${{ needs.version-check.outputs.version }} \ | |
| -e ENABLE_SENTRY=${{ matrix.bundle_sentry }} -e RELEASE_PIPELINE=${{ env.RELEASE_PIPELINE }} \ | |
| -e BUILD_DESTINATION=${{ matrix.distro }}${{ matrix.version }}_${{ matrix.arch }} -e UPLOAD_SENTRY=${{ env.UPLOAD_SENTRY }} \ | |
| -e SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_CLI_TOKEN }} -e NETDATA_SENTRY_DSN=${{ secrets.SENTRY_DSN }} \ | |
| --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }} | |
| - name: Save Packages | |
| id: artifacts | |
| if: needs.file-check.outputs.run == 'true' | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages | |
| path: ${{ github.workspace }}/artifacts/* | |
| - name: Test Packages | |
| id: test | |
| if: needs.file-check.outputs.run == 'true' | |
| shell: bash | |
| run: | | |
| docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e DISTRO=${{ matrix.distro }} \ | |
| -e VERSION=${{ needs.version-check.outputs.version }} -e DISTRO_VERSION=${{ matrix.version }} \ | |
| --platform=${{ matrix.platform }} -v "$PWD":/netdata ${{ matrix.base_image }} \ | |
| /netdata/.github/scripts/pkg-test.sh | |
| - name: Upload to PackageCloud | |
| id: upload | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' | |
| continue-on-error: true | |
| shell: bash | |
| env: | |
| PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} | |
| run: | | |
| printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})" | |
| for pkgfile in artifacts/*.${{ matrix.format }} ; do | |
| .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} \ | |
| "$(basename "${pkgfile}")" || true | |
| .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} "${pkgfile}" | |
| done | |
| - name: SSH setup | |
| id: ssh-setup | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} | |
| name: id_ecdsa | |
| known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} | |
| - name: Import GPG Keys | |
| id: import-keys | |
| if: needs.file-check.outputs.run == 'true' && matrix.format == 'deb' && github.event_name != 'pull_request' | |
| uses: crazy-max/ghaction-import-gpg@v6 | |
| with: | |
| gpg_private_key: ${{ secrets.NETDATABOT_PACKAGE_SIGNING_KEY }} | |
| - name: Sign DEB Packages | |
| id: sign-deb | |
| if: needs.file-check.outputs.run == 'true' && matrix.format == 'deb' && github.event_name != 'pull_request' | |
| shell: bash | |
| run: .github/scripts/deb-sign.sh artifacts ${{ steps.import-keys.outputs.fingerprint }} | |
| - name: Upload to packages.netdata.cloud | |
| id: package-upload | |
| continue-on-error: true | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' | |
| run: | | |
| .github/scripts/package-upload.sh \ | |
| packages.netdata.cloud \ | |
| ${{ matrix.repo_distro }} \ | |
| ${{ matrix.arch }} \ | |
| ${{ matrix.format }} \ | |
| ${{ needs.version-check.outputs.repo }} | |
| - name: Upload to packages2.netdata.cloud | |
| id: package2-upload | |
| if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' | |
| run: | | |
| .github/scripts/package-upload.sh \ | |
| packages2.netdata.cloud \ | |
| ${{ matrix.repo_distro }} \ | |
| ${{ matrix.arch }} \ | |
| ${{ matrix.format }} \ | |
| ${{ needs.version-check.outputs.repo }} | |
| - name: Failure Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_COLOR: 'danger' | |
| SLACK_ICON_EMOJI: ':github-actions:' | |
| SLACK_TITLE: 'Package Build failed:' | |
| SLACK_USERNAME: 'GitHub Actions' | |
| SLACK_MESSAGE: |- | |
| ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed. | |
| Checkout: ${{ steps.checkout.outcome }} | |
| Setup QEMU: ${{ steps.qemu.outcome }} | |
| Setup Docker: ${{ steps.docker-config.outcome }} | |
| Fetch images: ${{ steps.fetch-images.outcome }} | |
| Build: ${{ steps.build.outcome }} | |
| Test: ${{ steps.test.outcome }} | |
| Publish to PackageCloud: ${{ steps.upload.outcome }} | |
| Import SSH Key: ${{ steps.ssh-setup.outcome }} | |
| Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }} | |
| Import GPG Keys: ${{ steps.import-keys.outcome }} | |
| Sign DEB Packages: ${{ steps.sign-deb.outcome }} | |
| Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| if: >- | |
| ${{ | |
| failure() | |
| && github.event_name != 'pull_request' | |
| && startsWith(github.ref, 'refs/heads/master') | |
| && github.repository == 'netdata/netdata' | |
| && needs.file-check.outputs.run == 'true' | |
| }} |