Fix chef tests (#4993) #8
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: linux-package-test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - '.github/workflows/linux-package-test.yml' | |
| - 'cmd/otelcol/**' | |
| - 'internal/buildscripts/packaging/collect-libs.sh' | |
| - 'internal/buildscripts/packaging/docker-otelcol.sh' | |
| - 'internal/buildscripts/packaging/fpm/**' | |
| - 'internal/buildscripts/packaging/installer/install.sh' | |
| - 'internal/buildscripts/packaging/jmx-metric-gatherer-release.txt' | |
| - 'internal/buildscripts/packaging/tests/helpers/**' | |
| - 'internal/buildscripts/packaging/tests/images/**' | |
| - 'internal/buildscripts/packaging/tests/package_test.py' | |
| - 'internal/buildscripts/packaging/tests/requirements.txt' | |
| - 'internal/signalfx-agent/bundle/**' | |
| - 'Makefile' | |
| - 'Makefile.Common' | |
| - '!**.md' | |
| concurrency: | |
| group: linux-package-test-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| PIP_VERSION: '22.0.4' | |
| REQUIREMENTS_PATH: "internal/buildscripts/packaging/tests/requirements.txt" | |
| GO_VERSION: 1.21.11 | |
| jobs: | |
| setup-environment: | |
| # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Check out the codebase. | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache-dependency-path: '**/go.sum' | |
| - name: Installing dependency | |
| run: | | |
| make install-tools | |
| cross-compile: | |
| # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved | |
| runs-on: ubuntu-20.04 | |
| needs: [setup-environment] | |
| strategy: | |
| matrix: | |
| SYS_BINARIES: [ "binaries-linux_amd64", "binaries-linux_arm64", "binaries-linux_ppc64le" ] | |
| steps: | |
| - name: Check out the codebase. | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache-dependency-path: '**/go.sum' | |
| - name: Build Collector | |
| run: | | |
| make ${{ matrix.SYS_BINARIES }} | |
| - name: Uploading binaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.SYS_BINARIES }} | |
| path: | | |
| ./bin/* | |
| agent-bundle-linux: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| ARCH: ["amd64", "arm64"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| id: bundle-cache | |
| with: | |
| path: .cache/buildx/agent-bundle-${{ matrix.ARCH }} | |
| key: agent-bundle-buildx-${{ matrix.ARCH }}-${{ hashFiles('internal/signalfx-agent/bundle/**') }} | |
| restore-keys: | | |
| agent-bundle-buildx-${{ matrix.ARCH }}- | |
| - uses: docker/setup-qemu-action@v3 | |
| if: ${{ matrix.ARCH != 'amd64' }} | |
| with: | |
| platforms: ${{ matrix.ARCH }} | |
| image: tonistiigi/binfmt:qemu-v7.0.0 | |
| - run: make -C internal/signalfx-agent/bundle agent-bundle-linux ARCH=${{ matrix.ARCH }} | |
| env: | |
| BUNDLE_CACHE_HIT: "${{ steps.bundle-cache.outputs.cache-hit }}" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: agent-bundle-linux-${{ matrix.ARCH }} | |
| path: ./dist/agent-bundle_linux_${{ matrix.ARCH }}.tar.gz | |
| build-package: | |
| # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved | |
| runs-on: ubuntu-20.04 | |
| needs: [cross-compile, agent-bundle-linux] | |
| strategy: | |
| matrix: | |
| SYS_PACKAGE: [ "deb", "rpm", "tar" ] | |
| ARCH: [ "amd64", "arm64" ] | |
| fail-fast: false | |
| steps: | |
| - name: Check out the codebase. | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache-dependency-path: '**/go.sum' | |
| - name: Downloading binaries-linux_${{ matrix.ARCH }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: binaries-linux_${{ matrix.ARCH }} | |
| path: ./bin | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: agent-bundle-linux-${{ matrix.ARCH }} | |
| path: ./dist | |
| - name: Build ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package | |
| run: make ${{ matrix.SYS_PACKAGE }}-package SKIP_COMPILE=true SKIP_BUNDLE=true VERSION="" ARCH="${{ matrix.ARCH }}" | |
| - name: Uploading ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.SYS_PACKAGE }}-${{ matrix.ARCH }}-package | |
| path: ./dist/splunk-otel-collector* | |
| test-package-matrix: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Get matrix | |
| id: get-matrix | |
| run: | | |
| # create test matrix for distro and arch | |
| dockerfiles=$(find internal/buildscripts/packaging/tests/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u) | |
| if [ -z "$dockerfiles" ]; then | |
| echo "Failed to get dockerfiles from internal/buildscripts/packaging/tests/images!" >&2 | |
| exit 1 | |
| fi | |
| distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done) | |
| arch="\"amd64\", \"arm64\"" | |
| matrix="{\"DISTRO\": [${distro%,}], \"ARCH\": [${arch}]}" | |
| echo "$matrix" | jq | |
| echo "matrix=${matrix}" >> $GITHUB_OUTPUT | |
| outputs: | |
| matrix: ${{ steps.get-matrix.outputs.matrix }} | |
| test-package: | |
| # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved | |
| runs-on: ${{ fromJSON('["ubuntu-20.04", "ubuntu-22.04"]')[matrix.DISTRO == 'amazonlinux-2023'] }} | |
| timeout-minutes: 30 | |
| needs: [build-package, test-package-matrix] | |
| strategy: | |
| matrix: ${{ fromJSON(needs.test-package-matrix.outputs.matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Check out the codebase. | |
| uses: actions/checkout@v4 | |
| - name: Get package type for ${{ matrix.DISTRO }} | |
| run: | | |
| for pkg in "deb" "rpm" "tar"; do | |
| if [[ -f "internal/buildscripts/packaging/tests/images/${pkg}/Dockerfile.${{ matrix.DISTRO }}" ]]; then | |
| echo "SYS_PACKAGE=${pkg}" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| done | |
| echo "Unknown distro '${{ matrix.DISTRO }}'!" | |
| exit 1 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.SYS_PACKAGE }}-${{ matrix.ARCH }}-package | |
| path: ./dist | |
| - uses: docker/setup-qemu-action@v3 | |
| if: ${{ matrix.ARCH != 'amd64' }} | |
| with: | |
| platforms: ${{ matrix.ARCH }} | |
| image: tonistiigi/binfmt:qemu-v7.0.0 | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| cache-dependency-path: ${{ env.REQUIREMENTS_PATH }} | |
| - name: Install pytest | |
| run: | | |
| if which pip; then | |
| pip install --upgrade 'pip==${{ env.PIP_VERSION }}' | |
| else | |
| curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py | |
| python get-pip.py 'pip==${{ env.PIP_VERSION }}' | |
| fi | |
| pip install -r "${{ env.REQUIREMENTS_PATH }}" | |
| - name: Test ${{ env.SYS_PACKAGE }} on ${{ matrix.DISTRO }} ${{ matrix.ARCH }} | |
| id: pytest | |
| continue-on-error: true | |
| run: | | |
| distro="${{ matrix.DISTRO }}" | |
| if [[ "$distro" = "amazonlinux-2" ]]; then | |
| # workaround for pytest substring matching | |
| distro="amazonlinux-2 and not amazonlinux-2023" | |
| fi | |
| python3 -u -m pytest -s --verbose -m "${{ env.SYS_PACKAGE }}" \ | |
| -k "$distro and ${{ matrix.ARCH }}" \ | |
| internal/buildscripts/packaging/tests/package_test.py | |
| # qemu, networking, running systemd in containers, etc., can be flaky | |
| - name: Re-run failed tests | |
| if: ${{ steps.pytest.outcome == 'failure' }} | |
| run: | | |
| distro="${{ matrix.DISTRO }}" | |
| if [[ "$distro" = "amazonlinux-2" ]]; then | |
| # workaround for pytest substring matching | |
| distro="amazonlinux-2 and not amazonlinux-2023" | |
| fi | |
| python3 -u -m pytest -s --verbose -m "${{ env.SYS_PACKAGE }}" \ | |
| -k "$distro and ${{ matrix.ARCH }}" \ | |
| --last-failed \ | |
| internal/buildscripts/packaging/tests/package_test.py | |
| docker-otelcol: | |
| # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved | |
| runs-on: ubuntu-20.04 | |
| needs: [cross-compile, agent-bundle-linux] | |
| steps: | |
| # Multiarch images require more disk space | |
| - uses: jlumbroso/free-disk-space@v1.3.1 | |
| - name: Check out the codebase. | |
| uses: actions/checkout@v4 | |
| # Required to export a multiarch manifest and images to the local image store | |
| - name: Set up containerd image store | |
| uses: crazy-max/ghaction-setup-docker@v3 | |
| with: | |
| daemon-config: | | |
| { | |
| "features": { | |
| "containerd-snapshotter": true | |
| } | |
| } | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache-dependency-path: '**/go.sum' | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: arm64,ppc64le | |
| image: tonistiigi/binfmt:qemu-v7.0.0 | |
| - name: Downloading binaries-linux_amd64 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: binaries-linux_amd64 | |
| path: ./bin | |
| - name: Downloading binaries-linux_arm64 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: binaries-linux_arm64 | |
| path: ./bin | |
| - name: Downloading binaries-linux_ppc64le | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: binaries-linux_ppc64le | |
| path: ./bin | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: agent-bundle-linux-amd64 | |
| path: ./dist | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: agent-bundle-linux-arm64 | |
| path: ./dist | |
| - name: Build the multiarch docker image | |
| run: make docker-otelcol SKIP_COMPILE=true SKIP_BUNDLE=true ARCH=amd64,arm64,ppc64le | |
| - name: Save the multiarch image archive to be loaded by downstream jobs | |
| run: mkdir -p docker-otelcol && docker save -o ./docker-otelcol/image.tar otelcol:latest | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: otelcol | |
| path: ./docker-otelcol | |
| docker-otelcol-verify: | |
| # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved | |
| runs-on: ubuntu-20.04 | |
| needs: [docker-otelcol] | |
| strategy: | |
| matrix: | |
| ARCH: [ "amd64", "arm64", "ppc64le" ] | |
| fail-fast: false | |
| steps: | |
| - name: Check out the codebase. | |
| uses: actions/checkout@v4 | |
| # Required to load a multiarch archive to the local image store | |
| - name: Set up containerd image store | |
| uses: crazy-max/ghaction-setup-docker@v3 | |
| with: | |
| daemon-config: | | |
| { | |
| "features": { | |
| "containerd-snapshotter": true | |
| } | |
| } | |
| - name: Set up QEMU | |
| if: ${{ matrix.ARCH != 'amd64' }} | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: ${{ matrix.ARCH }} | |
| image: tonistiigi/binfmt:qemu-v7.0.0 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: otelcol | |
| path: ./docker-otelcol | |
| - name: Load multiarch docker image | |
| run: docker load -i ./docker-otelcol/image.tar | |
| - name: Run docker image | |
| run: | | |
| # ensure the collector can start with the default config file | |
| docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest | |
| sleep 10 | |
| if [ -z "$( docker ps --filter=status=running --filter=name=otelcol -q )" ]; then | |
| docker logs otelcol | |
| echo "Failing job execution: fail to start otelcol docker container with the default config in 10 seconds." | |
| exit 1 | |
| fi | |
| # get all provided config files from the container | |
| tmpdir=$(mktemp -d) | |
| docker cp otelcol:/etc/otel/collector $tmpdir | |
| docker rm -f otelcol | |
| # ensure the collector can start with all provided config files | |
| configs=$(ls ${tmpdir}/collector/ 2>/dev/null) | |
| if [ -z "$configs" ]; then | |
| echo "failed to get config files from otelcol:/etc/otel/collector" | |
| exit 1 | |
| fi | |
| for config in $configs; do | |
| docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_CONFIG=/etc/otel/collector/${config} -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest | |
| sleep 10 | |
| if [ -z "$( docker ps --filter=status=running --filter=name=otelcol -q )" ]; then | |
| docker logs otelcol | |
| echo "Failing job execution: fail to start otelcol docker container with ${config} in 10 seconds." | |
| exit 1 | |
| fi | |
| docker rm -f otelcol | |
| done | |
| - name: Check journalctl | |
| run: | | |
| # ensure journalctl can run with the collected libraries | |
| docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest | |
| docker exec otelcol /bin/journalctl | |
| docker rm -f otelcol | |
| - name: Check python and java | |
| if: ${{ matrix.ARCH != 'ppc64le' }} | |
| run: | | |
| # ensure python and java can run with the collected libraries | |
| docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest | |
| docker exec otelcol /usr/lib/splunk-otel-collector/agent-bundle/bin/python --version | |
| docker exec otelcol /usr/lib/splunk-otel-collector/agent-bundle/jre/bin/java -version | |
| # ensure collectd-python plugins were installed | |
| docker exec otelcol sh -c 'test -d /usr/lib/splunk-otel-collector/agent-bundle/collectd-python/' | |
| if [[ "$(docker exec otelcol ls /usr/lib/splunk-otel-collector/agent-bundle/collectd-python/ | wc -l)" -eq 0 ]]; then | |
| echo "/usr/lib/splunk-otel-collector/agent-bundle/collectd-python/ is empty!" >&2 | |
| exit 1 | |
| fi | |
| docker rm -f otelcol |