ci: build & test otelcol-config

Signed-off-by: Justin Kolberg <amd.prophet@gmail.com>
SumoLogic · Aug 15, 2024 · 10044b7 · 10044b7
1 parent f5fdc63
commit 10044b7
Show file tree

Hide file tree

Showing 18 changed files with 593 additions and 56 deletions.
diff --git a/.github/workflows/dev_builds.yml b/.github/workflows/dev_builds.yml
@@ -100,6 +100,25 @@ jobs:
       save-cache: true
       boringcrypto: ${{ matrix.boringcrypto == true }}
 
+  test-otelcol-config:
+    name: Test
+    uses: ./.github/workflows/workflow-test-otelcol-config.yml
+    strategy:
+      matrix:
+        include:
+          - arch_os: linux_amd64
+            runs-on: ubuntu-20.04
+          - arch_os: linux_amd64
+            runs-on: ubuntu-20.04
+            boringcrypto: true
+          - arch_os: darwin_amd64
+            runs-on: macos-latest
+    with:
+      arch_os: ${{ matrix.arch_os }}
+      runs-on: ${{ matrix.runs-on }}
+      save-cache: true
+      boringcrypto: ${{ matrix.boringcrypto == true }}
+
   build:
     name: Build
     uses: ./.github/workflows/workflow-build.yml
@@ -143,6 +162,39 @@ jobs:
       microsoft_certificate_name: ${{ secrets.MICROSOFT_CERTNAME }}
       microsoft_description: ${{ secrets.MICROSOFT_DESCRIPTION }}
 
+  build-otelcol-config:
+    name: Build (otelcol-config)
+    uses: ./.github/workflows/workflow-build-otelcol-config.yml
+    needs: [get-version]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch_os: darwin_amd64
+            runs-on: macos-latest
+          - arch_os: darwin_arm64
+            runs-on: macos-latest
+          - arch_os: linux_amd64
+            runs-on: ubuntu-20.04
+          - arch_os: linux_amd64
+            runs-on: ubuntu-20.04
+            fips: true
+          - arch_os: linux_arm64
+            runs-on: ubuntu-20.04
+          - arch_os: linux_arm64
+            runs-on: ubuntu-20.04
+            fips: true
+    with:
+      arch_os: ${{ matrix.arch_os }}
+      runs-on: ${{ matrix.runs-on }}
+      fips: ${{ matrix.fips == true }}
+      save-cache: true
+      sumo_component_gomod_version: "v${{ needs.get-version.outputs.version }}"
+    secrets:
+      apple_developer_certificate_p12_base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
+      apple_developer_certificate_password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
+      app_store_connect_password: ${{ secrets.AC_PASSWORD }}
+
   build-container-images:
     name: Build container
     runs-on: ubuntu-20.04
@@ -475,6 +527,8 @@ jobs:
   trigger-packaging:
     name: Trigger Packaging
     needs:
+      - build
+      - build-otelcol-config
       - lint
       - package-msi
       - push-docker-manifest

diff --git a/.github/workflows/workflow-build-otelcol-config.yml b/.github/workflows/workflow-build-otelcol-config.yml
@@ -0,0 +1,166 @@
+name: Build Otelcol Config Tool
+
+on:
+  workflow_call:
+    inputs:
+      arch_os:
+        description: Architecture and OS in the form "{arch}_{os}". See GOARCH and GOOS for accepted values.
+        default: linux_amd64
+        type: string
+      sumo_component_gomod_version:
+        description: Package version for components hosted in this repo. Normally, this is the v0.0.0-00010101000000-000000000000 placeholder.
+        type: string
+        required: false
+      fips:
+        description: Build binary with FIPS support
+        default: false
+        type: boolean
+      runs-on:
+        default: ubuntu-20.04
+        type: string
+      save-cache:
+        description: Save the module and build caches.
+        default: false
+        type: boolean
+    secrets:
+      apple_developer_certificate_p12_base64:
+        required: false
+      apple_developer_certificate_password:
+        required: false
+      app_store_connect_password:
+        required: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  GO_VERSION: "1.21.4"
+
+jobs:
+  build:
+    name: Build
+    runs-on: ${{ inputs.runs-on }}
+    env:
+      FIPS_SUFFIX: ${{ inputs.fips && '-fips' || '' }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fetch current branch
+        run: ./ci/fetch_current_branch.sh
+
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+
+      - name: Get Go env values
+        run: |
+          echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_ENV"
+          echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_ENV"
+          echo "GOARCH=$(go env GOARCH)" >> "$GITHUB_ENV"
+          echo "GOOS=$(go env GOOS)" >> "$GITHUB_ENV"
+          echo "ARCH_OS=$(go env GOOS)_$(go env GOARCH)" >> $GITHUB_ENV
+
+      - name: Get cache key
+        id: get-cache-key
+        run: |
+          echo "cache-key=go-build-${{ env.GO_VERSION }}-otelcol-config${FIPS_SUFFIX}-${{inputs.arch_os}}-${{ hashFiles('pkg/tools/otelcol-config/go.sum') }}" >> $GITHUB_OUTPUT
+          echo "restore-keys=go-build-${{ env.GO_VERSION }}-otelcol-config${FIPS_SUFFIX}-${{inputs.arch_os}}-" >> $GITHUB_OUTPUT
+          echo "toolchain-cache-key=toolchain-${{inputs.arch_os}}-${{ hashFiles('pkg/tools/otelcol-config/build-fips/config.mak', 'pkg/tools/otelcol-config/build-fips/Makefile') }}" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.GOMODCACHE }}/cache
+            ${{ env.GOCACHE }}
+          key: ${{ steps.get-cache-key.outputs.cache-key }}
+          restore-keys: |
+            ${{ steps.get-cache-key.outputs.restore-keys }}
+
+      - name: Build
+        if: '! inputs.fips'
+        run: make otelcol-config-${{inputs.arch_os}}
+        working-directory: ./pkg/tools/otelcol-config
+
+      - uses: actions/cache/restore@v4
+        id: restore-toolchain-cache
+        if: inputs.fips && contains(inputs.arch_os, 'linux')
+        with:
+          path: |
+            /opt/toolchain
+          key: ${{ steps.get-cache-key.outputs.toolchain-cache-key }}
+
+      # TODO: pass go version to toolchain make target
+      - name: Rebuild Toolchains
+        id: rebuild-toolchain
+        if: ${{ steps.restore-toolchain-cache.outcome == 'success' && steps.restore-toolchain-cache.outputs.cache-hit != 'true' }}
+        run: make toolchain-${{ inputs.arch_os }} OUTPUT=/opt/toolchain -j3
+        working-directory: ./toolchains
+
+      - name: Build (FIPS)
+        if: inputs.fips && contains(inputs.arch_os, 'linux')
+        run: |
+          CC=$(find /opt/toolchain/bin -type f -name "*-linux-musl-gcc")
+          test "$CC"
+          echo "Using toolchain: $CC"
+          make otelcol-config-${{inputs.arch_os}} \
+            FIPS_SUFFIX="-fips" \
+            CGO_ENABLED="1" \
+            CC="$CC" \
+            EXTRA_LDFLAGS="-linkmode external -extldflags '-static'"
+        working-directory: ./pkg/tools/otelcol-config
+
+      - name: Build (FIPS)
+        if: inputs.fips && contains(inputs.arch_os, 'linux')
+        run: |
+          CC=$(find /opt/toolchain/bin -type f -name "*-linux-musl-gcc")
+          test "$CC"
+          echo "Using toolchain: $CC"
+          make otelcol-config-${{inputs.arch_os}} \
+            FIPS_SUFFIX="-fips" \
+            CGO_ENABLED="1" \
+            CC="$CC" \
+            EXTRA_LDFLAGS="-linkmode external -extldflags '-static'"
+        working-directory: ./pkg/tools/otelcol-config
+
+      - name: Set binary name
+        id: set-binary-name
+        run: echo "binary_name=otelcol-config${FIPS_SUFFIX}-${{inputs.arch_os}}" >> $GITHUB_OUTPUT
+
+      - name: Show BoringSSL symbols
+        if: inputs.fips && contains(inputs.arch_os, 'linux')
+        working-directory: ./pkg/tools/otelcol-config
+        run: |
+          go tool nm ${{ steps.set-binary-name.outputs.binary_name }} | \
+          grep "_Cfunc__goboringcrypto_"
+
+      # TODO: find a way to test if the binary runs
+      # - name: Test binary
+      #   if: inputs.arch_os == env.ARCH_OS
+      #   working-directory: ./pkg/tools/otelcol-config
+      #   run: |
+      #     ./${{ steps.set-binary-name.outputs.binary_name }} -t foo
+
+      - name: Store binary as action artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.set-binary-name.outputs.binary_name }}
+          path: ./pkg/tools/otelcol-config/${{ steps.set-binary-name.outputs.binary_name }}
+          if-no-files-found: error
+
+      - uses: actions/cache/save@v4
+        if: ${{ steps.rebuild-toolchain.outcome == 'success' }}
+        with:
+          path: |
+            /opt/toolchain
+          key: ${{ steps.get-cache-key.outputs.toolchain-cache-key }}
+
+      - uses: actions/cache/save@v4
+        if: inputs.save-cache
+        with:
+          path: |
+            ${{ env.GOMODCACHE }}/cache
+            ${{ env.GOCACHE }}
+          key: ${{ steps.get-cache-key.outputs.cache-key }}
diff --git a/.github/workflows/workflow-test-otelcol-config.yml b/.github/workflows/workflow-test-otelcol-config.yml
@@ -0,0 +1,77 @@
+name: Run tests (otelcol-config)
+
+on:
+  workflow_call:
+    inputs:
+      arch_os:
+        description: Architecture and OS in the form "{arch}_{os}". See GOARCH and GOOS for accepted values.
+        default: linux_amd64
+        type: string
+      boringcrypto:
+        description: Run with BoringCrypto enabled
+        default: false
+        type: boolean
+      runs-on:
+        default: ubuntu-20.04
+        type: string
+      only-if-changed:
+        description: Run only if relevant files changed.
+        default: false
+        type: boolean
+      save-cache:
+        description: Save the module and build caches.
+        default: false
+        type: boolean
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  GO_VERSION: "1.21.4"
+
+jobs:
+  test:
+    name: Test (otelcol-config)
+    runs-on: ${{ inputs.runs-on }}
+    env:
+      BORINGCRYPTO_SUFFIX: ${{ inputs.boringcrypto && '-boringcrypto' || '' }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+
+      - name: Get GOCACHE and GOMODCACHE
+        run: |
+          echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_ENV"
+          echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_ENV"
+
+      - name: Get cache key
+        id: get-cache-key
+        run: |
+          echo "cache-key=go-test-${{ env.GO_VERSION }}${BORINGCRYPTO_SUFFIX}-${{inputs.arch_os}}-${{ hashFiles('pkg/tools/otelcol-config/go.sum') }}" >> $GITHUB_OUTPUT
+          echo "restore-keys=go-test-${{ env.GO_VERSION }}${BORINGCRYPTO_SUFFIX}-${{inputs.arch_os}}-" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.GOMODCACHE }}/cache
+            ${{ env.GOCACHE }}
+          key: ${{ steps.get-cache-key.outputs.cache-key }}
+          restore-keys: |
+            ${{ steps.get-cache-key.outputs.restore-keys }}
+
+      - name: Run tests
+        run: make test ${{ inputs.boringcrypto && 'CGO_ENABLED=1 GOEXPERIMENT=boringcrypto' || '' }}
+        working-directory: ./pkg/tools/otelcol-config
+
+      - uses: actions/cache/save@v4
+        with:
+          path: |
+            ${{ env.GOMODCACHE }}/cache
+            ${{ env.GOCACHE }}
+          key: ${{ steps.get-cache-key.outputs.cache-key }}
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # For FIPS binary, there are some debian runtime dependencies
-FROM debian:12.6 as otelcol
+FROM debian:12.6 AS otelcol
 COPY otelcol-sumo /
 # This shouldn't be necessary but sometimes we end up with execution bit not set.
 # ref: https://github.com/open-telemetry/opentelemetry-collector/issues/1317
@@ -12,13 +12,13 @@ RUN tar czhf otelcol.tar.gz /otelcol-sumo $(ldd /otelcol-sumo | grep -oP "\/.*?
 # however, we can copy full directory as root (/) to be base file structure for scratch image
 RUN mkdir /output && tar xf /otelcol.tar.gz --directory /output
 
-FROM alpine:3.20.2 as certs
+FROM alpine:3.20.2 AS certs
 RUN apk --update add ca-certificates
 
-FROM alpine:3.20.2 as directories
+FROM alpine:3.20.2 AS directories
 RUN mkdir /etc/otel/
 
-FROM debian:12.6 as systemd
+FROM debian:12.6 AS systemd
 RUN apt update && apt install -y systemd
 # prepare package with journald and it's dependencies keeping original paths
 # h stands for dereference of symbolic links
@@ -30,10 +30,10 @@ RUN mkdir /output && tar xf /journalctl.tar.gz --directory /output
 
 FROM scratch
 ARG BUILD_TAG=latest
-ENV TAG $BUILD_TAG
+ENV TAG=$BUILD_TAG
 ARG USER_UID=10001
 USER ${USER_UID}
-ENV HOME /etc/otel/
+ENV HOME=/etc/otel/
 
 # copy journalctl and it's dependencies as base structure
 COPY --from=systemd /output/ /

diff --git a/Dockerfile_local b/Dockerfile_local
@@ -1,18 +1,18 @@
-FROM golang:1.21.11-alpine as builder
+FROM golang:1.21.11-alpine AS builder
 ADD . /src
 WORKDIR /src/otelcolbuilder/
 ENV CGO_ENABLED=0
 RUN apk --update add make gcc g++ curl git
 RUN make install-builder BUILDER_BIN_PATH=/bin
 RUN make build BUILDER_BIN_PATH=/bin
 
-FROM alpine:3.20.2 as certs
+FROM alpine:3.20.2 AS certs
 RUN apk --update add ca-certificates
 
-FROM alpine:3.20.2 as directories
+FROM alpine:3.20.2 AS directories
 RUN mkdir /etc/otel/
 
-FROM debian:12.6 as systemd
+FROM debian:12.6 AS systemd
 RUN apt update && apt install -y systemd
 # prepare package with journald and it's dependencies keeping original paths
 # h stands for dereference of symbolic links
@@ -24,12 +24,12 @@ RUN mkdir /output && tar xf /journalctl.tar.gz --directory /output
 
 FROM scratch
 ARG BUILD_TAG=latest
-ENV TAG $BUILD_TAG
+ENV TAG=$BUILD_TAG
 ARG USER_UID=10001
 USER ${USER_UID}
-ENV HOME /etc/otel/
+ENV HOME=/etc/otel/
 
-# copy journalctl and it's dependencies as base structure
+# copy journalctl and it's dependencies AS base structure
 COPY --from=systemd /output/ /
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=builder /src/otelcolbuilder/cmd/otelcol-sumo /otelcol-sumo