From 10044b77ad78831814f84097d19f081e10dd30ce Mon Sep 17 00:00:00 2001 From: Justin Kolberg Date: Mon, 29 Jul 2024 17:27:09 -0700 Subject: [PATCH] ci: build & test otelcol-config Signed-off-by: Justin Kolberg --- .github/workflows/dev_builds.yml | 54 ++++++ .../workflow-build-otelcol-config.yml | 166 ++++++++++++++++ .../workflow-test-otelcol-config.yml | 77 ++++++++ Dockerfile | 12 +- Dockerfile_local | 14 +- Dockerfile_ubi | 4 +- Dockerfile_windows | 2 +- Makefile | 1 + otelcolbuilder/Makefile | 38 ++-- otelcolbuilder/build-fips/Dockerfile | 16 -- pkg/tools/otelcol-config/.gitignore | 1 + pkg/tools/otelcol-config/Makefile | 181 ++++++++++++++++++ .../otelcol-config/build-fips/entrypoint.sh | 13 ++ toolchains/.gitignore | 3 + toolchains/Dockerfile | 21 ++ toolchains/Makefile | 46 +++++ .../build-fips => toolchains}/README.md | 0 .../build-fips => toolchains}/config.mak | 0 18 files changed, 593 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/workflow-build-otelcol-config.yml create mode 100644 .github/workflows/workflow-test-otelcol-config.yml delete mode 100644 otelcolbuilder/build-fips/Dockerfile create mode 100644 pkg/tools/otelcol-config/.gitignore create mode 100755 pkg/tools/otelcol-config/build-fips/entrypoint.sh create mode 100644 toolchains/.gitignore create mode 100644 toolchains/Dockerfile create mode 100644 toolchains/Makefile rename {otelcolbuilder/build-fips => toolchains}/README.md (100%) rename {otelcolbuilder/build-fips => toolchains}/config.mak (100%) diff --git a/.github/workflows/dev_builds.yml b/.github/workflows/dev_builds.yml index fde9265609..95c004c78b 100644 --- a/.github/workflows/dev_builds.yml +++ b/.github/workflows/dev_builds.yml @@ -100,6 +100,25 @@ jobs: save-cache: true boringcrypto: ${{ matrix.boringcrypto == true }} + test-otelcol-config: + name: Test + uses: ./.github/workflows/workflow-test-otelcol-config.yml + strategy: + matrix: + include: + - arch_os: linux_amd64 + runs-on: ubuntu-20.04 + - arch_os: linux_amd64 + runs-on: ubuntu-20.04 + boringcrypto: true + - arch_os: darwin_amd64 + runs-on: macos-latest + with: + arch_os: ${{ matrix.arch_os }} + runs-on: ${{ matrix.runs-on }} + save-cache: true + boringcrypto: ${{ matrix.boringcrypto == true }} + build: name: Build uses: ./.github/workflows/workflow-build.yml @@ -143,6 +162,39 @@ jobs: microsoft_certificate_name: ${{ secrets.MICROSOFT_CERTNAME }} microsoft_description: ${{ secrets.MICROSOFT_DESCRIPTION }} + build-otelcol-config: + name: Build (otelcol-config) + uses: ./.github/workflows/workflow-build-otelcol-config.yml + needs: [get-version] + strategy: + fail-fast: false + matrix: + include: + - arch_os: darwin_amd64 + runs-on: macos-latest + - arch_os: darwin_arm64 + runs-on: macos-latest + - arch_os: linux_amd64 + runs-on: ubuntu-20.04 + - arch_os: linux_amd64 + runs-on: ubuntu-20.04 + fips: true + - arch_os: linux_arm64 + runs-on: ubuntu-20.04 + - arch_os: linux_arm64 + runs-on: ubuntu-20.04 + fips: true + with: + arch_os: ${{ matrix.arch_os }} + runs-on: ${{ matrix.runs-on }} + fips: ${{ matrix.fips == true }} + save-cache: true + sumo_component_gomod_version: "v${{ needs.get-version.outputs.version }}" + secrets: + apple_developer_certificate_p12_base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }} + apple_developer_certificate_password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }} + app_store_connect_password: ${{ secrets.AC_PASSWORD }} + build-container-images: name: Build container runs-on: ubuntu-20.04 @@ -475,6 +527,8 @@ jobs: trigger-packaging: name: Trigger Packaging needs: + - build + - build-otelcol-config - lint - package-msi - push-docker-manifest diff --git a/.github/workflows/workflow-build-otelcol-config.yml b/.github/workflows/workflow-build-otelcol-config.yml new file mode 100644 index 0000000000..92f171d3cd --- /dev/null +++ b/.github/workflows/workflow-build-otelcol-config.yml @@ -0,0 +1,166 @@ +name: Build Otelcol Config Tool + +on: + workflow_call: + inputs: + arch_os: + description: Architecture and OS in the form "{arch}_{os}". See GOARCH and GOOS for accepted values. + default: linux_amd64 + type: string + sumo_component_gomod_version: + description: Package version for components hosted in this repo. Normally, this is the v0.0.0-00010101000000-000000000000 placeholder. + type: string + required: false + fips: + description: Build binary with FIPS support + default: false + type: boolean + runs-on: + default: ubuntu-20.04 + type: string + save-cache: + description: Save the module and build caches. + default: false + type: boolean + secrets: + apple_developer_certificate_p12_base64: + required: false + apple_developer_certificate_password: + required: false + app_store_connect_password: + required: false + +defaults: + run: + shell: bash + +env: + GO_VERSION: "1.21.4" + +jobs: + build: + name: Build + runs-on: ${{ inputs.runs-on }} + env: + FIPS_SUFFIX: ${{ inputs.fips && '-fips' || '' }} + steps: + - uses: actions/checkout@v4 + + - name: Fetch current branch + run: ./ci/fetch_current_branch.sh + + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Get Go env values + run: | + echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_ENV" + echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_ENV" + echo "GOARCH=$(go env GOARCH)" >> "$GITHUB_ENV" + echo "GOOS=$(go env GOOS)" >> "$GITHUB_ENV" + echo "ARCH_OS=$(go env GOOS)_$(go env GOARCH)" >> $GITHUB_ENV + + - name: Get cache key + id: get-cache-key + run: | + echo "cache-key=go-build-${{ env.GO_VERSION }}-otelcol-config${FIPS_SUFFIX}-${{inputs.arch_os}}-${{ hashFiles('pkg/tools/otelcol-config/go.sum') }}" >> $GITHUB_OUTPUT + echo "restore-keys=go-build-${{ env.GO_VERSION }}-otelcol-config${FIPS_SUFFIX}-${{inputs.arch_os}}-" >> $GITHUB_OUTPUT + echo "toolchain-cache-key=toolchain-${{inputs.arch_os}}-${{ hashFiles('pkg/tools/otelcol-config/build-fips/config.mak', 'pkg/tools/otelcol-config/build-fips/Makefile') }}" >> $GITHUB_OUTPUT + + - uses: actions/cache/restore@v4 + with: + path: | + ${{ env.GOMODCACHE }}/cache + ${{ env.GOCACHE }} + key: ${{ steps.get-cache-key.outputs.cache-key }} + restore-keys: | + ${{ steps.get-cache-key.outputs.restore-keys }} + + - name: Build + if: '! inputs.fips' + run: make otelcol-config-${{inputs.arch_os}} + working-directory: ./pkg/tools/otelcol-config + + - uses: actions/cache/restore@v4 + id: restore-toolchain-cache + if: inputs.fips && contains(inputs.arch_os, 'linux') + with: + path: | + /opt/toolchain + key: ${{ steps.get-cache-key.outputs.toolchain-cache-key }} + + # TODO: pass go version to toolchain make target + - name: Rebuild Toolchains + id: rebuild-toolchain + if: ${{ steps.restore-toolchain-cache.outcome == 'success' && steps.restore-toolchain-cache.outputs.cache-hit != 'true' }} + run: make toolchain-${{ inputs.arch_os }} OUTPUT=/opt/toolchain -j3 + working-directory: ./toolchains + + - name: Build (FIPS) + if: inputs.fips && contains(inputs.arch_os, 'linux') + run: | + CC=$(find /opt/toolchain/bin -type f -name "*-linux-musl-gcc") + test "$CC" + echo "Using toolchain: $CC" + make otelcol-config-${{inputs.arch_os}} \ + FIPS_SUFFIX="-fips" \ + CGO_ENABLED="1" \ + CC="$CC" \ + EXTRA_LDFLAGS="-linkmode external -extldflags '-static'" + working-directory: ./pkg/tools/otelcol-config + + - name: Build (FIPS) + if: inputs.fips && contains(inputs.arch_os, 'linux') + run: | + CC=$(find /opt/toolchain/bin -type f -name "*-linux-musl-gcc") + test "$CC" + echo "Using toolchain: $CC" + make otelcol-config-${{inputs.arch_os}} \ + FIPS_SUFFIX="-fips" \ + CGO_ENABLED="1" \ + CC="$CC" \ + EXTRA_LDFLAGS="-linkmode external -extldflags '-static'" + working-directory: ./pkg/tools/otelcol-config + + - name: Set binary name + id: set-binary-name + run: echo "binary_name=otelcol-config${FIPS_SUFFIX}-${{inputs.arch_os}}" >> $GITHUB_OUTPUT + + - name: Show BoringSSL symbols + if: inputs.fips && contains(inputs.arch_os, 'linux') + working-directory: ./pkg/tools/otelcol-config + run: | + go tool nm ${{ steps.set-binary-name.outputs.binary_name }} | \ + grep "_Cfunc__goboringcrypto_" + + # TODO: find a way to test if the binary runs + # - name: Test binary + # if: inputs.arch_os == env.ARCH_OS + # working-directory: ./pkg/tools/otelcol-config + # run: | + # ./${{ steps.set-binary-name.outputs.binary_name }} -t foo + + - name: Store binary as action artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-binary-name.outputs.binary_name }} + path: ./pkg/tools/otelcol-config/${{ steps.set-binary-name.outputs.binary_name }} + if-no-files-found: error + + - uses: actions/cache/save@v4 + if: ${{ steps.rebuild-toolchain.outcome == 'success' }} + with: + path: | + /opt/toolchain + key: ${{ steps.get-cache-key.outputs.toolchain-cache-key }} + + - uses: actions/cache/save@v4 + if: inputs.save-cache + with: + path: | + ${{ env.GOMODCACHE }}/cache + ${{ env.GOCACHE }} + key: ${{ steps.get-cache-key.outputs.cache-key }} diff --git a/.github/workflows/workflow-test-otelcol-config.yml b/.github/workflows/workflow-test-otelcol-config.yml new file mode 100644 index 0000000000..0aa09eca44 --- /dev/null +++ b/.github/workflows/workflow-test-otelcol-config.yml @@ -0,0 +1,77 @@ +name: Run tests (otelcol-config) + +on: + workflow_call: + inputs: + arch_os: + description: Architecture and OS in the form "{arch}_{os}". See GOARCH and GOOS for accepted values. + default: linux_amd64 + type: string + boringcrypto: + description: Run with BoringCrypto enabled + default: false + type: boolean + runs-on: + default: ubuntu-20.04 + type: string + only-if-changed: + description: Run only if relevant files changed. + default: false + type: boolean + save-cache: + description: Save the module and build caches. + default: false + type: boolean + +defaults: + run: + shell: bash + +env: + GO_VERSION: "1.21.4" + +jobs: + test: + name: Test (otelcol-config) + runs-on: ${{ inputs.runs-on }} + env: + BORINGCRYPTO_SUFFIX: ${{ inputs.boringcrypto && '-boringcrypto' || '' }} + steps: + - uses: actions/checkout@v4 + + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Get GOCACHE and GOMODCACHE + run: | + echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_ENV" + echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_ENV" + + - name: Get cache key + id: get-cache-key + run: | + echo "cache-key=go-test-${{ env.GO_VERSION }}${BORINGCRYPTO_SUFFIX}-${{inputs.arch_os}}-${{ hashFiles('pkg/tools/otelcol-config/go.sum') }}" >> $GITHUB_OUTPUT + echo "restore-keys=go-test-${{ env.GO_VERSION }}${BORINGCRYPTO_SUFFIX}-${{inputs.arch_os}}-" >> $GITHUB_OUTPUT + + - uses: actions/cache/restore@v4 + with: + path: | + ${{ env.GOMODCACHE }}/cache + ${{ env.GOCACHE }} + key: ${{ steps.get-cache-key.outputs.cache-key }} + restore-keys: | + ${{ steps.get-cache-key.outputs.restore-keys }} + + - name: Run tests + run: make test ${{ inputs.boringcrypto && 'CGO_ENABLED=1 GOEXPERIMENT=boringcrypto' || '' }} + working-directory: ./pkg/tools/otelcol-config + + - uses: actions/cache/save@v4 + with: + path: | + ${{ env.GOMODCACHE }}/cache + ${{ env.GOCACHE }} + key: ${{ steps.get-cache-key.outputs.cache-key }} diff --git a/Dockerfile b/Dockerfile index 9a3ad07075..3b2f18ebc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # For FIPS binary, there are some debian runtime dependencies -FROM debian:12.6 as otelcol +FROM debian:12.6 AS otelcol COPY otelcol-sumo / # This shouldn't be necessary but sometimes we end up with execution bit not set. # ref: https://github.com/open-telemetry/opentelemetry-collector/issues/1317 @@ -12,13 +12,13 @@ RUN tar czhf otelcol.tar.gz /otelcol-sumo $(ldd /otelcol-sumo | grep -oP "\/.*? # however, we can copy full directory as root (/) to be base file structure for scratch image RUN mkdir /output && tar xf /otelcol.tar.gz --directory /output -FROM alpine:3.20.2 as certs +FROM alpine:3.20.2 AS certs RUN apk --update add ca-certificates -FROM alpine:3.20.2 as directories +FROM alpine:3.20.2 AS directories RUN mkdir /etc/otel/ -FROM debian:12.6 as systemd +FROM debian:12.6 AS systemd RUN apt update && apt install -y systemd # prepare package with journald and it's dependencies keeping original paths # h stands for dereference of symbolic links @@ -30,10 +30,10 @@ RUN mkdir /output && tar xf /journalctl.tar.gz --directory /output FROM scratch ARG BUILD_TAG=latest -ENV TAG $BUILD_TAG +ENV TAG=$BUILD_TAG ARG USER_UID=10001 USER ${USER_UID} -ENV HOME /etc/otel/ +ENV HOME=/etc/otel/ # copy journalctl and it's dependencies as base structure COPY --from=systemd /output/ / diff --git a/Dockerfile_local b/Dockerfile_local index 54be5ddb62..776cd4460d 100644 --- a/Dockerfile_local +++ b/Dockerfile_local @@ -1,4 +1,4 @@ -FROM golang:1.21.11-alpine as builder +FROM golang:1.21.11-alpine AS builder ADD . /src WORKDIR /src/otelcolbuilder/ ENV CGO_ENABLED=0 @@ -6,13 +6,13 @@ RUN apk --update add make gcc g++ curl git RUN make install-builder BUILDER_BIN_PATH=/bin RUN make build BUILDER_BIN_PATH=/bin -FROM alpine:3.20.2 as certs +FROM alpine:3.20.2 AS certs RUN apk --update add ca-certificates -FROM alpine:3.20.2 as directories +FROM alpine:3.20.2 AS directories RUN mkdir /etc/otel/ -FROM debian:12.6 as systemd +FROM debian:12.6 AS systemd RUN apt update && apt install -y systemd # prepare package with journald and it's dependencies keeping original paths # h stands for dereference of symbolic links @@ -24,12 +24,12 @@ RUN mkdir /output && tar xf /journalctl.tar.gz --directory /output FROM scratch ARG BUILD_TAG=latest -ENV TAG $BUILD_TAG +ENV TAG=$BUILD_TAG ARG USER_UID=10001 USER ${USER_UID} -ENV HOME /etc/otel/ +ENV HOME=/etc/otel/ -# copy journalctl and it's dependencies as base structure +# copy journalctl and it's dependencies AS base structure COPY --from=systemd /output/ / COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /src/otelcolbuilder/cmd/otelcol-sumo /otelcol-sumo diff --git a/Dockerfile_ubi b/Dockerfile_ubi index c935d4be99..289defe40b 100644 --- a/Dockerfile_ubi +++ b/Dockerfile_ubi @@ -6,7 +6,7 @@ ARG BUILD_ARCH FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10 ARG BUILD_TAG -ENV TAG $BUILD_TAG +ENV TAG=$BUILD_TAG ENV SUMMARY="UBI based sumologic-otel-collector" \ DESCRIPTION="Sumo Logic Distribution for OpenTelemetry Collector is a Sumo Logic-supported distribution of the OpenTelemetry Collector. It is a single agent to send logs, metrics and traces to Sumo Logic." @@ -32,7 +32,7 @@ RUN chmod 755 /otelcol-sumo ARG USER_UID=10001 USER ${USER_UID} -ENV HOME /etc/otel/ +ENV HOME=/etc/otel/ ENTRYPOINT ["/otelcol-sumo"] CMD ["--config", "/etc/otel/config.yaml"] diff --git a/Dockerfile_windows b/Dockerfile_windows index 87887b953a..7870391fe5 100644 --- a/Dockerfile_windows +++ b/Dockerfile_windows @@ -1,7 +1,7 @@ ARG BASE_IMAGE_TAG=ltsc2022 FROM mcr.microsoft.com/windows/servercore:${BASE_IMAGE_TAG} ARG BUILD_TAG=latest -ENV TAG $BUILD_TAG +ENV TAG=$BUILD_TAG ADD /otelcol-sumo.exe /otelcol-sumo.exe ENTRYPOINT ["/otelcol-sumo.exe"] diff --git a/Makefile b/Makefile index 09b5649ba1..96a1ff57f4 100644 --- a/Makefile +++ b/Makefile @@ -225,6 +225,7 @@ prepare-tag: install-gsed .PHONY: build build: @$(MAKE) -C ./otelcolbuilder/ build + @$(MAKE) -C ./pkg/tools/otelcol-config/ build .PHONY: install-builder install-builder: diff --git a/otelcolbuilder/Makefile b/otelcolbuilder/Makefile index 938b5cf6e0..2c230d1f86 100644 --- a/otelcolbuilder/Makefile +++ b/otelcolbuilder/Makefile @@ -1,3 +1,5 @@ +include ../toolchains/Makefile + BINARY_NAME ?= otelcol-sumo BUILDER_VERSION ?= 0.103.1 FIPS_SUFFIX ?= "" @@ -148,10 +150,14 @@ build-debug: ensure-correct-builder-version @$(MAKE) _gobuild_debug .PHONY: build-fips -build-fips: - docker volume create build-fips - docker build --platform linux/amd64 -t otelcol-sumo-builder-fips -f ./build-fips/Dockerfile ./build-fips - docker run -it --platform linux/amd64 --rm -v build-fips:/root/build -v $(PWD)/../:/root/workspace -w /root/workspace --tmpfs /tmp otelcol-sumo-builder-fips +build-fips: docker-golang-cross-builder + docker run -it --rm \ + -v $(DOCKER_CROSS_BUILDER_VOLUME):/root/build \ + -v $(PWD)/../:/root/workspace \ + -w /root/workspace \ + --tmpfs /tmp \ + --entrypoint /root/workspace/otelcolbuilder/build-fips/entrypoint.sh \ + golang-cross-builder .PHONY: generate-sources generate-sources: @@ -221,36 +227,20 @@ otelcol-sumo-linux_arm: otelcol-sumo-windows_amd64: GOOS=windows GOARCH=amd64 $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-windows_amd64 -MUSL_X_MAKE_REPO=https://github.com/richfelker/musl-cross-make.git -MUSL_X_MAKE_REF=fe915821b652a7fa37b34a596f47d8e20bc72338 - -musl-cross-make: - git clone $(MUSL_X_MAKE_REPO); - cd musl-cross-make && git checkout $(MUSL_X_MAKE_REF); - cp ./build-fips/config.mak ./musl-cross-make/config.mak; - -.PHONY: toolchain-linux_arm64 -toolchain-linux_arm64: musl-cross-make - TARGET=aarch64-linux-musl $(MAKE) -C musl-cross-make install - -.PHONY: toolchain-linux_amd64 -toolchain-linux_amd64: musl-cross-make - TARGET=x86_64-linux-musl $(MAKE) -C musl-cross-make install - .PHONY: otelcol-sumo-linux-fips_arm64 otelcol-sumo-linux-fips_arm64: - $(MAKE) toolchain-linux_arm64 OUTPUT=$(PWD)/toolchain_arm64 -j3 + $(MAKE) toolchain-linux_arm64 $(MAKE) otelcol-sumo-linux_arm64 \ FIPS_SUFFIX="-fips" \ CGO_ENABLED="1" \ - CC="$(PWD)/toolchain_arm64/bin/aarch64-linux-musl-gcc" \ + CC="$(TOOLCHAINS_DIR)/toolchain_arm64/bin/aarch64-linux-musl-gcc" \ EXTRA_LDFLAGS="-linkmode external -extldflags '-static'" .PHONY: otelcol-sumo-linux-fips_amd64 otelcol-sumo-linux-fips_amd64: - $(MAKE) toolchain-linux_amd64 OUTPUT=$(PWD)/toolchain_amd64 -j3 + $(MAKE) toolchain-linux_amd64 $(MAKE) otelcol-sumo-linux_amd64 \ FIPS_SUFFIX="-fips" \ CGO_ENABLED="1" \ - CC="$(PWD)/toolchain_amd64/bin/x86_64-linux-musl-gcc" \ + CC="$(TOOLCHAINS_DIR)/toolchain_amd64/bin/x86_64-linux-musl-gcc" \ EXTRA_LDFLAGS="-linkmode external -extldflags '-static'" diff --git a/otelcolbuilder/build-fips/Dockerfile b/otelcolbuilder/build-fips/Dockerfile deleted file mode 100644 index b100530659..0000000000 --- a/otelcolbuilder/build-fips/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:20.04 - -ARG DEBIAN_VERSION="bullseye" -ARG UBUNTU_VERSION="20.04" -ARG GO_VERSION=1.21.11 - -RUN apt-get update -RUN apt-get install -y git wget xz-utils build-essential curl - -RUN curl -Lo go.tar.gz https://go.dev/dl/go1.21.11.linux-amd64.tar.gz -RUN tar -zxvf go.tar.gz -C /usr/local -ENV PATH="/usr/local/go/bin:${PATH}" - -COPY entrypoint.sh /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/pkg/tools/otelcol-config/.gitignore b/pkg/tools/otelcol-config/.gitignore new file mode 100644 index 0000000000..da4717d56f --- /dev/null +++ b/pkg/tools/otelcol-config/.gitignore @@ -0,0 +1 @@ +otelcol-config* diff --git a/pkg/tools/otelcol-config/Makefile b/pkg/tools/otelcol-config/Makefile index ded7a36092..7c79ea5ba9 100644 --- a/pkg/tools/otelcol-config/Makefile +++ b/pkg/tools/otelcol-config/Makefile @@ -1 +1,182 @@ include ../../Makefile.Common +include ../../../toolchains/Makefile + +TOOLCHAINS_DIR ?= "$(shell '$(MAKE)' print-toolchains-dir)" + +BINARY_NAME ?= otelcol-config +GO ?= go + +OS ?= $(shell uname -s | tr A-Z a-z) +ARCH ?= $(shell uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + +# Builds for darwin need to be built with CGO_ENABLED set to 1 because some telegraf +# plugins that are used within the telegrafreceiver are implemented with CGO. +# Example of this might be the cpu input plugin using gopsutil to the cpu stats. +# +# https://github.com/shirou/gopsutil/blob/7ea80628/cpu/cpu_darwin_nocgo.go +# https://github.com/shirou/gopsutil/blob/7ea80628/cpu/cpu_darwin.go +ifeq ($(shell go env GOOS),darwin) +CGO_ENABLED ?= 1 +else +# CGO_ENABLED=0 is used becuase we don't want to rely on c libs and opentelemetry +# also builds their binaries this way. +# ref: https://github.com/open-telemetry/opentelemetry-collector/blob/4c503ddc/Makefile#L254-L256 +CGO_ENABLED ?= 0 +endif + +# We don't want to strip symbols for FIPS builds, as they're useful for verifying the build +ifeq ($(FIPS_SUFFIX),"") +GOEXPERIMENT="" +LDFLAGS+= -s -w +# We want to use boringcrypto for fips setup +else +ifeq ($(shell go env GOOS),windows) +GOEXPERIMENT=systemcrypto +GOTAGS := $(GOTAGS) requirefips +LDFLAGS+= -w +else +GOEXPERIMENT=boringcrypto +LDFLAGS+= -w +endif +endif + +# We want to enforce go dns for all types of binaries. The FIPS binaries are built with CGO enabled and use +# the CGO resolver. This has caused a DNS resolution error for .local domains in our K8s containers. +# ref: https://pkg.go.dev/net#hdr-Name_Resolution +GOTAGS := $(GOTAGS) netgo + +################################################################################ +# Functions +################################################################################ + +# Check that given variables are set and all have non-empty values, +# die with an error otherwise. +# +# PARAMS: +# 1. Variable name(s) to test. +# 2. (optional) Error message to print. +# +# EXAMPLE: +# @:$(call check_defined, ENV_REGION, you must set ENV_REGION=usc1|awsuse2) +# +check_defined = \ + $(strip $(foreach 1,$1, \ + $(call __check_defined,$1,$(strip $(value 2))))) +__check_defined = \ + $(if $(value $1),, \ + $(error Undefined $1$(if $2, ($2))$(if $(value @), \ + required by target `$@'))) + +.PHONY: build +build: + CGO_ENABLED=$(CGO_ENABLED) \ + GOEXPERIMENT=$(GOEXPERIMENT) \ + $(GO) build -v \ + -tags "$(GOTAGS)" \ + -ldflags="$(LDFLAGS)" \ + -trimpath \ + -o $(BINARY_NAME) . + chmod +x $(BINARY_NAME) + +.PHONY: build-fips +build-fips: docker-golang-cross-builder + docker run -it --rm \ + -v golang-cross-builder:/root/build \ + -v $(PWD)/../../../:/root/workspace \ + -w /root/workspace \ + --tmpfs /tmp \ + --entrypoint ./pkg/tools/otelcol-config/build-fips/entrypoint.sh \ + golang-cross-builder + +################################################################################ +# Binary targets +################################################################################ + +.PHONY: otelcol-config-darwin_amd64 +otelcol-config-darwin_amd64: + GOOS=darwin GOARCH=amd64 $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-darwin_amd64 + +.PHONY: otelcol-config-darwin_arm64 +otelcol-config-darwin_arm64: + GOOS=darwin GOARCH=arm64 $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-darwin_arm64 + +.PHONY: otelcol-config-linux_amd64 +otelcol-config-linux_amd64: + GOOS=linux GOARCH=amd64 $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-linux_amd64 + +.PHONY: otelcol-config-linux_arm64 +otelcol-config-linux_arm64: + GOOS=linux GOARCH=arm64 $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-linux_arm64 + +.PHONY: otelcol-config-linux_arm +otelcol-config-linux_arm: + GOOS=linux GOARCH=arm $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-linux_arm + +.PHONY: otelcol-config-windows_amd64 +otelcol-config-windows_amd64: + GOOS=windows GOARCH=amd64 $(MAKE) build BINARY_NAME=$(BINARY_NAME)$(FIPS_SUFFIX)-windows_amd64 + +################################################################################ +# FIPS binary targets +################################################################################ + +.PHONY: otelcol-config-linux-fips_amd64 +otelcol-config-linux-fips_amd64: + $(MAKE) toolchain-linux_amd64 + $(MAKE) otelcol-config-linux_amd64 \ + FIPS_SUFFIX="-fips" \ + CGO_ENABLED="1" \ + CC="$(TOOLCHAINS_DIR)toolchain_amd64/bin/x86_64-linux-musl-gcc" \ + LDFLAGS="-linkmode external -extldflags '-static'" + +.PHONY: otelcol-config-linux-fips_arm64 +otelcol-config-linux-fips_arm64: + $(MAKE) toolchain-linux_arm64 + $(MAKE) otelcol-config-linux_arm64 \ + FIPS_SUFFIX="-fips" \ + CGO_ENABLED="1" \ + CC="$(TOOLCHAINS_DIR)toolchain_arm64/bin/aarch64-linux-musl-gcc" \ + LDFLAGS="-linkmode external -extldflags '-static'" + +################################################################################ +# Disk imaging targets +################################################################################ + +# otelcol-sumo-dmg: APP_NAME=otelcol-sumo +# otelcol-sumo-dmg: DMG_VOLUME_NAME=Sumo Logic Distribution for OpenTelemetry Collector +# otelcol-sumo-dmg: DMG_PATH=test.dmg + +.PHONY: otelcol-sumo-dmg +otelcol-sumo-dmg: + $(call check_defined, OTELCOL_BIN_PATH, you must set OTELCOL_BIN_PATH=path/to/otelcol_sumo) + $(call check_defined, OTELCOL_CONFIG_BIN_PATH, you must set OTELCOL_CONFIG_BIN_PATH=path/to/otelcol-config) + $(eval TMP_DIR ?= $(shell mktemp -d)) + cp otelcol-sumo "$(DMG_SOURCE_DIR)" +# $(eval DMG_SOURCE_DIR = $(shell $(MAKE) darwin-dmg-prepare)) +# $(MAKE) darwin-dmg-add-file DMG_SOURCE_DIR="$(DMG_SOURCE_DIR)" SOURCE_FILE_PATH="flag.go" +# $(MAKE) darwin-dmg-add-file DMG_SOURCE_DIR="$(DMG_SOURCE_DIR)" SOURCE_FILE_PATH="flag_actions.go" + hdiutil create "$(DMG_PATH)" -ov -volname "$(DMG_VOLUME_NAME)" -fs APFS -format UDZO -srcfolder "$(DMG_SOURCE_DIR)" + +################################################################################ +# Code signing targets +################################################################################ + +.PHONY: darwin-sign-binary +darwin-sign-binary: +darwin-sign-binary: + codesign --timestamp --options=runtime -s "$(DEVELOPER_SIGNING_IDENTITY)" -v "$(BINARY_PATH)" + +.PHONY: darwin-sign-dmg +darwin-sign: BINARY_PATH = cmd/$(BINARY_NAME) +darwin-sign: DMG_PATH = $(BINARY_PATH).dmg +darwin-sign: + xcrun notarytool submit --apple-id "$(AC_USERNAME)" --password "$(AC_PASSWORD)" --team-id "$(DEVELOPER_TEAM_ID)" --progress --wait "$(DMG_PATH)" + xcrun stapler staple "$(DMG_PATH)" + +.PHONY: darwin_amd64-sign +darwin_amd64-sign: BINARY_NAME=otelcol-config-darwin_amd64 +darwin_amd64-sign: darwin-dmg-and-sign + +.PHONY: darwin_arm64-sign +darwin_arm64-sign: BINARY_NAME=otelcol-config-darwin_arm64 +darwin_arm64-sign: darwin-dmg-and-sign diff --git a/pkg/tools/otelcol-config/build-fips/entrypoint.sh b/pkg/tools/otelcol-config/build-fips/entrypoint.sh new file mode 100755 index 0000000000..6f3dea98f1 --- /dev/null +++ b/pkg/tools/otelcol-config/build-fips/entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +# Mac security suite is interferring, can't build in bind mount workspace +cp -r /root/workspace/* /root/build/ +cd /root/build + +# Build otelcol-config +cd pkg/tools/otelcol-config || exit 1 +make otelcol-config-linux-fips_amd64 +make otelcol-config-linux-fips_arm64 + +# Copy produced binaries to bind mount workspace +cp otelcol-config-fips-linux_* /root/workspace/pkg/tools/otelcol-config diff --git a/toolchains/.gitignore b/toolchains/.gitignore new file mode 100644 index 0000000000..422c93929d --- /dev/null +++ b/toolchains/.gitignore @@ -0,0 +1,3 @@ +# FIPS Static +musl-cross-make +toolchain_* diff --git a/toolchains/Dockerfile b/toolchains/Dockerfile new file mode 100644 index 0000000000..fa2ab69cf1 --- /dev/null +++ b/toolchains/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_VERSION="bullseye" +ARG UBUNTU_VERSION="20.04" +ARG GO_VERSION=1.21.11 + +RUN apt-get update +RUN apt-get install -yq tzdata +ENV TZ="UTC" + +RUN apt-get install -y git wget xz-utils build-essential curl texinfo file + +RUN curl -Lo go.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz +RUN tar -zxvf go.tar.gz -C /usr/local +ENV PATH="/usr/local/go/bin:${PATH}" + +COPY config.mak /config.mak +COPY Makefile /Makefile + +RUN make toolchain-linux_arm64 +RUN make toolchain-linux_amd64 diff --git a/toolchains/Makefile b/toolchains/Makefile new file mode 100644 index 0000000000..736a8b7bff --- /dev/null +++ b/toolchains/Makefile @@ -0,0 +1,46 @@ +# Determine the absolute path of the toolchains directory that contains this +# Makefile. +TOOLCHAINS_DIR := $(realpath $(dir $(realpath $(lastword $(MAKEFILE_LIST))))) + +DOCKER_CROSS_BUILDER_NAME=golang-cross-builder +DOCKER_CROSS_BUILDER_VOLUME=golang-cross-builder + +MUSL_X_MAKE_REPO=https://github.com/richfelker/musl-cross-make.git +MUSL_X_MAKE_REF=fe915821b652a7fa37b34a596f47d8e20bc72338 + +.PHONY: print-toolchains-dir +print-toolchains-dir: + @echo $(TOOLCHAINS_DIR) + +.PHONY: print-docker-cross-builder-name +print-docker-cross-builder-name: + @echo $(DOCKER_CROSS_BUILDER_NAME) + +.PHONY: print-docker-cross-builder-volume +print-docker-cross-builder-volume: + @echo $(DOCKER_CROSS_BUILDER_VOLUME) + +#.PHONY: musl-cross-make +#musl-cross-make: MUSL_REPO_PATH=$(TOOLCHAINS_DIR)/musl-cross-make +musl-cross-make: + git clone $(MUSL_X_MAKE_REPO); + cd musl-cross-make && git checkout $(MUSL_X_MAKE_REF); + cp $(TOOLCHAINS_DIR)/config.mak musl-cross-make/config.mak; + +.PHONY: toolchain-linux_arm64 +toolchain-linux_arm64: OUTPUT=$(TOOLCHAINS_DIR)/toolchain_arm64 +toolchain-linux_arm64: musl-cross-make + TARGET=aarch64-linux-musl $(MAKE) -C musl-cross-make install -j3 + +.PHONY: toolchain-linux_amd64 +toolchain-linux_amd64: OUTPUT=$(TOOLCHAINS_DIR)/toolchain_amd64 +toolchain-linux_amd64: musl-cross-make + TARGET=x86_64-linux-musl $(MAKE) -C musl-cross-make install -j3 + +.PHONY: docker-golang-cross-builder +docker-golang-cross-builder: + docker volume create $(DOCKER_CROSS_BUILDER_VOLUME) + docker build \ + -t $(DOCKER_CROSS_BUILDER_NAME) \ + -f $(TOOLCHAINS_DIR)/Dockerfile \ + $(TOOLCHAINS_DIR) diff --git a/otelcolbuilder/build-fips/README.md b/toolchains/README.md similarity index 100% rename from otelcolbuilder/build-fips/README.md rename to toolchains/README.md diff --git a/otelcolbuilder/build-fips/config.mak b/toolchains/config.mak similarity index 100% rename from otelcolbuilder/build-fips/config.mak rename to toolchains/config.mak