diff --git a/.circleci/config.yml b/.circleci/config.yml index c13f4fc72..33daa2ee4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,13 +4,7 @@ version: "2.1" executors: cross-builder: docker: - # NOTE: To upgrade the Go version, first push the upgrade to the cross-builder Dockerfile in the edge repo, - # then update the version here to match. Until we finish the migration to using the cross-builder image, - # you'll also need to update references to `cimg/go` and `GO_VERSION` in this file. - # - # NOTE 2: Remove `gem install dotenv -v 2.8.1` from the `install_dependencies` step once - # the cross-builder supports Ruby v3. - - image: quay.io/influxdb/cross-builder:go1.21.9-latest + - image: quay.io/influxdb/builder:kapacitor-20240531 resource_class: large linux-amd64: machine: @@ -27,28 +21,6 @@ commands: name: Install additional dependencies command: | set -x - # APT packages - apt-get -qq update && apt-get -qq install -y \ - software-properties-common \ - unzip \ - mercurial \ - make \ - ruby \ - ruby-dev \ - rpm \ - zip \ - python \ - python-setuptools \ - python3 \ - python3-setuptools \ - python3-boto \ - autoconf \ - automake \ - libtool - - # Ruby dependencies - gem install dotenv -v 2.8.1 - gem install fpm # Protobuf wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protobuf-python-${PROTO_VERSION}.tar.gz diff --git a/Dockerfile_build_ubuntu64 b/Dockerfile_build_ubuntu64 index dac4e156a..f405da33c 100644 --- a/Dockerfile_build_ubuntu64 +++ b/Dockerfile_build_ubuntu64 @@ -1,34 +1,12 @@ -FROM quay.io/influxdb/cross-builder:go1.21.9-latest +FROM quay.io/influxdb/builder:kapacitor-20240531 # This dockerfile is capabable of performing all # build/test/package/deploy actions needed for Kapacitor. MAINTAINER support@influxdb.com -RUN apt-get -qq update && apt-get -qq install -y \ - software-properties-common \ - unzip \ - mercurial \ - make \ - ruby \ - ruby-dev \ - rpm \ - zip \ - python \ - python-setuptools \ - python3 \ - python3-setuptools \ - python3-boto \ - autoconf \ - automake \ - libtool - -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 - -RUN gem install fpm - # Install protobuf3 python library -# NOTE: PROTO_VERSION env var is inherited from the cross-builder image. +# NOTE: PROTO_VERSION env var is inherited from the parent builder image. RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protobuf-python-${PROTO_VERSION}.tar.gz \ && tar -xf protobuf-python-${PROTO_VERSION}.tar.gz \ && cd protobuf-${PROTO_VERSION}/python \ diff --git a/builder/Dockerfile_build b/builder/Dockerfile_build new file mode 100644 index 000000000..651a70049 --- /dev/null +++ b/builder/Dockerfile_build @@ -0,0 +1,105 @@ +FROM ubuntu:20.04 + +# This builder image is base image for building Kapacitor binaries. +# It is used by the CI/CD pipeline to use same version of Go, Protobuf and other dependencies. + +MAINTAINER support@influxdb.com + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get -qq update && apt-get -qq install -y \ + software-properties-common \ + unzip \ + mercurial \ + make \ + ruby \ + ruby-dev \ + rpm \ + zip \ + python \ + python-setuptools \ + python3 \ + python3-setuptools \ + python3-boto \ + autoconf \ + automake \ + libtool \ + wget \ + git \ + pkg-config + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 + +# Remove `gem install dotenv -v 2.8.1` once the base image will support Ruby v3. +RUN gem install dotenv -v 2.8.1 +RUN gem install fpm + +# +# Install GoLang +# +# src: https://github.com/influxdata/chronograf/blob/9c4b1aa1a458ed86716985ed93783c35cc6411a7/etc/Dockerfile_build#L35 +# +ENV GOPATH /root/go +ENV GO_VERSION 1.21.10 +ENV GO_ARCH amd64 +ENV GO111MODULES ON +RUN wget https://golang.org/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \ + tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \ + rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz +ENV PATH /usr/local/go/bin:$PATH +ENV PATH $GOPATH/bin:$PATH + +# +# Install Rust +# +# src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/install-rust.sh#L7 +# +ENV RUST_LATEST_VERSION=1.78.0 +# For security, we specify a particular rustup version and a SHA256 hash, computed +# ourselves and hardcoded here. When updating `RUSTUP_LATEST_VERSION`: +# 1. Download the new rustup script from https://github.com/rust-lang/rustup/releases. +# 2. Audit the script and changes to it. You might want to grep for strange URLs... +# 3. Update `OUR_RUSTUP_SHA` with the result of running `sha256sum rustup-init.sh`. +ENV RUSTUP_LATEST_VERSION=1.25.1 +ENV OUR_RUSTUP_SHA="173f4881e2de99ba9ad1acb59e65be01b2a44979d83b6ec648d0d22f8654cbce" +# Download rustup script +RUN wget https://raw.githubusercontent.com/rust-lang/rustup/${RUSTUP_LATEST_VERSION}/rustup-init.sh +# Verify checksum of rustup script. Exit with error if check fails. +RUN echo "${OUR_RUSTUP_SHA} rustup-init.sh" | sha256sum --check -- \ + || { echo "Checksum problem!"; exit 1; } +# Run rustup +RUN sh rustup-init.sh --default-toolchain "$RUST_LATEST_VERSION" -y +# Ensure cargo is runnable +RUN . $HOME/.cargo/env && cargo help +RUN . $HOME/.cargo/env && rustup target add \ + aarch64-unknown-linux-musl \ + x86_64-apple-darwin \ + x86_64-pc-windows-gnu \ + x86_64-unknown-linux-musl +ENV PATH=/root/.cargo/bin:${PATH} + +# +# Install protobuf3 runtime and protoc binary +# +# src: https://github.com/influxdata/edge/blob/cb1343dd74ecba8ec07fe810195530a0b9055aa9/dockerfiles/cross-builder/Dockerfile#L93 +# +# NOTE: PROTO_VERSION env var can be overridden to install a different version +ARG PROTO_VERSION=3.17.3 +ENV PROTO_VERSION=${PROTO_VERSION} +ENV PROTO_BUILD_TIME=2021100120071633118879 +RUN PROTO_ARCHIVE=protoc-${PROTO_VERSION}-${PROTO_BUILD_TIME}.tar.gz && \ + wget https://edge-xcc-archives.s3-us-west-2.amazonaws.com/${PROTO_ARCHIVE} && \ + tar xzf ${PROTO_ARCHIVE} -C /usr && \ + rm ${PROTO_ARCHIVE} + +# +# Install pkg-config +# +# src: https://github.com/influxdata/edge/blob/4677c285014ac27727e5a1ae9bf2c1633afc6ea6/dockerfiles/cross-builder/Dockerfile#L104 +# +RUN go install github.com/influxdata/pkg-config@v0.2.11 + +# Configure local git +RUN git config --global user.email "support@influxdb.com" +RUN git config --global user.Name "Docker Builder" + diff --git a/builder/Dockerfile_build_push.sh b/builder/Dockerfile_build_push.sh new file mode 100755 index 000000000..b04a232fc --- /dev/null +++ b/builder/Dockerfile_build_push.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# This script is used to build and push a Docker builder image to quay.io. +# You have to set credentials for quay.io in your environment by setting the following variables: +# - QUAY_CD_USER +# - QUAY_CD_PASSWORD + +set -x + +DOCKER_TAG="kapacitor-$(date +%Y%m%d)" + +docker build --rm=false --platform linux/amd64 -f ./Dockerfile_build -t builder:"$DOCKER_TAG" . +docker tag builder:"$DOCKER_TAG" quay.io/influxdb/builder:"$DOCKER_TAG" + +docker push quay.io/influxdb/builder:"$DOCKER_TAG" diff --git a/builder/README.md b/builder/README.md new file mode 100644 index 000000000..190435be7 --- /dev/null +++ b/builder/README.md @@ -0,0 +1,32 @@ +# Builds + +Our CI/CD pipelines utilize a Docker build image configured with support for GoLang, Rust, and Protobuf. The `circle.yml` file references this Docker container to handle building, testing, and creating release packages. + +## Custom Builder + +The necessity for a custom builder arises from compatibility issues between the `protobuf` library and Chronograf's Python UDFs. The `cross-builder` was updated to `protobuf` version `26.1` in [PR #669](https://github.com/influxdata/edge/pull/669), introducing breaking changes in the Python protobuf library. Specifically, [protobuf 5.26.1 on PyPI](https://pypi.org/project/protobuf/5.26.1/) does not support Python 2. Consequently, using the newest `cross-builder` would result in the loss of Python v2 support in UDFs. + +## Updating Component Versions + +To update component versions like GoLang, Rust, and Protobuf, modifications must be made in `Dockerfile_build`. After updates, a new Docker image needs to be built, published, and then utilized in CI. + +### Step 1: Authenticate with Quay.io + +```sh +export QUAY_CD_USER= +export QUAY_CD_PASSWORD= +``` + +### Step 2: Build and Push the New Docker Image to Quay + +Navigate to the builder directory and execute the build script: + +```sh +cd $KAPACITOR_REPOSITORY_ROOT/builder +./Dockerfile_build_push.sh +``` + +### Step 3: Update Scripts and CircleCI Configuration + +1. Update the `cross-builder` tag in `.circleci/config.yml` to the new version. +2. Update the `quay.io/influxdb/builder` tag in `Dockerfile_build_ubuntu64` to reflect the new version.