[8.13] Pipeline Creation for Building and Pushing Connectors Docker Images (#2152)

.buildkite/publish/build-docker.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+########
+# Builds the docker image and saves it to an archive file
+# so it can be stored as an artifact in Buildkite
+########
+
+set -exu
+set -o pipefail
+
+if [[ "${ARCHITECTURE:-}" == "" ]]; then
+  echo "!! ARCHITECTURE is not set. Exiting."
+  exit 2
+fi
+
+# Load our common environment variables for publishing
+export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $CURDIR/publish-common.sh
+
+pushd $PROJECT_ROOT
+
+# set our complete tag name and build the image
+TAG_NAME="$BASE_TAG_NAME-${ARCHITECTURE}:${VERSION}"
+docker build -t $TAG_NAME .
+
+# save the image to an archive file
+OUTPUT_PATH="$PROJECT_ROOT/.artifacts"
+OUTPUT_FILE="$OUTPUT_PATH/elastic-connectors-docker-${VERSION}-${ARCHITECTURE}.tar.gz"
+mkdir -p $OUTPUT_PATH
+docker save $TAG_NAME | gzip > $OUTPUT_FILE
+
+popd

.buildkite/publish/build-multiarch-docker.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+########
+# Builds the multiarch docker image and pushes it to the docker registry
+########
+
+set -exu
+set -o pipefail
+
+# Load our common environment variables for publishing
+export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $CURDIR/publish-common.sh
+
+# Set our tag name as well as the tag names of the indiividual platform images
+TAG_NAME="${BASE_TAG_NAME}:${VERSION}"
+AMD64_TAG="${BASE_TAG_NAME}-amd64:${VERSION}"
+ARM64_TAG="${BASE_TAG_NAME}-arm64:${VERSION}"
+
+# ensure +x is set to avoid writing any sensitive information to the console
+set +x
+
+DOCKER_PASSWORD=$(vault read -address "${VAULT_ADDR}" -field secret_20230609 secret/ci/elastic-connectors/${VAULT_USER})
+
+# Log into Docker
+echo "Logging into docker..."
+DOCKER_USER=$(vault read -address "${VAULT_ADDR}" -field user_20230609 secret/ci/elastic-connectors/${VAULT_USER})
+vault read -address "${VAULT_ADDR}" -field secret_20230609 secret/ci/elastic-connectors/${VAULT_USER} | \
+  buildah login --username="${DOCKER_USER}" --password-stdin docker.elastic.co
+
+# Create the manifest for the multiarch image
+echo "Creating manifest..."
+buildah manifest create $TAG_NAME \
+  $AMD64_TAG \
+  $ARM64_TAG
+
+# ... and push it
+echo "Pushing manifest..."
+buildah manifest push $TAG_NAME docker://$TAG_NAME
+
+# Write out the final manifest for debugging purposes
+echo "Built and pushed multiarch image... dumping final manifest..."
+buildah manifest inspect $TAG_NAME

.buildkite/publish/publish-common.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if [[ "${CURDIR:-}" == "" ]]; then
+  echo "!! CURDIR is not set. Exiting."
+  exit 2
+fi
+
+function realpath {
+  echo "$(cd "$(dirname "$1")"; pwd)"/"$(basename "$1")";
+}
+
+export SCRIPT_DIR="$CURDIR"
+export BUILDKITE_DIR=$(realpath "$(dirname "$SCRIPT_DIR")")
+export PROJECT_ROOT=$(realpath "$(dirname "$BUILDKITE_DIR")")
+
+VERSION_PATH="$PROJECT_ROOT/connectors/VERSION"
+export VERSION=$(cat $VERSION_PATH)
+
+if [[ "${USE_SNAPSHOT:-}" == "true" ]]; then
+  echo "Adding SNAPSHOT labeling"
+  export VERSION="${VERSION}-SNAPSHOT"
+fi
+
+export BASE_TAG_NAME="docker.elastic.co/enterprise-search/elastic-connectors"
+export VAULT_ADDR=${VAULT_ADDR:-https://vault-ci-prod.elastic.dev}
+export VAULT_USER="docker-swiftypeadmin"

.buildkite/publish/push-docker.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+########
+# Pushes the docker image to the docker registry
+########
+
+set -exu
+set -o pipefail
+
+if [[ "${ARCHITECTURE:-}" == "" ]]; then
+  echo "!! ARCHITECTURE is not set. Exiting."
+  exit 2
+fi
+
+# Load our common environment variables for publishing
+export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $CURDIR/publish-common.sh
+
+# Load the image from the artifact created in build-docker.sh
+echo "Loading image from archive file..."
+docker load < "$PROJECT_ROOT/.artifacts/elastic-connectors-docker-${VERSION}-${ARCHITECTURE}.tar.gz"
+
+# ensure +x is set to avoid writing any sensitive information to the console
+set +x
+
+# Log into Docker
+echo "Logging into docker..."
+DOCKER_USER=$(vault read -address "${VAULT_ADDR}" -field user_20230609 secret/ci/elastic-connectors/${VAULT_USER})
+vault read -address "${VAULT_ADDR}" -field secret_20230609 secret/ci/elastic-connectors/${VAULT_USER} | \
+  docker login -u $DOCKER_USER --password-stdin docker.elastic.co
+
+# Set our tag name and push the image
+TAG_NAME="$BASE_TAG_NAME-${ARCHITECTURE}:${VERSION}"
+echo "Pushing image to docker with tag: $TAG_NAME"
+docker push $TAG_NAME

.buildkite/publish/test-docker.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+########
+# Loads the docker image and tests the structure
+########
+
+set -exu
+set -o pipefail
+
+if [[ "${ARCHITECTURE:-}" == "" ]]; then
+  echo "!! ARCHITECTURE is not set. Exiting."
+  exit 2
+fi
+
+# Load our common environment variables for publishing
+export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $CURDIR/publish-common.sh
+
+# Detect the platform we are running on (needed for container-structure-test)
+arch_name=`uname -sr`
+case "$arch_name" in
+    Darwin*)
+        echo "detected MacOS platform"
+        LOCAL_MACHINE_ARCH="MacOS"
+        ;;
+    Linux*)
+        echo "detected Linux platform"
+        LOCAL_MACHINE_ARCH="Linux"
+        ;;
+    *)
+        echo "Unsupported platform: $arch_name"
+        exit 2
+        ;;
+esac
+
+# Load the image from the artifact created in build-docker.sh
+echo "Loading image from archive file..."
+docker load < "$PROJECT_ROOT/.artifacts/elastic-connectors-docker-${VERSION}-${ARCHITECTURE}.tar.gz"
+
+# Ensure we have container-structure-test installed
+echo "Ensuring test environment is set up"
+
+BIN_DIR="$PROJECT_ROOT/bin"
+TEST_EXEC="$BIN_DIR/container-structure-test"
+if [[ ! -f "$TEST_EXEC" ]]; then
+  mkdir -p "$BIN_DIR"
+
+  pushd "$BIN_DIR"
+  if [[ "$LOCAL_MACHINE_ARCH" == "MacOS" ]]; then
+    curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-darwin-$ARCHITECTURE
+    mv container-structure-test-darwin-$ARCHITECTURE container-structure-test
+  else
+    curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-$ARCHITECTURE
+    mv container-structure-test-linux-$ARCHITECTURE container-structure-test
+  fi
+
+  chmod +x container-structure-test
+  popd
+fi
+
+# Generate our config file
+TEST_CONFIG_FILE="$PROJECT_ROOT/.buildkite/publish/container-structure-test.yaml"
+
+# The config file needs escaped dots - we'll do that here
+ESCAPED_VERSION=${VERSION//./\\\\.}
+
+# Generate the config file text
+TEST_CONFIG_TEXT='
+schemaVersion: "2.0.0"
+
+commandTests:
+  # ensure Python 3.10.* is installed
+  - name: "Python 3 Installation 3.10.*"
+    command: "python3"
+    args: ["--version"]
+    expectedOutput: ["Python\\s3\\.10\\.*"]
+  - name: "Connectors Installation"
+    command: "/app/bin/elastic-ingest"
+    args: ["--version"]
+    expectedOutput: ["'"${ESCAPED_VERSION}"'*"]
+'
+# ... and save the config file
+printf '%s\n' "$TEST_CONFIG_TEXT" > "$TEST_CONFIG_FILE"
+
+# Finally, run the tests
+echo "Running container-structure-test"
+TAG_NAME="$BASE_TAG_NAME-${ARCHITECTURE}:${VERSION}"
+"$TEST_EXEC" test --image "$TAG_NAME" --config "$TEST_CONFIG_FILE"

.buildkite/release-pipeline.yml

@@ -0,0 +1,90 @@
+## 🏠/.buildkite/pipeline-release.yml
+# Manual triggered pipeline to build and publish Docker images
+
+agents:
+  provider: "gcp"
+  machineType: "n1-standard-8"
+  useVault: true
+
+steps:
+  # ----
+  # Docker builds for amd64
+  # ----
+  - group: ":package: amd64 Build and Test"
+    key: "build_and_test_amd64"
+    if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/)"
+    env:
+      - ARCHITECTURE="amd64"
+    agents:
+      provider: aws
+      instanceType: m6i.xlarge
+      imagePrefix: ci-amazonlinux-2
+    steps:
+      - label: "Building amd64 Docker image"
+        command: ".buildkite/publish/build-docker.sh"
+        key: "build_docker_image_amd64"
+        artifact_paths: ".artifacts/elastic-connectors-docker-*.tar.gz"
+      - label: "Testing amd64 Docker image"
+        depends_on: "build_docker_image_amd64"
+        command: ".buildkite/publish/test-docker.sh"
+  # ----
+  # Docker builds for arm64
+  # ----
+  - group: ":package: arm64 Build and Test"
+    key: "build_and_test_arm64"
+    if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/)"
+    env:
+      - ARCHITECTURE="arm64"
+    agents:
+      provider: aws
+      instanceType: m6g.xlarge
+      imagePrefix: ci-amazonlinux-2-aarch64
+      diskSizeGb: 40
+      diskName: '/dev/xvda'
+    steps:
+      - label: "Building arm64 Docker image"
+        command: ".buildkite/publish/build-docker.sh"
+        key: "build_docker_image_arm64"
+        artifact_paths: ".artifacts/elastic-connectors-docker-*.tar.gz"
+      - label: "Testing arm64 Docker image"
+        depends_on: "build_docker_image_arm64"
+        command: ".buildkite/publish/test-docker.sh"
+  # ----
+  # Multiarch Docker image build and push
+  # ----
+  - group: ":truck: Publish images"
+    depends_on:
+      - "build_and_test_amd64"
+      - "build_and_test_arm64"
+    steps:
+      - label: "Push amd64 Docker image"
+        key: "push_amd64_docker_image"
+        env:
+          - ARCHITECTURE="amd64"
+        agents:
+          provider: aws
+          instanceType: m6i.xlarge
+          imagePrefix: ci-amazonlinux-2
+        commands:
+          - "mkdir -p .artifacts"
+          - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_amd64
+          - ".buildkite/publish/push-docker.sh"
+      - label: "Push arm64 Docker image"
+        key: "push_arm64_docker_image"
+        env:
+          - ARCHITECTURE="arm64"
+        agents:
+          provider: aws
+          instanceType: m6g.xlarge
+          imagePrefix: ci-amazonlinux-2-aarch64
+          diskSizeGb: 40
+          diskName: '/dev/xvda'
+        commands:
+          - "mkdir -p .artifacts"
+          - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_arm64
+          - ".buildkite/publish/push-docker.sh"
+      - label: "Build and push multiarch Docker image"
+        command: ".buildkite/publish/build-multiarch-docker.sh"
+        depends_on:
+          - "push_amd64_docker_image"
+          - "push_arm64_docker_image"

catalog-info.yaml

@@ -182,3 +182,35 @@ spec:
           access_level: "READ_ONLY"
         ingestion-team: {}
         search-productivity-team: {}
+
+########
+# Docker image build and publish - manual release
+########
+---
+apiVersion: "backstage.io/v1alpha1"
+kind: "Resource"
+metadata:
+  name: "connectors-docker-build-publish"
+  description: "Docker image build and publish for Elastic connectors"
+  links:
+    - title: "Connectors Docker Build and Publish"
+      url: "https://buildkite.com/elastic/connectors-docker-build-publish"
+spec:
+  type: "buildkite-pipeline"
+  owner: "group:ingestion-team"
+  system: "buildkite"
+  implementation:
+    apiVersion: "buildkite.elastic.dev/v1"
+    kind: "Pipeline"
+    metadata:
+      name: "connectors-docker-build-publish"
+    spec:
+      repository: "elastic/connectors"
+      pipeline_file: ".buildkite/release-pipeline.yml"
+      provider_settings:
+        trigger_mode: "none"
+      teams:
+        ingestion-team: {}
+        search-productivity-team: {}
+        everyone:
+          access_level: "READ_ONLY"