Skip to content

Commit c6a5334

Browse files
markjhoyelastic
markjhoy
authored and
elastic
committed
Pipeline Creation for Building and Pushing Connectors Docker Images (#2152)
1 parent 2dda7cc commit c6a5334

7 files changed

+345
-0
lines changed

‎.buildkite/publish/build-docker.sh

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
########
4+
# Builds the docker image and saves it to an archive file
5+
# so it can be stored as an artifact in Buildkite
6+
########
7+
8+
set -exu
9+
set -o pipefail
10+
11+
if [[ "${ARCHITECTURE:-}" == "" ]]; then
12+
echo "!! ARCHITECTURE is not set. Exiting."
13+
exit 2
14+
fi
15+
16+
# Load our common environment variables for publishing
17+
export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
18+
source $CURDIR/publish-common.sh
19+
20+
pushd $PROJECT_ROOT
21+
22+
# set our complete tag name and build the image
23+
TAG_NAME="$BASE_TAG_NAME-${ARCHITECTURE}:${VERSION}"
24+
docker build -t $TAG_NAME .
25+
26+
# save the image to an archive file
27+
OUTPUT_PATH="$PROJECT_ROOT/.artifacts"
28+
OUTPUT_FILE="$OUTPUT_PATH/elastic-connectors-docker-${VERSION}-${ARCHITECTURE}.tar.gz"
29+
mkdir -p $OUTPUT_PATH
30+
docker save $TAG_NAME | gzip > $OUTPUT_FILE
31+
32+
popd

‎.buildkite/publish/build-multiarch-docker.sh

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
3+
########
4+
# Builds the multiarch docker image and pushes it to the docker registry
5+
########
6+
7+
set -exu
8+
set -o pipefail
9+
10+
# Load our common environment variables for publishing
11+
export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
12+
source $CURDIR/publish-common.sh
13+
14+
# Set our tag name as well as the tag names of the indiividual platform images
15+
TAG_NAME="${BASE_TAG_NAME}:${VERSION}"
16+
AMD64_TAG="${BASE_TAG_NAME}-amd64:${VERSION}"
17+
ARM64_TAG="${BASE_TAG_NAME}-arm64:${VERSION}"
18+
19+
# ensure +x is set to avoid writing any sensitive information to the console
20+
set +x
21+
22+
DOCKER_PASSWORD=$(vault read -address "${VAULT_ADDR}" -field secret_20230609 secret/ci/elastic-connectors/${VAULT_USER})
23+
24+
# Log into Docker
25+
echo "Logging into docker..."
26+
DOCKER_USER=$(vault read -address "${VAULT_ADDR}" -field user_20230609 secret/ci/elastic-connectors/${VAULT_USER})
27+
vault read -address "${VAULT_ADDR}" -field secret_20230609 secret/ci/elastic-connectors/${VAULT_USER} | \
28+
buildah login --username="${DOCKER_USER}" --password-stdin docker.elastic.co
29+
30+
# Create the manifest for the multiarch image
31+
echo "Creating manifest..."
32+
buildah manifest create $TAG_NAME \
33+
$AMD64_TAG \
34+
$ARM64_TAG
35+
36+
# ... and push it
37+
echo "Pushing manifest..."
38+
buildah manifest push $TAG_NAME docker://$TAG_NAME
39+
40+
# Write out the final manifest for debugging purposes
41+
echo "Built and pushed multiarch image... dumping final manifest..."
42+
buildah manifest inspect $TAG_NAME

‎.buildkite/publish/publish-common.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
if [[ "${CURDIR:-}" == "" ]]; then
4+
echo "!! CURDIR is not set. Exiting."
5+
exit 2
6+
fi
7+
8+
function realpath {
9+
echo "$(cd "$(dirname "$1")"; pwd)"/"$(basename "$1")";
10+
}
11+
12+
export SCRIPT_DIR="$CURDIR"
13+
export BUILDKITE_DIR=$(realpath "$(dirname "$SCRIPT_DIR")")
14+
export PROJECT_ROOT=$(realpath "$(dirname "$BUILDKITE_DIR")")
15+
16+
VERSION_PATH="$PROJECT_ROOT/connectors/VERSION"
17+
export VERSION=$(cat $VERSION_PATH)
18+
19+
if [[ "${USE_SNAPSHOT:-}" == "true" ]]; then
20+
echo "Adding SNAPSHOT labeling"
21+
export VERSION="${VERSION}-SNAPSHOT"
22+
fi
23+
24+
export BASE_TAG_NAME="docker.elastic.co/enterprise-search/elastic-connectors"
25+
export VAULT_ADDR=${VAULT_ADDR:-https://vault-ci-prod.elastic.dev}
26+
export VAULT_USER="docker-swiftypeadmin"

‎.buildkite/publish/push-docker.sh

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
3+
########
4+
# Pushes the docker image to the docker registry
5+
########
6+
7+
set -exu
8+
set -o pipefail
9+
10+
if [[ "${ARCHITECTURE:-}" == "" ]]; then
11+
echo "!! ARCHITECTURE is not set. Exiting."
12+
exit 2
13+
fi
14+
15+
# Load our common environment variables for publishing
16+
export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
17+
source $CURDIR/publish-common.sh
18+
19+
# Load the image from the artifact created in build-docker.sh
20+
echo "Loading image from archive file..."
21+
docker load < "$PROJECT_ROOT/.artifacts/elastic-connectors-docker-${VERSION}-${ARCHITECTURE}.tar.gz"
22+
23+
# ensure +x is set to avoid writing any sensitive information to the console
24+
set +x
25+
26+
# Log into Docker
27+
echo "Logging into docker..."
28+
DOCKER_USER=$(vault read -address "${VAULT_ADDR}" -field user_20230609 secret/ci/elastic-connectors/${VAULT_USER})
29+
vault read -address "${VAULT_ADDR}" -field secret_20230609 secret/ci/elastic-connectors/${VAULT_USER} | \
30+
docker login -u $DOCKER_USER --password-stdin docker.elastic.co
31+
32+
# Set our tag name and push the image
33+
TAG_NAME="$BASE_TAG_NAME-${ARCHITECTURE}:${VERSION}"
34+
echo "Pushing image to docker with tag: $TAG_NAME"
35+
docker push $TAG_NAME

‎.buildkite/publish/test-docker.sh

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
3+
########
4+
# Loads the docker image and tests the structure
5+
########
6+
7+
set -exu
8+
set -o pipefail
9+
10+
if [[ "${ARCHITECTURE:-}" == "" ]]; then
11+
echo "!! ARCHITECTURE is not set. Exiting."
12+
exit 2
13+
fi
14+
15+
# Load our common environment variables for publishing
16+
export CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
17+
source $CURDIR/publish-common.sh
18+
19+
# Detect the platform we are running on (needed for container-structure-test)
20+
arch_name=`uname -sr`
21+
case "$arch_name" in
22+
Darwin*)
23+
echo "detected MacOS platform"
24+
LOCAL_MACHINE_ARCH="MacOS"
25+
;;
26+
Linux*)
27+
echo "detected Linux platform"
28+
LOCAL_MACHINE_ARCH="Linux"
29+
;;
30+
*)
31+
echo "Unsupported platform: $arch_name"
32+
exit 2
33+
;;
34+
esac
35+
36+
# Load the image from the artifact created in build-docker.sh
37+
echo "Loading image from archive file..."
38+
docker load < "$PROJECT_ROOT/.artifacts/elastic-connectors-docker-${VERSION}-${ARCHITECTURE}.tar.gz"
39+
40+
# Ensure we have container-structure-test installed
41+
echo "Ensuring test environment is set up"
42+
43+
BIN_DIR="$PROJECT_ROOT/bin"
44+
TEST_EXEC="$BIN_DIR/container-structure-test"
45+
if [[ ! -f "$TEST_EXEC" ]]; then
46+
mkdir -p "$BIN_DIR"
47+
48+
pushd "$BIN_DIR"
49+
if [[ "$LOCAL_MACHINE_ARCH" == "MacOS" ]]; then
50+
curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-darwin-$ARCHITECTURE
51+
mv container-structure-test-darwin-$ARCHITECTURE container-structure-test
52+
else
53+
curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-$ARCHITECTURE
54+
mv container-structure-test-linux-$ARCHITECTURE container-structure-test
55+
fi
56+
57+
chmod +x container-structure-test
58+
popd
59+
fi
60+
61+
# Generate our config file
62+
TEST_CONFIG_FILE="$PROJECT_ROOT/.buildkite/publish/container-structure-test.yaml"
63+
64+
# The config file needs escaped dots - we'll do that here
65+
ESCAPED_VERSION=${VERSION//./\\\\.}
66+
67+
# Generate the config file text
68+
TEST_CONFIG_TEXT='
69+
schemaVersion: "2.0.0"
70+
71+
commandTests:
72+
# ensure Python 3.10.* is installed
73+
- name: "Python 3 Installation 3.10.*"
74+
command: "python3"
75+
args: ["--version"]
76+
expectedOutput: ["Python\\s3\\.10\\.*"]
77+
- name: "Connectors Installation"
78+
command: "/app/bin/elastic-ingest"
79+
args: ["--version"]
80+
expectedOutput: ["'"${ESCAPED_VERSION}"'*"]
81+
'
82+
# ... and save the config file
83+
printf '%s\n' "$TEST_CONFIG_TEXT" > "$TEST_CONFIG_FILE"
84+
85+
# Finally, run the tests
86+
echo "Running container-structure-test"
87+
TAG_NAME="$BASE_TAG_NAME-${ARCHITECTURE}:${VERSION}"
88+
"$TEST_EXEC" test --image "$TAG_NAME" --config "$TEST_CONFIG_FILE"

‎.buildkite/release-pipeline.yml

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
## 🏠/.buildkite/pipeline-release.yml
2+
# Manual triggered pipeline to build and publish Docker images
3+
4+
agents:
5+
provider: "gcp"
6+
machineType: "n1-standard-8"
7+
useVault: true
8+
9+
steps:
10+
# ----
11+
# Docker builds for amd64
12+
# ----
13+
- group: ":package: amd64 Build and Test"
14+
key: "build_and_test_amd64"
15+
if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/)"
16+
env:
17+
- ARCHITECTURE="amd64"
18+
agents:
19+
provider: aws
20+
instanceType: m6i.xlarge
21+
imagePrefix: ci-amazonlinux-2
22+
steps:
23+
- label: "Building amd64 Docker image"
24+
command: ".buildkite/publish/build-docker.sh"
25+
key: "build_docker_image_amd64"
26+
artifact_paths: ".artifacts/elastic-connectors-docker-*.tar.gz"
27+
- label: "Testing amd64 Docker image"
28+
depends_on: "build_docker_image_amd64"
29+
command: ".buildkite/publish/test-docker.sh"
30+
# ----
31+
# Docker builds for arm64
32+
# ----
33+
- group: ":package: arm64 Build and Test"
34+
key: "build_and_test_arm64"
35+
if: "build.branch =~ /^[0-9]+\\.[0-9x]+.*/)"
36+
env:
37+
- ARCHITECTURE="arm64"
38+
agents:
39+
provider: aws
40+
instanceType: m6g.xlarge
41+
imagePrefix: ci-amazonlinux-2-aarch64
42+
diskSizeGb: 40
43+
diskName: '/dev/xvda'
44+
steps:
45+
- label: "Building arm64 Docker image"
46+
command: ".buildkite/publish/build-docker.sh"
47+
key: "build_docker_image_arm64"
48+
artifact_paths: ".artifacts/elastic-connectors-docker-*.tar.gz"
49+
- label: "Testing arm64 Docker image"
50+
depends_on: "build_docker_image_arm64"
51+
command: ".buildkite/publish/test-docker.sh"
52+
# ----
53+
# Multiarch Docker image build and push
54+
# ----
55+
- group: ":truck: Publish images"
56+
depends_on:
57+
- "build_and_test_amd64"
58+
- "build_and_test_arm64"
59+
steps:
60+
- label: "Push amd64 Docker image"
61+
key: "push_amd64_docker_image"
62+
env:
63+
- ARCHITECTURE="amd64"
64+
agents:
65+
provider: aws
66+
instanceType: m6i.xlarge
67+
imagePrefix: ci-amazonlinux-2
68+
commands:
69+
- "mkdir -p .artifacts"
70+
- buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_amd64
71+
- ".buildkite/publish/push-docker.sh"
72+
- label: "Push arm64 Docker image"
73+
key: "push_arm64_docker_image"
74+
env:
75+
- ARCHITECTURE="arm64"
76+
agents:
77+
provider: aws
78+
instanceType: m6g.xlarge
79+
imagePrefix: ci-amazonlinux-2-aarch64
80+
diskSizeGb: 40
81+
diskName: '/dev/xvda'
82+
commands:
83+
- "mkdir -p .artifacts"
84+
- buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_docker_image_arm64
85+
- ".buildkite/publish/push-docker.sh"
86+
- label: "Build and push multiarch Docker image"
87+
command: ".buildkite/publish/build-multiarch-docker.sh"
88+
depends_on:
89+
- "push_amd64_docker_image"
90+
- "push_arm64_docker_image"

‎catalog-info.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -182,3 +182,35 @@ spec:
182182
access_level: "READ_ONLY"
183183
ingestion-team: {}
184184
search-productivity-team: {}
185+
186+
########
187+
# Docker image build and publish - manual release
188+
########
189+
---
190+
apiVersion: "backstage.io/v1alpha1"
191+
kind: "Resource"
192+
metadata:
193+
name: "connectors-docker-build-publish"
194+
description: "Docker image build and publish for Elastic connectors"
195+
links:
196+
- title: "Connectors Docker Build and Publish"
197+
url: "https://buildkite.com/elastic/connectors-docker-build-publish"
198+
spec:
199+
type: "buildkite-pipeline"
200+
owner: "group:ingestion-team"
201+
system: "buildkite"
202+
implementation:
203+
apiVersion: "buildkite.elastic.dev/v1"
204+
kind: "Pipeline"
205+
metadata:
206+
name: "connectors-docker-build-publish"
207+
spec:
208+
repository: "elastic/connectors"
209+
pipeline_file: ".buildkite/release-pipeline.yml"
210+
provider_settings:
211+
trigger_mode: "none"
212+
teams:
213+
ingestion-team: {}
214+
search-productivity-team: {}
215+
everyone:
216+
access_level: "READ_ONLY"

0 commit comments

Comments
 (0)