Merge pull request #3035 from buildkite/parallel-docker-images

Parallel container image uploads
buildkite · Oct 10, 2024 · 5a696c8 · 5a696c8
2 parents 04155d4 + e17e218
commit 5a696c8
Show file tree

Hide file tree

Showing 5 changed files with 151 additions and 97 deletions.
diff --git a/.buildkite/pipeline.release-experimental.yml b/.buildkite/pipeline.release-experimental.yml
@@ -136,13 +136,22 @@ steps:
             - with: { pkg_arch: "SKIP_FAKE_ARCH" }
               skip: true
 
-  - name: ":docker: Publish Edge Docker Images"
-    command: ".buildkite/steps/publish-docker-images.sh"
-    env:
-      CODENAME: "experimental"
-    plugins:
-      - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-edge
-      - ecr#v2.7.0:
-          login: true
-          account-ids: "445615400570"
+  - group: ":docker: Publish Edge Docker Images"
+    steps:
+      - name: ":docker: Publish Edge Images to {{matrix.registry}}"
+        command: ".buildkite/steps/publish-docker-images.sh"
+        env:
+          CODENAME: "experimental"
+          REGISTRY: "{{matrix.registry}}"
+        plugins:
+          - aws-assume-role-with-web-identity:
+              role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-edge
+          - ecr#v2.7.0:
+              login: true
+              account-ids: "445615400570"
+        matrix:
+          setup:
+            registry:
+              - docker.io
+              - ghcr.io
+              - packages.buildkite.com
diff --git a/.buildkite/pipeline.release-stable.yml b/.buildkite/pipeline.release-stable.yml
@@ -160,16 +160,25 @@ steps:
             - with: { pkg_arch: "SKIP_FAKE_ARCH" }
               skip: true
 
-  - name: ":docker: Publish Docker Images"
-    command: ".buildkite/steps/publish-docker-images.sh"
-    env:
-      CODENAME: "stable"
-    plugins:
-      - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-stable
-      - ecr#v2.7.0:
-          login: true
-          account-ids: "445615400570"
+  - group: ":docker: Publish Docker Images"
+    steps:
+      - name: ":docker: Publish Docker Images to {{matrix.registry}}"
+        command: ".buildkite/steps/publish-docker-images.sh"
+        env:
+          CODENAME: "stable"
+          REGISTRY: "{{matrix.registry}}"
+        plugins:
+          - aws-assume-role-with-web-identity:
+              role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-stable
+          - ecr#v2.7.0:
+              login: true
+              account-ids: "445615400570"
+        matrix:
+          setup:
+            registry:
+              - docker.io
+              - ghcr.io
+              - packages.buildkite.com
 
   - wait
 

diff --git a/.buildkite/pipeline.release-unstable.yml b/.buildkite/pipeline.release-unstable.yml
@@ -160,16 +160,25 @@ steps:
             - with: { pkg_arch: "SKIP_FAKE_ARCH" }
               skip: true
 
-  - name: ":docker: Publish Unstable Docker Images"
-    command: ".buildkite/steps/publish-docker-images.sh"
-    env:
-      CODENAME: "unstable"
-    plugins:
-      - aws-assume-role-with-web-identity:
-          role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-beta
-      - ecr#v2.7.0:
-          login: true
-          account-ids: "445615400570"
+  - group: ":docker: Publish Unstable Docker Images"
+    steps:
+      - name: ":docker: Publish Unstable Images to {{matrix.registry}}"
+        command: ".buildkite/steps/publish-docker-images.sh"
+        env:
+          CODENAME: "unstable"
+          REGISTRY: "{{matrix.registry}}"
+        plugins:
+          - aws-assume-role-with-web-identity:
+              role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-agent-release-beta
+          - ecr#v2.7.0:
+              login: true
+              account-ids: "445615400570"
+        matrix:
+          setup:
+            registry:
+              - docker.io
+              - ghcr.io
+              - packages.buildkite.com
 
   - wait
 

diff --git a/.buildkite/steps/publish-docker-image.sh b/.buildkite/steps/publish-docker-image.sh
@@ -1,10 +1,11 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 ## This script can be run locally like this:
 ##
-## .buildkite/steps/publish-docker-image.sh (alpine|ubuntu) imagename (stable|experimental|unstable) <version> <build>
-## .buildkite/steps/publish-docker-image.sh alpine buildkiteci/agent:lox-manual-build stable 3.1.1
+## .buildkite/steps/publish-docker-image.sh (docker.io|ghcr.io|packages.buildkite.com) (alpine|ubuntu) imagename (stable|experimental|unstable) <version> <build>
+## e.g.
+## .buildkite/steps/publish-docker-image.sh docker.io alpine buildkiteci/agent:lox-manual-build stable 3.1.1
 
 dry_run() {
   if [[ "${DRY_RUN:-}" == "false" ]] ; then
@@ -36,68 +37,81 @@ parse_version() {
 
 release_image() {
   local tag="$1"
-  echo "--- :docker: Copying ${target_image}:${tag} to Docker Hub"
-  dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://docker.io/buildkite/${target_image}:${tag}"
-  echo "--- :github: Copying ${target_image}:${tag} to GHCR"
-  dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://ghcr.io/buildkite/${target_image}:${tag}"
-
-  # OIDC tokens only last 5 minutes, and issuing them is cheap, so log in as close as possible to the push
-  buildkite-agent oidc request-token \
-    --audience "https://packages.buildkite.com/buildkite/agent-docker" \
-    --lifetime 300 \
-    | docker login packages.buildkite.com/buildkite/agent-docker --username=buildkite --password-stdin
-
-  echo "--- :buildkite: Copying ${target_image}:${tag} to Buildkite Packages"
-  dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://packages.buildkite.com/buildkite/agent-docker/${target_image}:${tag}"
+
+  case "${registry}" in
+  docker.io)
+    echo "--- :docker: Copying ${target_image}:${tag} to Docker Hub"
+    dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://docker.io/buildkite/${target_image}:${tag}"
+    ;;
+  ghcr.io)
+    echo "--- :github: Copying ${target_image}:${tag} to GHCR"
+    dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://ghcr.io/buildkite/${target_image}:${tag}"
+    ;;
+  packages.buildkite.com)
+    # OIDC tokens only last 5 minutes, and issuing them is cheap, so log in as close as possible to the push
+    buildkite-agent oidc request-token \
+      --audience "https://packages.buildkite.com/buildkite/agent-docker" \
+      --lifetime 300 \
+      | docker login packages.buildkite.com/buildkite/agent-docker --username=buildkite --password-stdin
+
+    echo "--- :buildkite: Copying ${target_image}:${tag} to Buildkite Packages"
+    dry_run skopeo copy --multi-arch all "docker://${source_image}" "docker://packages.buildkite.com/buildkite/agent-docker/${target_image}:${tag}"
+    ;;
+  *)
+    echo "+++ Registry '${registry}' is not supported\!"
+    exit 1
+    ;;
+  esac
 }
 
-variant="${1:-}"
-source_image="${2:-}"
-codename="${3:-}"
-version="${4:-}"
-build="${5:-dev}"
+registry="${1:-}"
+variant="${2:-}"
+source_image="${3:-}"
+codename="${4:-}"
+version="${5:-}"
+build="${6:-dev}"
 
 target_image="agent"
 variant_suffix=""
 
-if [[ "$variant" != "alpine" ]] ; then
-  variant_suffix="-$variant"
+if [[ "${variant}" != "alpine" ]] ; then
+  variant_suffix="-${variant}"
 fi
 
 echo "Tagging docker images for $variant/$codename (version $version build $build)"
 
 # variants of edge/experimental
-if [[ "$codename" == "experimental" ]] ; then
+if [[ "${codename}" == "experimental" ]] ; then
   release_image "edge-build-${build}${variant_suffix}"
   release_image "edge${variant_suffix}"
 fi
 
 # variants of stable - e.g 2.3.2
-if [[ "$codename" == "stable" ]] ; then
-  for tag in $(parse_version "$version") ; do
+if [[ "${codename}" == "stable" ]] ; then
+  for tag in $(parse_version "${version}") ; do
     release_image "${tag}${variant_suffix}"
   done
   release_image "${variant}"
 
   # publish bare 'ubuntu' only from ubuntu-22.04
-  if [[ "$variant" == "ubuntu-22.04" ]] ; then
-    for tag in $(parse_version "$version") ; do
+  if [[ "${variant}" == "ubuntu-22.04" ]] ; then
+    for tag in $(parse_version "${version}") ; do
       release_image "${tag}-ubuntu"
     done
     release_image "ubuntu"
   fi
 
   # publish latest and stable only from alpine
-  if [[ "$variant" == "alpine" ]] ; then
+  if [[ "${variant}" == "alpine" ]] ; then
     release_image "latest"
     release_image "stable"
   fi
 fi
 
 # variants of beta/unstable - e.g 3.0-beta.16
-if [[ "$codename" == "unstable" ]] ; then
+if [[ "${codename}" == "unstable" ]] ; then
   release_image "beta${variant_suffix}"
-  if [[ "$version" =~ -(alpha|beta|rc)\.[0-9]+$ ]] ; then
+  if [[ "${version}" =~ -(alpha|beta|rc)\.[0-9]+$ ]] ; then
     release_image "${version}${variant_suffix}"
   fi
 fi
diff --git a/.buildkite/steps/publish-docker-images.sh b/.buildkite/steps/publish-docker-images.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 dry_run() {
@@ -9,51 +9,64 @@ dry_run() {
   fi
 }
 
-if [[ "$CODENAME" == "" ]]; then
+if [[ "${CODENAME:-}" == "" ]]; then
   echo "Error: Missing \$CODENAME (stable, experimental or unstable)"
   exit 1
 fi
 
-echo "--- docker login to Docker Hub"
-
-dockerhub_user="$(aws ssm get-parameter \
-  --name /pipelines/agent/DOCKER_HUB_USER \
-  --with-decryption \
-  --output text \
-  --query Parameter.Value \
-  --region us-east-1\
-)"
-
-aws ssm get-parameter \
-  --name /pipelines/agent/DOCKER_HUB_PASSWORD \
-  --with-decryption \
-  --output text \
-  --query Parameter.Value \
-  --region us-east-1 \
-  | docker login --username="${dockerhub_user}" --password-stdin
-
+if [[ "${REGISTRY:-}" == "" ]]; then
+    echo "Error: Missing \$REGISTRY (docker.io, ghcr.io, or packages.buildkite.com)"
+    exit 1
+fi
 
-echo "--- docker login to GitHub"
+case "${REGISTRY}" in
+docker.io)
+  echo "--- docker login to Docker Hub"
+  dockerhub_user="$(aws ssm get-parameter \
+    --name /pipelines/agent/DOCKER_HUB_USER \
+    --with-decryption \
+    --output text \
+    --query Parameter.Value \
+    --region us-east-1\
+  )"
+  aws ssm get-parameter \
+    --name /pipelines/agent/DOCKER_HUB_PASSWORD \
+    --with-decryption \
+    --output text \
+    --query Parameter.Value \
+    --region us-east-1 \
+    | docker login --username="${dockerhub_user}" --password-stdin
+  ;;
+ghcr.io)
+  echo "--- docker login to GitHub"
 
-ghcr_user=buildkite-agent-releaser
-aws ssm get-parameter \
-  --name /pipelines/agent/GITHUB_RELEASE_ACCESS_TOKEN \
-  --with-decryption \
-  --output text \
-  --query Parameter.Value \
-  --region us-east-1 \
-  | docker login ghcr.io --username="${ghcr_user}" --password-stdin
+  ghcr_user=buildkite-agent-releaser
+  aws ssm get-parameter \
+    --name /pipelines/agent/GITHUB_RELEASE_ACCESS_TOKEN \
+    --with-decryption \
+    --output text \
+    --query Parameter.Value \
+    --region us-east-1 \
+    | docker login ghcr.io --username="${ghcr_user}" --password-stdin
 
-echo "--- docker login to Buildkite Packages"
+  ;;
+packages.buildkite.com)
+  echo "--- Uploading images to Buildkite Packages"
+  ;;
+*)
+  echo "+++ Registry '${REGISTRY}' is not supported\!"
+  exit 1
+  ;;
+esac
 
-version=$(buildkite-agent meta-data get "agent-version")
-build=$(buildkite-agent meta-data get "agent-version-build")
+version="$(buildkite-agent meta-data get "agent-version")"
+build="$(buildkite-agent meta-data get "agent-version-build")"
 
 for variant in "alpine" "alpine-k8s" "ubuntu-18.04" "ubuntu-20.04" "ubuntu-22.04" "sidecar" ; do
   echo "--- Getting docker image tag for $variant from build meta data"
-  source_image=$(buildkite-agent meta-data get "agent-docker-image-$variant")
-  echo "Docker Image Tag for $variant: $source_image"
+  source_image="$(buildkite-agent meta-data get "agent-docker-image-${variant}")"
+  echo "Docker Image Tag for ${variant}: ${source_image}"
 
-  echo "--- :docker: Publishing images for $variant"
-  .buildkite/steps/publish-docker-image.sh "$variant" "$source_image" "$CODENAME" "$version" "$build"
+  echo "--- :docker: Publishing images for ${variant}"
+  .buildkite/steps/publish-docker-image.sh "${REGISTRY}" "${variant}" "${source_image}" "${CODENAME}" "${version}" "${build}"
 done