From 2d21679d230a1fa4146ccb8862f1209c32851ade Mon Sep 17 00:00:00 2001 From: yordanovsstoyan Date: Tue, 17 Sep 2024 10:35:38 +0300 Subject: [PATCH 1/3] Update Docker Workflow --- .../workflows/docker-build-and-publish.yaml | 122 ++++++++++++++++-- 1 file changed, 112 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-build-and-publish.yaml b/.github/workflows/docker-build-and-publish.yaml index 3c6ea8710..1f8c17561 100644 --- a/.github/workflows/docker-build-and-publish.yaml +++ b/.github/workflows/docker-build-and-publish.yaml @@ -68,9 +68,13 @@ concurrency: cancel-in-progress: ${{ github.ref_name != github.event.repository.default_branch }} jobs: - build-and-publish: - name: Build and Publish - runs-on: ubuntu-22.04 + build-and-push: + strategy: + matrix: + platform: ${{ fromJson(inputs.platforms) }} + + name: Build for ${{ matrix.platform }} + runs-on: ${{ fromJson(inputs.runs-on)[matrix.platform] }} steps: - name: Check out repository uses: bakdata/ci-templates/actions/checkout@1.44.0 @@ -114,15 +118,113 @@ jobs: event=tag,type=semver,pattern={{ version }} env: DOCKER_METADATA_PR_HEAD_SHA: true # set correct sha for PRs + + - name: Prepare build args + id: args + run: | + { + echo "build-args<> "$GITHUB_OUTPUT" - - name: Build and push - uses: docker/build-push-action@v5 + - name: Build and push by digest + uses: docker/build-push-action@v6 + id: build with: context: ${{ inputs.docker-context }} - file: ${{ inputs.dockerfile-path }} - build-args: ${{ inputs.docker-build-args }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + file: ${{ inputs.docker-context }}/${{ inputs.dockerfile-path }} + platforms: ${{ matrix.platform }} + build-args: ${{ steps.args.outputs.build-args }} cache-from: type=gha cache-to: type=gha,mode=max + # don't specify 'tags' here (error "get can't push tagged ref by digest") + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + # HACK We upload the digest so the other jobs can reference the files properly, because there are no proper matrix job outputs + # https://github.com/orgs/community/discussions/26639#discussioncomment-3838014 + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || 'linux-arm64' }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # publish merged manifest list of all previously build and pushed images + publish: + name: Publish Manifest List + needs: + - build-and-push + runs-on: ubuntu-latest + steps: + # We download the digest,because there are no proper outputs from the previous matrix job + # https://github.com/orgs/community/discussions/26639#discussioncomment-3838014 + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Login to the Registry + uses: "docker/login-action@v3" + with: + registry: "${{ inputs.docker-registry }}" + username: "${{ secrets.docker-user }}" + password: "${{ secrets.docker-password }}" + + - name: Set image name + run: | + fullImageName="${{ inputs.image-name }}" + if [[ -n "${{ inputs.image-namespace }}" ]]; then + fullImageName="${{ inputs.image-namespace }}/${fullImageName}" + fi + if [[ -n "${{ inputs.docker-registry }}" ]]; then + fullImageName="${{ inputs.docker-registry }}/${fullImageName}" + fi + + echo "IMAGE_NAME=${fullImageName}" >> "$GITHUB_ENV" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Collect Docker metadata + id: metadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + event=push,type=raw,value=${{ inputs.image-tag }} + event=pr,type=raw,value=${{ inputs.image-tag }} + event=tag,type=semver,pattern={{ version }} + env: + DOCKER_METADATA_PR_HEAD_SHA: true # set correct sha for PRs + + - name: Create and push merged manifest list + working-directory: /tmp/digests + run: | + # HACK that reads file names from downloaded artifacts as digest outputs from matrix jobs: + # https://github.com/orgs/community/discussions/26639#discussioncomment-3838014 + images=() + for file in *; do + images+=("${{ env.IMAGE_NAME }}@sha256:${file}") + done + + # read tags from metadata step output and create CLI params by adding -t for each tag + readarray -t tags < <(jq -cr '.tags | .[]' <<< "${DOCKER_METADATA_OUTPUT_JSON}") + tag_params=() + for tag in "${tags[@]}"; do + tag_params+=("-t") + tag_params+=("${tag}") + done + + docker buildx imagetools create "${tag_params[@]}" "${images[@]}" From 2345693f9302b813e82b2d62e3f17dc01ce2b574 Mon Sep 17 00:00:00 2001 From: yordanovsstoyan Date: Tue, 17 Sep 2024 10:38:03 +0300 Subject: [PATCH 2/3] Update Docker Workflow --- .../workflows/docker-build-and-publish.yaml | 5 ++++ .../docker-build-and-publish/README.md | 25 ++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-build-and-publish.yaml b/.github/workflows/docker-build-and-publish.yaml index 1f8c17561..f9eca4287 100644 --- a/.github/workflows/docker-build-and-publish.yaml +++ b/.github/workflows/docker-build-and-publish.yaml @@ -54,6 +54,11 @@ on: required: false type: boolean default: false + runs-on: + description: "Runner config as JSON object string. Each key is one of the input `platforms` and each value is the list of runner tags on which the Docker image for that platform should be natively built on. Defaults to a mapping from expected possible `platforms` to GitHub runners native to that architecture." + required: false + default: "{ 'linux/amd64': ['ubuntu-22.04'], 'linux/arm64': ['linux-arm-latest'] }" + type: string secrets: docker-user: diff --git a/docs/workflows/docker-build-and-publish/README.md b/docs/workflows/docker-build-and-publish/README.md index 1d87e8e77..b26cf132a 100644 --- a/docs/workflows/docker-build-and-publish/README.md +++ b/docs/workflows/docker-build-and-publish/README.md @@ -49,18 +49,19 @@ jobs: -| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | -| ------------------- | ------- | -------- | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| checkout-lfs-files | boolean | false | `false` | Whether the Git checkout action should resolve LFS files or not. (Default is false) | -| checkout-submodules | string | false | `"false"` | Whether to checkout submodules: `true` to checkout submodules or `recursive` to recursively checkout submodules. | -| docker-build-args | string | false | | List of build-time variables (see https://github.com/docker/build-push-action?tab=readme-ov-file#inputs) | -| docker-context | string | false | `"."` | The docker context. | -| docker-registry | string | false | `"docker.io"` | Host where the image should be pushed to. | -| dockerfile-path | string | false | `"./Dockerfile"` | Path to the Dockerfile. | -| image-name | string | false | `"${{ github.event.repository.name }}"` | Name of Docker image. | -| image-namespace | string | false | `"bakdata"` | Namespace of Docker image. | -| image-tag | string | false | `"pipeline-${{ github.run_id }}-git-{{ sha }}"` | Tag of Docker image. | -| ref | string | false | | Ref name to checkout | +| INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | +| ------------------- | ------- | -------- | ---------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| checkout-lfs-files | boolean | false | `false` | Whether the Git checkout action should resolve LFS files or not. (Default is false) | +| checkout-submodules | string | false | `"false"` | Whether to checkout submodules: `true` to checkout submodules or `recursive` to recursively checkout submodules. | +| docker-build-args | string | false | | List of build-time variables (see https://github.com/docker/build-push-action?tab=readme-ov-file#inputs) | +| docker-context | string | false | `"."` | The docker context. | +| docker-registry | string | false | `"docker.io"` | Host where the image should be pushed to. | +| dockerfile-path | string | false | `"./Dockerfile"` | Path to the Dockerfile. | +| image-name | string | false | `"${{ github.event.repository.name }}"` | Name of Docker image. | +| image-namespace | string | false | `"bakdata"` | Namespace of Docker image. | +| image-tag | string | false | `"pipeline-${{ github.run_id }}-git-{{ sha }}"` | Tag of Docker image. | +| ref | string | false | | Ref name to checkout | +| runs-on | string | false | `"{ 'linux/amd64': ['ubuntu-22.04'], 'linux/arm64': ['linux-arm-latest'] }"` | Runner config as JSON object string. Each key is one of the input `platforms` and each value is the list of runner tags on which the Docker image for that platform should be natively built on. Defaults to a mapping from expected possible `platforms` to GitHub runners native to that architecture. | From 5e1b2570bcc697c337487ba79a9063311de1bc2a Mon Sep 17 00:00:00 2001 From: yordanovsstoyan Date: Tue, 17 Sep 2024 10:42:52 +0300 Subject: [PATCH 3/3] Update Docker Workflow --- .github/workflows/docker-build-and-publish.yaml | 5 +++++ docs/workflows/docker-build-and-publish/README.md | 1 + 2 files changed, 6 insertions(+) diff --git a/.github/workflows/docker-build-and-publish.yaml b/.github/workflows/docker-build-and-publish.yaml index f9eca4287..d5834654c 100644 --- a/.github/workflows/docker-build-and-publish.yaml +++ b/.github/workflows/docker-build-and-publish.yaml @@ -59,6 +59,11 @@ on: required: false default: "{ 'linux/amd64': ['ubuntu-22.04'], 'linux/arm64': ['linux-arm-latest'] }" type: string + platforms: + description: "Architectures for the created image (comma separated)" + required: false + default: "['linux/amd64']" + type: string secrets: docker-user: diff --git a/docs/workflows/docker-build-and-publish/README.md b/docs/workflows/docker-build-and-publish/README.md index b26cf132a..d7a7bb9b9 100644 --- a/docs/workflows/docker-build-and-publish/README.md +++ b/docs/workflows/docker-build-and-publish/README.md @@ -60,6 +60,7 @@ jobs: | image-name | string | false | `"${{ github.event.repository.name }}"` | Name of Docker image. | | image-namespace | string | false | `"bakdata"` | Namespace of Docker image. | | image-tag | string | false | `"pipeline-${{ github.run_id }}-git-{{ sha }}"` | Tag of Docker image. | +| platforms | string | false | `"['linux/amd64']"` | Architectures for the created image (comma separated) | | ref | string | false | | Ref name to checkout | | runs-on | string | false | `"{ 'linux/amd64': ['ubuntu-22.04'], 'linux/arm64': ['linux-arm-latest'] }"` | Runner config as JSON object string. Each key is one of the input `platforms` and each value is the list of runner tags on which the Docker image for that platform should be natively built on. Defaults to a mapping from expected possible `platforms` to GitHub runners native to that architecture. |