From f395322c5df9df2922ff5a4b03dd4e2e660a6f11 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Thu, 20 Jul 2023 15:00:38 -0700 Subject: [PATCH 01/30] add GitHub Action workflow for Docker deploy --- .../workflows/build-then-deploy-docker.yml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/build-then-deploy-docker.yml diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml new file mode 100644 index 0000000000..b4a4b5436e --- /dev/null +++ b/.github/workflows/build-then-deploy-docker.yml @@ -0,0 +1,62 @@ +# Copyright 2023 Democratized Data Foundation +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# This workflow builds a Docker container image, if the build is successful +# then it will deploy the image to DockerHub & GitHub container registries. +name: Build Then Deploy Docker Workflow + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + push-to-registries: + name: Push Docker image to multiple registries + + runs-on: ubuntu-latest + + permissions: + packages: write + contents: read + + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: | + sourcenetwork/defradb + ghcr.io/${{ github.repository }} + + - name: Build and push Docker images + uses: docker/build-push-action@v4 + with: + context: . + path: tools/defradb.containerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From 9ddc073351944a8fd3b6913fe87ea522296965a9 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 10:30:45 -0700 Subject: [PATCH 02/30] temporarily run on push --- .github/workflows/build-then-deploy-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index b4a4b5436e..c16777ab6a 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -14,8 +14,8 @@ name: Build Then Deploy Docker Workflow on: push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' + # tags: + # - 'v[0-9]+.[0-9]+.[0-9]+' jobs: push-to-registries: From 96881df20b039895238d0566e48d1d1ad7f7fd25 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 10:38:22 -0700 Subject: [PATCH 03/30] update workflow triggers --- .github/workflows/build-then-deploy-docker.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index c16777ab6a..5f8dc98f8e 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -13,9 +13,17 @@ name: Build Then Deploy Docker Workflow on: + pull_request: + branches: + - master + - develop + push: - # tags: - # - 'v[0-9]+.[0-9]+.[0-9]+' + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - master + - develop jobs: push-to-registries: From 5e2bdfc41ab98d5f28e257d43f7d6d770ca41dba Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 13:21:25 -0700 Subject: [PATCH 04/30] temporarily use pull_request_target in workflow --- .github/workflows/build-then-deploy-docker.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index 5f8dc98f8e..d3181bd386 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -12,18 +12,14 @@ # then it will deploy the image to DockerHub & GitHub container registries. name: Build Then Deploy Docker Workflow -on: - pull_request: - branches: - - master - - develop +on: pull_request_target - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - branches: - - master - - develop + # push: + # tags: + # - 'v[0-9]+.[0-9]+.[0-9]+' + # branches: + # - master + # - develop jobs: push-to-registries: From d5569e79dec8f6e6538da3f8a9e49de5a20bfcff Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 13:24:08 -0700 Subject: [PATCH 05/30] add types edited to pull_request_target --- .github/workflows/build-then-deploy-docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index d3181bd386..8271c271b4 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -12,7 +12,9 @@ # then it will deploy the image to DockerHub & GitHub container registries. name: Build Then Deploy Docker Workflow -on: pull_request_target +on: + pull_request_target: + types: [edited] # push: # tags: From fcce941ecee5930027efc59a0d8a2fc538dc61c5 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 13:32:28 -0700 Subject: [PATCH 06/30] update workflow triggers --- .github/workflows/build-then-deploy-docker.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index 8271c271b4..e85d058cf0 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -14,14 +14,16 @@ name: Build Then Deploy Docker Workflow on: pull_request_target: - types: [edited] + branches: + - master + - develop - # push: - # tags: - # - 'v[0-9]+.[0-9]+.[0-9]+' - # branches: - # - master - # - develop + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - master + - develop jobs: push-to-registries: From 495f6a6b65f8cc9da96a2ec0bde790d7345ca739 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 13:42:29 -0700 Subject: [PATCH 07/30] revert to pull_request trigger --- .github/workflows/build-then-deploy-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index e85d058cf0..1bc601c5c5 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -13,7 +13,7 @@ name: Build Then Deploy Docker Workflow on: - pull_request_target: + pull_request: branches: - master - develop From 3fdf2411137412dcbae766a858de35fa2f708e61 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 14:29:52 -0700 Subject: [PATCH 08/30] use file instead of path in docker/build-push-action step --- .github/workflows/build-then-deploy-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index 1bc601c5c5..6bdeb4d5c8 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -64,7 +64,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . - path: tools/defradb.containerfile + file: tools/defradb.containerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From 0c4f58a8a758d12d4b6bb897b2095ac035a2ef0a Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 14:39:38 -0700 Subject: [PATCH 09/30] remove pull_request workflow trigger --- .github/workflows/build-then-deploy-docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index 6bdeb4d5c8..53bed9e914 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -13,11 +13,6 @@ name: Build Then Deploy Docker Workflow on: - pull_request: - branches: - - master - - develop - push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' From 6084321cbff5470743e44122cfd8653813d5ca52 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 15:30:07 -0700 Subject: [PATCH 10/30] remove push branch trigger --- .github/workflows/build-then-deploy-docker.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/build-then-deploy-docker.yml index 53bed9e914..2fc3f6f1d5 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/build-then-deploy-docker.yml @@ -16,9 +16,6 @@ on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' - branches: - - master - - develop jobs: push-to-registries: From 10bf16032abed478fc7886305a1b22439908a156 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Fri, 21 Jul 2023 16:29:12 -0700 Subject: [PATCH 11/30] workflow fixes --- ...eploy-docker.yml => push-docker-image-to-registries.yml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{build-then-deploy-docker.yml => push-docker-image-to-registries.yml} (92%) diff --git a/.github/workflows/build-then-deploy-docker.yml b/.github/workflows/push-docker-image-to-registries.yml similarity index 92% rename from .github/workflows/build-then-deploy-docker.yml rename to .github/workflows/push-docker-image-to-registries.yml index 2fc3f6f1d5..a0473586c6 100644 --- a/.github/workflows/build-then-deploy-docker.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -10,7 +10,7 @@ # This workflow builds a Docker container image, if the build is successful # then it will deploy the image to DockerHub & GitHub container registries. -name: Build Then Deploy Docker Workflow +name: Push Docker Image To Registries Workflow on: push: @@ -19,7 +19,7 @@ on: jobs: push-to-registries: - name: Push Docker image to multiple registries + name: Push Docker image to registries job runs-on: ubuntu-latest @@ -52,7 +52,7 @@ jobs: sourcenetwork/defradb ghcr.io/${{ github.repository }} - - name: Build and push Docker images + - name: Push docker images as build succeeds uses: docker/build-push-action@v4 with: context: . From ab98758e4667910f2ce78928f89543aa6c3520db Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 09:45:22 -0700 Subject: [PATCH 12/30] add wasmer build step to docker file --- tools/defradb.containerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/defradb.containerfile b/tools/defradb.containerfile index 9bb62d6d53..e2ac089b2b 100644 --- a/tools/defradb.containerfile +++ b/tools/defradb.containerfile @@ -2,6 +2,13 @@ # An image to run defradb. +# Stage: BUILD_WASMER +# Build wasmer to fix linking issue https://github.com/wasmerio/wasmer-go/issues/281 +FROM docker.io/rust:1.71 AS BUILD_WASMER +WORKDIR /wasmer/ +RUN git clone --depth 1 --branch v4.0.0 https://github.com/wasmerio/wasmer.git . +RUN make build-capi + # Stage: BUILD # Several steps are involved to enable caching and because of the behavior of COPY regarding directories. FROM docker.io/golang:1.20 AS BUILD @@ -9,11 +16,13 @@ WORKDIR /repo/ COPY go.mod go.sum Makefile ./ RUN make deps:modules COPY . . +COPY --from=BUILD_WASMER /wasmer/target/release/libwasmer.so /lib/libwasmer.so RUN make build # Stage: RUN -FROM gcr.io/distroless/base-debian11 +FROM debian:bookworm-slim COPY --from=BUILD /repo/build/defradb /defradb +COPY --from=BUILD_WASMER /wasmer/target/release/libwasmer.so /lib/libwasmer.so # Documents which ports are normally used. # To publish the ports: `docker run -p 9181:9181` ... From 038698478f32b569a1a718ad46b205313c9654d0 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 09:56:47 -0700 Subject: [PATCH 13/30] update workflow to test Docker image --- .../push-docker-image-to-registries.yml | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index a0473586c6..98a974f5e7 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -12,11 +12,15 @@ # then it will deploy the image to DockerHub & GitHub container registries. name: Push Docker Image To Registries Workflow -on: +on: + pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' +env: + TEST_TAG: sourcenetwork/defradb:test + jobs: push-to-registries: name: Push Docker image to registries job @@ -31,6 +35,17 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 + - name: Build Docker image + uses: docker/build-push-action@v4 + with: + context: . + file: tools/defradb.containerfile + load: true + tags: ${{ env.TEST_TAG }} + + - name: Test Docker image + run: docker run --rm ${{ env.TEST_TAG }} + - name: Log in to Docker Hub uses: docker/login-action@v2 with: @@ -43,7 +58,7 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 @@ -52,7 +67,7 @@ jobs: sourcenetwork/defradb ghcr.io/${{ github.repository }} - - name: Push docker images as build succeeds + - name: Push Docker images uses: docker/build-push-action@v4 with: context: . From 841f0b8f898cfc741a091f2bc41f9d043912f382 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 10:09:48 -0700 Subject: [PATCH 14/30] remove pull_request workflow trigger --- .github/workflows/push-docker-image-to-registries.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 98a974f5e7..fceb70c6eb 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,7 +13,6 @@ name: Push Docker Image To Registries Workflow on: - pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' From ceb837430daf6eaedcf322936af408c429d13e0d Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 10:59:43 -0700 Subject: [PATCH 15/30] remove wasmer build from Dockerfile --- .../workflows/push-docker-image-to-registries.yml | 1 + tools/defradb.containerfile | 13 ++++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index fceb70c6eb..98a974f5e7 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,6 +13,7 @@ name: Push Docker Image To Registries Workflow on: + pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' diff --git a/tools/defradb.containerfile b/tools/defradb.containerfile index e2ac089b2b..48639e495e 100644 --- a/tools/defradb.containerfile +++ b/tools/defradb.containerfile @@ -2,13 +2,6 @@ # An image to run defradb. -# Stage: BUILD_WASMER -# Build wasmer to fix linking issue https://github.com/wasmerio/wasmer-go/issues/281 -FROM docker.io/rust:1.71 AS BUILD_WASMER -WORKDIR /wasmer/ -RUN git clone --depth 1 --branch v4.0.0 https://github.com/wasmerio/wasmer.git . -RUN make build-capi - # Stage: BUILD # Several steps are involved to enable caching and because of the behavior of COPY regarding directories. FROM docker.io/golang:1.20 AS BUILD @@ -16,13 +9,15 @@ WORKDIR /repo/ COPY go.mod go.sum Makefile ./ RUN make deps:modules COPY . . -COPY --from=BUILD_WASMER /wasmer/target/release/libwasmer.so /lib/libwasmer.so +# manually copy libwasmer.so to fix linking issue https://github.com/wasmerio/wasmer-go/issues/281 +# NOTE: when building on arm64 change linux-amd64 to linux-aarch64 +RUN cp /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-amd64/libwasmer.so /lib/libwasmer.so RUN make build # Stage: RUN FROM debian:bookworm-slim COPY --from=BUILD /repo/build/defradb /defradb -COPY --from=BUILD_WASMER /wasmer/target/release/libwasmer.so /lib/libwasmer.so +COPY --from=BUILD /lib/libwasmer.so /lib/libwasmer.so # Documents which ports are normally used. # To publish the ports: `docker run -p 9181:9181` ... From 735dd595be39ae138c154db75ba1387b38d190a2 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 11:04:09 -0700 Subject: [PATCH 16/30] remove pull_request trigger --- .github/workflows/push-docker-image-to-registries.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 98a974f5e7..fceb70c6eb 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,7 +13,6 @@ name: Push Docker Image To Registries Workflow on: - pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' From 766070f74e924e6bfab64c9333ccf53a7ca41acb Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 16:07:36 -0700 Subject: [PATCH 17/30] add pull-docker-image and validate-containerfile workflows --- .github/workflows/pull-docker-image.yml | 54 ++++++++++++++++++++ .github/workflows/validate-containerfile.yml | 48 +++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 .github/workflows/pull-docker-image.yml create mode 100644 .github/workflows/validate-containerfile.yml diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml new file mode 100644 index 0000000000..3143457114 --- /dev/null +++ b/.github/workflows/pull-docker-image.yml @@ -0,0 +1,54 @@ +# Copyright 2023 Democratized Data Foundation +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# This workflow validates that the images pushed to the container +# registry can be pulled and run sucessfully. +name: Pull Docker Image Workflow + +on: + workflow_run: + # Warning: this workflow must NOT: + # - interact with any new code. + # - checkout new code. + # - build/compile anything (only pull). + # - make any indirect calls (i.e. make xyz, or npm install, etc.) + # Note this workflow: + # - will use the base's (or default) workflow file's state. + # - doesn't run on the PR or the branch coming in, it runs on the default branch. + # - has read-write repo token + # - has access to secrets + workflows: ["Push Docker Image To Registries Workflow"] + types: + - completed + +env: + DOCKERHUB_IMAGE_TAG: sourcenetwork/defradb:latest + GITHUB_IMAGE_TAG: ghcr.io/sourcenetwork/defradb:latest + +jobs: + pull-docker-image: + name: Pull docker image job + + if: ${{ github.event.workflow_run.conclusion == 'success' }} + + runs-on: ubuntu-latest + + steps: + - name: Pull DockerHub image + run: docker pull ${{ env.DOCKERHUB_IMAGE_TAG }} + + - name: Test DockerHub image + run: docker run --rm ${{ env.DOCKERHUB_IMAGE_TAG }} + + - name: Pull GitHub image + run: docker pull ${{ env.GITHUB_IMAGE_TAG }} + + - name: Test GitHub image + run: docker run --rm ${{ env.GITHUB_IMAGE_TAG }} diff --git a/.github/workflows/validate-containerfile.yml b/.github/workflows/validate-containerfile.yml new file mode 100644 index 0000000000..a9dbb985dd --- /dev/null +++ b/.github/workflows/validate-containerfile.yml @@ -0,0 +1,48 @@ +# Copyright 2023 Democratized Data Foundation +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# This workflow tests that the container build is successful and +# that the built container runs successfully. +name: Validate Containerfile Workflow + +on: + pull_request: + branches: + - master + - develop + paths: + - '.github/workflows/validate-containerfile.yml' + - 'tools/defradb.containerfile' + + push: + branches: + - master + - develop + +jobs: + validate-containerfile: + name: Validate containerfile job + + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Build Docker image + uses: docker/build-push-action@v4 + with: + context: . + file: tools/defradb.containerfile + load: true + tags: ${{ env.TEST_TAG }} + + - name: Test Docker image + run: docker run --rm ${{ env.TEST_TAG }} From a2a9c339e02fb014b91d8a5f7b52511a36c2a6b4 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 16:40:34 -0700 Subject: [PATCH 18/30] build docker for amd64 and arm64. update workflow to test both arm64 and amd64 --- .../workflows/push-docker-image-to-registries.yml | 12 +++++++++--- .github/workflows/validate-containerfile.yml | 11 +++++++++-- tools/defradb.containerfile | 5 +++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index fceb70c6eb..18a73d68ab 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,6 +13,7 @@ name: Push Docker Image To Registries Workflow on: + pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' @@ -39,11 +40,15 @@ jobs: with: context: . file: tools/defradb.containerfile + platforms: linux/amd64,linux/arm64 load: true tags: ${{ env.TEST_TAG }} - - name: Test Docker image - run: docker run --rm ${{ env.TEST_TAG }} + - name: Test Docker linux/amd64 image + run: docker run --rm --platform linux/amd64 ${{ env.TEST_TAG }} + + - name: Test Docker linux/arm64 image + run: docker run --rm --platform linux/arm64 ${{ env.TEST_TAG }} - name: Log in to Docker Hub uses: docker/login-action@v2 @@ -71,6 +76,7 @@ jobs: with: context: . file: tools/defradb.containerfile + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/validate-containerfile.yml b/.github/workflows/validate-containerfile.yml index a9dbb985dd..65bf3ec397 100644 --- a/.github/workflows/validate-containerfile.yml +++ b/.github/workflows/validate-containerfile.yml @@ -26,6 +26,9 @@ on: - master - develop +env: + TEST_TAG: sourcenetwork/defradb:test + jobs: validate-containerfile: name: Validate containerfile job @@ -41,8 +44,12 @@ jobs: with: context: . file: tools/defradb.containerfile + platforms: linux/amd64,linux/arm64 load: true tags: ${{ env.TEST_TAG }} - - name: Test Docker image - run: docker run --rm ${{ env.TEST_TAG }} + - name: Test Docker linux/amd64 image + run: docker run --rm --platform linux/amd64 ${{ env.TEST_TAG }} + + - name: Test Docker linux/arm64 image + run: docker run --rm --platform linux/arm64 ${{ env.TEST_TAG }} diff --git a/tools/defradb.containerfile b/tools/defradb.containerfile index 48639e495e..c5674f10db 100644 --- a/tools/defradb.containerfile +++ b/tools/defradb.containerfile @@ -10,8 +10,9 @@ COPY go.mod go.sum Makefile ./ RUN make deps:modules COPY . . # manually copy libwasmer.so to fix linking issue https://github.com/wasmerio/wasmer-go/issues/281 -# NOTE: when building on arm64 change linux-amd64 to linux-aarch64 -RUN cp /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-amd64/libwasmer.so /lib/libwasmer.so +RUN export WASMER_ARCH=$(go env GOHOSTARCH | sed "s/arm64/aarch64/") && \ + export WASMER_PATH=$(go env GOMODCACHE)/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-$WASMER_ARCH/libwasmer.so && \ + cp $WASMER_PATH /lib/libwasmer.so RUN make build # Stage: RUN From 323a3bd3692beca9845ad2c2f7b2a3b5d8e289f7 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 16:46:35 -0700 Subject: [PATCH 19/30] use buildx in docker workflows --- .github/workflows/push-docker-image-to-registries.yml | 6 ++++++ .github/workflows/validate-containerfile.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 18a73d68ab..3f1c0b49a3 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -34,6 +34,12 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Build Docker image uses: docker/build-push-action@v4 diff --git a/.github/workflows/validate-containerfile.yml b/.github/workflows/validate-containerfile.yml index 65bf3ec397..56d6790d14 100644 --- a/.github/workflows/validate-containerfile.yml +++ b/.github/workflows/validate-containerfile.yml @@ -38,6 +38,12 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Build Docker image uses: docker/build-push-action@v4 From d20cf4e2cd6e7fd557b544ec5b8aab34827fe17d Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 16:56:08 -0700 Subject: [PATCH 20/30] remove docker test from push workflow --- .../workflows/push-docker-image-to-registries.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 3f1c0b49a3..7445fcf701 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -40,21 +40,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - - name: Build Docker image - uses: docker/build-push-action@v4 - with: - context: . - file: tools/defradb.containerfile - platforms: linux/amd64,linux/arm64 - load: true - tags: ${{ env.TEST_TAG }} - - - name: Test Docker linux/amd64 image - run: docker run --rm --platform linux/amd64 ${{ env.TEST_TAG }} - - - name: Test Docker linux/arm64 image - run: docker run --rm --platform linux/arm64 ${{ env.TEST_TAG }} - name: Log in to Docker Hub uses: docker/login-action@v2 From 78341c00c78fc681501162572d5e1b99cd5d57bf Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 17:19:49 -0700 Subject: [PATCH 21/30] use matrix for docker test workflows --- .github/workflows/pull-docker-image.yml | 15 +++++++++++---- .../workflows/push-docker-image-to-registries.yml | 1 - .github/workflows/validate-containerfile.yml | 15 ++++++++++----- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index 3143457114..13ef78214d 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -40,15 +40,22 @@ jobs: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + steps: - name: Pull DockerHub image - run: docker pull ${{ env.DOCKERHUB_IMAGE_TAG }} + run: docker pull --platform ${{ matrix.platform }} ${{ env.DOCKERHUB_IMAGE_TAG }} - name: Test DockerHub image - run: docker run --rm ${{ env.DOCKERHUB_IMAGE_TAG }} + run: docker run --platform ${{ matrix.platform }} --rm ${{ env.DOCKERHUB_IMAGE_TAG }} - name: Pull GitHub image - run: docker pull ${{ env.GITHUB_IMAGE_TAG }} + run: docker pull --platform ${{ matrix.platform }} ${{ env.GITHUB_IMAGE_TAG }} - name: Test GitHub image - run: docker run --rm ${{ env.GITHUB_IMAGE_TAG }} + run: docker run --platform ${{ matrix.platform }} --rm ${{ env.GITHUB_IMAGE_TAG }} diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 7445fcf701..2c34fd94a0 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,7 +13,6 @@ name: Push Docker Image To Registries Workflow on: - pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' diff --git a/.github/workflows/validate-containerfile.yml b/.github/workflows/validate-containerfile.yml index 56d6790d14..0cc8a6ce4a 100644 --- a/.github/workflows/validate-containerfile.yml +++ b/.github/workflows/validate-containerfile.yml @@ -35,6 +35,13 @@ jobs: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + steps: - name: Check out the repo uses: actions/checkout@v3 @@ -50,12 +57,10 @@ jobs: with: context: . file: tools/defradb.containerfile - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} load: true tags: ${{ env.TEST_TAG }} - - name: Test Docker linux/amd64 image - run: docker run --rm --platform linux/amd64 ${{ env.TEST_TAG }} + - name: Test Docker image + run: docker run --rm --platform ${{ matrix.platform }} ${{ env.TEST_TAG }} - - name: Test Docker linux/arm64 image - run: docker run --rm --platform linux/arm64 ${{ env.TEST_TAG }} From 000e4f015b04577f93f4a92bef1a856d5bf8e711 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 17:31:56 -0700 Subject: [PATCH 22/30] simplify pull docker test matrix --- .github/workflows/pull-docker-image.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index 13ef78214d..3ca32be3aa 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -9,7 +9,7 @@ # licenses/APL.txt. # This workflow validates that the images pushed to the container -# registry can be pulled and run sucessfully. +# registries can be pulled then run sucessfully. name: Pull Docker Image Workflow on: @@ -28,10 +28,6 @@ on: types: - completed -env: - DOCKERHUB_IMAGE_TAG: sourcenetwork/defradb:latest - GITHUB_IMAGE_TAG: ghcr.io/sourcenetwork/defradb:latest - jobs: pull-docker-image: name: Pull docker image job @@ -46,16 +42,13 @@ jobs: platform: - linux/amd64 - linux/arm64 + image_tag: + - sourcenetwork/defradb:latest + - ghcr.io/sourcenetwork/defradb:latest steps: - - name: Pull DockerHub image - run: docker pull --platform ${{ matrix.platform }} ${{ env.DOCKERHUB_IMAGE_TAG }} - - - name: Test DockerHub image - run: docker run --platform ${{ matrix.platform }} --rm ${{ env.DOCKERHUB_IMAGE_TAG }} - - - name: Pull GitHub image - run: docker pull --platform ${{ matrix.platform }} ${{ env.GITHUB_IMAGE_TAG }} + - name: Pull Docker image + run: docker pull --platform ${{ matrix.platform }} ${{ matrix.image_tag }} - - name: Test GitHub image - run: docker run --platform ${{ matrix.platform }} --rm ${{ env.GITHUB_IMAGE_TAG }} + - name: Test Docker image + run: docker run --platform ${{ matrix.platform }} --rm ${{ matrix.image_tag }} From 75926f756b4bcc2007f03afb5197762bd2ca09ef Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 17:33:19 -0700 Subject: [PATCH 23/30] remove unused env from push-docker-image-to-registries workflow --- .github/workflows/push-docker-image-to-registries.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 2c34fd94a0..7a523ef542 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -17,9 +17,6 @@ on: tags: - 'v[0-9]+.[0-9]+.[0-9]+' -env: - TEST_TAG: sourcenetwork/defradb:test - jobs: push-to-registries: name: Push Docker image to registries job From 45ef91f420cb2a25f7f005f383665adc4eeb4d19 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 19:14:38 -0700 Subject: [PATCH 24/30] add workflow dispatch --- .github/workflows/pull-docker-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index 3ca32be3aa..5d6f7ca5e4 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -13,6 +13,7 @@ name: Pull Docker Image Workflow on: + workflow_dispatch: workflow_run: # Warning: this workflow must NOT: # - interact with any new code. From 3d24a729ca09de86ebe553680afc7f50d646967f Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 19:20:40 -0700 Subject: [PATCH 25/30] revert Docker platforms workflows --- .github/workflows/pull-docker-image.yml | 3 --- .../push-docker-image-to-registries.yml | 17 ++++++++++++++++- .github/workflows/validate-containerfile.yml | 10 +--------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index 5d6f7ca5e4..ff21c52281 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -40,9 +40,6 @@ jobs: strategy: fail-fast: false matrix: - platform: - - linux/amd64 - - linux/arm64 image_tag: - sourcenetwork/defradb:latest - ghcr.io/sourcenetwork/defradb:latest diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 7a523ef542..c2427a707d 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,10 +13,14 @@ name: Push Docker Image To Registries Workflow on: + pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' +env: + TEST_TAG: sourcenetwork/defradb:test + jobs: push-to-registries: name: Push Docker image to registries job @@ -37,6 +41,18 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Build Docker image + uses: docker/build-push-action@v4 + with: + context: . + file: tools/defradb.containerfile + load: true + tags: ${{ env.TEST_TAG }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Test Docker image + run: docker run --rm ${{ env.TEST_TAG }} + - name: Log in to Docker Hub uses: docker/login-action@v2 with: @@ -63,7 +79,6 @@ jobs: with: context: . file: tools/defradb.containerfile - platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/validate-containerfile.yml b/.github/workflows/validate-containerfile.yml index 0cc8a6ce4a..5b300f4d30 100644 --- a/.github/workflows/validate-containerfile.yml +++ b/.github/workflows/validate-containerfile.yml @@ -35,13 +35,6 @@ jobs: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - steps: - name: Check out the repo uses: actions/checkout@v3 @@ -57,10 +50,9 @@ jobs: with: context: . file: tools/defradb.containerfile - platforms: ${{ matrix.platform }} load: true tags: ${{ env.TEST_TAG }} - name: Test Docker image - run: docker run --rm --platform ${{ matrix.platform }} ${{ env.TEST_TAG }} + run: docker run --rm ${{ env.TEST_TAG }} From 6554ba6c37732c3a58ca3d6c8693fcc6201245ee Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 19:29:07 -0700 Subject: [PATCH 26/30] trigger pull-docker-image on pr --- .github/workflows/pull-docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index ff21c52281..65d08ceed7 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -13,7 +13,7 @@ name: Pull Docker Image Workflow on: - workflow_dispatch: + pull_request: workflow_run: # Warning: this workflow must NOT: # - interact with any new code. @@ -33,7 +33,7 @@ jobs: pull-docker-image: name: Pull docker image job - if: ${{ github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest @@ -46,7 +46,7 @@ jobs: steps: - name: Pull Docker image - run: docker pull --platform ${{ matrix.platform }} ${{ matrix.image_tag }} + run: docker pull ${{ matrix.image_tag }} - name: Test Docker image - run: docker run --platform ${{ matrix.platform }} --rm ${{ matrix.image_tag }} + run: docker run --rm ${{ matrix.image_tag }} From a6e8f89d038d4e8d6aef772ea7180132475e7cbd Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 19:30:47 -0700 Subject: [PATCH 27/30] update pull-docker-image tag --- .github/workflows/pull-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index 65d08ceed7..c71c7023df 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -41,8 +41,8 @@ jobs: fail-fast: false matrix: image_tag: - - sourcenetwork/defradb:latest - - ghcr.io/sourcenetwork/defradb:latest + - sourcenetwork/defradb:pr-1684 + - ghcr.io/sourcenetwork/defradb:pr-1684 steps: - name: Pull Docker image From 4eab4d22f32a5ad7616d0101e9ee4790623b6981 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 19:33:16 -0700 Subject: [PATCH 28/30] remove test workflow triggers --- .github/workflows/pull-docker-image.yml | 7 +++---- .github/workflows/push-docker-image-to-registries.yml | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull-docker-image.yml b/.github/workflows/pull-docker-image.yml index c71c7023df..eb0170b7ef 100644 --- a/.github/workflows/pull-docker-image.yml +++ b/.github/workflows/pull-docker-image.yml @@ -13,7 +13,6 @@ name: Pull Docker Image Workflow on: - pull_request: workflow_run: # Warning: this workflow must NOT: # - interact with any new code. @@ -33,7 +32,7 @@ jobs: pull-docker-image: name: Pull docker image job - # if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest @@ -41,8 +40,8 @@ jobs: fail-fast: false matrix: image_tag: - - sourcenetwork/defradb:pr-1684 - - ghcr.io/sourcenetwork/defradb:pr-1684 + - sourcenetwork/defradb:latest + - ghcr.io/sourcenetwork/defradb:latest steps: - name: Pull Docker image diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index c2427a707d..7d90ce8072 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -13,7 +13,6 @@ name: Push Docker Image To Registries Workflow on: - pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' From 3ab92892542aff672dcc455e0024848012ec7e9f Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 19:44:28 -0700 Subject: [PATCH 29/30] set push-docker workflow trigger to push --- .github/workflows/push-docker-image-to-registries.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index 7d90ce8072..e1c3268fc5 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -14,8 +14,8 @@ name: Push Docker Image To Registries Workflow on: push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' + # tags: + # - 'v[0-9]+.[0-9]+.[0-9]+' env: TEST_TAG: sourcenetwork/defradb:test From d30d76185ba7a1426cbfd8d841a7c2d1381fbba6 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 24 Jul 2023 20:35:30 -0700 Subject: [PATCH 30/30] fix push-docker-image-to-registries job name --- .github/workflows/push-docker-image-to-registries.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-docker-image-to-registries.yml b/.github/workflows/push-docker-image-to-registries.yml index e1c3268fc5..d7d00d14aa 100644 --- a/.github/workflows/push-docker-image-to-registries.yml +++ b/.github/workflows/push-docker-image-to-registries.yml @@ -14,14 +14,14 @@ name: Push Docker Image To Registries Workflow on: push: - # tags: - # - 'v[0-9]+.[0-9]+.[0-9]+' + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' env: TEST_TAG: sourcenetwork/defradb:test jobs: - push-to-registries: + push-docker-image-to-registries: name: Push Docker image to registries job runs-on: ubuntu-latest