From 6dd64fcce93e87497f0e3f6594e74f030b4538b4 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Tue, 18 Jun 2024 08:19:21 -0700 Subject: [PATCH] Build image in github actions (#6026) * Build image in github actions Signed-off-by: Friedrich Gonzalez * Bug fixes Signed-off-by: Friedrich Gonzalez * Target master Signed-off-by: Friedrich Gonzalez * Fix save-multiarch-build-image Signed-off-by: Friedrich Gonzalez * Include QEMU and buildx action Signed-off-by: Friedrich Gonzalez * Let's simplify and tests first Signed-off-by: Friedrich Gonzalez * test push Signed-off-by: Friedrich Gonzalez * Push intermediate images Signed-off-by: Friedrich Gonzalez * Re-enable full build, update docs and make sure push is only possible from master Signed-off-by: Friedrich Gonzalez * Fetch tags and use Makefile in build-image Signed-off-by: Friedrich Gonzalez --------- Signed-off-by: Friedrich Gonzalez --- .github/workflows/build-image.yml | 68 +++++++++++++++++++ .github/workflows/test-build-deploy.yml | 6 ++ .gitignore | 3 + Makefile | 9 +-- build-image/Makefile | 17 +++++ .../how-to-update-the-build-image.md | 8 +-- 6 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/build-image.yml create mode 100644 build-image/Makefile diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 0000000000..3eb6312a18 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,68 @@ +name: Build Image + +on: + push: + branches: [ master ] + paths: + - 'build-image/**' + - '.github/workflows/build-image.yml' + pull_request: + branches: [ master ] + paths: + - 'build-image/**' + - '.github/workflows/build-image.yml' + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + name: Checkout + with: + fetch-depth: 0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Save image + run: make save-multiarch-build-image + + - name: Upload Docker Images Artifacts + uses: actions/upload-artifact@v4 + with: + name: build-image + path: | + ./build-image-amd64.tar + ./build-image-arm64.tar + if-no-files-found: error + + push: + needs: build + if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + name: Checkout + with: + fetch-depth: 0 + + - name: Download Docker Images Artifacts + uses: actions/download-artifact@v4 + with: + name: build-image + + - name: Load image + run: make load-multiarch-build-image + + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{secrets.QUAY_REGISTRY_USER}} + password: ${{secrets.QUAY_REGISTRY_PASSWORD}} + + - name: Push image + run: make push-multiarch-build-image diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index dfa752c596..de7eade218 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -5,7 +5,13 @@ on: branches: [master] tags: - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions + paths-ignore: + - 'build-image/**' + - '.github/workflows/build-image.yml' pull_request: + paths-ignore: + - 'build-image/**' + - '.github/workflows/build-image.yml' jobs: lint: diff --git a/.gitignore b/.gitignore index ac025027c1..5749235286 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ Makefile.local .vscode compose compose-simple + +/build-image-arm64.tar +/build-image-amd64.tar diff --git a/Makefile b/Makefile index 7c0c10a337..b1af41119a 100644 --- a/Makefile +++ b/Makefile @@ -53,14 +53,7 @@ fetch-build-image: docker tag $(BUILD_IMAGE):$(LATEST_BUILD_IMAGE_TAG) $(BUILD_IMAGE):latest touch build-image/.uptodate -push-multiarch-build-image: - @echo - # Build image for each platform separately... it tends to generate fewer errors. - $(SUDO) docker buildx build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) build-image/ - $(SUDO) docker buildx build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) build-image/ - # This command will run the same build as above, but it will reuse existing platform-specific images, - # put them together and push to registry. - $(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG) build-image/ +-include build-image/Makefile # We don't want find to scan inside a bunch of directories, to accelerate the # 'make: Entering directory '/go/src/github.com/cortexproject/cortex' phase. diff --git a/build-image/Makefile b/build-image/Makefile new file mode 100644 index 0000000000..9844b681ab --- /dev/null +++ b/build-image/Makefile @@ -0,0 +1,17 @@ +save-multiarch-build-image: + @echo + # Build image for each platform separately... it tends to generate fewer errors. + $(SUDO) docker buildx build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG)-amd64 --output type=docker,dest=./build-image-amd64.tar build-image/ + $(SUDO) docker buildx build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)build-image:$(IMAGE_TAG)-arm64 --output type=docker,dest=./build-image-arm64.tar build-image/ + +load-multiarch-build-image: + $(SUDO) docker load -i build-image-amd64.tar + $(SUDO) docker load -i build-image-arm64.tar + +push-multiarch-build-image: + # This command will run the same build as multiarch-build-image, but it will reuse existing platform-specific images, + # put them together and push to registry. + $(SUDO) docker push $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-amd64 + $(SUDO) docker push $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-arm64 + $(SUDO) docker manifest create $(IMAGE_PREFIX)build-image:$(IMAGE_TAG) --amend $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-amd64 --amend $(IMAGE_PREFIX)build-image:${IMAGE_TAG}-arm64 + $(SUDO) docker manifest push $(IMAGE_PREFIX)build-image:$(IMAGE_TAG) diff --git a/docs/contributing/how-to-update-the-build-image.md b/docs/contributing/how-to-update-the-build-image.md index 497ab00a00..df0c62a2f1 100644 --- a/docs/contributing/how-to-update-the-build-image.md +++ b/docs/contributing/how-to-update-the-build-image.md @@ -5,12 +5,10 @@ weight: 5 slug: how-to-update-the-build-image --- -The build image currently can only be updated by a Cortex maintainer. If you're not a maintainer you can still open a PR with the changes, asking a maintainer to assist you publishing the updated image. The procedure is: +The procedure is: 1. Update `build-image/Dockerfile` -1. Run `go env` and make sure `GOPROXY=https://proxy.golang.org,direct` (Go's default). Some environment may required `GOPROXY=direct`, and if you push a build image with this, build workflow on GitHub will take a lot longer to download modules. -1. `docker login quay.io`. Note that pushing to `quay.io/cortexproject/build-image` repository can only be done by a maintainer. -1. Build the and publish the image by using `make push-multiarch-build-image`. This will build and push multi-platform docker image (for linux/amd64 and linux/arm64). Running this step successfully requires [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/), but does not require a specific platform. -1. Replace the image tag in `.github/workflows/*` (_there may be multiple references_) and Makefile (variable `LATEST_BUILD_IMAGE_TAG`). +1. Create a PR to master with that changed, after the PR is merged to master, the new build image is available in the quay.io repository. Check github action logs [here](https://github.com/cortexproject/cortex/actions/workflows/build-image.yml) for to find the image tag. +1. Create another PR to replace the image tag in `.github/workflows/*` (_there may be multiple references_) and Makefile (variable `LATEST_BUILD_IMAGE_TAG`). 1. If you are updating Go's runtime version be sure to change `actions/setup-go`'s `go-version` in ``.github/workflows/*`. 1. Open a PR and make sure the CI with new build-image passes