From 23dcbcf6540acf4c3a8bebbf2e5e38aa62961ce1 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 22 Feb 2024 10:14:41 -0600 Subject: [PATCH] feat(ci): no more docker builds on PR-related events (#27146) --- .asf.yaml | 8 ---- .github/workflows/docker.yml | 21 +++++----- .github/workflows/no-op.yml | 19 ++++++++- docs/docs/installation/docker.mdx | 66 +++++++++++++++---------------- 4 files changed, 61 insertions(+), 53 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 180414f24aa59..e1904d082f1a0 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -73,14 +73,6 @@ github: - test-postgres (3.9) - test-postgres (3.10) - test-sqlite (3.9) - - docker-build (dev, linux/amd64) - - docker-build (lean, linux/amd64) - - docker-build (py310, linux/arm64) - - docker-build (py310, linux/amd64) - - docker-build (websocket, linux/arm64) - - docker-build (websocket, linux/amd64) - - docker-build (dockerize, linux/arm64) - - docker-build (dockerize, linux/amd64) required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae54835e0fb73..75da2e1442dd7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,8 @@ on: branches: - 'master' pull_request: - types: [synchronize, opened, reopened, ready_for_review] + branches: + - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} @@ -17,15 +18,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] - platform: ["linux/amd64", "linux/arm64"] - exclude: - # disabling because slow! no python wheels for arm/py39 and - # QEMU is slow! - - build_preset: "dev" - platform: "linux/arm64" - - build_preset: "lean" - platform: "linux/arm64" + build_preset: | + ${{ fromJson(github.event_name == 'pull_request' ? '["ci"]' : '["dev", "lean", "py310", "websocket", "dockerize"]') }} fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -46,4 +40,9 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | pip install click - ./scripts/build_docker.py ${{ matrix.build_preset }} ${{ github.event_name }} --platform ${{ matrix.platform }} + ./scripts/build_docker.py \ + ${{ matrix.build_preset }} \ + ${{ github.event_name }} \ + --build_context_ref "$RELEASE" $FORCE_LATEST \ + --platform "linux/arm64" \ + --platform "linux/amd64" diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 835063640aa96..afea397e00635 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -19,7 +19,7 @@ # - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows. # - This workflow should be kept as-is, without path-specific conditions. -name: No Operation Checks +name: no-op Checks on: pull_request jobs: @@ -52,3 +52,20 @@ jobs: run: | echo "This is a no-op step for python-lint to ensure a successful status." exit 0 + # section to be deleted after master merge + docker-build: + strategy: + matrix: + build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] + platform: ["linux/amd64", "linux/arm64"] + exclude: + - build_preset: "dev" + platform: "linux/arm64" + - build_preset: "lean" + platform: "linux/arm64" + runs-on: ubuntu-latest + steps: + - name: No-op for docker + run: | + echo "No-op to get rid of the docker-build checks reqs in .asf.yml" + exit 0 diff --git a/docs/docs/installation/docker.mdx b/docs/docs/installation/docker.mdx index 407bb74c4a2b8..170ca1461aa88 100644 --- a/docs/docs/installation/docker.mdx +++ b/docs/docs/installation/docker.mdx @@ -1,47 +1,47 @@ -# Docker Images and Tags +# Docker builds, images and tags The Apache Superset community extensively uses Docker for development, release, and productionizing Superset. This page details our Docker builds and tag naming schemes to help users navigate our offerings. Images are built and pushed to the [Superset Docker Hub repository]( -https://hub.docker.com/r/apache/superset). Different sets of images are created for: - -- **Published releases** (`release`): with tags like `3.0.0` and the `latest` tag. - Those are multi-platform (arm+amd). More on that later. -- **Pull request iterations** (`pull_request`):, each identified by tags starting with a SHA like - `8a2f7d378ab13c156fa183d9284b607ed69f5ecc`, and `pr-3454`, referencing the pull - request ID. +https://hub.docker.com/r/apache/superset) using GitHub Actions. +Different sets of images are built and/or published at different times: + +- **Published releases** (`release`): published using + tags like `3.0.0` and the `latest` tag. +- **Pull request iterations** (`pull_request`): for each pull request, while + we actively build the docker to validate the build, we do + not publish those images for security reasons, we simply `docker build --load` - **Merges to the main branch** (`push`): resulting in new SHAs, with tags prefixed with `master` for the latest `master` version. -Each CI build run has multiple builds for different purposes, identified by suffixes: -- **Build preset:** We offer various images for different needs: - - `lean`: The default Docker image, including both frontend and backend. Tags - without a build_preset are lean builds, e.g., `latest`. - - `dev`: For development, with a headless browser and root access. - - `py310`, e.g., Py310: Similar to lean but with a different Python version (in this example, 3.10). - - `ci`: For certain CI workloads. - - `websocket`: For Superset clusters supporting advanced features. - - `dockerize`: Used by Helm. -- **Platform:** We build for `linux/arm64` and `linux/amd64`. The `-arm` suffix - indicates ARM builds (e.g., `latest-arm`), while tags without a suffix are for - AMD (e.g., `latest`). - -## Key Image Tags and Examples - -- `latest`: The latest official release build, implicitly the lean build on - `linux/amd64`. +# Build presets + +We have a set of build "presets" that each represent a combination of +parameters for the build, mostly pointing to either different target layer +for the build, and/or base image. + +Here are the build presets that are exposed through the `build_docker.py` script: +- `lean`: The default Docker image, including both frontend and backend. Tags +without a build_preset are lean builds, e.g., `latest`. +- `dev`: For development, with a headless browser, dev-related utilities and root access. +- `py310`, e.g., Py310: Similar to lean but with a different Python version (in this example, 3.10). +- `ci`: For certain CI workloads. +- `websocket`: For Superset clusters supporting advanced features. +- `dockerize`: Used by Helm. + +## Key tags examples + +- `latest`: The latest official release build - `latest-dev`: the `-dev` image of the latest official release build, with a headless browser and root access. -- `master`: The latest build from the `master` branch, implicitly lean on - `linux/amd64`. +- `master`: The latest build from the `master` branch, implicitly the lean build + preset - `master-dev`: Similar to `master` but includes a headless browser and root access. - `pr-5252`: The latest commit in PR 5252. -- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev-arm`: A `linux/arm64` build for - this specific SHA, which could be from a pull request, master merge, or release. -- `30948dc-dev-arm`: Same as above, but SHA truncated to 7 characters for a - shorter handle on the same image +- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev`: A build for + this specific SHA, which could be from a `master` merge, or release. - `websocket-latest`: The WebSocket image for use in a Superset cluster. For insights or modifications to the build matrix and tagging conventions, @@ -64,9 +64,9 @@ build times, larger images, lower layer cache hit rate, ...). For production use cases, we recommend that you derive our `lean` image(s) and add database support for the database you need. -## On supporting arm64 AND amd64 +## On supporting different platforms (namely arm64 AND amd64) -Only the release builds are multi-platform, supporting `linux/arm64` +Currently all automated builds are multi-platform, supporting both `linux/arm64` and `linux/amd64`. This enables higher level constructs like `helm` and docker-compose to point to these images and effectively be multi-platform as well.