From f3605aa942c9c0daad62ff00c331d38ece0ba97b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 16 Feb 2024 10:43:27 -0800 Subject: [PATCH 1/5] feat(ci): no more docker builds on PR-related events Speaking with the devexp committee, and in the recent community hall meeting, it's pretty clear we don't need to build the whole docker matrix on every PR event. It's somewhat expensive and isn't useful. One of the reason we did it, from what I've gathered, is we want to catch if some change breaks the docker build for whatever reason. Now we may not need to actively prevent this pre-merge as this is unlikely, yet we can still catch it on `master` as I'm keeping the push events on master. Note that: - I'm also moving the master push build to be multi-platform (amd+arm) as we started doing this for the release builds, which is nice as the same image can be used on different archs - The PR events were not "pushed" or cached in our shared, remote dockerhub-hosted repo anyways, they were just kind of dry-runs Oh one more thing, notice the no-op.yml changes? That's because the required checks related to branch protection are always based on what's defined on .asf.yml ON `master`, meaning i need to add dummy/no-op checks to get this merged, and those can be cleaned up later on in a follow up PR. Let me know if you know a better way to deal with this. --- .asf.yaml | 8 -------- .github/workflows/docker.yml | 17 ++++++----------- .github/workflows/no-op.yml | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 180414f24aa59..e1904d082f1a0 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -73,14 +73,6 @@ github: - test-postgres (3.9) - test-postgres (3.10) - test-sqlite (3.9) - - docker-build (dev, linux/amd64) - - docker-build (lean, linux/amd64) - - docker-build (py310, linux/arm64) - - docker-build (py310, linux/amd64) - - docker-build (websocket, linux/arm64) - - docker-build (websocket, linux/amd64) - - docker-build (dockerize, linux/arm64) - - docker-build (dockerize, linux/amd64) required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae54835e0fb73..0a46872b39a2c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,8 +4,6 @@ on: push: branches: - 'master' - pull_request: - types: [synchronize, opened, reopened, ready_for_review] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} @@ -18,14 +16,6 @@ jobs: strategy: matrix: build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] - platform: ["linux/amd64", "linux/arm64"] - exclude: - # disabling because slow! no python wheels for arm/py39 and - # QEMU is slow! - - build_preset: "dev" - platform: "linux/arm64" - - build_preset: "lean" - platform: "linux/arm64" fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -46,4 +36,9 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | pip install click - ./scripts/build_docker.py ${{ matrix.build_preset }} ${{ github.event_name }} --platform ${{ matrix.platform }} + ./scripts/build_docker.py \ + ${{ matrix.build_preset }} \ + ${{ github.event_name }} \ + --build_context_ref "$RELEASE" $FORCE_LATEST \ + --platform "linux/arm64" \ + --platform "linux/amd64" diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index 835063640aa96..c114a27009405 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -43,6 +43,23 @@ jobs: exit 0 python-lint: + strategy: + matrix: + build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] + platform: ["linux/amd64", "linux/arm64"] + exclude: + - build_preset: "dev" + platform: "linux/arm64" + - build_preset: "lean" + platform: "linux/arm64" + runs-on: ubuntu-latest + steps: + - name: No-op for python-lint + run: | + echo "This is a no-op step for python-lint to ensure a successful status." + exit 0 + # section to be deleted after master merge + docker: strategy: matrix: python-version: ["3.9"] @@ -52,3 +69,11 @@ jobs: run: | echo "This is a no-op step for python-lint to ensure a successful status." exit 0 + - docker-build (dev, linux/amd64) + - docker-build (lean, linux/amd64) + - docker-build (py310, linux/arm64) + - docker-build (py310, linux/amd64) + - docker-build (websocket, linux/arm64) + - docker-build (websocket, linux/amd64) + - docker-build (dockerize, linux/arm64) + - docker-build (dockerize, linux/amd64) From 9ae83d24491727877b1e8e310ea940c282bd67fb Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 16 Feb 2024 10:54:08 -0800 Subject: [PATCH 2/5] oops --- .github/workflows/no-op.yml | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index c114a27009405..ae8371e5a2ea0 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -45,13 +45,7 @@ jobs: python-lint: strategy: matrix: - build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] - platform: ["linux/amd64", "linux/arm64"] - exclude: - - build_preset: "dev" - platform: "linux/arm64" - - build_preset: "lean" - platform: "linux/arm64" + python-version: ["3.9"] runs-on: ubuntu-latest steps: - name: No-op for python-lint @@ -62,18 +56,16 @@ jobs: docker: strategy: matrix: - python-version: ["3.9"] + build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] + platform: ["linux/amd64", "linux/arm64"] + exclude: + - build_preset: "dev" + platform: "linux/arm64" + - build_preset: "lean" + platform: "linux/arm64" runs-on: ubuntu-latest steps: - - name: No-op for python-lint + - name: No-op for docker run: | echo "This is a no-op step for python-lint to ensure a successful status." exit 0 - - docker-build (dev, linux/amd64) - - docker-build (lean, linux/amd64) - - docker-build (py310, linux/arm64) - - docker-build (py310, linux/amd64) - - docker-build (websocket, linux/arm64) - - docker-build (websocket, linux/amd64) - - docker-build (dockerize, linux/arm64) - - docker-build (dockerize, linux/amd64) From 009241b349a1fcfda6595ba253ea881ebdff933b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 16 Feb 2024 10:56:35 -0800 Subject: [PATCH 3/5] oops --- .github/workflows/no-op.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml index ae8371e5a2ea0..afea397e00635 100644 --- a/.github/workflows/no-op.yml +++ b/.github/workflows/no-op.yml @@ -19,7 +19,7 @@ # - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows. # - This workflow should be kept as-is, without path-specific conditions. -name: No Operation Checks +name: no-op Checks on: pull_request jobs: @@ -53,7 +53,7 @@ jobs: echo "This is a no-op step for python-lint to ensure a successful status." exit 0 # section to be deleted after master merge - docker: + docker-build: strategy: matrix: build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] @@ -67,5 +67,5 @@ jobs: steps: - name: No-op for docker run: | - echo "This is a no-op step for python-lint to ensure a successful status." + echo "No-op to get rid of the docker-build checks reqs in .asf.yml" exit 0 From 1e0fcc190c238cb7524bf9dd3174f5542f269646 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 20 Feb 2024 12:36:41 -0600 Subject: [PATCH 4/5] incorporating feedback --- .github/workflows/docker.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0a46872b39a2c..f6be6c525c73b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,9 @@ on: push: branches: - 'master' + pull_request: + branches: + - 'master' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} @@ -15,7 +18,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] + build_preset: | + ${{ fromJson(github.event_name == 'pull_request' && github.base_ref == 'master' ? '["dev"]' : '["dev", "lean", "py310", "websocket", "dockerize"]') }} fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" From 263f58ce8149cb69efd3be5592254e54cdcfc55b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 20 Feb 2024 13:02:02 -0600 Subject: [PATCH 5/5] update the docs --- .github/workflows/docker.yml | 2 +- docs/docs/installation/docker.mdx | 66 +++++++++++++++---------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f6be6c525c73b..75da2e1442dd7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: build_preset: | - ${{ fromJson(github.event_name == 'pull_request' && github.base_ref == 'master' ? '["dev"]' : '["dev", "lean", "py310", "websocket", "dockerize"]') }} + ${{ fromJson(github.event_name == 'pull_request' ? '["ci"]' : '["dev", "lean", "py310", "websocket", "dockerize"]') }} fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/docs/docs/installation/docker.mdx b/docs/docs/installation/docker.mdx index 407bb74c4a2b8..170ca1461aa88 100644 --- a/docs/docs/installation/docker.mdx +++ b/docs/docs/installation/docker.mdx @@ -1,47 +1,47 @@ -# Docker Images and Tags +# Docker builds, images and tags The Apache Superset community extensively uses Docker for development, release, and productionizing Superset. This page details our Docker builds and tag naming schemes to help users navigate our offerings. Images are built and pushed to the [Superset Docker Hub repository]( -https://hub.docker.com/r/apache/superset). Different sets of images are created for: - -- **Published releases** (`release`): with tags like `3.0.0` and the `latest` tag. - Those are multi-platform (arm+amd). More on that later. -- **Pull request iterations** (`pull_request`):, each identified by tags starting with a SHA like - `8a2f7d378ab13c156fa183d9284b607ed69f5ecc`, and `pr-3454`, referencing the pull - request ID. +https://hub.docker.com/r/apache/superset) using GitHub Actions. +Different sets of images are built and/or published at different times: + +- **Published releases** (`release`): published using + tags like `3.0.0` and the `latest` tag. +- **Pull request iterations** (`pull_request`): for each pull request, while + we actively build the docker to validate the build, we do + not publish those images for security reasons, we simply `docker build --load` - **Merges to the main branch** (`push`): resulting in new SHAs, with tags prefixed with `master` for the latest `master` version. -Each CI build run has multiple builds for different purposes, identified by suffixes: -- **Build preset:** We offer various images for different needs: - - `lean`: The default Docker image, including both frontend and backend. Tags - without a build_preset are lean builds, e.g., `latest`. - - `dev`: For development, with a headless browser and root access. - - `py310`, e.g., Py310: Similar to lean but with a different Python version (in this example, 3.10). - - `ci`: For certain CI workloads. - - `websocket`: For Superset clusters supporting advanced features. - - `dockerize`: Used by Helm. -- **Platform:** We build for `linux/arm64` and `linux/amd64`. The `-arm` suffix - indicates ARM builds (e.g., `latest-arm`), while tags without a suffix are for - AMD (e.g., `latest`). - -## Key Image Tags and Examples - -- `latest`: The latest official release build, implicitly the lean build on - `linux/amd64`. +# Build presets + +We have a set of build "presets" that each represent a combination of +parameters for the build, mostly pointing to either different target layer +for the build, and/or base image. + +Here are the build presets that are exposed through the `build_docker.py` script: +- `lean`: The default Docker image, including both frontend and backend. Tags +without a build_preset are lean builds, e.g., `latest`. +- `dev`: For development, with a headless browser, dev-related utilities and root access. +- `py310`, e.g., Py310: Similar to lean but with a different Python version (in this example, 3.10). +- `ci`: For certain CI workloads. +- `websocket`: For Superset clusters supporting advanced features. +- `dockerize`: Used by Helm. + +## Key tags examples + +- `latest`: The latest official release build - `latest-dev`: the `-dev` image of the latest official release build, with a headless browser and root access. -- `master`: The latest build from the `master` branch, implicitly lean on - `linux/amd64`. +- `master`: The latest build from the `master` branch, implicitly the lean build + preset - `master-dev`: Similar to `master` but includes a headless browser and root access. - `pr-5252`: The latest commit in PR 5252. -- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev-arm`: A `linux/arm64` build for - this specific SHA, which could be from a pull request, master merge, or release. -- `30948dc-dev-arm`: Same as above, but SHA truncated to 7 characters for a - shorter handle on the same image +- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev`: A build for + this specific SHA, which could be from a `master` merge, or release. - `websocket-latest`: The WebSocket image for use in a Superset cluster. For insights or modifications to the build matrix and tagging conventions, @@ -64,9 +64,9 @@ build times, larger images, lower layer cache hit rate, ...). For production use cases, we recommend that you derive our `lean` image(s) and add database support for the database you need. -## On supporting arm64 AND amd64 +## On supporting different platforms (namely arm64 AND amd64) -Only the release builds are multi-platform, supporting `linux/arm64` +Currently all automated builds are multi-platform, supporting both `linux/arm64` and `linux/amd64`. This enables higher level constructs like `helm` and docker-compose to point to these images and effectively be multi-platform as well.