From 07dfc2b0da35d0767c56b0e647aa4e25de1b730c Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 12:47:26 -0800 Subject: [PATCH 01/13] feat(docker): allow for docker release builds to be multi-platform Allow for release builds to be multi-platform --- .github/workflows/docker-release.yml | 14 ++---- scripts/build_docker.py | 34 +++++++++----- tests/unit_tests/scripts/docker_build.py | 60 ++++++++++++++---------- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 3ba7b6bf42fd0..ae572c910e354 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -43,14 +43,6 @@ jobs: strategy: matrix: build_preset: ["dev", "lean", "py310", "websocket", "dockerize"] - platform: ["linux/amd64", "linux/arm64"] - exclude: - # disabling because slow! no python wheels for arm/py39 and - # QEMU is slow! - - build_preset: "dev" - platform: "linux/arm64" - - build_preset: "lean" - platform: "linux/arm64" fail-fast: false steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -88,8 +80,10 @@ jobs: EVENT="release" fi pip install click + # Make a multi-platform image ./scripts/build_docker.py \ ${{ matrix.build_preset }} \ "$EVENT" \ - --build_context_ref "$RELEASE" \ - --platform ${{ matrix.platform }} $FORCE_LATEST + --build_context_ref "$RELEASE" $FORCE_LATEST \ + --platform "linux/arm64" + --platform "linux/amd64" diff --git a/scripts/build_docker.py b/scripts/build_docker.py index de6b8444f8ef8..095a0f188d236 100755 --- a/scripts/build_docker.py +++ b/scripts/build_docker.py @@ -79,7 +79,7 @@ def make_docker_tag(l: list[str]) -> str: def get_docker_tags( build_preset: str, - build_platform: str, + build_platforms: list[str], sha: str, build_context: str, build_context_ref: str, @@ -91,17 +91,18 @@ def get_docker_tags( tags: set[str] = set() tag_chunks: list[str] = [] - short_build_platform = build_platform.replace("linux/", "").replace("64", "") - is_latest = is_latest_release(build_context_ref) if build_preset != "lean": # Always add the preset_build name if different from default (lean) tag_chunks += [build_preset] - if short_build_platform != "amd": - # Always a platform indicator if different from default (amd) - tag_chunks += [short_build_platform] + if len(build_platforms) == 1: + build_platform = build_platforms[0] + short_build_platform = build_platform.replace("linux/", "").replace("64", "") + if short_build_platform != "amd": + # Always a platform indicator if different from default (amd) + tag_chunks += [short_build_platform] # Always craft a tag for the SHA tags.add(make_docker_tag([sha] + tag_chunks)) @@ -123,7 +124,7 @@ def get_docker_tags( def get_docker_command( build_preset: str, - build_platform: str, + build_platforms: list[str], is_authenticated: bool, sha: str, build_context: str, @@ -160,7 +161,7 @@ def get_docker_command( tags = get_docker_tags( build_preset, - build_platform, + build_platforms, sha, build_context, build_context_ref, @@ -170,8 +171,14 @@ def get_docker_command( docker_args = "--load" if not is_authenticated else "--push" target_argument = f"--target {build_target}" if build_target else "" - short_build_platform = build_platform.replace("linux/", "").replace("64", "") - cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" + + cache_ref = f"{CACHE_REPO}:{py_ver}" + if len(build_platforms) == 1: + build_platform = build_platforms[0] + short_build_platform = build_platform.replace("linux/", "").replace("64", "") + cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}" + platform_arg = "--platform " + ",".join(build_platforms) + cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}" cache_to_arg = ( f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else "" @@ -187,7 +194,7 @@ def get_docker_command( {cache_from_arg} \\ {cache_to_arg} \\ {build_arg} \\ - --platform {build_platform} \\ + {platform_arg} \\ --label sha={sha} \\ --label target={build_target} \\ --label build_trigger={build_context} \\ @@ -206,7 +213,8 @@ def get_docker_command( @click.option( "--platform", type=click.Choice(["linux/arm64", "linux/amd64"]), - default="linux/amd64", + default=["linux/amd64"], + multiple=True, ) @click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") @@ -217,7 +225,7 @@ def main( build_preset: str, build_context: str, build_context_ref: str, - platform: str, + platform: list[str], dry_run: bool, force_latest: bool, ) -> None: diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py index ee9ad66ead212..1f3fd708f3bb1 100644 --- a/tests/unit_tests/scripts/docker_build.py +++ b/tests/unit_tests/scripts/docker_build.py @@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool): @pytest.mark.parametrize( - "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags", + "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags", [ # PRs ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "pull_request", PR_ID, @@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool): ), ( "ci", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "pull_request", PR_ID, @@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "pull_request", PR_ID, @@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool): # old releases ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "release", OLD_REL, @@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "release", OLD_REL, @@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "release", OLD_REL, @@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "release", OLD_REL, @@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool): # new releases ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "release", NEW_REL, @@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "release", NEW_REL, @@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "release", NEW_REL, @@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "release", NEW_REL, @@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool): # merge on master ( "lean", - "linux/arm64", + ["linux/arm64"], SHA, "push", "master", @@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool): ), ( "lean", - "linux/amd64", + ["linux/amd64"], SHA, "push", "master", @@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/arm64", + ["linux/arm64"], SHA, "push", "master", @@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool): ), ( "dev", - "linux/amd64", + ["linux/amd64"], SHA, "push", "master", @@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool): ], ) def test_get_docker_tags( - build_preset, build_platform, sha, build_context, build_context_ref, expected_tags + build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags ): tags = docker_utils.get_docker_tags( - build_preset, build_platform, sha, build_context, build_context_ref + build_preset, build_platforms, sha, build_context, build_context_ref ) for tag in expected_tags: assert tag in tags @pytest.mark.parametrize( - "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains", + "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains", [ ( "lean", - "linux/amd64", + ["linux/amd64"], True, SHA, "push", @@ -251,18 +251,28 @@ def test_get_docker_tags( ), ( "dev", - "linux/amd64", + ["linux/amd64"], False, SHA, "push", "master", ["--load", f"-t {REPO}:master-dev "], ), + # multi-platform + ( + "lean", + ["linux/arm64", "linux/amd64"], + True, + SHA, + "push", + "master", + [f"--platform linux/arm64,linux/amd64"], + ), ], ) def test_get_docker_command( build_preset, - build_platform, + build_platforms, is_authenticated, sha, build_context, @@ -271,7 +281,7 @@ def test_get_docker_command( ): cmd = docker_utils.get_docker_command( build_preset, - build_platform, + build_platforms, is_authenticated, sha, build_context, From 68f71d47289ee957790ca3e77623bc8b9ac7b334 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 12:58:55 -0800 Subject: [PATCH 02/13] missing slash --- .github/workflows/docker-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index ae572c910e354..22ee833f3eb7f 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -85,5 +85,5 @@ jobs: ${{ matrix.build_preset }} \ "$EVENT" \ --build_context_ref "$RELEASE" $FORCE_LATEST \ - --platform "linux/arm64" + --platform "linux/arm64" \ --platform "linux/amd64" From 2e6bba1fa5b2278edf42b3c870926437ebb1d61b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 16:15:31 -0800 Subject: [PATCH 03/13] exit with error on subprocess error --- scripts/build_docker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/build_docker.py b/scripts/build_docker.py index 095a0f188d236..5ef0825e06c66 100755 --- a/scripts/build_docker.py +++ b/scripts/build_docker.py @@ -40,6 +40,10 @@ def run_cmd(command: str) -> str: output += line process.wait() # Wait for the subprocess to finish + + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command, output) + return output From f58fbb56677f01bdc01a8453fd9e36a600d06076 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 16:47:54 -0800 Subject: [PATCH 04/13] trying to resolve cache conflicts --- Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc3e66703783e..c618e8ed75887 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,9 @@ FROM --platform=${BUILDPLATFORM} node:16-bookworm-slim AS superset-node ARG NPM_BUILD_CMD="build" -RUN apt-get update -qq \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ build-essential \ python3 @@ -61,7 +63,9 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 -RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ && apt-get update -qq && apt-get install -yqq --no-install-recommends \ build-essential \ @@ -111,7 +115,9 @@ ARG GECKODRIVER_VERSION=v0.33.0 \ USER root -RUN apt-get update -qq \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ libnss3 \ libdbus-glib-1-2 \ From 91dc94ea36e21e7a3a17d4c043955671ae8ce651 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 17:41:45 -0800 Subject: [PATCH 05/13] pointing to the right place --- .github/workflows/ephemeral-env.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 8cc7966e00853..05f5ca414404c 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -101,6 +101,7 @@ jobs: } core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`) const pr = await github.rest.pulls.get(request); + console.log(pr.data); return pr.data; - name: Debug @@ -143,10 +144,9 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: superset-ci - SHA: ${{ steps.get-sha.outputs.sha }} IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci run: | - docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:$SHA + docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci docker push -a $ECR_REGISTRY/$ECR_REPOSITORY ephemeral-env-up: @@ -181,7 +181,7 @@ jobs: aws ecr describe-images \ --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ --repository-name superset-ci \ - --image-ids imageTag=${{ steps.get-sha.outputs.sha }} + --image-ids imageTag=pr-${{ github.event.issue.number }}-ci - name: Fail on missing container image if: steps.check-image.outcome == 'failure' @@ -204,7 +204,7 @@ jobs: with: task-definition: .github/workflows/ecs-task-definition.json container-name: superset-ci - image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }} + image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci - name: Update env vars in the Amazon ECS task definition run: | From 71fd8c3ba3dcbcbec0b7bb11fc712ac48890e4de Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 19:14:23 -0800 Subject: [PATCH 06/13] try no-cache --- Dockerfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c618e8ed75887..1b8958ab03439 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,7 @@ FROM --platform=${BUILDPLATFORM} node:16-bookworm-slim AS superset-node ARG NPM_BUILD_CMD="build" -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ +RUN --no-cache \ apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ build-essential \ @@ -63,8 +62,7 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ +RUN --no-cache \ mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ && apt-get update -qq && apt-get install -yqq --no-install-recommends \ @@ -115,8 +113,7 @@ ARG GECKODRIVER_VERSION=v0.33.0 \ USER root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ +RUN --no-cache \ apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ libnss3 \ From 4af4b8a555e470c5bbf9da092316d73056b5ff0b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 19:38:15 -0800 Subject: [PATCH 07/13] CACHE --- Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1b8958ab03439..3604f35e2dae5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,9 @@ FROM --platform=${BUILDPLATFORM} node:16-bookworm-slim AS superset-node ARG NPM_BUILD_CMD="build" -RUN --no-cache \ +RUN \ + --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ + --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ build-essential \ @@ -62,7 +64,9 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 -RUN --no-cache \ +RUN \ + --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ + --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ && apt-get update -qq && apt-get install -yqq --no-install-recommends \ @@ -113,7 +117,9 @@ ARG GECKODRIVER_VERSION=v0.33.0 \ USER root -RUN --no-cache \ +RUN \ + --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ + --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ libnss3 \ From 02fb8215579d3db3de8b91f4a7a3ba7661c3d177 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 19:43:50 -0800 Subject: [PATCH 08/13] verbose --- Dockerfile | 8 +++++--- scripts/build_docker.py | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3604f35e2dae5..bd46527f0b808 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,12 +64,14 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 +RUN \ + mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ + && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset + RUN \ --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ - mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ - && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ - && apt-get update -qq && apt-get install -yqq --no-install-recommends \ + apt-get update -qq && apt-get install -yqq --no-install-recommends \ build-essential \ curl \ default-libmysqlclient-dev \ diff --git a/scripts/build_docker.py b/scripts/build_docker.py index 5ef0825e06c66..2a9e676d9263b 100755 --- a/scripts/build_docker.py +++ b/scripts/build_docker.py @@ -222,6 +222,7 @@ def get_docker_command( ) @click.option("--build_context_ref", help="a reference to the pr, release or branch") @click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.") +@click.option("--verbose", is_flag=True, help="Print more info") @click.option( "--force-latest", is_flag=True, help="Force the 'latest' tag on the release" ) @@ -232,6 +233,7 @@ def main( platform: list[str], dry_run: bool, force_latest: bool, + verbose: bool, ) -> None: """ This script executes docker build and push commands based on given arguments. @@ -274,6 +276,8 @@ def main( """ ) script = script + docker_build_command + if verbose: + run_cmd("cat Dockerfile") stdout = run_cmd(script) else: print("Dry Run - Docker Build Command:") From 6a9211fb422cfd8f87f7c694dfabb4890129af75 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 19:50:11 -0800 Subject: [PATCH 09/13] using current dockerfile --- .github/workflows/docker-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 22ee833f3eb7f..5a6c826d9ed69 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -75,8 +75,10 @@ jobs: fi # build_docker.py may not exist on that SHA, let's switcharoo to /tmp cp ./scripts/build_docker.py /tmp + cp ./Dockerfile /tmp git checkout "${{ github.event.inputs.git-ref }}" cp /tmp/build_docker.py scripts/ + cp /Dockerfile ./ EVENT="release" fi pip install click From 4f2a11b2ab49546c3f55384ad979b253e502c177 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 19:52:15 -0800 Subject: [PATCH 10/13] using current dockerfile --- .github/workflows/docker-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 5a6c826d9ed69..6bcac3e934388 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -78,7 +78,7 @@ jobs: cp ./Dockerfile /tmp git checkout "${{ github.event.inputs.git-ref }}" cp /tmp/build_docker.py scripts/ - cp /Dockerfile ./ + cp /tmp/Dockerfile ./ EVENT="release" fi pip install click From 57c1f485586afa5f7d4ffd3d4558321ef48f2c5f Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 19:55:09 -0800 Subject: [PATCH 11/13] rm --- .github/workflows/docker-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 6bcac3e934388..22ee833f3eb7f 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -75,10 +75,8 @@ jobs: fi # build_docker.py may not exist on that SHA, let's switcharoo to /tmp cp ./scripts/build_docker.py /tmp - cp ./Dockerfile /tmp git checkout "${{ github.event.inputs.git-ref }}" cp /tmp/build_docker.py scripts/ - cp /tmp/Dockerfile ./ EVENT="release" fi pip install click From c7a36ee9462f3a39d74c4a16fb44b5616a79295f Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 20:02:37 -0800 Subject: [PATCH 12/13] triggering docker when releases are edited --- .github/workflows/docker-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 22ee833f3eb7f..73f4133848f81 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -2,7 +2,7 @@ name: Docker Publish Release on: release: - types: [published] + types: [published, edited] # Can be triggered manually workflow_dispatch: From 0655ac23729e7fadcea8d53d5cce8c04792ab888 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 8 Feb 2024 20:04:50 -0800 Subject: [PATCH 13/13] roll back dockerfile changes --- .github/workflows/ephemeral-env.yml | 1 - Dockerfile | 21 +++++---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 05f5ca414404c..8f804f741ce9d 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -101,7 +101,6 @@ jobs: } core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`) const pr = await github.rest.pulls.get(request); - console.log(pr.data); return pr.data; - name: Debug diff --git a/Dockerfile b/Dockerfile index bd46527f0b808..fc3e66703783e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,10 +26,7 @@ FROM --platform=${BUILDPLATFORM} node:16-bookworm-slim AS superset-node ARG NPM_BUILD_CMD="build" -RUN \ - --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ - --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ - apt-get update -qq \ +RUN apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ build-essential \ python3 @@ -64,14 +61,9 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 -RUN \ - mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ - && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset - -RUN \ - --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ - --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ - apt-get update -qq && apt-get install -yqq --no-install-recommends \ +RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ + && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ + && apt-get update -qq && apt-get install -yqq --no-install-recommends \ build-essential \ curl \ default-libmysqlclient-dev \ @@ -119,10 +111,7 @@ ARG GECKODRIVER_VERSION=v0.33.0 \ USER root -RUN \ - --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/lib/apt/lists \ - --mount=type=cache,id=apt-cache-${TARGETARCH}-${TARGETVARIANT},sharing=locked,target=/var/cache/apt \ - apt-get update -qq \ +RUN apt-get update -qq \ && apt-get install -yqq --no-install-recommends \ libnss3 \ libdbus-glib-1-2 \