Only run separate per-platform build when preparing build cache (#24023)

Apparently pushing multi-platform images when building cache on CI has some problems recently, connected with ghcr.io being more vulnerable to race condition described in this issue: containerd/containerd#5978 Apparently when two, different platform layers are pushed about the same time to ghcr.io, the error "cannot reuse body, request must be retried" is generated. However we actually do not even need to build the multiplatform latest images because as of recently we have separate cache for each platform, and the ghcr.io/:latest images are not used any more not even for docker builds. We we always build images rather than pull and we use --from-cache for that - specific per platform. The only image pulling we do is when we pull the :COMMIT_HASH images in CI- but those are single-platform images (amd64) and even if we add tests for arm, they will have different tag. Hopefully we can still build release images without causing the race condition too frequently - this is more likely because when we build images for cache we use machines with different performance characteristics and the same layers are pushed at different times from different platforms. (cherry picked from commit 5d05fcd)
apache · Jun 1, 2022 · 871915c · 871915c
1 parent d2b766f
commit 871915c
Show file tree

Hide file tree

Showing 14 changed files with 296 additions and 340 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1664,6 +1664,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       fail-fast: false
       matrix:
         python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
+        platform: ["linux/amd64", "linux/arm64"]
     env:
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
@@ -1685,11 +1686,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         run: breeze free-space
       - name: "Start ARM instance"
         run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+        if: matrix.platform == 'linux/arm64'
       - name: "Build & Push CI image ${{ matrix.python-version }}:latest"
         run: >
           breeze build-image
           --prepare-buildx-cache
-          --platform linux/amd64,linux/arm64
+          --platform ${{ matrix.platform }}
         env:
           PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       - name: >
@@ -1722,12 +1724,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           --install-packages-from-context
           --prepare-buildx-cache
           --disable-airflow-repo-cache
-          --platform linux/amd64,linux/arm64
+          --platform ${{ matrix.platform }}
         env:
           PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh
-        if: always()
+        if: always() && matrix.platform == 'linux/arm64'
       - name: "Fix ownership"
         run: breeze fix-ownership
         if: always()
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -72,7 +72,7 @@
     build_cache,
     perform_environment_checks,
     prepare_docker_build_command,
-    prepare_empty_docker_build_command,
+    prepare_docker_build_from_input,
 )
 from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest
 from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed
@@ -143,13 +143,13 @@
         {
             "name": "Preparing cache and push (for maintainers and CI)",
             "options": [
-                "--platform",
-                "--prepare-buildx-cache",
-                "--push-image",
-                "--empty-image",
                 "--github-token",
                 "--github-username",
+                "--platform",
                 "--login-to-github-registry",
+                "--push-image",
+                "--empty-image",
+                "--prepare-buildx-cache",
             ],
         },
     ],
@@ -460,54 +460,45 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams)
             return 0, f"Image build: {ci_image_params.python}"
     if ci_image_params.prepare_buildx_cache or ci_image_params.push_image:
         login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose)
-    cmd = prepare_docker_build_command(
-        image_params=ci_image_params,
-        verbose=verbose,
-    )
-    if ci_image_params.empty_image:
-        env = os.environ.copy()
-        env['DOCKER_BUILDKIT'] = "1"
-        get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n")
-        cmd = prepare_empty_docker_build_command(image_params=ci_image_params)
-        build_command_result = run_command(
-            cmd,
-            input="FROM scratch\n",
-            verbose=verbose,
-            dry_run=dry_run,
-            cwd=AIRFLOW_SOURCES_ROOT,
-            text=True,
-            env=env,
-        )
+    if ci_image_params.prepare_buildx_cache:
+        build_command_result = build_cache(image_params=ci_image_params, dry_run=dry_run, verbose=verbose)
     else:
-        get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n")
-        build_command_result = run_command(
-            cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False
-        )
-        if build_command_result.returncode == 0:
-            if ci_image_params.prepare_buildx_cache:
-                build_command_result = build_cache(
-                    image_params=ci_image_params, dry_run=dry_run, verbose=verbose
-                )
-
-    if not ci_image_params.prepare_buildx_cache:
-        if not dry_run:
+        if ci_image_params.empty_image:
+            env = os.environ.copy()
+            env['DOCKER_BUILDKIT'] = "1"
+            get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n")
+            build_command_result = run_command(
+                prepare_docker_build_from_input(image_params=ci_image_params),
+                input="FROM scratch\n",
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                text=True,
+                env=env,
+            )
+        else:
+            get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n")
+            build_command_result = run_command(
+                prepare_docker_build_command(
+                    image_params=ci_image_params,
+                    verbose=verbose,
+                ),
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                text=True,
+                check=False,
+            )
             if build_command_result.returncode == 0:
                 if ci_image_params.tag_as_latest:
                     build_command_result = tag_image_as_latest(ci_image_params, dry_run, verbose)
-                if (
-                    ci_image_params.airflow_image_name == ci_image_params.airflow_image_name_with_tag
-                    or ci_image_params.tag_as_latest
-                    and build_command_result.returncode == 0
-                ):
-                    mark_image_as_refreshed(ci_image_params)
-            else:
-                get_console().print("[error]Error when building image![/]")
-                return (
-                    build_command_result.returncode,
-                    f"Image build: {ci_image_params.python}",
-                )
-        else:
-            get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]")
+                if ci_image_params.preparing_latest_image():
+                    if dry_run:
+                        get_console().print(
+                            "[info]Not updating build hash because we are in `dry_run` mode.[/]"
+                        )
+                    else:
+                        mark_image_as_refreshed(ci_image_params)
     return build_command_result.returncode, f"Image build: {ci_image_params.python}"
 
 

diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
@@ -70,7 +70,7 @@
     build_cache,
     perform_environment_checks,
     prepare_docker_build_command,
-    prepare_empty_docker_build_command,
+    prepare_docker_build_from_input,
 )
 from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest
 from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR
@@ -148,11 +148,11 @@
             "options": [
                 "--github-token",
                 "--github-username",
+                "--platform",
                 "--login-to-github-registry",
                 "--push-image",
-                "--prepare-buildx-cache",
-                "--platform",
                 "--empty-image",
+                "--prepare-buildx-cache",
             ],
         },
     ],
@@ -497,36 +497,36 @@ def build_production_image(
     if prod_image_params.prepare_buildx_cache or prod_image_params.push_image:
         login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose)
     get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n")
-    if prod_image_params.empty_image:
-        env = os.environ.copy()
-        env['DOCKER_BUILDKIT'] = "1"
-        get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n")
-        cmd = prepare_empty_docker_build_command(image_params=prod_image_params)
-        build_command_result = run_command(
-            cmd,
-            input="FROM scratch\n",
-            verbose=verbose,
-            dry_run=dry_run,
-            cwd=AIRFLOW_SOURCES_ROOT,
-            check=False,
-            text=True,
-            env=env,
-        )
+    if prod_image_params.prepare_buildx_cache:
+        build_command_result = build_cache(image_params=prod_image_params, dry_run=dry_run, verbose=verbose)
     else:
-        cmd = prepare_docker_build_command(
-            image_params=prod_image_params,
-            verbose=verbose,
-        )
-        build_command_result = run_command(
-            cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True
-        )
-        if build_command_result.returncode == 0:
-            if prod_image_params.prepare_buildx_cache:
-                build_command_result = build_cache(
-                    image_params=prod_image_params, dry_run=dry_run, verbose=verbose
-                )
-            else:
+        if prod_image_params.empty_image:
+            env = os.environ.copy()
+            env['DOCKER_BUILDKIT'] = "1"
+            get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n")
+            build_command_result = run_command(
+                prepare_docker_build_from_input(image_params=prod_image_params),
+                input="FROM scratch\n",
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                check=False,
+                text=True,
+                env=env,
+            )
+        else:
+            build_command_result = run_command(
+                prepare_docker_build_command(
+                    image_params=prod_image_params,
+                    verbose=verbose,
+                ),
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                check=False,
+                text=True,
+            )
+            if build_command_result.returncode == 0:
                 if prod_image_params.tag_as_latest:
                     build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose)
-
     return build_command_result.returncode, f"Image build: {prod_image_params.python}"
diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
@@ -20,13 +20,12 @@
 
 from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
 from airflow_breeze.global_constants import get_airflow_version
-from airflow_breeze.params._common_build_params import _CommonBuildParams
-from airflow_breeze.utils.console import get_console
+from airflow_breeze.params.common_build_params import CommonBuildParams
 from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
 
 
 @dataclass
-class BuildCiParams(_CommonBuildParams):
+class BuildCiParams(CommonBuildParams):
     """
     CI build parameters. Those parameters are used to determine command issued to build CI image.
     """
@@ -96,6 +95,4 @@ def optional_image_args(self) -> List[str]:
         ]
 
     def __post_init__(self):
-        if self.prepare_buildx_cache:
-            get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]")
-            self.push_image = True
+        pass
diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
@@ -30,12 +30,12 @@
     get_airflow_extras,
     get_airflow_version,
 )
-from airflow_breeze.params._common_build_params import _CommonBuildParams
+from airflow_breeze.params.common_build_params import CommonBuildParams
 from airflow_breeze.utils.console import get_console
 
 
 @dataclass
-class BuildProdParams(_CommonBuildParams):
+class BuildProdParams(CommonBuildParams):
     """
     PROD build parameters. Those parameters are used to determine command issued to build PROD image.
     """

diff --git a/...low_breeze/params/_common_build_params.py → ...flow_breeze/params/common_build_params.py b/...low_breeze/params/_common_build_params.py → ...flow_breeze/params/common_build_params.py
@@ -27,7 +27,7 @@
 
 
 @dataclass
-class _CommonBuildParams:
+class CommonBuildParams:
     """
     Common build parameters. Those parameters are common parameters for CI And PROD build.
     """
@@ -147,6 +147,9 @@ def get_cache(self, single_platform: str) -> str:
     def is_multi_platform(self) -> bool:
         return "," in self.platform
 
+    def preparing_latest_image(self) -> bool:
+        return self.tag_as_latest or self.airflow_image_name == self.airflow_image_name_with_tag
+
     @property
     def platforms(self) -> List[str]:
         return self.platform.split(",")
@@ -160,6 +163,4 @@ def optional_image_args(self) -> List[str]:
         raise NotImplementedError()
 
     def __post_init__(self):
-        if self.prepare_buildx_cache:
-            get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]")
-            self.push_image = True
+        pass
diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py
@@ -284,8 +284,7 @@
 )
 option_prepare_buildx_cache = click.option(
     '--prepare-buildx-cache',
-    help='Prepares build cache additionally to building images (this is done as two separate steps after'
-    'the images are build). Implies --push-image flag',
+    help='Prepares build cache (this is done as separate per-platform steps instead of building the image).',
     is_flag=True,
     envvar='PREPARE_BUILDX_CACHE',
 )