Only run separate per-platform build when preparing build cache (#24023)

Apparently pushing multi-platform images when building cache on CI
has some problems recently, connected with ghcr.io being more
vulnerable to race condition described in this issue:

containerd/containerd#5978

Apparently when two, different platform layers are pushed about
the same time to ghcr.io, the error
"cannot reuse body, request must be retried" is generated.

However we actually do not even need to build the multiplatform
latest images because as of recently we have separate cache for each
platform, and the ghcr.io/:latest images are not used any more
not even for docker builds. We we always build images rather than
pull and we use --from-cache for that - specific per platform. The only
image pulling we do is when we pull the :COMMIT_HASH images in CI- but
those are single-platform images (amd64) and even if we add tests for
arm, they will have different tag.

Hopefully we can still build release images without causing the
race condition too frequently - this is more likely because when
we build images for cache we use machines with different performance
characteristics and the same layers are pushed at different times
from different platforms.

.github/workflows/ci.yml

@@ -1664,6 +1664,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       fail-fast: false
       matrix:
         python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
+        platform: ["linux/amd64", "linux/arm64"]
     env:
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
@@ -1685,11 +1686,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         run: breeze free-space
       - name: "Start ARM instance"
         run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+        if: matrix.platform == 'linux/arm64'
       - name: "Build & Push CI image ${{ matrix.python-version }}:latest"
         run: >
           breeze build-image
           --prepare-buildx-cache
-          --platform linux/amd64,linux/arm64
+          --platform ${{ matrix.platform }}
         env:
           PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       - name: >
@@ -1722,12 +1724,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
           --install-packages-from-context
           --prepare-buildx-cache
           --disable-airflow-repo-cache
-          --platform linux/amd64,linux/arm64
+          --platform ${{ matrix.platform }}
         env:
           PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh
-        if: always()
+        if: always() && matrix.platform == 'linux/arm64'
       - name: "Fix ownership"
         run: breeze fix-ownership
         if: always()

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -71,7 +71,7 @@
     build_cache,
     perform_environment_checks,
     prepare_docker_build_command,
-    prepare_empty_docker_build_command,
+    prepare_docker_build_from_input,
 )
 from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest
 from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed
@@ -142,13 +142,13 @@
         {
             "name": "Preparing cache and push (for maintainers and CI)",
             "options": [
-                "--platform",
-                "--prepare-buildx-cache",
-                "--push-image",
-                "--empty-image",
                 "--github-token",
                 "--github-username",
+                "--platform",
                 "--login-to-github-registry",
+                "--push-image",
+                "--empty-image",
+                "--prepare-buildx-cache",
             ],
         },
     ],
@@ -457,54 +457,45 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams)
             return 0, f"Image build: {ci_image_params.python}"
     if ci_image_params.prepare_buildx_cache or ci_image_params.push_image:
         login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose)
-    cmd = prepare_docker_build_command(
-        image_params=ci_image_params,
-        verbose=verbose,
-    )
-    if ci_image_params.empty_image:
-        env = os.environ.copy()
-        env['DOCKER_BUILDKIT'] = "1"
-        get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n")
-        cmd = prepare_empty_docker_build_command(image_params=ci_image_params)
-        build_command_result = run_command(
-            cmd,
-            input="FROM scratch\n",
-            verbose=verbose,
-            dry_run=dry_run,
-            cwd=AIRFLOW_SOURCES_ROOT,
-            text=True,
-            env=env,
-        )
+    if ci_image_params.prepare_buildx_cache:
+        build_command_result = build_cache(image_params=ci_image_params, dry_run=dry_run, verbose=verbose)
     else:
-        get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n")
-        build_command_result = run_command(
-            cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False
-        )
-        if build_command_result.returncode == 0:
-            if ci_image_params.prepare_buildx_cache:
-                build_command_result = build_cache(
-                    image_params=ci_image_params, dry_run=dry_run, verbose=verbose
-                )
-
-    if not ci_image_params.prepare_buildx_cache:
-        if not dry_run:
+        if ci_image_params.empty_image:
+            env = os.environ.copy()
+            env['DOCKER_BUILDKIT'] = "1"
+            get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n")
+            build_command_result = run_command(
+                prepare_docker_build_from_input(image_params=ci_image_params),
+                input="FROM scratch\n",
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                text=True,
+                env=env,
+            )
+        else:
+            get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n")
+            build_command_result = run_command(
+                prepare_docker_build_command(
+                    image_params=ci_image_params,
+                    verbose=verbose,
+                ),
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                text=True,
+                check=False,
+            )
             if build_command_result.returncode == 0:
                 if ci_image_params.tag_as_latest:
                     build_command_result = tag_image_as_latest(ci_image_params, dry_run, verbose)
-                if (
-                    ci_image_params.airflow_image_name == ci_image_params.airflow_image_name_with_tag
-                    or ci_image_params.tag_as_latest
-                    and build_command_result.returncode == 0
-                ):
-                    mark_image_as_refreshed(ci_image_params)
-            else:
-                get_console().print("[error]Error when building image![/]")
-                return (
-                    build_command_result.returncode,
-                    f"Image build: {ci_image_params.python}",
-                )
-        else:
-            get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]")
+                if ci_image_params.preparing_latest_image():
+                    if dry_run:
+                        get_console().print(
+                            "[info]Not updating build hash because we are in `dry_run` mode.[/]"
+                        )
+                    else:
+                        mark_image_as_refreshed(ci_image_params)
     return build_command_result.returncode, f"Image build: {ci_image_params.python}"
 
 

dev/breeze/src/airflow_breeze/commands/production_image_commands.py

@@ -70,7 +70,7 @@
     build_cache,
     perform_environment_checks,
     prepare_docker_build_command,
-    prepare_empty_docker_build_command,
+    prepare_docker_build_from_input,
 )
 from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest
 from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR
@@ -148,11 +148,11 @@
             "options": [
                 "--github-token",
                 "--github-username",
+                "--platform",
                 "--login-to-github-registry",
                 "--push-image",
-                "--prepare-buildx-cache",
-                "--platform",
                 "--empty-image",
+                "--prepare-buildx-cache",
             ],
         },
     ],
@@ -497,36 +497,36 @@ def build_production_image(
     if prod_image_params.prepare_buildx_cache or prod_image_params.push_image:
         login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose)
     get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n")
-    if prod_image_params.empty_image:
-        env = os.environ.copy()
-        env['DOCKER_BUILDKIT'] = "1"
-        get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n")
-        cmd = prepare_empty_docker_build_command(image_params=prod_image_params)
-        build_command_result = run_command(
-            cmd,
-            input="FROM scratch\n",
-            verbose=verbose,
-            dry_run=dry_run,
-            cwd=AIRFLOW_SOURCES_ROOT,
-            check=False,
-            text=True,
-            env=env,
-        )
+    if prod_image_params.prepare_buildx_cache:
+        build_command_result = build_cache(image_params=prod_image_params, dry_run=dry_run, verbose=verbose)
     else:
-        cmd = prepare_docker_build_command(
-            image_params=prod_image_params,
-            verbose=verbose,
-        )
-        build_command_result = run_command(
-            cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True
-        )
-        if build_command_result.returncode == 0:
-            if prod_image_params.prepare_buildx_cache:
-                build_command_result = build_cache(
-                    image_params=prod_image_params, dry_run=dry_run, verbose=verbose
-                )
-            else:
+        if prod_image_params.empty_image:
+            env = os.environ.copy()
+            env['DOCKER_BUILDKIT'] = "1"
+            get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n")
+            build_command_result = run_command(
+                prepare_docker_build_from_input(image_params=prod_image_params),
+                input="FROM scratch\n",
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                check=False,
+                text=True,
+                env=env,
+            )
+        else:
+            build_command_result = run_command(
+                prepare_docker_build_command(
+                    image_params=prod_image_params,
+                    verbose=verbose,
+                ),
+                verbose=verbose,
+                dry_run=dry_run,
+                cwd=AIRFLOW_SOURCES_ROOT,
+                check=False,
+                text=True,
+            )
+            if build_command_result.returncode == 0:
                 if prod_image_params.tag_as_latest:
                     build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose)
-
     return build_command_result.returncode, f"Image build: {prod_image_params.python}"

dev/breeze/src/airflow_breeze/params/build_ci_params.py

@@ -20,13 +20,12 @@
 
 from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
 from airflow_breeze.global_constants import get_airflow_version
-from airflow_breeze.params._common_build_params import _CommonBuildParams
-from airflow_breeze.utils.console import get_console
+from airflow_breeze.params.common_build_params import CommonBuildParams
 from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
 
 
 @dataclass
-class BuildCiParams(_CommonBuildParams):
+class BuildCiParams(CommonBuildParams):
     """
     CI build parameters. Those parameters are used to determine command issued to build CI image.
     """
@@ -96,6 +95,4 @@ def optional_image_args(self) -> List[str]:
         ]
 
     def __post_init__(self):
-        if self.prepare_buildx_cache:
-            get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]")
-            self.push_image = True
+        pass

dev/breeze/src/airflow_breeze/params/build_prod_params.py

@@ -30,12 +30,12 @@
     get_airflow_extras,
     get_airflow_version,
 )
-from airflow_breeze.params._common_build_params import _CommonBuildParams
+from airflow_breeze.params.common_build_params import CommonBuildParams
 from airflow_breeze.utils.console import get_console
 
 
 @dataclass
-class BuildProdParams(_CommonBuildParams):
+class BuildProdParams(CommonBuildParams):
     """
     PROD build parameters. Those parameters are used to determine command issued to build PROD image.
     """

dev/breeze/src/airflow_breeze/params/_common_build_params.py → dev/breeze/src/airflow_breeze/params/common_build_params.py

@@ -27,7 +27,7 @@
 
 
 @dataclass
-class _CommonBuildParams:
+class CommonBuildParams:
     """
     Common build parameters. Those parameters are common parameters for CI And PROD build.
     """
@@ -147,6 +147,9 @@ def get_cache(self, single_platform: str) -> str:
     def is_multi_platform(self) -> bool:
         return "," in self.platform
 
+    def preparing_latest_image(self) -> bool:
+        return self.tag_as_latest or self.airflow_image_name == self.airflow_image_name_with_tag
+
     @property
     def platforms(self) -> List[str]:
         return self.platform.split(",")
@@ -160,6 +163,4 @@ def optional_image_args(self) -> List[str]:
         raise NotImplementedError()
 
     def __post_init__(self):
-        if self.prepare_buildx_cache:
-            get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]")
-            self.push_image = True
+        pass

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -284,8 +284,7 @@
 )
 option_prepare_buildx_cache = click.option(
     '--prepare-buildx-cache',
-    help='Prepares build cache additionally to building images (this is done as two separate steps after'
-    'the images are build). Implies --push-image flag',
+    help='Prepares build cache (this is done as separate per-platform steps instead of building the image).',
     is_flag=True,
     envvar='PREPARE_BUILDX_CACHE',
 )