diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8de7a7f9b184..779b4f8c0f083 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1689,6 +1689,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: > breeze build-image --prepare-buildx-cache + --max-retries 3 --platform linux/amd64,linux/arm64 env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} @@ -1722,6 +1723,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --install-packages-from-context --prepare-buildx-cache --disable-airflow-repo-cache + --max-retries 3 --platform linux/amd64,linux/arm64 env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 08c39adb456f9..230e1752e40b2 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -18,6 +18,7 @@ import os import sys from pathlib import Path +from subprocess import CompletedProcess from typing import List, Optional, Tuple, Union import click @@ -50,6 +51,7 @@ option_image_name, option_image_tag, option_install_providers_from_sources, + option_max_retries, option_parallelism, option_platform, option_prepare_buildx_cache, @@ -87,6 +89,7 @@ instruct_build_image, is_repo_rebased, run_command, + run_result_contains, ) CI_IMAGE_TOOLS_COMMANDS = { @@ -110,6 +113,7 @@ "--tag-as-latest", "--docker-cache", "--force-build", + "--max-retries", ], }, { @@ -202,6 +206,7 @@ @option_docker_cache @option_image_tag @option_prepare_buildx_cache +@option_max_retries @option_push_image @option_empty_image @option_install_providers_from_sources @@ -477,15 +482,34 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) ) else: get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n") - build_command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False - ) - if build_command_result.returncode == 0: - if ci_image_params.prepare_buildx_cache: + num_tries = 1 if ci_image_params.max_retries is None else ci_image_params.max_retries + build_command_result = CompletedProcess(args=[], returncode=1, stdout="This should never happen.") + while num_tries > 0: + build_command_result = run_command( + cmd, + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + check=False, + text=True, + capture_output=True, + ) + if ci_image_params.prepare_buildx_cache and build_command_result.returncode == 0: build_command_result = build_cache( image_params=ci_image_params, dry_run=dry_run, verbose=verbose ) - + if build_command_result.returncode == 0: + break + num_tries -= 1 + if run_result_contains(build_command_result, "cannot reuse body, request must be retried"): + if num_tries > 0: + get_console().print( + "[info]Retrying failed command on retryable condition. " + f"There are {num_tries} left[/]" + ) + continue + else: + break if not ci_image_params.prepare_buildx_cache: if not dry_run: if build_command_result.returncode == 0: @@ -504,7 +528,9 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) f"Image build: {ci_image_params.python}", ) else: - get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]") + get_console().print( + "[info]Not tagging/marking image as refreshed because we are in `dry_run` mode.[/]" + ) return build_command_result.returncode, f"Image build: {ci_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 32aed85d9ed14..5253edef55b8f 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -17,6 +17,7 @@ import contextlib import os import sys +from subprocess import CompletedProcess from typing import Optional, Tuple import click @@ -49,6 +50,7 @@ option_image_name, option_image_tag, option_install_providers_from_sources, + option_max_retries, option_parallelism, option_platform, option_prepare_buildx_cache, @@ -77,7 +79,12 @@ from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image -from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command +from airflow_breeze.utils.run_utils import ( + filter_out_none, + fix_group_permissions, + run_command, + run_result_contains, +) PRODUCTION_IMAGE_TOOLS_COMMANDS = { "name": "Production Image tools", @@ -99,6 +106,7 @@ "--image-tag", "--tag-as-latest", "--docker-cache", + "--max-retries", ], }, { @@ -206,6 +214,7 @@ @option_docker_cache @option_image_tag @option_prepare_buildx_cache +@option_max_retries @option_push_image @option_empty_image @option_airflow_constraints_mode_prod @@ -517,16 +526,36 @@ def build_production_image( image_params=prod_image_params, verbose=verbose, ) - build_command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True - ) - if build_command_result.returncode == 0: - if prod_image_params.prepare_buildx_cache: - build_command_result = build_cache( - image_params=prod_image_params, dry_run=dry_run, verbose=verbose - ) + num_tries = 1 if prod_image_params.max_retries is None else prod_image_params.max_retries + build_command_result = CompletedProcess(args=[], returncode=1, stdout="This should never happen.") + while num_tries > 0: + build_command_result = run_command( + cmd, + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + check=False, + text=True, + capture_output=True, + ) + if build_command_result.returncode == 0: + if prod_image_params.prepare_buildx_cache: + build_command_result = build_cache( + image_params=prod_image_params, dry_run=dry_run, verbose=verbose + ) + else: + if prod_image_params.tag_as_latest: + build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose) + if build_command_result.returncode == 0: + break + num_tries -= 1 + if run_result_contains(build_command_result, "cannot reuse body, request must be retried"): + if num_tries > 0: + get_console().print( + "[info]Retrying failed command on retryable condition. " + f"There are {num_tries} left[/]" + ) + continue else: - if prod_image_params.tag_as_latest: - build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose) - + break return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/params/_common_build_params.py b/dev/breeze/src/airflow_breeze/params/_common_build_params.py index 4f2139d20e723..78ab06b36992d 100644 --- a/dev/breeze/src/airflow_breeze/params/_common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/_common_build_params.py @@ -55,6 +55,7 @@ class _CommonBuildParams: github_token: str = os.environ.get('GITHUB_TOKEN', "") github_username: str = "" image_tag: Optional[str] = None + max_retries: Optional[int] = None install_providers_from_sources: bool = False platform: str = f"linux/{os.uname().machine}" prepare_buildx_cache: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 545a6ec149912..78adf7aa60e77 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -289,6 +289,12 @@ is_flag=True, envvar='PREPARE_BUILDX_CACHE', ) +option_max_retries = click.option( + '--max-retries', + help='Maximum number of retries for the operation for "retryable" intermittent problems.', + type=click.IntRange(min=2), + envvar='MAX_RETRIES', +) option_push_image = click.option( '--push-image', help='Push image after building it.', diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 86b84be4c076f..f297ecca46bed 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -405,3 +405,11 @@ def get_runnable_ci_image(verbose: bool, dry_run: bool) -> str: instruction=f"breeze build-image --python {python_version}", ) return airflow_image + + +def run_result_contains(result: RunCommandResult, message: str) -> bool: + if result.stdout and message in result.stdout: + return True + if result.stderr and message in result.stderr: + return True + return False diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 91028e25d8dde..568a03defea45 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ - + - Command: build-image + Command: build-image -                                                                                                                          - Usage: breeze build-image [OPTIONS]                                                                                     -                                                                                                                         - Build CI image. Include building multiple images for all python versions (sequentially).                                -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ -│                                       Useful when you build or pull image with --image-tag.                          │ -│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ -│                                       [default: registry]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ -│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ -│                                      [default: constraints-source-providers]                                         │ -│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ -│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ -│                            afterthe images are build). Implies --push-image flag                                     │ -│  --push-image              Push image after building it.                                                             │ -│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ -│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ -│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-image [OPTIONS]                                                                                     +                                                                                                                         + Build CI image. Include building multiple images for all python versions (sequentially).                                +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ +│                                       Useful when you build or pull image with --image-tag.                          │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +│  --max-retries                        Maximum number of retries for the operation for "retryable" intermittent       │ +│                                       problems.                                                                      │ +│                                       (INTEGER RANGE)                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ +│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ +│                                      [default: constraints-source-providers]                                         │ +│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ +│                            afterthe images are build). Implies --push-image flag                                     │ +│  --push-image              Push image after building it.                                                             │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index d90abce791153..9a03880a5b054 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - Command: build-prod-image + Command: build-prod-image -                                                                                                                          - Usage: breeze build-prod-image [OPTIONS]                                                                                -                                                                                                                         - Build Production image. Include building multiple images for all or selected Python versions sequentially.              -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --install-airflow-version        -V  Install version of Airflow from PyPI. (TEXT)                                   │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ -│                                       Useful when you build or pull image with --image-tag.                          │ -│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ -│                                       [default: registry]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --airflow-extras                    Extras to install by default.                                                   │ -│                                      (TEXT)                                                                          │ -│                                      [default:                                                                       │ -│                                      amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,goog…  │ -│  --airflow-constraints-mode          Mode of constraints for PROD image building                                     │ -│                                      (constraints | constraints-no-providers | constraints-source-providers)         │ -│                                      [default: constraints]                                                          │ -│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│  --install-packages-from-context           Install wheels from local docker-context-files when building image.       │ -│  --airflow-is-in-context                   If set Airflow is installed from docker-context-files only rather than    │ -│                                            from PyPI or sources.                                                     │ -│  --cleanup-context                         Clean up docker context files before running build (cannot be used        │ -│                                            together with --install-packages-from-context).                           │ -│  --disable-mysql-client-installation       Do not install MySQL client.                                              │ -│  --disable-mssql-client-installation       Do not install MsSQl client.                                              │ -│  --disable-postgres-client-installation    Do not install Postgres client.                                           │ -│  --disable-airflow-repo-cache              Disable cache from Airflow repository during building.                    │ -│  --install-airflow-reference               Install Airflow using GitHub tag or branch. (TEXT)                        │ -│  --installation-method                     Install Airflow from: sources or PyPI. (. | apache-airflow)               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ -│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ -│  --push-image              Push image after building it.                                                             │ -│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ -│                            afterthe images are build). Implies --push-image flag                                     │ -│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ -│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-prod-image [OPTIONS]                                                                                +                                                                                                                         + Build Production image. Include building multiple images for all or selected Python versions sequentially.              +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --install-airflow-version        -V  Install version of Airflow from PyPI. (TEXT)                                   │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ +│                                       Useful when you build or pull image with --image-tag.                          │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +│  --max-retries                        Maximum number of retries for the operation for "retryable" intermittent       │ +│                                       problems.                                                                      │ +│                                       (INTEGER RANGE)                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-extras                    Extras to install by default.                                                   │ +│                                      (TEXT)                                                                          │ +│                                      [default:                                                                       │ +│                                      amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,goog…  │ +│  --airflow-constraints-mode          Mode of constraints for PROD image building                                     │ +│                                      (constraints | constraints-no-providers | constraints-source-providers)         │ +│                                      [default: constraints]                                                          │ +│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│  --install-packages-from-context           Install wheels from local docker-context-files when building image.       │ +│  --airflow-is-in-context                   If set Airflow is installed from docker-context-files only rather than    │ +│                                            from PyPI or sources.                                                     │ +│  --cleanup-context                         Clean up docker context files before running build (cannot be used        │ +│                                            together with --install-packages-from-context).                           │ +│  --disable-mysql-client-installation       Do not install MySQL client.                                              │ +│  --disable-mssql-client-installation       Do not install MsSQl client.                                              │ +│  --disable-postgres-client-installation    Do not install Postgres client.                                           │ +│  --disable-airflow-repo-cache              Disable cache from Airflow repository during building.                    │ +│  --install-airflow-reference               Install Airflow using GitHub tag or branch. (TEXT)                        │ +│  --installation-method                     Install Airflow from: sources or PyPI. (. | apache-airflow)               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +│  --push-image              Push image after building it.                                                             │ +│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ +│                            afterthe images are build). Implies --push-image flag                                     │ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 4d58a2969e4be..05e27f016cbc3 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -969f1b4101773456e832c03b8d4e89ef +25a28c794bb5fee96137f0aa67b40d7f