From 107e6a000f971d6c4ea336524640271c09c7d4ca Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 1 May 2025 14:24:06 +0200 Subject: [PATCH] Bump min versions of crucial providers There are a few min-versions of providers that should make its way into apache-airflow pyproject.toml - because they contain crucial Airflow 3 fixes and users should not be installng those providers in lower versions. This PR adds such limits and makes it clearer when updates happen what happens and where to override them. --- .pre-commit-config.yaml | 2 +- .../src/airflow_breeze/utils/packages.py | 29 +++++++++++-------- pyproject.toml | 12 ++++---- .../update_airflow_pyproject_toml.py | 21 ++++++++------ 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c85cc059e7774..8e5c8cfdab09b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -889,7 +889,7 @@ repos: name: Update Airflow's meta-package pyproject.toml language: python entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py - files: ^pyproject\.toml$ + files: ^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$ pass_filenames: false require_serial: true additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1', 'packaging>=23.2' ] diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index ea4564022c8e6..3eb6e1e43d0eb 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -1040,22 +1040,27 @@ def update_version_suffix_in_non_provider_pyproject_toml(version_suffix: str, py lines = pyproject_toml_path.read_text().splitlines() updated_lines = [] for line in lines: - if line.startswith("version = "): + base_line, comment = line.split(" #", 1) if " #" in line else (line, "") + if comment: + comment = " #" + comment + if base_line.startswith("version = "): get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.") - line = line.rstrip('"') + f'{version_suffix}"' + base_line = base_line.rstrip('"') + f'{version_suffix}"' # do not modify references for .post prefixes if not version_suffix.startswith(".post"): - if line.strip().startswith('"apache-airflow-') and ">=" in line: + if base_line.strip().startswith('"apache-airflow-') and ">=" in base_line: floored_version_suffix = floor_version_suffix(version_suffix) - get_console().print(f"[info]Updating version suffix to {floored_version_suffix} for {line}.") - line = line.rstrip('",') + f'{floored_version_suffix}",' - if line.strip().startswith('"apache-airflow-core') and "==" in line: - get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.") - line = line.rstrip('",') + f'{version_suffix}",' - if line.strip().startswith('"apache-airflow-task-sdk') and "==" in line: - get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.") - line = line.rstrip('",') + f'{version_suffix}",' - updated_lines.append(line) + get_console().print( + f"[info]Updating version suffix to {floored_version_suffix} for {base_line}." + ) + base_line = base_line.rstrip('",') + f'{floored_version_suffix}",' + if base_line.strip().startswith('"apache-airflow-core') and "==" in base_line: + get_console().print(f"[info]Updating version suffix to {version_suffix} for {base_line}.") + base_line = base_line.rstrip('",') + f'{version_suffix}",' + if base_line.strip().startswith('"apache-airflow-task-sdk') and "==" in base_line: + get_console().print(f"[info]Updating version suffix to {version_suffix} for {base_line}.") + base_line = base_line.rstrip('",') + f'{version_suffix}",' + updated_lines.append(f"{base_line}{comment}") new_content = "\n".join(updated_lines) + "\n" get_console().print(f"[info]Writing updated content to {pyproject_toml_path}.\n") pyproject_toml_path.write_text(new_content) diff --git a/pyproject.toml b/pyproject.toml index 75fff4ef8a19e..85df1a667114e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,7 +212,7 @@ packages = [] "apache-airflow-providers-exasol>=4.6.1" ] "fab" = [ - "apache-airflow-providers-fab>=2.0.0" + "apache-airflow-providers-fab>=2.0.2" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py ] "facebook" = [ "apache-airflow-providers-facebook>=3.7.0" @@ -221,7 +221,7 @@ packages = [] "apache-airflow-providers-ftp>=3.12.0" ] "git" = [ - "apache-airflow-providers-git>=0.0.1" + "apache-airflow-providers-git>=0.0.2" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py ] "github" = [ "apache-airflow-providers-github>=2.8.0" @@ -281,7 +281,7 @@ packages = [] "apache-airflow-providers-openfaas>=3.7.0" ] "openlineage" = [ - "apache-airflow-providers-openlineage>=2.1.3" + "apache-airflow-providers-openlineage>=2.1.3" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py ] "opensearch" = [ "apache-airflow-providers-opensearch>=1.5.0" @@ -422,10 +422,10 @@ packages = [] "apache-airflow-providers-edge3>=1.0.0", "apache-airflow-providers-elasticsearch>=5.5.2", "apache-airflow-providers-exasol>=4.6.1", - "apache-airflow-providers-fab>=2.0.0", + "apache-airflow-providers-fab>=2.0.2", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-facebook>=3.7.0", "apache-airflow-providers-ftp>=3.12.0", - "apache-airflow-providers-git>=0.0.1", + "apache-airflow-providers-git>=0.0.2", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-github>=2.8.0", "apache-airflow-providers-google>=10.24.0", "apache-airflow-providers-grpc>=3.7.0", @@ -445,7 +445,7 @@ packages = [] "apache-airflow-providers-odbc>=4.8.0", "apache-airflow-providers-openai>=1.5.0", "apache-airflow-providers-openfaas>=3.7.0", - "apache-airflow-providers-openlineage>=2.1.3", + "apache-airflow-providers-openlineage>=2.1.3", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-opensearch>=1.5.0", "apache-airflow-providers-opsgenie>=5.8.0", "apache-airflow-providers-oracle>=3.12.0", diff --git a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py index e4c56ea7d8c53..224ecdaac8727 100755 --- a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py +++ b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py @@ -56,9 +56,9 @@ # minimum versions for compatibility with Airflow 3 MIN_VERSION_OVERRIDE: dict[str, Version] = { "amazon": parse_version("2.1.3"), - "fab": parse_version("2.0.0"), + "fab": parse_version("2.0.2"), "openlineage": parse_version("2.1.3"), - "git": parse_version("0.0.1"), + "git": parse_version("0.0.2"), "common.messaging": parse_version("1.0.0"), } @@ -88,7 +88,7 @@ def provider_path(provider_id: str) -> str: all_providers_metadata = json.loads(PROVIDER_METADATA_FILE_PATH.read_text()) -def find_min_provider_version(provider_id: str) -> Version | None: +def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]: metadata = all_providers_metadata.get(provider_id) # We should periodically update the starting date to avoid pip install resolution issues # TODO: when min Python version is 3.11 change back the code to fromisoformat @@ -101,7 +101,7 @@ def find_min_provider_version(provider_id: str) -> Version | None: min_version_override = MIN_VERSION_OVERRIDE.get(provider_id) if not metadata: if not min_version_override: - return None + return None, "" last_version_newer_than_cutoff = min_version_override else: versions: list[Version] = sorted([parse_version(version) for version in metadata], reverse=True) @@ -117,14 +117,17 @@ def find_min_provider_version(provider_id: str) -> Version | None: f"[bright_blue]Provider id {provider_id} min version found:[/] " f"{last_version_newer_than_cutoff} (date {date_released}" ) + override_comment = "" if last_version_newer_than_cutoff: if min_version_override and min_version_override > last_version_newer_than_cutoff: console.print( f"[yellow]Overriding provider id {provider_id} min version:[/] {min_version_override} " - f"overridden from hard-coded versions." + f"set from hard-coded versions.\n\n" + f"[yellow]Modify MIN_VERSION_OVERRIDE in {__file__} to set different version![/]\n" ) last_version_newer_than_cutoff = min_version_override - return last_version_newer_than_cutoff + override_comment = f" # Set from MIN_VERSION_OVERRIDE in {Path(__file__).name}" + return last_version_newer_than_cutoff, override_comment PROVIDER_MIN_VERSIONS: dict[str, str | None] = {} @@ -141,11 +144,11 @@ def find_min_provider_version(provider_id: str) -> Version | None: all_provider_lines = [] for provider_id in all_providers: distribution_name = provider_distribution_name(provider_id) - min_provider_version = find_min_provider_version(provider_id) + min_provider_version, comment = find_min_provider_version(provider_id) if min_provider_version: - all_provider_lines.append(f' "{distribution_name}>={min_provider_version}",\n') + all_provider_lines.append(f' "{distribution_name}>={min_provider_version}",{comment}\n') all_optional_dependencies.append( - f'"{provider_id}" = [\n "{distribution_name}>={min_provider_version}"\n]\n' + f'"{provider_id}" = [\n "{distribution_name}>={min_provider_version}"{comment}\n]\n' ) else: all_optional_dependencies.append(f'"{provider_id}" = [\n "{distribution_name}"\n]\n')