diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c85cc059e7774..8e5c8cfdab09b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -889,7 +889,7 @@ repos: name: Update Airflow's meta-package pyproject.toml language: python entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py - files: ^pyproject\.toml$ + files: ^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$ pass_filenames: false require_serial: true additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1', 'packaging>=23.2' ] diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index ea4564022c8e6..3eb6e1e43d0eb 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -1040,22 +1040,27 @@ def update_version_suffix_in_non_provider_pyproject_toml(version_suffix: str, py lines = pyproject_toml_path.read_text().splitlines() updated_lines = [] for line in lines: - if line.startswith("version = "): + base_line, comment = line.split(" #", 1) if " #" in line else (line, "") + if comment: + comment = " #" + comment + if base_line.startswith("version = "): get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.") - line = line.rstrip('"') + f'{version_suffix}"' + base_line = base_line.rstrip('"') + f'{version_suffix}"' # do not modify references for .post prefixes if not version_suffix.startswith(".post"): - if line.strip().startswith('"apache-airflow-') and ">=" in line: + if base_line.strip().startswith('"apache-airflow-') and ">=" in base_line: floored_version_suffix = floor_version_suffix(version_suffix) - get_console().print(f"[info]Updating version suffix to {floored_version_suffix} for {line}.") - line = line.rstrip('",') + f'{floored_version_suffix}",' - if line.strip().startswith('"apache-airflow-core') and "==" in line: - get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.") - line = line.rstrip('",') + f'{version_suffix}",' - if line.strip().startswith('"apache-airflow-task-sdk') and "==" in line: - get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.") - line = line.rstrip('",') + f'{version_suffix}",' - updated_lines.append(line) + get_console().print( + f"[info]Updating version suffix to {floored_version_suffix} for {base_line}." + ) + base_line = base_line.rstrip('",') + f'{floored_version_suffix}",' + if base_line.strip().startswith('"apache-airflow-core') and "==" in base_line: + get_console().print(f"[info]Updating version suffix to {version_suffix} for {base_line}.") + base_line = base_line.rstrip('",') + f'{version_suffix}",' + if base_line.strip().startswith('"apache-airflow-task-sdk') and "==" in base_line: + get_console().print(f"[info]Updating version suffix to {version_suffix} for {base_line}.") + base_line = base_line.rstrip('",') + f'{version_suffix}",' + updated_lines.append(f"{base_line}{comment}") new_content = "\n".join(updated_lines) + "\n" get_console().print(f"[info]Writing updated content to {pyproject_toml_path}.\n") pyproject_toml_path.write_text(new_content) diff --git a/pyproject.toml b/pyproject.toml index 75fff4ef8a19e..85df1a667114e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,7 +212,7 @@ packages = [] "apache-airflow-providers-exasol>=4.6.1" ] "fab" = [ - "apache-airflow-providers-fab>=2.0.0" + "apache-airflow-providers-fab>=2.0.2" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py ] "facebook" = [ "apache-airflow-providers-facebook>=3.7.0" @@ -221,7 +221,7 @@ packages = [] "apache-airflow-providers-ftp>=3.12.0" ] "git" = [ - "apache-airflow-providers-git>=0.0.1" + "apache-airflow-providers-git>=0.0.2" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py ] "github" = [ "apache-airflow-providers-github>=2.8.0" @@ -281,7 +281,7 @@ packages = [] "apache-airflow-providers-openfaas>=3.7.0" ] "openlineage" = [ - "apache-airflow-providers-openlineage>=2.1.3" + "apache-airflow-providers-openlineage>=2.1.3" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py ] "opensearch" = [ "apache-airflow-providers-opensearch>=1.5.0" @@ -422,10 +422,10 @@ packages = [] "apache-airflow-providers-edge3>=1.0.0", "apache-airflow-providers-elasticsearch>=5.5.2", "apache-airflow-providers-exasol>=4.6.1", - "apache-airflow-providers-fab>=2.0.0", + "apache-airflow-providers-fab>=2.0.2", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-facebook>=3.7.0", "apache-airflow-providers-ftp>=3.12.0", - "apache-airflow-providers-git>=0.0.1", + "apache-airflow-providers-git>=0.0.2", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-github>=2.8.0", "apache-airflow-providers-google>=10.24.0", "apache-airflow-providers-grpc>=3.7.0", @@ -445,7 +445,7 @@ packages = [] "apache-airflow-providers-odbc>=4.8.0", "apache-airflow-providers-openai>=1.5.0", "apache-airflow-providers-openfaas>=3.7.0", - "apache-airflow-providers-openlineage>=2.1.3", + "apache-airflow-providers-openlineage>=2.1.3", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-opensearch>=1.5.0", "apache-airflow-providers-opsgenie>=5.8.0", "apache-airflow-providers-oracle>=3.12.0", diff --git a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py index e4c56ea7d8c53..224ecdaac8727 100755 --- a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py +++ b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py @@ -56,9 +56,9 @@ # minimum versions for compatibility with Airflow 3 MIN_VERSION_OVERRIDE: dict[str, Version] = { "amazon": parse_version("2.1.3"), - "fab": parse_version("2.0.0"), + "fab": parse_version("2.0.2"), "openlineage": parse_version("2.1.3"), - "git": parse_version("0.0.1"), + "git": parse_version("0.0.2"), "common.messaging": parse_version("1.0.0"), } @@ -88,7 +88,7 @@ def provider_path(provider_id: str) -> str: all_providers_metadata = json.loads(PROVIDER_METADATA_FILE_PATH.read_text()) -def find_min_provider_version(provider_id: str) -> Version | None: +def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]: metadata = all_providers_metadata.get(provider_id) # We should periodically update the starting date to avoid pip install resolution issues # TODO: when min Python version is 3.11 change back the code to fromisoformat @@ -101,7 +101,7 @@ def find_min_provider_version(provider_id: str) -> Version | None: min_version_override = MIN_VERSION_OVERRIDE.get(provider_id) if not metadata: if not min_version_override: - return None + return None, "" last_version_newer_than_cutoff = min_version_override else: versions: list[Version] = sorted([parse_version(version) for version in metadata], reverse=True) @@ -117,14 +117,17 @@ def find_min_provider_version(provider_id: str) -> Version | None: f"[bright_blue]Provider id {provider_id} min version found:[/] " f"{last_version_newer_than_cutoff} (date {date_released}" ) + override_comment = "" if last_version_newer_than_cutoff: if min_version_override and min_version_override > last_version_newer_than_cutoff: console.print( f"[yellow]Overriding provider id {provider_id} min version:[/] {min_version_override} " - f"overridden from hard-coded versions." + f"set from hard-coded versions.\n\n" + f"[yellow]Modify MIN_VERSION_OVERRIDE in {__file__} to set different version![/]\n" ) last_version_newer_than_cutoff = min_version_override - return last_version_newer_than_cutoff + override_comment = f" # Set from MIN_VERSION_OVERRIDE in {Path(__file__).name}" + return last_version_newer_than_cutoff, override_comment PROVIDER_MIN_VERSIONS: dict[str, str | None] = {} @@ -141,11 +144,11 @@ def find_min_provider_version(provider_id: str) -> Version | None: all_provider_lines = [] for provider_id in all_providers: distribution_name = provider_distribution_name(provider_id) - min_provider_version = find_min_provider_version(provider_id) + min_provider_version, comment = find_min_provider_version(provider_id) if min_provider_version: - all_provider_lines.append(f' "{distribution_name}>={min_provider_version}",\n') + all_provider_lines.append(f' "{distribution_name}>={min_provider_version}",{comment}\n') all_optional_dependencies.append( - f'"{provider_id}" = [\n "{distribution_name}>={min_provider_version}"\n]\n' + f'"{provider_id}" = [\n "{distribution_name}>={min_provider_version}"{comment}\n]\n' ) else: all_optional_dependencies.append(f'"{provider_id}" = [\n "{distribution_name}"\n]\n')