Skip to content

Bump min versions of crucial providers #50076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
May 1, 2025
Merged

Bump min versions of crucial providers #50076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -889,7 +889,7 @@ repos:
name: Update Airflow's meta-package pyproject.toml
language: python
entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py
files: ^pyproject\.toml$
files: ^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$
pass_filenames: false
require_serial: true
additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1', 'packaging>=23.2' ]
Expand Down
29 changes: 17 additions & 12 deletions dev/breeze/src/airflow_breeze/utils/packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1040,22 +1040,27 @@ def update_version_suffix_in_non_provider_pyproject_toml(version_suffix: str, py
lines = pyproject_toml_path.read_text().splitlines()
updated_lines = []
for line in lines:
if line.startswith("version = "):
base_line, comment = line.split(" #", 1) if " #" in line else (line, "")
if comment:
comment = " #" + comment
if base_line.startswith("version = "):
get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.")
line = line.rstrip('"') + f'{version_suffix}"'
base_line = base_line.rstrip('"') + f'{version_suffix}"'
# do not modify references for .post prefixes
if not version_suffix.startswith(".post"):
if line.strip().startswith('"apache-airflow-') and ">=" in line:
if base_line.strip().startswith('"apache-airflow-') and ">=" in base_line:
floored_version_suffix = floor_version_suffix(version_suffix)
get_console().print(f"[info]Updating version suffix to {floored_version_suffix} for {line}.")
line = line.rstrip('",') + f'{floored_version_suffix}",'
if line.strip().startswith('"apache-airflow-core') and "==" in line:
get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.")
line = line.rstrip('",') + f'{version_suffix}",'
if line.strip().startswith('"apache-airflow-task-sdk') and "==" in line:
get_console().print(f"[info]Updating version suffix to {version_suffix} for {line}.")
line = line.rstrip('",') + f'{version_suffix}",'
updated_lines.append(line)
get_console().print(
f"[info]Updating version suffix to {floored_version_suffix} for {base_line}."
)
base_line = base_line.rstrip('",') + f'{floored_version_suffix}",'
if base_line.strip().startswith('"apache-airflow-core') and "==" in base_line:
get_console().print(f"[info]Updating version suffix to {version_suffix} for {base_line}.")
base_line = base_line.rstrip('",') + f'{version_suffix}",'
if base_line.strip().startswith('"apache-airflow-task-sdk') and "==" in base_line:
get_console().print(f"[info]Updating version suffix to {version_suffix} for {base_line}.")
base_line = base_line.rstrip('",') + f'{version_suffix}",'
updated_lines.append(f"{base_line}{comment}")
new_content = "\n".join(updated_lines) + "\n"
get_console().print(f"[info]Writing updated content to {pyproject_toml_path}.\n")
pyproject_toml_path.write_text(new_content)
Expand Down
12 changes: 6 additions & 6 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ packages = []
"apache-airflow-providers-exasol>=4.6.1"
]
"fab" = [
"apache-airflow-providers-fab>=2.0.0"
"apache-airflow-providers-fab>=2.0.2" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"facebook" = [
"apache-airflow-providers-facebook>=3.7.0"
Expand All @@ -221,7 +221,7 @@ packages = []
"apache-airflow-providers-ftp>=3.12.0"
]
"git" = [
"apache-airflow-providers-git>=0.0.1"
"apache-airflow-providers-git>=0.0.2" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"github" = [
"apache-airflow-providers-github>=2.8.0"
Expand Down Expand Up @@ -281,7 +281,7 @@ packages = []
"apache-airflow-providers-openfaas>=3.7.0"
]
"openlineage" = [
"apache-airflow-providers-openlineage>=2.1.3"
"apache-airflow-providers-openlineage>=2.1.3" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"opensearch" = [
"apache-airflow-providers-opensearch>=1.5.0"
Expand Down Expand Up @@ -422,10 +422,10 @@ packages = []
"apache-airflow-providers-edge3>=1.0.0",
"apache-airflow-providers-elasticsearch>=5.5.2",
"apache-airflow-providers-exasol>=4.6.1",
"apache-airflow-providers-fab>=2.0.0",
"apache-airflow-providers-fab>=2.0.2", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-facebook>=3.7.0",
"apache-airflow-providers-ftp>=3.12.0",
"apache-airflow-providers-git>=0.0.1",
"apache-airflow-providers-git>=0.0.2", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-github>=2.8.0",
"apache-airflow-providers-google>=10.24.0",
"apache-airflow-providers-grpc>=3.7.0",
Expand All @@ -445,7 +445,7 @@ packages = []
"apache-airflow-providers-odbc>=4.8.0",
"apache-airflow-providers-openai>=1.5.0",
"apache-airflow-providers-openfaas>=3.7.0",
"apache-airflow-providers-openlineage>=2.1.3",
"apache-airflow-providers-openlineage>=2.1.3", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-opensearch>=1.5.0",
"apache-airflow-providers-opsgenie>=5.8.0",
"apache-airflow-providers-oracle>=3.12.0",
Expand Down
21 changes: 12 additions & 9 deletions scripts/ci/pre_commit/update_airflow_pyproject_toml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@
# minimum versions for compatibility with Airflow 3
MIN_VERSION_OVERRIDE: dict[str, Version] = {
"amazon": parse_version("2.1.3"),
"fab": parse_version("2.0.0"),
"fab": parse_version("2.0.2"),
"openlineage": parse_version("2.1.3"),
"git": parse_version("0.0.1"),
"git": parse_version("0.0.2"),
"common.messaging": parse_version("1.0.0"),
}

Expand Down Expand Up @@ -88,7 +88,7 @@ def provider_path(provider_id: str) -> str:
all_providers_metadata = json.loads(PROVIDER_METADATA_FILE_PATH.read_text())


def find_min_provider_version(provider_id: str) -> Version | None:
def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
metadata = all_providers_metadata.get(provider_id)
# We should periodically update the starting date to avoid pip install resolution issues
# TODO: when min Python version is 3.11 change back the code to fromisoformat
Expand All @@ -101,7 +101,7 @@ def find_min_provider_version(provider_id: str) -> Version | None:
min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
if not metadata:
if not min_version_override:
return None
return None, ""
last_version_newer_than_cutoff = min_version_override
else:
versions: list[Version] = sorted([parse_version(version) for version in metadata], reverse=True)
Expand All @@ -117,14 +117,17 @@ def find_min_provider_version(provider_id: str) -> Version | None:
f"[bright_blue]Provider id {provider_id} min version found:[/] "
f"{last_version_newer_than_cutoff} (date {date_released}"
)
override_comment = ""
if last_version_newer_than_cutoff:
if min_version_override and min_version_override > last_version_newer_than_cutoff:
console.print(
f"[yellow]Overriding provider id {provider_id} min version:[/] {min_version_override} "
f"overridden from hard-coded versions."
f"set from hard-coded versions.\n\n"
f"[yellow]Modify MIN_VERSION_OVERRIDE in {__file__} to set different version![/]\n"
)
last_version_newer_than_cutoff = min_version_override
return last_version_newer_than_cutoff
override_comment = f" # Set from MIN_VERSION_OVERRIDE in {Path(__file__).name}"
return last_version_newer_than_cutoff, override_comment


PROVIDER_MIN_VERSIONS: dict[str, str | None] = {}
Expand All @@ -141,11 +144,11 @@ def find_min_provider_version(provider_id: str) -> Version | None:
all_provider_lines = []
for provider_id in all_providers:
distribution_name = provider_distribution_name(provider_id)
min_provider_version = find_min_provider_version(provider_id)
min_provider_version, comment = find_min_provider_version(provider_id)
if min_provider_version:
all_provider_lines.append(f' "{distribution_name}>={min_provider_version}",\n')
all_provider_lines.append(f' "{distribution_name}>={min_provider_version}",{comment}\n')
all_optional_dependencies.append(
f'"{provider_id}" = [\n "{distribution_name}>={min_provider_version}"\n]\n'
f'"{provider_id}" = [\n "{distribution_name}>={min_provider_version}"{comment}\n]\n'
)
else:
all_optional_dependencies.append(f'"{provider_id}" = [\n "{distribution_name}"\n]\n')
Expand Down