Skip to content

Improve dependency report and uppgrading #52619

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Jul 1, 2025
Merged

Improve dependency report and uppgrading #52619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion dev/breeze/doc/ci/04_selective_checks.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ GitHub Actions to pass the list of parameters to a command to execute
| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.10 3.11 | * |
| all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | |
| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | |
| build_system_changed_in_pyproject_toml | When builds system dependencies changed in pyproject.toml changed in the PR. | false | |
| ci-image-build | Whether CI image build is needed | true | |
| core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core | API Always Providers | * |
| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | |
Expand Down
50 changes: 9 additions & 41 deletions dev/breeze/src/airflow_breeze/utils/selective_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ class FileGroupForCi(Enum):
TASK_SDK_FILES = "task_sdk_files"
GO_SDK_FILES = "go_sdk_files"
AIRFLOW_CTL_FILES = "airflow_ctl_files"
ALL_PYPROJECT_TOML_FILES = "all_pyproject_toml_files"
ALL_PYTHON_FILES = "all_python_files"
ALL_SOURCE_FILES = "all_sources_for_tests"
ALL_AIRFLOW_PYTHON_FILES = "all_airflow_python_files"
Expand Down Expand Up @@ -186,9 +187,6 @@ def __hash__(self):
r"^airflow-core/tests/unit/kubernetes",
r"^helm-tests",
],
FileGroupForCi.DEPENDENCY_FILES: [
r"^generated/provider_dependencies.json$",
],
FileGroupForCi.DOC_FILES: [
r"^docs",
r"^devel-common/src/docs",
Expand Down Expand Up @@ -266,6 +264,9 @@ def __hash__(self):
FileGroupForCi.ALL_PROVIDER_YAML_FILES: [
r".*/provider\.yaml$",
],
FileGroupForCi.ALL_PYPROJECT_TOML_FILES: [
r".*pyproject\.toml$",
],
FileGroupForCi.TESTS_UTILS_FILES: [
r"^airflow-core/tests/unit/utils/",
r"^devel-common/.*\.py$",
Expand Down Expand Up @@ -505,13 +506,8 @@ def _should_run_all_tests_and_versions(self) -> bool:
if not self._commit_ref:
get_console().print("[warning]Running everything in all versions as commit is missing[/]")
return True
if self.hatch_build_changed:
get_console().print("[warning]Running everything with all versions: hatch_build.py changed[/]")
return True
if self.pyproject_toml_changed and self.build_system_changed_in_pyproject_toml:
get_console().print(
"[warning]Running everything with all versions: build-system changed in pyproject.toml[/]"
)
if self.pyproject_toml_changed:
get_console().print("[warning]Running everything with all versions: changed pyproject.toml[/]")
return True
if self.generated_dependencies_changed:
get_console().print(
Expand Down Expand Up @@ -926,6 +922,8 @@ def _get_core_test_types_to_run(self) -> list[str]:
def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = False) -> list[str]:
if self._default_branch != "main":
return []
if self.upgrade_to_newer_dependencies:
return ["Providers"]
if self.full_tests_needed or self.run_task_sdk_tests:
if split_to_individual_providers:
return list(providers_test_type())
Expand Down Expand Up @@ -1069,10 +1067,6 @@ def _print_diff(old_lines: list[str], new_lines: list[str]):
def generated_dependencies_changed(self) -> bool:
return "generated/provider_dependencies.json" in self._files

@cached_property
def hatch_build_changed(self) -> bool:
return "hatch_build.py" in self._files

@cached_property
def any_provider_yaml_or_pyproject_toml_changed(self) -> bool:
if not self._commit_ref:
Expand Down Expand Up @@ -1126,37 +1120,11 @@ def pyproject_toml_changed(self) -> bool:
self._old_toml = tomllib.loads(old_result.stdout)
return True

@cached_property
def build_system_changed_in_pyproject_toml(self) -> bool:
if not self.pyproject_toml_changed:
return False
new_build_backend = self._new_toml["build-system"]["build-backend"]
old_build_backend = self._old_toml["build-system"]["build-backend"]
if new_build_backend != old_build_backend:
get_console().print("[warning]Build backend changed in pyproject.toml [/]")
self._print_diff([old_build_backend], [new_build_backend])
return True
new_requires = self._new_toml["build-system"]["requires"]
old_requires = self._old_toml["build-system"]["requires"]
if new_requires != old_requires:
get_console().print("[warning]Build system changed in pyproject.toml [/]")
self._print_diff(old_requires, new_requires)
return True
return False

@cached_property
def upgrade_to_newer_dependencies(self) -> bool:
if len(self._matching_files(FileGroupForCi.DEPENDENCY_FILES, CI_FILE_GROUP_MATCHES)) > 0:
if len(self._matching_files(FileGroupForCi.ALL_PYPROJECT_TOML_FILES, CI_FILE_GROUP_MATCHES)) > 0:
get_console().print("[warning]Upgrade to newer dependencies: Dependency files changed[/]")
return True
if self.hatch_build_changed:
get_console().print("[warning]Upgrade to newer dependencies: hatch_build.py changed[/]")
return True
if self.build_system_changed_in_pyproject_toml:
get_console().print(
"[warning]Upgrade to newer dependencies: Build system changed in pyproject.toml[/]"
)
return True
if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]:
get_console().print("[warning]Upgrade to newer dependencies: Push or Schedule event[/]")
return True
Expand Down
136 changes: 1 addition & 135 deletions dev/breeze/tests/test_selective_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -805,34 +805,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
),
(
pytest.param(
("generated/provider_dependencies.json",),
{
"selected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
"all-python-versions": "['3.10', '3.11', '3.12']",
"all-python-versions-list-as-string": ALL_PYTHON_VERSIONS_AS_STRING,
"python-versions": "['3.10', '3.11', '3.12']",
"python-versions-list-as-string": ALL_PYTHON_VERSIONS_AS_STRING,
"ci-image-build": "true",
"prod-image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
"full-tests-needed": "true",
"skip-pre-commits": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED,
"upgrade-to-newer-dependencies": "true",
"core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
"needs-mypy": "true",
"mypy-checks": ALL_MYPY_CHECKS,
},
id="Everything should run - including all providers and upgrading to "
"newer requirements as pyproject.toml changed and all Python versions",
)
),
(
pytest.param(
("generated/provider_dependencies.json",),
("pyproject.toml",),
{
"selected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
"all-python-versions": "['3.10', '3.11', '3.12']",
Expand All @@ -856,83 +829,6 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
id="Everything should run and upgrading to newer requirements as dependencies change",
)
),
pytest.param(
("providers/amazon/src/airflow/providers/amazon/__init__.py",),
{
"selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
"common.compat common.messaging common.sql exasol ftp google http imap microsoft.azure "
"mongo mysql openlineage postgres salesforce ssh teradata",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"ci-image-build": "true",
"prod-image-build": "false",
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
"skip-pre-commits": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS,
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"run-amazon-tests": "true",
"core-test-types-list-as-strings-in-json": json.dumps(
[{"description": "Always", "test_types": "Always"}]
),
"providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "amazon...google",
"test_types": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,"
"common.compat,common.messaging,common.sql,exasol,ftp,http,imap,"
"microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] "
"Providers[google]",
}
]
),
"needs-mypy": "true",
"mypy-checks": "['mypy-providers']",
},
id="Providers tests run including amazon tests if amazon provider files changed",
),
pytest.param(
("providers/amazon/src/airflow/providers/amazon/pyproject.toml",),
{
"selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
"common.compat common.messaging common.sql exasol ftp google http imap microsoft.azure "
"mongo mysql openlineage postgres salesforce ssh teradata",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"ci-image-build": "true",
"prod-image-build": "false",
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
# no python files changed so flynt should not run
"skip-pre-commits": "flynt," + ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS,
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"run-amazon-tests": "true",
"core-test-types-list-as-strings-in-json": json.dumps(
[{"description": "Always", "test_types": "Always"}]
),
"providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "amazon...google",
"test_types": "Providers[amazon] Providers[apache.hive,cncf.kubernetes,"
"common.compat,common.messaging,common.sql,exasol,ftp,http,imap,"
"microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] "
"Providers[google]",
}
]
),
"needs-mypy": "true",
"mypy-checks": "['mypy-providers']",
},
id="Providers tests run including amazon tests if only amazon pyproject.toml files changed",
),
pytest.param(
("providers/amazon/src/airflow/providers/amazon/provider.yaml",),
{
Expand Down Expand Up @@ -1276,17 +1172,6 @@ def test_expected_output_pull_request_main(
@pytest.mark.parametrize(
"files, commit_ref, expected_outputs",
[
(
pytest.param(
("pyproject.toml",),
"2bc8e175b3a4cc84fe33e687f1a00d2a49563090",
{
"full-tests-needed": "false",
"all-versions": "false",
},
id="No full tests needed / all versions when pyproject.toml changes in insignificant way",
)
),
(
pytest.param(
("pyproject.toml",),
Expand Down Expand Up @@ -2183,16 +2068,6 @@ def test_files_provided_trigger_full_build_for_any_event_type(github_event):
None,
id="Regular source changed",
),
pytest.param(
("pyproject.toml",),
{
"upgrade-to-newer-dependencies": "false",
},
(),
# In this commit only ruff configuration changed
"2bc8e175b3a4cc84fe33e687f1a00d2a49563090",
id="pyproject.toml changed but no dependency change",
),
pytest.param(
("providers/microsoft/azure/src/airflow/providers/microsoft/azure/provider.yaml",),
{
Expand All @@ -2202,15 +2077,6 @@ def test_files_provided_trigger_full_build_for_any_event_type(github_event):
None,
id="Provider.yaml changed",
),
pytest.param(
("generated/provider_dependencies.json",),
{
"upgrade-to-newer-dependencies": "true",
},
(),
"None",
id="Generated provider_dependencies changed",
),
pytest.param(
("airflow-core/src/airflow/models/dag.py",),
{
Expand Down
21 changes: 19 additions & 2 deletions dev/constraints-updated-version-check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,15 @@ def main(
print_table_header(format_str, headers, total_width)

outdated_count, skipped_count, explanations = process_packages(
packages, constraints_date, mode, explain_why, verbose, col_widths, format_str, python_version
packages,
constraints_date,
mode,
explain_why,
verbose,
col_widths,
format_str,
python_version,
airflow_constraints_mode,
)

print_table_footer(total_width, len(packages), outdated_count, skipped_count, mode)
Expand Down Expand Up @@ -275,6 +283,7 @@ def process_packages(
col_widths: dict,
format_str: str,
python_version: str,
airflow_constraints_mode: str,
) -> tuple[int, int, list[str]]:
import subprocess
import tempfile
Expand Down Expand Up @@ -382,6 +391,7 @@ def update_pyproject_dependency(pyproject_path: Path, pkg: str, latest_version:
update_pyproject_dependency,
verbose,
python_version,
airflow_constraints_mode,
)
explanations.append(explanation)
except HTTPError as e:
Expand Down Expand Up @@ -444,6 +454,7 @@ def explain_package_upgrade(
update_pyproject_dependency,
verbose: bool,
python_version: str,
airflow_constraints_mode: str,
) -> str:
explanation = (
f"[bold blue]\n--- Explaining for {pkg} (current: {pinned_version}, latest: {latest_version}) ---[/]"
Expand All @@ -457,12 +468,18 @@ def preserve_pyproject_file(pyproject_path: Path):
finally:
pyproject_path.write_text(original_content)

additional_args = []
if airflow_constraints_mode == "constraints-source-providers":
# In case of source constraints we also need to add all development dependencies
# to reflect exactly what is installed in the CI image by default
additional_args.extend(["--group dev", "--group docs", "--group docs-gen", "--group leveldb"])
with preserve_pyproject_file(airflow_pyproject):
before_result = run_uv_sync(
[
"uv",
"sync",
"--all-packages",
*additional_args,
"--resolution",
"highest",
"--refresh",
Expand All @@ -472,7 +489,7 @@ def preserve_pyproject_file(pyproject_path: Path):
cwd=repo_root,
)
(temp_dir_path / "uv_sync_before.txt").write_text(before_result.stdout + before_result.stderr)
update_pyproject_dependency(airflow_pyproject, pkg, latest_version)
update_pyproject_dependency(airflow_pyproject, pkg, latest_version, airflow_constraints_mode)
if verbose:
syntax = Syntax(
airflow_pyproject.read_text(), "toml", theme="monokai", line_numbers=True, word_wrap=False
Expand Down
Loading