diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 615bee819b782..7f0b64dcbc726 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -170,8 +170,7 @@ GitHub Actions to pass the list of parameters to a command to execute | all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.9 3.10 | * | | all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | -| build_system_changed_in_pyproject_toml | When builds system dependencies changed in pyproject.toml changed in the PR. | false | | -| ci-image-build | Whether CI image build is needed | true | | +| ci-image-build | Whether CI image build is needed | true | | | core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core | API Always Providers | * | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | | default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index e41cf992cdb01..68b5782262b17 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -111,6 +111,7 @@ class FileGroupForCi(Enum): TASK_SDK_FILES = "task_sdk_files" GO_SDK_FILES = "go_sdk_files" AIRFLOW_CTL_FILES = "airflow_ctl_files" + ALL_PYPROJECT_TOML_FILES = "all_pyproject_toml_files" ALL_PYTHON_FILES = "all_python_files" ALL_SOURCE_FILES = "all_sources_for_tests" ALL_AIRFLOW_PYTHON_FILES = "all_airflow_python_files" @@ -186,9 +187,6 @@ def __hash__(self): r"^airflow-core/tests/unit/kubernetes", r"^helm-tests", ], - FileGroupForCi.DEPENDENCY_FILES: [ - r"^generated/provider_dependencies.json$", - ], FileGroupForCi.DOC_FILES: [ r"^docs", r"^devel-common/src/docs", @@ -266,6 +264,9 @@ def __hash__(self): FileGroupForCi.ALL_PROVIDER_YAML_FILES: [ r".*/provider\.yaml$", ], + FileGroupForCi.ALL_PYPROJECT_TOML_FILES: [ + r".*pyproject\.toml$", + ], FileGroupForCi.TESTS_UTILS_FILES: [ r"^airflow-core/tests/unit/utils/", r"^devel-common/.*\.py$", @@ -505,13 +506,8 @@ def _should_run_all_tests_and_versions(self) -> bool: if not self._commit_ref: get_console().print("[warning]Running everything in all versions as commit is missing[/]") return True - if self.hatch_build_changed: - get_console().print("[warning]Running everything with all versions: hatch_build.py changed[/]") - return True - if self.pyproject_toml_changed and self.build_system_changed_in_pyproject_toml: - get_console().print( - "[warning]Running everything with all versions: build-system changed in pyproject.toml[/]" - ) + if self.pyproject_toml_changed: + get_console().print("[warning]Running everything with all versions: changed pyproject.toml[/]") return True if self.generated_dependencies_changed: get_console().print( @@ -926,6 +922,8 @@ def _get_core_test_types_to_run(self) -> list[str]: def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = False) -> list[str]: if self._default_branch != "main": return [] + if self.upgrade_to_newer_dependencies: + return ["Providers"] if self.full_tests_needed or self.run_task_sdk_tests: if split_to_individual_providers: return list(providers_test_type()) @@ -1069,10 +1067,6 @@ def _print_diff(old_lines: list[str], new_lines: list[str]): def generated_dependencies_changed(self) -> bool: return "generated/provider_dependencies.json" in self._files - @cached_property - def hatch_build_changed(self) -> bool: - return "hatch_build.py" in self._files - @cached_property def any_provider_yaml_or_pyproject_toml_changed(self) -> bool: if not self._commit_ref: @@ -1126,37 +1120,11 @@ def pyproject_toml_changed(self) -> bool: self._old_toml = tomllib.loads(old_result.stdout) return True - @cached_property - def build_system_changed_in_pyproject_toml(self) -> bool: - if not self.pyproject_toml_changed: - return False - new_build_backend = self._new_toml["build-system"]["build-backend"] - old_build_backend = self._old_toml["build-system"]["build-backend"] - if new_build_backend != old_build_backend: - get_console().print("[warning]Build backend changed in pyproject.toml [/]") - self._print_diff([old_build_backend], [new_build_backend]) - return True - new_requires = self._new_toml["build-system"]["requires"] - old_requires = self._old_toml["build-system"]["requires"] - if new_requires != old_requires: - get_console().print("[warning]Build system changed in pyproject.toml [/]") - self._print_diff(old_requires, new_requires) - return True - return False - @cached_property def upgrade_to_newer_dependencies(self) -> bool: - if len(self._matching_files(FileGroupForCi.DEPENDENCY_FILES, CI_FILE_GROUP_MATCHES)) > 0: + if len(self._matching_files(FileGroupForCi.ALL_PYPROJECT_TOML_FILES, CI_FILE_GROUP_MATCHES)) > 0: get_console().print("[warning]Upgrade to newer dependencies: Dependency files changed[/]") return True - if self.hatch_build_changed: - get_console().print("[warning]Upgrade to newer dependencies: hatch_build.py changed[/]") - return True - if self.build_system_changed_in_pyproject_toml: - get_console().print( - "[warning]Upgrade to newer dependencies: Build system changed in pyproject.toml[/]" - ) - return True if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]: get_console().print("[warning]Upgrade to newer dependencies: Push or Schedule event[/]") return True diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index fa2d3b53070ba..f41eabd3611bc 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -813,34 +813,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ), ( pytest.param( - ("generated/provider_dependencies.json",), - { - "selected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.9', '3.10', '3.11', '3.12']", - "all-python-versions-list-as-string": ALL_PYTHON_VERSIONS_AS_STRING, - "python-versions": "['3.9', '3.10', '3.11', '3.12']", - "python-versions-list-as-string": ALL_PYTHON_VERSIONS_AS_STRING, - "ci-image-build": "true", - "prod-image-build": "true", - "needs-helm-tests": "true", - "run-tests": "true", - "run-amazon-tests": "true", - "docs-build": "true", - "full-tests-needed": "true", - "skip-pre-commits": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, - "upgrade-to-newer-dependencies": "true", - "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "needs-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, - }, - id="Everything should run - including all providers and upgrading to " - "newer requirements as pyproject.toml changed and all Python versions", - ) - ), - ( - pytest.param( - ("generated/provider_dependencies.json",), + ("pyproject.toml",), { "selected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, "all-python-versions": "['3.9', '3.10', '3.11', '3.12']", @@ -864,83 +837,6 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Everything should run and upgrading to newer requirements as dependencies change", ) ), - pytest.param( - ("providers/amazon/src/airflow/providers/amazon/__init__.py",), - { - "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.compat common.messaging common.sql exasol ftp google http imap microsoft.azure " - "mongo mysql openlineage postgres salesforce ssh teradata", - "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", - "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", - "python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "docs-build": "true", - "skip-pre-commits": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS, - "run-kubernetes-tests": "false", - "upgrade-to-newer-dependencies": "false", - "run-amazon-tests": "true", - "core-test-types-list-as-strings-in-json": json.dumps( - [{"description": "Always", "test_types": "Always"}] - ), - "providers-test-types-list-as-strings-in-json": json.dumps( - [ - { - "description": "amazon...google", - "test_types": "Providers[amazon] Providers[apache.hive,cncf.kubernetes," - "common.compat,common.messaging,common.sql,exasol,ftp,http,imap," - "microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] " - "Providers[google]", - } - ] - ), - "needs-mypy": "true", - "mypy-checks": "['mypy-providers']", - }, - id="Providers tests run including amazon tests if amazon provider files changed", - ), - pytest.param( - ("providers/amazon/src/airflow/providers/amazon/pyproject.toml",), - { - "selected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.compat common.messaging common.sql exasol ftp google http imap microsoft.azure " - "mongo mysql openlineage postgres salesforce ssh teradata", - "all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", - "all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - "python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']", - "python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION, - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "docs-build": "true", - # no python files changed so flynt should not run - "skip-pre-commits": "flynt," + ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS, - "run-kubernetes-tests": "false", - "upgrade-to-newer-dependencies": "false", - "run-amazon-tests": "true", - "core-test-types-list-as-strings-in-json": json.dumps( - [{"description": "Always", "test_types": "Always"}] - ), - "providers-test-types-list-as-strings-in-json": json.dumps( - [ - { - "description": "amazon...google", - "test_types": "Providers[amazon] Providers[apache.hive,cncf.kubernetes," - "common.compat,common.messaging,common.sql,exasol,ftp,http,imap," - "microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] " - "Providers[google]", - } - ] - ), - "needs-mypy": "true", - "mypy-checks": "['mypy-providers']", - }, - id="Providers tests run including amazon tests if only amazon pyproject.toml files changed", - ), pytest.param( ("providers/amazon/src/airflow/providers/amazon/provider.yaml",), { @@ -1284,17 +1180,6 @@ def test_expected_output_pull_request_main( @pytest.mark.parametrize( "files, commit_ref, expected_outputs", [ - ( - pytest.param( - ("pyproject.toml",), - "2bc8e175b3a4cc84fe33e687f1a00d2a49563090", - { - "full-tests-needed": "false", - "all-versions": "false", - }, - id="No full tests needed / all versions when pyproject.toml changes in insignificant way", - ) - ), ( pytest.param( ("pyproject.toml",), @@ -2188,16 +2073,6 @@ def test_files_provided_trigger_full_build_for_any_event_type(github_event): None, id="Regular source changed", ), - pytest.param( - ("pyproject.toml",), - { - "upgrade-to-newer-dependencies": "false", - }, - (), - # In this commit only ruff configuration changed - "2bc8e175b3a4cc84fe33e687f1a00d2a49563090", - id="pyproject.toml changed but no dependency change", - ), pytest.param( ("providers/microsoft/azure/src/airflow/providers/microsoft/azure/provider.yaml",), { @@ -2207,15 +2082,6 @@ def test_files_provided_trigger_full_build_for_any_event_type(github_event): None, id="Provider.yaml changed", ), - pytest.param( - ("generated/provider_dependencies.json",), - { - "upgrade-to-newer-dependencies": "true", - }, - (), - "None", - id="Generated provider_dependencies changed", - ), pytest.param( ("airflow-core/src/airflow/models/dag.py",), { diff --git a/dev/constraints-updated-version-check.py b/dev/constraints-updated-version-check.py index 35630854b1392..52476f26e9a95 100755 --- a/dev/constraints-updated-version-check.py +++ b/dev/constraints-updated-version-check.py @@ -188,7 +188,15 @@ def main( print_table_header(format_str, headers, total_width) outdated_count, skipped_count, explanations = process_packages( - packages, constraints_date, mode, explain_why, verbose, col_widths, format_str, python_version + packages, + constraints_date, + mode, + explain_why, + verbose, + col_widths, + format_str, + python_version, + airflow_constraints_mode, ) print_table_footer(total_width, len(packages), outdated_count, skipped_count, mode) @@ -275,6 +283,7 @@ def process_packages( col_widths: dict, format_str: str, python_version: str, + airflow_constraints_mode: str, ) -> tuple[int, int, list[str]]: import subprocess import tempfile @@ -382,6 +391,7 @@ def update_pyproject_dependency(pyproject_path: Path, pkg: str, latest_version: update_pyproject_dependency, verbose, python_version, + airflow_constraints_mode, ) explanations.append(explanation) except HTTPError as e: @@ -444,6 +454,7 @@ def explain_package_upgrade( update_pyproject_dependency, verbose: bool, python_version: str, + airflow_constraints_mode: str, ) -> str: explanation = ( f"[bold blue]\n--- Explaining for {pkg} (current: {pinned_version}, latest: {latest_version}) ---[/]" @@ -457,12 +468,18 @@ def preserve_pyproject_file(pyproject_path: Path): finally: pyproject_path.write_text(original_content) + additional_args = [] + if airflow_constraints_mode == "constraints-source-providers": + # In case of source constraints we also need to add all development dependencies + # to reflect exactly what is installed in the CI image by default + additional_args.extend(["--group dev", "--group docs", "--group docs-gen", "--group leveldb"]) with preserve_pyproject_file(airflow_pyproject): before_result = run_uv_sync( [ "uv", "sync", "--all-packages", + *additional_args, "--resolution", "highest", "--refresh", @@ -472,7 +489,7 @@ def preserve_pyproject_file(pyproject_path: Path): cwd=repo_root, ) (temp_dir_path / "uv_sync_before.txt").write_text(before_result.stdout + before_result.stderr) - update_pyproject_dependency(airflow_pyproject, pkg, latest_version) + update_pyproject_dependency(airflow_pyproject, pkg, latest_version, airflow_constraints_mode) if verbose: syntax = Syntax( airflow_pyproject.read_text(), "toml", theme="monokai", line_numbers=True, word_wrap=False