diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index 30a3367710a92..d2aeecd397470 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -24,10 +24,10 @@ inputs: default: "3.9" uv-version: description: 'uv version to use' - default: "0.5.20" # Keep this comment to allow automatic replacement of uv version + default: "0.5.23" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' - default: "4.0.1" # Keep this comment to allow automatic replacement of pre-commit version + default: "4.1.0" # Keep this comment to allow automatic replacement of pre-commit version pre-commit-uv-version: description: 'pre-commit-uv version to use' default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 2c97219a4e5bd..ddcc4fefaca25 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -22,10 +22,7 @@ labelPRBasedOnFilePath: - providers/airbyte/** provider:alibaba: - - providers/src/airflow/providers/alibaba/**/* - - docs/apache-airflow-providers-alibaba/**/* - - providers/tests/alibaba/**/* - - providers/tests/system/alibaba/**/* + - providers/alibaba/** provider:amazon: - providers/src/airflow/providers/amazon/**/* @@ -173,15 +170,10 @@ labelPRBasedOnFilePath: - providers/tests/system/common/io/**/* provider:common-sql: - - providers/src/airflow/providers/common/sql/**/* - - docs/apache-airflow-providers-common-sql/**/* - - providers/tests/common/sql/**/* - - providers/tests/system/common/sql/**/* + - providers/common/sql/** provider:standard: - - providers/src/airflow/providers/standard/**/* - - docs/apache-airflow-providers-standard/**/* - - providers/tests/standard/**/* + - providers/standard/** provider:databricks: - providers/src/airflow/providers/databricks/**/* @@ -647,12 +639,12 @@ labelPRBasedOnFilePath: - airflow/cli/commands/local_commands/triggerer_command.py - airflow/jobs/triggerer_job_runner.py - airflow/models/trigger.py - - providers/src/airflow/providers/standard/triggers/**/* + - providers/standard/src/airflow/providers/standard/triggers/**/* - tests/cli/commands/local_commands/test_triggerer_command.py - tests/jobs/test_triggerer_job.py - tests/models/test_trigger.py - tests/jobs/test_triggerer_job_logging.py - - providers/tests/standard/triggers/**/* + - providers/standard/tests/provider_tests/standard/triggers/**/* area:Serialization: - airflow/serialization/**/* diff --git a/.github/workflows/test-provider-packages.yml b/.github/workflows/test-provider-packages.yml index b0912fa6dfe37..7cdeab6be887b 100644 --- a/.github/workflows/test-provider-packages.yml +++ b/.github/workflows/test-provider-packages.yml @@ -109,6 +109,10 @@ jobs: run: > breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 --package-format ${{ matrix.package-format }} + - name: "Prepare task-sdk package: ${{ matrix.package-format }}" + run: > + breeze release-management prepare-task-sdk-package --version-suffix-for-pypi dev0 + --package-format ${{ matrix.package-format }} - name: "Verify ${{ matrix.package-format }} packages with twine" run: | uv tool uninstall twine || true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8431755955c12..3c04521b32b50 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -180,14 +180,6 @@ repos: additional_dependencies: ['rich>=12.4.4', "ruff==0.8.1"] files: ^providers/src/airflow/providers/.*\.py$ require_serial: true - - id: update-common-sql-api-stubs - name: Check and update common.sql API stubs - entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py - language: python - files: ^scripts/ci/pre_commit/update_common_sql_api\.py|^providers/src/airflow/providers/common/sql/.*\.pyi?$ - additional_dependencies: ['rich>=12.4.4', 'mypy==1.9.0', 'black==24.10.0', 'jinja2'] - pass_filenames: false - require_serial: true - id: update-black-version name: Update black versions everywhere (manual) entry: ./scripts/ci/pre_commit/update_black_version.py @@ -475,9 +467,7 @@ repos: language: python files: | (?x) - ^providers/src/airflow/providers/.*\.py$ | - ^providers/src/airflow/providers/.*/provider\.yaml$ | - ^providers/tests/.*\.py$ | + ^providers/.*/provider\.yaml$| ^scripts/ci/pre_commit/update_providers_dependencies\.py$ pass_filenames: false additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml', 'tomli'] @@ -517,6 +507,14 @@ repos: files: ^docs/.*/diagram_[^/]*\.py$ pass_filenames: true additional_dependencies: ['rich>=12.4.4', "diagrams>=0.23.4"] + - id: generate-volumes-for-sources + name: Generate volumes for docker compose + entry: ./scripts/ci/pre_commit/generate_volumes_for_sources.py + language: python + files: ^providers/.*/provider.yaml$ + pass_filenames: false + require_serial: true + additional_dependencies: ['rich>=12.4.4'] - id: update-supported-versions name: Updates supported versions in documentation entry: ./scripts/ci/pre_commit/supported_versions.py @@ -623,6 +621,8 @@ repos: ^airflow/config_templates/| ^airflow/models/baseoperator.py$| ^airflow/operators/__init__.py$| + ^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute.py$| + ^providers/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$| ^providers/src/airflow/providers/amazon/aws/hooks/emr.py$| ^providers/src/airflow/providers/amazon/aws/operators/emr.py$| ^providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py$| @@ -672,8 +672,11 @@ repos: ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint.sh$| ^tests/| ^providers/tests/| + ^providers/.*/tests/| ^.pre-commit-config\.yaml$| ^.*CHANGELOG\.(rst|txt)$| + ^.*changelog\.(rst|txt)$| + ^.*commits\.(rst|txt)$| ^.*RELEASE_NOTES\.rst$| ^contributing-docs/03_contributors_quick_start.rst$| ^.*\.(png|gif|jp[e]?g|tgz|lock)$| @@ -724,6 +727,8 @@ repos: ^airflow/operators/.*$| ^providers/src/airflow/providers/.*$| ^providers/src/airflow/providers/standard/sensors/.*$| + ^providers/.*/src/airflow/providers/.*$| + ^providers/.*/src/airflow/providers/standard/sensors/.*$| ^dev/provider_packages/.*$ - id: check-base-operator-usage language: pygrep @@ -733,8 +738,9 @@ repos: pass_filenames: true files: > (?x) - ^providers/src/airflow/providers/.*\.py$ - exclude: providers/src/airflow/providers/standard/operators/bash.py|providers/src/airflow/providers/standard/operators/python.py|providers/src/airflow/providers/standard/sensors/external_task.py + ^providers/src/airflow/providers/.*\.py$| + ^providers/.*/src/airflow/providers/.*\.py$ + exclude: providers/standard/.*/.*\.py$ - id: check-get-lineage-collector-providers language: python name: Check providers import hook lineage code from compat @@ -1205,10 +1211,12 @@ repos: ^airflow/utils/file.py$ | ^airflow/utils/helpers.py$ | ^airflow/utils/log/secrets_masker.py$ | - ^providers/src/airflow/providers/ | - ^(providers/)?tests/ | - task_sdk/src/airflow/sdk/definitions/dag.py$ | - task_sdk/src/airflow/sdk/definitions/_internal/node.py$ | + ^providers/ | + ^tests/ | + ^providers/tests/ | + ^providers/.*/tests/ | + ^task_sdk/src/airflow/sdk/definitions/dag.py$ | + ^task_sdk/src/airflow/sdk/definitions/_internal/node.py$ | ^dev/.*\.py$ | ^scripts/.*\.py$ | ^docker_tests/.*$ | @@ -1343,7 +1351,7 @@ repos: name: Run mypy for providers language: python entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages - files: ^providers/src/airflow/providers/.*\.py$|^providers/tests//.*\.py$|^providers/.*/src/.*\.py$|^providers/.*/tests/.*\.py$ + files: ^providers/.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] - id: mypy-providers @@ -1392,7 +1400,7 @@ repos: name: Validate provider.yaml files entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py language: python - files: ^providers/src/airflow/providers/.*/provider\.yaml$|^providers/.*/src/provider\.yaml$ + files: ^providers/.*/provider\.yaml$ additional_dependencies: ['rich>=12.4.4'] require_serial: true - id: check-template-fields-valid diff --git a/Dockerfile b/Dockerfile index 9b7e8a4391f3e..8c6cf1397ddd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=24.3.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.20 +ARG AIRFLOW_UV_VERSION=0.5.23 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index 4e80ff1050abd..fc10f521bed93 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -915,13 +915,6 @@ function determine_airflow_to_use() { # Some packages might leave legacy typing module which causes test issues # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} typing || true - if [[ ${LINK_PROVIDERS_TO_AIRFLOW_PACKAGE=} == "true" ]]; then - echo - echo "${COLOR_BLUE}Linking providers to airflow package as we are using them from mounted sources.${COLOR_RESET}" - echo - rm -rf /usr/local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow/providers - ln -s "${AIRFLOW_SOURCES}/providers/src/airflow/providers" "/usr/local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow/providers" - fi fi if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then @@ -1268,9 +1261,9 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=24.3.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.20 +ARG AIRFLOW_UV_VERSION=0.5.23 # TODO(potiuk): automate with upgrade check (possibly) -ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1" +ARG AIRFLOW_PRE_COMMIT_VERSION="4.1.0" ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ diff --git a/airflow/utils/dag_parsing_context.py b/airflow/utils/dag_parsing_context.py index 1ba90097366a5..4c534cea116bd 100644 --- a/airflow/utils/dag_parsing_context.py +++ b/airflow/utils/dag_parsing_context.py @@ -19,6 +19,7 @@ import warnings +from airflow.sdk.definitions._internal.dag_parsing_context import _airflow_parsing_context_manager from airflow.sdk.definitions.context import get_parsing_context # TODO: Remove this module in Airflow 3.2 @@ -30,4 +31,4 @@ stacklevel=2, ) -__all__ = ["get_parsing_context"] +__all__ = ["get_parsing_context", "_airflow_parsing_context_manager"] diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index a0bff418b0762..b8ec98923166f 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -280,6 +280,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | generate-pypi-readme | Generate PyPI README | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ +| generate-volumes-for-sources | Generate volumes for docker compose | | ++-----------------------------------------------------------+--------------------------------------------------------+---------+ | identity | Print checked files | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | insert-license | * Add license for all SQL files | | @@ -364,8 +366,6 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-chart-dependencies | Update chart dependencies to latest (manual) | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-common-sql-api-stubs | Check and update common.sql API stubs | | -+-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-er-diagram | Update ER diagram | * | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-extras | Update extras in documentation | | diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 84f71f34c3f1d..1301ad33edfa4 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -443,8 +443,8 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.3.1` | `pip` version used. | -| `AIRFLOW_UV_VERSION` | `0.5.20` | `uv` version used. | -| `AIRFLOW_PRE_COMMIT_VERSION` | `4.0.1` | `pre-commit` version used. | +| `AIRFLOW_UV_VERSION` | `0.5.23` | `uv` version used. | +| `AIRFLOW_PRE_COMMIT_VERSION` | `4.1.0` | `pre-commit` version used. | | `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` | `pre-commit-uv` version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg b/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg index 63bb39a5189e8..3237bf9a944b7 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-task-sdk-package.svg @@ -1,4 +1,4 @@ - +