diff --git a/.asf.yaml b/.asf.yaml index 3130630e80c6b..59dfd3c401ac4 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -134,6 +134,11 @@ github: required_linear_history: true required_conversation_resolution: true required_signatures: false + v3-0-stable: + required_pull_request_reviews: + required_approving_review_count: 1 + required_linear_history: true + required_signatures: false providers-fab/v1-5: required_pull_request_reviews: required_approving_review_count: 1 diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index dd7944841510a..3a427ee12519e 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -24,7 +24,7 @@ inputs: default: "3.9" uv-version: description: 'uv version to use' - default: "0.6.13" # Keep this comment to allow automatic replacement of uv version + default: "0.7.2" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' default: "4.2.0" # Keep this comment to allow automatic replacement of pre-commit version @@ -61,6 +61,16 @@ runs: key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} path: /tmp/ id: restore-pre-commit-cache + - name: "Check if pre-commit cache tarball exists" + shell: bash + run: | + if [ -f /tmp/cache-pre-commit.tar.gz ]; then + echo "✅ Cache tarball found: /tmp/cache-pre-commit.tar.gz" + else + echo "❌ Cache tarball missing. Expected /tmp/cache-pre-commit.tar.gz" + exit 1 + fi + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' - name: "Restore .cache from the tar file" run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz shell: bash diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index e9529bd0d08d2..296e58fa23ad2 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -20,16 +20,12 @@ name: Additional CI image checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." - required: true - type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions: @@ -37,7 +33,7 @@ on: # yamllint disable-line rule:truthy required: true type: string branch: - description: "Branch used to run the CI jobs in (main/v2_*_test)." + description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string constraints-branch: @@ -103,12 +99,11 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} + runners: ${{ inputs.runners }} cache-type: "Early" include-prod-images: "false" push-latest-images: "false" - platform: "linux/amd64" + platform: ${{ inputs.platform }} python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} @@ -116,13 +111,15 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} - if: inputs.branch == 'main' + if: > + inputs.canary-run == 'true' && + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') # Check that after earlier cache push, breeze command will build quickly check-that-image-builds-quickly: timeout-minutes: 11 name: Check that image builds quickly - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} @@ -131,6 +128,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} VERBOSE: "true" + PLATFORM: ${{ inputs.platform }} if: inputs.branch == 'main' steps: - name: "Cleanup repo" @@ -152,30 +150,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin - name: "Check that image builds quickly" - run: breeze shell --max-time 600 --platform "linux/amd64" - -# # This is only a check if ARM images are successfully building when committer runs PR from -# # Apache repository. This is needed in case you want to fix failing cache job in "canary" run -# # There is no point in running this one in "canary" run, because the above step is doing the -# # same build anyway. -# build-ci-arm-images: -# name: Build CI ARM images -# uses: ./.github/workflows/ci-image-build.yml -# permissions: -# contents: read -# packages: write -# with: -# platform: "linux/arm64" -# push-image: "false" -# upload-image-artifact: "true" -# upload-mount-cache-artifact: ${{ inputs.canary-run }} -# runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} -# runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} -# python-versions: ${{ inputs.python-versions }} -# branch: ${{ inputs.branch }} -# constraints-branch: ${{ inputs.constraints-branch }} -# use-uv: ${{ inputs.use-uv }} -# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} -# docker-cache: ${{ inputs.docker-cache }} -# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} -# + run: breeze shell --max-time 600 --platform "${PLATFORM}" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index e656f48a5a300..489b2967af3d1 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -20,8 +20,12 @@ name: Additional PROD image tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string default-branch: @@ -36,10 +40,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies (true/false)" required: true type: string - chicken-egg-providers: - description: "Whether to build chicken-egg provider distributions in the same run (true/false)" - required: true - type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -67,14 +67,13 @@ jobs: name: PROD image extra checks (main) uses: ./.github/workflows/prod-image-extra-checks.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} use-uv: "false" - build-provider-distributions: ${{ inputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -84,14 +83,13 @@ jobs: name: PROD image extra checks (release) uses: ./.github/workflows/prod-image-extra-checks.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} use-uv: "false" - build-provider-distributions: ${{ inputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -100,7 +98,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of PROD image building" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -120,7 +118,7 @@ jobs: - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} image-type: "prod" python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} @@ -138,7 +136,7 @@ jobs: test-docker-compose-quick-start: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -157,7 +155,7 @@ jobs: - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} image-type: "prod" python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index c7071c5f34d7c..a7156ef98dfb7 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -21,6 +21,14 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: # Static inputs defined to choose which distribution to test to run + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string distribution-name: description: "The name of the distribution to test" required: true @@ -33,11 +41,6 @@ on: # yamllint disable-line rule:truthy description: "distribution test type" # eg task-sdk-tests required: true type: string - # Environment inputs - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." - required: true - type: string default-python-version: description: "Which version of python should be used by default" required: true @@ -60,7 +63,7 @@ jobs: distributions-tests: timeout-minutes: 80 name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -83,7 +86,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 2eca2eec2474e..5d931b001865a 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,8 +20,8 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string run-ui-tests: @@ -66,7 +66,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} steps: - name: "Cleanup repo" shell: bash @@ -87,7 +87,7 @@ jobs: tests-ui: timeout-minutes: 15 name: React UI tests - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.run-ui-tests == 'true' steps: - name: "Cleanup repo" @@ -155,7 +155,7 @@ jobs: install-pre-commit: timeout-minutes: 5 name: "Install pre-commit for cache" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" steps: @@ -183,7 +183,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit if: inputs.basic-checks-only == 'true' steps: @@ -236,7 +236,7 @@ jobs: upgrade-check: timeout-minutes: 45 name: "Upgrade checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -306,7 +306,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -337,24 +337,26 @@ jobs: - name: Install twine run: pip install twine - name: "Check Airflow create minor branch command" - run: | - ./scripts/ci/testing/run_breeze_command_with_retries.sh \ - release-management create-minor-branch --version-branch 2-8 --answer yes + run: > + breeze release-management create-minor-branch + --version-branch 3-1 --answer yes --dry-run - name: "Check Airflow RC process command" - run: | - ./scripts/ci/testing/run_breeze_command_with_retries.sh \ - release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0 --answer yes + run: > + breeze release-management start-rc-process + --version 3.1.0rc1 --previous-version 3.0.0 --answer yes --dry-run - name: "Check Airflow release process command" - run: | - ./scripts/ci/testing/run_breeze_command_with_retries.sh \ - release-management start-release --release-candidate 2.8.3rc1 --previous-release 2.8.0 --answer yes + run: > + breeze release-management start-release --release-candidate 3.1.0rc1 + --previous-release 3.0.0 --answer yes --dry-run - name: "Test providers metadata generation" run: | - ./scripts/ci/testing/run_breeze_command_with_retries.sh \ - release-management generate-providers-metadata --refresh-constraints - - name: "Fetch all git tags" + git remote add apache https://github.com/apache/airflow.git + git fetch apache --tags + breeze release-management generate-providers-metadata --refresh-constraints + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Fetch all git tags for origin" run: git fetch --tags >/dev/null 2>&1 || true - name: "Test airflow core issue generation automatically" run: | - ./scripts/ci/testing/run_breeze_command_with_retries.sh \ - release-management generate-issue-content-core --limit-pr-count 25 --latest --verbose + breeze release-management generate-issue-content-core --limit-pr-count 25 --latest --verbose diff --git a/.github/workflows/ci.yml b/.github/workflows/ci-amd.yml similarity index 89% rename from .github/workflows/ci.yml rename to .github/workflows/ci-amd.yml index 406132c3a0b73..4859adf0dfec1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci-amd.yml @@ -16,7 +16,7 @@ # under the License. # --- -name: Tests +name: Tests AMD on: # yamllint disable-line rule:truthy schedule: - cron: '28 1,7,13,19 * * *' @@ -43,7 +43,7 @@ env: VERBOSE: "true" concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ci-amd-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -59,7 +59,6 @@ jobs: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} canary-run: ${{ steps.source-run-info.outputs.canary-run }} - chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} core-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} @@ -84,12 +83,6 @@ jobs: include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} individual-providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} - is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} - is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} - is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} - is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} - is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} - is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} kubernetes-combos-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} @@ -124,11 +117,8 @@ jobs: run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} - runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} - runs-on-as-json-docs-build: ${{ steps.selective-checks.outputs.runs-on-as-json-docs-build }} - runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted-asf: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted-asf }} - runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }} + amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} + arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} @@ -183,7 +173,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/basic-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runners: ${{ needs.build-info.outputs.amd-runners }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} run-www-tests: ${{needs.build-info.outputs.run-www-tests}} needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} @@ -204,17 +194,16 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} use-uv: ${{ needs.build-info.outputs.use-uv }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} if: needs.build-info.outputs.ci-image-build == 'true' @@ -229,9 +218,8 @@ jobs: id-token: write if: needs.build-info.outputs.canary-run == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} @@ -252,12 +240,12 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.ci-image-build == 'true' with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} # generate no providers constraints only in canary builds - they take quite some time to generate # they are not needed for regular builds, they are only needed to update constraints in canaries generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -269,8 +257,8 @@ jobs: id-token: write contents: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} - runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -282,7 +270,6 @@ jobs: basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -306,7 +293,8 @@ jobs: needs.build-info.outputs.skip-providers-tests != 'true' && needs.build-info.outputs.latest-versions-only != 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -328,8 +316,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -346,7 +334,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -373,7 +362,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -400,7 +390,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -427,7 +418,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -455,7 +447,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -484,7 +477,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -514,7 +508,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "" test-name-separator: "" @@ -542,7 +537,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "" test-name-separator: "" @@ -576,7 +572,8 @@ jobs: needs.build-info.outputs.full-tests-needed == 'true') with: default-branch: ${{ needs.build-info.outputs.default-branch }} - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" core-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > @@ -601,7 +598,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} run-system-tests: ${{ needs.build-info.outputs.run-tests }} @@ -624,7 +622,8 @@ jobs: if: > needs.build-info.outputs.run-tests == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -652,7 +651,8 @@ jobs: packages: read if: needs.build-info.outputs.run-tests == 'true' with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -681,9 +681,9 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - build-type: "Regular" + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" + build-type: "Regular" push-image: "false" upload-image-artifact: "true" upload-package-artifact: "true" @@ -691,9 +691,7 @@ jobs: default-python-version: ${{ needs.build-info.outputs.default-python-version }} branch: ${{ needs.build-info.outputs.default-branch }} use-uv: ${{ needs.build-info.outputs.use-uv }} - build-provider-distributions: ${{ needs.build-info.outputs.default-branch == 'main' }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} @@ -704,11 +702,11 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" default-branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} @@ -724,8 +722,8 @@ jobs: contents: read packages: read with: + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -743,7 +741,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -764,7 +763,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -785,23 +785,35 @@ jobs: # This will fire when all the jobs from "needs" are either successful or skipped if: always() && !failure() && !cancelled() needs: + - additional-ci-image-checks + - additional-prod-image-tests + - basic-tests - build-info + - basic-tests - generate-constraints + - build-prod-images - ci-image-checks - - tests-sqlite-core - - tests-sqlite-providers + - providers + - tests-helm + - tests-integration-system + - tests-kubernetes - tests-mysql-core - tests-mysql-providers - - tests-postgres-core - - tests-postgres-providers - tests-non-db-core - tests-non-db-providers - - tests-integration-system - - build-prod-images + - tests-postgres-core + - tests-postgres-providers + - tests-special + - tests-sqlite-core + - tests-sqlite-providers + - tests-task-sdk + - tests-airflow-ctl + - tests-with-lowest-direct-resolution-core + - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: - runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -818,17 +830,6 @@ jobs: notify-slack-failure: name: "Notify Slack on Failure" needs: - - basic-tests - - additional-ci-image-checks - - providers - - tests-helm - - tests-special - - tests-with-lowest-direct-resolution-core - - tests-with-lowest-direct-resolution-providers - - additional-prod-image-tests - - tests-kubernetes - - tests-task-sdk - - tests-airflow-ctl - finalize-tests if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 runs-on: ["ubuntu-22.04"] @@ -842,10 +843,51 @@ jobs: # yamllint disable rule:line-length payload: | channel: "internal-airflow-ci-cd" - text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) 🕒🚨\n\n*Details:* " blocks: - type: "section" text: type: "mrkdwn" - text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) 🕒🚨\n\n*Details:* " # yamllint enable rule:line-length + + summarize-warnings: + timeout-minutes: 15 + name: "Summarize warnings" + runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} + if: needs.build-info.outputs.run-tests == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh + - name: "Download all test warning artifacts from the current build" + uses: actions/download-artifact@v4 + with: + path: ./artifacts + pattern: test-warnings-* + - name: "Setup python" + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.default-python-version }} + - name: "Summarize all warnings" + run: | + ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ + --pattern "**/warnings-*.txt" \ + --output ./files + - name: "Upload artifact for summarized warnings" + uses: actions/upload-artifact@v4 + with: + name: test-summarized-amd-runner-warnings + path: ./files/warn-summary-*.txt + retention-days: 7 + if-no-files-found: ignore + overwrite: true diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml new file mode 100644 index 0000000000000..5b8cee0a432ca --- /dev/null +++ b/.github/workflows/ci-arm.yml @@ -0,0 +1,572 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Tests ARM +on: # yamllint disable-line rule:truthy + schedule: + - cron: '28 3,9,15,21 * * *' + push: + branches: + - v[0-9]+-[0-9]+-test + - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ + workflow_dispatch: +permissions: + # All other permissions are set to none by default + contents: read +env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + VERBOSE: "true" + +concurrency: + group: ci-arm-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + build-info: + name: "Build info" + # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners + runs-on: ["ubuntu-22.04"] + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + outputs: + all-python-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} + canary-run: ${{ steps.source-run-info.outputs.canary-run }} + ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} + core-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} + debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} + default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} + default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} + default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} + default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} + default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} + docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} + docs-build: ${{ steps.selective-checks.outputs.docs-build }} + docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} + excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }} + force-pip: ${{ steps.selective-checks.outputs.force-pip }} + full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} + has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} + helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} + include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} + individual-providers-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} + kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} + kubernetes-combos-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} + kubernetes-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} + latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} + mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} + mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} + mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} + needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} + needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} + needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} + needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} + only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }} + postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} + postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} + prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} + providers-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} + run-airflow-ctl-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-tests }} + run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} + run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} + run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} + run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} + run-tests: ${{ steps.selective-checks.outputs.run-tests }} + run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} + run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} + amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} + arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} + selected-providers-list-as-string: >- + ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} + skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} + source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} + sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} + testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} + testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: Fetch incoming commit ${{ github.sha }} with its parent + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Get information about the Workflow" + id: source-run-info + run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} + env: + SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: env + run: printenv + env: + PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} + GITHUB_CONTEXT: ${{ toJson(github) }} + + basic-tests: + name: "Basic tests" + needs: [build-info] + uses: ./.github/workflows/basic-tests.yml + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} + run-www-tests: ${{needs.build-info.outputs.run-www-tests}} + needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} + default-python-version: ${{needs.build-info.outputs.default-python-version}} + basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} + skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + canary-run: ${{needs.build-info.outputs.canary-run}} + latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} + use-uv: ${{needs.build-info.outputs.use-uv}} + + build-ci-images: + name: Build CI images + needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + if: needs.build-info.outputs.ci-image-build == 'true' + + additional-ci-image-checks: + name: "Additional CI image checks" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/additional-ci-image-checks.yml + permissions: + contents: read + packages: write + id-token: write + if: needs.build-info.outputs.canary-run == 'true' + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + if: needs.build-info.outputs.ci-image-build == 'true' + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + # generate no providers constraints only in canary builds - they take quite some time to generate + # they are not needed for regular builds, they are only needed to update constraints in canaries + generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + + providers: + name: "provider distributions tests" + uses: ./.github/workflows/test-providers.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + if: > + needs.build-info.outputs.skip-providers-tests != 'true' && + needs.build-info.outputs.latest-versions-only != 'true' + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + canary-run: ${{ needs.build-info.outputs.canary-run }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + providers-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + tests-helm: + name: "Helm tests" + uses: ./.github/workflows/helm-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: > + needs.build-info.outputs.needs-helm-tests == 'true' && + needs.build-info.outputs.default-branch == 'main' && + needs.build-info.outputs.latest-versions-only != 'true' + + tests-postgres-core: + name: "Postgres tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "postgres" + test-name: "Postgres" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.postgres-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.postgres-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-migration-tests: "true" + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-postgres-providers: + name: "Postgres tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "postgres" + test-name: "Postgres" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.postgres-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.postgres-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-migration-tests: "true" + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-sqlite-core: + name: "Sqlite tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "Sqlite" + test-name-separator: "" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for sqlite + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-sqlite-providers: + name: "Sqlite tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "Sqlite" + test-name-separator: "" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for sqlite + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + + tests-non-db-core: + name: "Non-DB tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "" + test-name-separator: "" + test-scope: "Non-DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for non-db + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-non-db-providers: + name: "Non-DB tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "" + test-name-separator: "" + test-scope: "Non-DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for non-db + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + build-prod-images: + name: Build PROD images + needs: [build-info, build-ci-images, generate-constraints] + uses: ./.github/workflows/prod-image-build.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + build-type: "Regular" + push-image: "false" + upload-image-artifact: "true" + upload-package-artifact: "true" + python-versions: ${{ needs.build-info.outputs.python-versions }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + branch: ${{ needs.build-info.outputs.default-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + + tests-kubernetes: + name: "Kubernetes tests" + uses: ./.github/workflows/k8s-tests.yml + needs: [build-info, build-prod-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} + if: > + ( needs.build-info.outputs.run-kubernetes-tests == 'true' || + needs.build-info.outputs.needs-helm-tests == 'true') + + finalize-tests: + name: Finalize tests + permissions: + contents: write + packages: write + # This will fire when all the jobs from "needs" are either successful or skipped + if: always() && !failure() && !cancelled() + needs: + - additional-ci-image-checks + - basic-tests + - build-info + - basic-tests + - generate-constraints + - build-prod-images + - providers + - tests-helm + - tests-kubernetes + - tests-non-db-core + - tests-non-db-providers + - tests-postgres-core + - tests-postgres-providers + - tests-sqlite-core + - tests-sqlite-providers + uses: ./.github/workflows/finalize-tests.yml + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions: ${{ needs.build-info.outputs.python-versions }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + + notify-slack-failure: + name: "Notify Slack on Failure" + needs: + - finalize-tests + if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 + runs-on: ["ubuntu-22.04"] + steps: + - name: Notify Slack + id: slack + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + with: + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} + # yamllint disable rule:line-length + payload: | + channel: "internal-airflow-ci-cd" + text: "🚨🕒 Scheduled CI Failure Alert (ARM) 🕒🚨\n\n*Details:* " + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "🚨🕒 Scheduled CI Failure Alert (ARM) 🕒🚨\n\n*Details:* " + # yamllint enable rule:line-length diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index c695778b87b99..d531ffebebf2d 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -20,12 +20,8 @@ name: Build CI images on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." - required: true - type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string target-commit-sha: @@ -77,7 +73,7 @@ on: # yamllint disable-line rule:truthy required: true type: string branch: - description: "Branch used to run the CI jobs in (main/v2_*_test)." + description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string constraints-branch: @@ -106,17 +102,12 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 110 name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}" - # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! - # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" - # instead of an array of strings. - # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} - VERSION_SUFFIX_FOR_PYPI: "dev0" GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 3e7a3bba9088a..8c80213c255f5 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -20,12 +20,12 @@ name: CI Image Checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - runs-on-as-json-docs-build: - description: "The array of labels (in json form) determining the labels used for docs build." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string needs-mypy: @@ -41,7 +41,7 @@ on: # yamllint disable-line rule:truthy required: true type: string branch: - description: "Branch used to run the CI jobs in (main/v2_*_test)." + description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string canary-run: @@ -60,10 +60,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies (true/false)" required: true type: string - chicken-egg-providers: - description: "List of providers that should be prepared from sources" - required: false - type: string basic-checks-only: description: "Whether to run only basic checks (true/false)" required: true @@ -121,7 +117,7 @@ jobs: install-pre-commit: timeout-minutes: 5 name: "Install pre-commit for cache (only canary runs)" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.basic-checks-only == 'false' @@ -165,7 +161,7 @@ jobs: static-checks: timeout-minutes: 45 name: "Static checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -183,7 +179,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} id: breeze @@ -205,7 +201,7 @@ jobs: mypy: timeout-minutes: 45 name: "MyPy checks" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit if: inputs.needs-mypy == 'true' strategy: @@ -226,7 +222,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} id: breeze @@ -249,7 +245,7 @@ jobs: build-docs: timeout-minutes: 150 name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.docs-build == 'true' strategy: fail-fast: false @@ -274,7 +270,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Restore docs inventory cache" @@ -324,7 +320,9 @@ jobs: INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" VERBOSE: "true" - if: inputs.canary-run == 'true' && inputs.branch == 'main' + if: > + inputs.canary-run == 'true' && + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') steps: - name: "Cleanup repo" shell: bash @@ -351,7 +349,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Publish docs" @@ -389,7 +387,7 @@ jobs: test-python-api-client: timeout-minutes: 60 name: "Test Python API client" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.needs-api-codegen == 'true' env: BACKEND: "postgres" @@ -423,13 +421,13 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Generate airflow python client" run: > breeze release-management prepare-python-client --distribution-format both - --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python + --python-client-repo ./airflow-client-python - name: "Show diff" run: git diff --color HEAD working-directory: ./airflow-client-python diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 47db38269b35f..082744fd6faea 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -20,12 +20,12 @@ name: Finalize tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions: @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} timeout-minutes: 80 name: "Update constraints" permissions: @@ -93,7 +93,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} VERBOSE: "true" - if: inputs.upgrade-to-newer-dependencies != 'false' + if: inputs.upgrade-to-newer-dependencies != 'false' && inputs.platform == 'linux/amd64' steps: - name: "Cleanup repo" shell: bash @@ -127,25 +127,26 @@ jobs: run: ./scripts/ci/constraints/ci_commit_constraints.sh if: inputs.canary-run == 'true' - name: "Push changes" - if: inputs.canary-run == 'true' + if: inputs.canary-run == 'true' && github.event_name != 'pull_request' working-directory: "constraints" run: git push - push-buildx-cache-to-github-registry-amd: - name: Push Regular AMD Image Cache + push-buildx-cache-to-github-registry: + name: Push Regular Image Cache ${{ inputs.platform }} needs: [update-constraints] uses: ./.github/workflows/push-image-cache.yml permissions: contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} + runners: ${{ inputs.runners }} + platform: ${ { inputs.platform }} cache-type: "Regular AMD" include-prod-images: "true" push-latest-images: "true" - platform: "linux/amd64" python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} @@ -153,66 +154,6 @@ jobs: include-success-outputs: ${{ inputs.include-success-outputs }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} - if: inputs.canary-run == 'true' - - # push-buildx-cache-to-github-registry-arm: - # name: Push Regular ARM Image Cache - # needs: [update-constraints] - # uses: ./.github/workflows/push-image-cache.yml - # permissions: - # contents: read - # packages: write - # with: - # runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - # runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} - # cache-type: "Regular ARM" - # include-prod-images: "true" - # push-latest-images: "true" - # platform: "linux/arm64" - # python-versions: ${{ inputs.python-versions }} - # branch: ${{ inputs.branch }} - # constraints-branch: ${{ inputs.constraints-branch }} - # use-uv: "true" - # include-success-outputs: ${{ inputs.include-success-outputs }} - # docker-cache: ${{ inputs.docker-cache }} - # if: inputs.canary-run == 'true' - - summarize-warnings: - timeout-minutes: 15 - name: "Summarize warnings" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - - name: "Download all test warning artifacts from the current build" - uses: actions/download-artifact@v4 - with: - path: ./artifacts - pattern: test-warnings-* - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.default-python-version }} - - name: "Summarize all warnings" - run: | - ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ - --pattern "**/warnings-*.txt" \ - --output ./files - - name: "Upload artifact for summarized warnings" - uses: actions/upload-artifact@v4 - with: - name: test-summarized-warnings - path: ./files/warn-summary-*.txt - retention-days: 7 - if-no-files-found: ignore - overwrite: true + if: > + inputs.canary-run == 'true' && + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 0f74081686316..363888ff29718 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -20,8 +20,12 @@ name: Generate constraints on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions-list-as-string: @@ -32,10 +36,6 @@ on: # yamllint disable-line rule:truthy description: "Whether to generate constraints without providers (true/false)" required: true type: string - chicken-egg-providers: - description: "Space-separated list of providers that should be installed from context files" - required: true - type: string debug-resources: description: "Whether to run in debug mode (true/false)" required: true @@ -50,7 +50,7 @@ jobs: contents: read timeout-minutes: 70 name: Generate constraints ${{ inputs.python-versions-list-as-string }} - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: DEBUG_RESOURCES: ${{ inputs.debug-resources }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -59,7 +59,6 @@ jobs: INCLUDE_SUCCESS_OUTPUTS: "true" PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: "Cleanup repo" shell: bash @@ -79,7 +78,7 @@ jobs: - name: "Prepare all CI images: ${{ inputs.python-versions-list-as-string}}" uses: ./.github/actions/prepare_all_ci_images with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }} docker-volume-location: "" # TODO(jscheffl): Understand why it fails here and fix it - name: "Verify all CI images ${{ inputs.python-versions-list-as-string }}" @@ -98,39 +97,27 @@ jobs: # The no providers constraints are only needed when we want to update constraints (in canary builds) # They slow down the start of PROD image builds so we want to only run them when needed. if: inputs.generate-no-providers-constraints == 'true' - - name: "Prepare chicken-eggs provider distributions" - # In case of provider distributions which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + - name: "Prepare updated provider distributions" + # In case of provider distributions which are not yet released, we build them from sources shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }} run: > breeze release-management prepare-provider-distributions --include-not-ready-providers - --distribution-format wheel --version-suffix-for-pypi dev0 - ${CHICKEN_EGG_PROVIDERS} - if: inputs.chicken-egg-providers != '' + --distribution-format wheel - name: "Prepare airflow distributions" shell: bash run: > - breeze release-management prepare-airflow-distributions - --distribution-format wheel --version-suffix-for-pypi dev0 + breeze release-management prepare-airflow-distributions --distribution-format wheel - name: "Prepare task-sdk distribution" shell: bash run: > - breeze release-management prepare-task-sdk-distributions - --distribution-format wheel --version-suffix-for-pypi dev0 + breeze release-management prepare-task-sdk-distributions --distribution-format wheel - name: "PyPI constraints" shell: bash timeout-minutes: 25 - env: - CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }} run: | for PYTHON in $PYTHON_VERSIONS; do - breeze release-management generate-constraints \ - --airflow-constraints-mode constraints --answer yes \ - --chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}" \ - --python "${PYTHON}" + breeze release-management generate-constraints --airflow-constraints-mode constraints \ + --answer yes --python "${PYTHON}" done - name: "Dependency upgrade summary" shell: bash diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1b4aa19cbe595..e00ac14506ab4 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -20,12 +20,12 @@ name: Helm tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string helm-test-packages: @@ -46,7 +46,7 @@ jobs: tests-helm: timeout-minutes: 80 name: "Unit tests Helm: ${{ matrix.helm-test-package }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -74,7 +74,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" @@ -85,7 +85,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" steps: diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index fc3159223e8fb..bf75cc87f31a3 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -20,10 +20,14 @@ name: Integration and system tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: + runners: description: "The array of labels (in json form) determining public runners." required: true type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string testable-core-integrations: description: "The list of testable core integrations as JSON array." required: true @@ -71,7 +75,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -99,7 +103,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Integration: core ${{ matrix.integration }}" @@ -120,7 +124,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true' name: "Integration: providers ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -148,7 +152,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Integration: providers ${{ matrix.integration }}" @@ -168,7 +172,7 @@ jobs: timeout-minutes: 30 if: inputs.run-system-tests == 'true' name: "System Tests" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: BACKEND: "postgres" BACKEND_VERSION: ${{ inputs.default-postgres-version }}" @@ -192,7 +196,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "System Tests" diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 40f73e3c59c66..6ed0c79d187c5 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -20,12 +20,12 @@ name: K8s tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions-list-as-string: @@ -54,7 +54,7 @@ jobs: tests-kubernetes: timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: matrix: executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] @@ -103,27 +103,26 @@ jobs: USE_STANDARD_NAMING: ${{ matrix.use-standard-naming }} VERBOSE: "false" - name: "\ - Upload KinD logs on failure ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ + Print logs ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ ${{ matrix.use-standard-naming }}" - uses: actions/upload-artifact@v4 - if: failure() || cancelled() - with: - name: "\ - kind-logs-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ - ${{ matrix.use-standard-naming }}" - path: /tmp/kind_logs_* - retention-days: '7' + run: | + for file in `find /tmp/kind_logs_*/ -type f` ; do + echo "::group::${file}" + cat $file + echo "::endgroup::" + done + if: failure() || cancelled() || inputs.include-success-outputs == 'true' - name: "\ - Upload test resource logs on failure ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ + Upload KinD logs ${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-\ ${{ matrix.use-standard-naming }}" uses: actions/upload-artifact@v4 - if: failure() || cancelled() with: name: "\ - k8s-test-resources-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ + kind-logs-${{ matrix.kubernetes-combo }}-${{ matrix.executor }}-\ ${{ matrix.use-standard-naming }}" - path: /tmp/k8s_test_resources_* + path: /tmp/kind_logs_* retention-days: '7' + if: failure() || cancelled() || inputs.include-success-outputs == 'true' - name: "Delete clusters just in case they are left" run: breeze k8s delete-cluster --all if: always() diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index a335576d4bcf5..721158f097856 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,8 +20,8 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string build-type: @@ -85,25 +85,17 @@ on: # yamllint disable-line rule:truthy required: true type: string branch: - description: "Branch used to run the CI jobs in (main/v2_*_test)." + description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true type: string - build-provider-distributions: - description: "Whether to build provider distributions (true/false). If false providers are from PyPI" - required: true - type: string upgrade-to-newer-dependencies: description: "Whether to attempt to upgrade image to newer dependencies (true/false)" required: true type: string - chicken-egg-providers: - description: "Space-separated list of providers that should be installed from context files" - required: true - type: string docker-cache: description: "Docker cache specification to build the image (registry, local, disabled)." required: true @@ -122,11 +114,10 @@ jobs: build-prod-packages: name: "Build Airflow and provider distributions" timeout-minutes: 10 - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - VERSION_SUFFIX_FOR_PYPI: ${{ inputs.branch == 'main' && 'dev0' || '' }} steps: - name: "Cleanup repo" shell: bash @@ -148,25 +139,22 @@ jobs: with: use-uv: ${{ inputs.use-uv }} if: inputs.upload-package-artifact == 'true' - - name: "Prepare providers packages" + - name: "Prepare providers packages - all providers built from sources" shell: bash run: > breeze release-management prepare-provider-distributions --distributions-list-file ./prod_image_installed_providers.txt - --distribution-format wheel --include-not-ready-providers + --distribution-format wheel --include-not-ready-providers --skip-tag-check if: > - inputs.upload-package-artifact == 'true' && - inputs.build-provider-distributions == 'true' - - name: "Prepare chicken-eggs provider distributions" + inputs.upload-package-artifact == 'true' && inputs.branch == 'main' + - name: "Prepare providers packages with only new versions of providers" shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ inputs.chicken-egg-providers }} run: > breeze release-management prepare-provider-distributions - --distribution-format wheel ${CHICKEN_EGG_PROVIDERS} + --distributions-list-file ./prod_image_installed_providers.txt + --distribution-format wheel --include-not-ready-providers if: > - inputs.upload-package-artifact == 'true' && - inputs.chicken-egg-providers != '' + inputs.upload-package-artifact == 'true' && inputs.branch != 'main' - name: "Prepare airflow package" shell: bash run: > @@ -198,7 +186,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 80 name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: - build-prod-packages env: @@ -206,7 +194,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} - VERSION_SUFFIX_FOR_PYPI: ${{ inputs.branch == 'main' && 'dev0' || '' }} INCLUDE_NOT_READY_PROVIDERS: "true" # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the # Airflow one is going to be used @@ -217,6 +204,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} + PLATFORM: ${{ inputs.platform }} VERBOSE: "true" steps: - name: "Cleanup repo" @@ -240,11 +228,23 @@ jobs: with: name: prod-packages path: ./docker-context-files + - name: "Show downloaded packages" + run: ls -la ./docker-context-files - name: "Download constraints" uses: actions/download-artifact@v4 with: name: constraints path: ./docker-context-files + - name: "Show constraints" + run: | + for file in ./docker-context-files/constraints*/constraints*.txt + do + echo "=== ${file} ===" + echo + cat ${file} + echo + echo "=== END ${file} ===" + done - name: "Login to ghcr.io" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -267,25 +267,6 @@ jobs: INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" - if: inputs.build-provider-distributions == 'true' - - name: "Build PROD images with PyPi providers ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - shell: bash - run: > - breeze prod-image build - --builder airflow_cache - --commit-sha "${{ github.sha }}" - --install-distributions-from-context - --airflow-constraints-mode constraints - --use-constraints-for-context-distributions - env: - PUSH: ${{ inputs.push-image }} - DOCKER_CACHE: ${{ inputs.docker-cache }} - DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }} - DEBIAN_VERSION: ${{ inputs.debian-version }} - INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} - INCLUDE_NOT_READY_PROVIDERS: "true" - if: inputs.build-provider-distributions != 'true' - name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" run: breeze prod-image verify - name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 0b208cb552059..4f6ce6c762964 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -20,8 +20,12 @@ name: PROD images extra checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions: @@ -33,25 +37,17 @@ on: # yamllint disable-line rule:truthy required: true type: string branch: - description: "Branch used to run the CI jobs in (main/v2_*_test)." + description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string use-uv: description: "Whether to use uv to build the image (true/false)" required: true type: string - build-provider-distributions: - description: "Whether to build provider distributions (true/false). If false providers are from PyPI" - required: true - type: string upgrade-to-newer-dependencies: description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)" required: true type: string - chicken-egg-providers: - description: "Space-separated list of providers that should be installed from context files" - required: true - type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true @@ -70,21 +66,19 @@ jobs: myssql-client-image: uses: ./.github/workflows/prod-image-build.yml with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} build-type: "MySQL Client" upload-image-artifact: "false" upload-package-artifact: "false" install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" use-uv: ${{ inputs.use-uv }} - build-provider-distributions: ${{ inputs.build-provider-distributions }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} @@ -92,24 +86,20 @@ jobs: pip-image: uses: ./.github/workflows/prod-image-build.yml - # Skip testing PIP image on release branches as all images there are built with pip - if: ${{ inputs.use-uv == 'true' }} with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} build-type: "pip" upload-image-artifact: "false" upload-package-artifact: "false" install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" use-uv: "false" - build-provider-distributions: ${{ inputs.build-provider-distributions }} upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - chicken-egg-providers: ${{ inputs.chicken-egg-providers }} constraints-branch: ${{ inputs.constraints-branch }} docker-cache: ${{ inputs.docker-cache }} disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml new file mode 100644 index 0000000000000..23e644a286de6 --- /dev/null +++ b/.github/workflows/publish-docs-to-s3.yml @@ -0,0 +1,258 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Publish Docs to S3 +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + ref: + description: "The branch or tag to checkout for the docs publishing" + required: true + type: string + exclude-docs: + description: "Comma separated list of docs to exclude" + required: false + default: "NO_DOCS" + type: string + destination-location: + description: "The destination location in S3" + required: false + default: "s3://staging-docs-airflow-apache-org/docs" + type: string + docs-list-as-string: + description: "Space separated list of docs to build" + required: false + default: "" + type: string +env: + AIRFLOW_ROOT_PATH: "/home/runner/work/temp-airflow-repo-reference" # checkout dir for referenced tag +permissions: + contents: read +jobs: + build-info: + timeout-minutes: 10 + name: "Build Info" + runs-on: ["ubuntu-24.04"] + outputs: + runners: ${{ steps.selective-checks.outputs.amd-runners }} + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + VERBOSE: true + REF: ${{ inputs.ref }} + EXCLUDE_DOCS: ${{ inputs.exclude-docs }} + DESTINATION_LOCATION: ${{ inputs.destination-location }} + DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + if: contains(fromJSON('[ + "ashb", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "kaxil", + "pierrejeambrun", + "potiuk", + "utkarsharma2" + ]'), github.event.sender.login) + steps: + - name: "Input parameters summary" + shell: bash + run: | + echo "Input parameters summary" + echo "=========================" + echo "Ref: '${REF}'" + echo "Exclude docs: '${EXCLUDE_DOCS}'" + echo "Destination location: '${DESTINATION_LOCATION}'" + echo "Docs list as string: '${DOCS_LIST_AS_STRING}'" + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Install uv" + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: "true" + - name: Selective checks + id: selective-checks + env: + VERBOSE: "false" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + + build-ci-images: + name: Build CI images + uses: ./.github/workflows/ci-image-build.yml + needs: [build-info] + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: false + python-versions: "['3.9']" + branch: ${{ inputs.ref }} + use-uv: true + upgrade-to-newer-dependencies: false + constraints-branch: "constraints-main" + docker-cache: registry + disable-airflow-repo-cache: false + + build-docs: + needs: [build-ci-images] + timeout-minutes: 150 + name: "Build documentation" + runs-on: ubuntu-latest + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + INCLUDE_NOT_READY_PROVIDERS: "true" + INCLUDE_SUCCESS_OUTPUTS: false + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} " + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Checkout from ${{ inputs.ref }} to build docs" + run: | + git clone https://github.com/apache/airflow.git "${AIRFLOW_ROOT_PATH}" + cd "${AIRFLOW_ROOT_PATH}" && git checkout ${REF} + env: + REF: ${{ inputs.ref }} + - name: "Prepare breeze & CI image: 3.9" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: "linux/amd64" + python: 3.9 + use-uv: true + - name: "Building docs with --docs-only flag" + env: + DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + run: > + breeze build-docs ${DOCS_LIST_AS_STRING} --docs-only + - name: "Upload build docs" + uses: actions/upload-artifact@v4 + with: + name: airflow-docs + path: ${{ env.AIRFLOW_ROOT_PATH }}/generated/_build + retention-days: '7' + if-no-files-found: 'error' + overwrite: 'true' + + publish-docs-to-s3: + needs: [build-docs] + name: "Publish documentation to S3" + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + INCLUDE_NOT_READY_PROVIDERS: "true" + INCLUDE_SUCCESS_OUTPUTS: false + PYTHON_MAJOR_MINOR_VERSION: 3.9 + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} " + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Checkout ${{ inputs.ref }}" + run: | + git clone https://github.com/apache/airflow.git "${AIRFLOW_ROOT_PATH}" + cd "${AIRFLOW_ROOT_PATH}" && git checkout ${REF} + env: + REF: ${{ inputs.ref }} + - name: "Download docs prepared as artifacts" + uses: actions/download-artifact@v4 + with: + name: airflow-docs + path: ${{ env.AIRFLOW_ROOT_PATH }}/generated/_build + - name: Check disk space available + run: df -h + # Here we will create temp airflow-site dir to publish + # docs and for back-references + - name: Create /mnt/airflow-site directory + run: | + sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}" /mnt/airflow-site + echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/" >> "$GITHUB_ENV" + - name: "Prepare breeze & CI image: 3.9" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: "linux/amd64" + python: 3.9 + use-uv: true + - name: "Publish docs to tmp directory" + env: + DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }} + run: > + breeze release-management publish-docs --override-versioned --run-in-parallel + ${DOCS_LIST_AS_STRING} + - name: Check disk space available + run: df -h + - name: "Generate back references for providers" + run: breeze release-management add-back-references all-providers + - name: "Generate back references for apache-airflow" + run: breeze release-management add-back-references apache-airflow + - name: "Generate back references for docker-stack" + run: breeze release-management add-back-references docker-stack + - name: "Generate back references for helm-chart" + run: breeze release-management add-back-references helm-chart + - name: Install AWS CLI v2 + run: | + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip + unzip -q /tmp/awscliv2.zip -d /tmp + rm /tmp/awscliv2.zip + sudo /tmp/aws/install --update + rm -rf /tmp/aws/ + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + with: + aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: "Syncing docs to S3" + env: + DESTINATION_LOCATION: "${{ inputs.destination-location }}" + SOURCE_DIR_PATH: "/mnt/airflow-site/docs-archive/" + EXCLUDE_DOCS: "${{ inputs.exclude-docs }}" + run: | + breeze release-management publish-docs-to-s3 --source-dir-path ${SOURCE_DIR_PATH} \ + --destination-location ${DESTINATION_LOCATION} --stable-versions \ + --exclude-docs ${EXCLUDE_DOCS} diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index cf8097cebd768..d562f798c7791 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -20,12 +20,8 @@ name: Push image cache on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-public: - description: "The array of labels (in json form) determining public runners." - required: true - type: string - runs-on-as-json-self-hosted: - description: "The array of labels (in json form) determining self-hosted runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string cache-type: @@ -57,7 +53,7 @@ on: # yamllint disable-line rule:truthy required: true type: string branch: - description: "Branch used to run the CI jobs in (main/v2_*_test)." + description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string constraints-branch: @@ -83,11 +79,7 @@ on: # yamllint disable-line rule:truthy jobs: push-ci-image-cache: name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " - # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! - # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" - # instead of an array of strings. - # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + runs-on: ${{ fromJSON(inputs.runners) }} permissions: contents: read packages: write @@ -116,7 +108,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" UPGRADE_TO_NEWER_DEPENDENCIES: "false" VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: "Cleanup repo" shell: bash @@ -159,11 +150,7 @@ jobs: push-prod-image-cache: name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" - # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! - # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" - # instead of an array of strings. - # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + runs-on: ${{ fromJSON(inputs.runners) }} permissions: contents: read packages: write @@ -191,7 +178,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python }}" UPGRADE_TO_NEWER_DEPENDENCIES: "false" VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" if: inputs.include-prod-images == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index c850bef580889..9d8655ecaead3 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -21,12 +21,12 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: airflowVersion: - description: 'Airflow version' + description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)' required: true - skipLatest: - description: 'Skip Latest: Set to true if not latest.' - default: '' - required: false + amdOnly: + type: boolean + description: 'Limit to amd64 only (faster testing)' + default: false permissions: contents: read packages: read @@ -40,18 +40,39 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-24.04"] outputs: pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} - skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }} - limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }} + platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} + airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }} + skipLatest: ${{ steps.selective-checks.outputs.skipLatest }} + amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} + arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} env: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true + AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} + AMD_ONLY: ${{ github.event.inputs.amdOnly }} + if: contains(fromJSON('[ + "ashb", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "kaxil", + "pierrejeambrun", + "potiuk", + "utkarsharma2" + ]'), github.event.sender.login) steps: + - name: "Input parameters summary" + shell: bash + run: | + echo "Input parameters summary" + echo "=========================" + echo "Airflow version: '${AIRFLOW_VERSION}'" + echo "AMD only: '${AMD_ONLY}'" - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -61,35 +82,49 @@ jobs: persist-credentials: false - name: "Cleanup docker" run: ./scripts/ci/cleanup_docker.sh + - name: "Install uv" + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: "Check airflow version" + id: check-airflow-version + shell: bash + run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}" - name: "Install Breeze" uses: ./.github/actions/breeze with: - use-uv: "false" + use-uv: "true" - name: Selective checks id: selective-checks env: VERBOSE: "false" run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: "Determine build matrix" + shell: bash + id: determine-matrix + run: | + if [[ "${AMD_ONLY}" = "true" ]]; then + echo 'platformMatrix=["linux/amd64"]' >> "${GITHUB_OUTPUT}" + else + echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >> "${GITHUB_OUTPUT}" + fi - release-images: - timeout-minutes: 120 - name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - runs-on: ["ubuntu-22.04"] + build-images: + timeout-minutes: 50 + # yamllint disable rule:line-length + name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }}" + runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(needs.build-info.outputs.amd-runners) || fromJSON(needs.build-info.outputs.arm-runners) }} needs: [build-info] strategy: fail-fast: false matrix: python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} - if: contains(fromJSON('[ - "ashb", - "eladkal", - "ephraimbuddy", - "jedcunningham", - "kaxil", - "pierrejeambrun", - "potiuk", - "utkarsharma2" - ]'), github.event.sender.login) + platform: ${{ fromJSON(needs.build-info.outputs.platformMatrix) }} + env: + AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + PLATFORM: ${{ matrix.platform }} + SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} + COMMIT_SHA: ${{ github.sha }} + REPOSITORY: ${{ github.repository }} steps: - name: "Cleanup repo" shell: bash @@ -112,6 +147,12 @@ jobs: run: > echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: "Get env vars for metadata" + shell: bash + run: | + echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}" + echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" + echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" - name: Login to ghcr.io env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -133,90 +174,127 @@ jobs: sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt install docker-buildx-plugin - - name: "Install regctl" - # yamllint disable rule:line-length - run: | - mkdir -p ~/bin - curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl - chmod 755 ${HOME}/bin/regctl - echo "${HOME}/bin" >>${GITHUB_PATH} - - name: "Install emulation support" - run: docker run --privileged --rm tonistiigi/binfmt --install all - name: "Create airflow_cache builder" - run: docker buildx create --name airflow_cache - - name: "Prepare chicken-eggs provider distributions" - # In case of provider distributions which use latest rc1 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.rc0 - shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} - run: > - breeze release-management prepare-provider-distributions - --distribution-format wheel - --version-suffix-for-pypi rc1 ${CHICKEN_EGG_PROVIDERS} - if: needs.build-info.outputs.chicken-egg-providers != '' - - name: "Copy dist packages to docker-context files" - shell: bash - run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - if: needs.build-info.outputs.chicken-egg-providers != '' + run: docker buildx create --name airflow_cache --driver docker-container - name: > - Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} - env: - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }} - LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }} - CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} + Build regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} run: > - breeze release-management release-prod-images - --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" - ${SKIP_LATEST} - ${LIMIT_PLATFORM} - --limit-python ${PYTHON_VERSION} - --chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}" + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} + --metadata-folder dist - name: > - Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} - env: - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }} - LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }} + Verify regular image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} run: > - breeze release-management release-prod-images - --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" - ${SKIP_LATEST} - ${LIMIT_PLATFORM} - --limit-python ${PYTHON_VERSION} --slim-images + breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME} - name: > - Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - REPOSITORY: ${{ github.repository }} + Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} run: > - breeze prod-image verify - --pull - --image-name - ${REPOSITORY}:${AIRFLOW_VERSION}-python${PYTHON_VERSION} + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images + --metadata-folder dist - name: > - Verify slim AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} + Verify slim image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} + run: > + breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME} + - name: "List upload-able artifacts" + shell: bash + run: find ./dist -name '*.json' + - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}" + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ./dist/metadata-* + retention-days: 7 + if-no-files-found: error + - name: "Docker logout" + run: docker logout + if: always() + + merge-images: + timeout-minutes: 5 + name: "Merge: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" + runs-on: ["ubuntu-22.04"] + needs: [build-info, build-images] + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} + env: + AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} + COMMIT_SHA: ${{ github.sha }} + REPOSITORY: ${{ github.repository }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: "false" + - name: Free space + run: breeze ci free-space --answer yes + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + - name: "Login to hub.docker.com" + run: > + echo ${{ secrets.DOCKERHUB_TOKEN }} | + docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: Login to ghcr.io env: - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin + - name: "Download metadata artifacts" + uses: actions/download-artifact@v4 + with: + path: ./dist + pattern: metadata-${{ matrix.python-version }}-* + - name: "List downloaded artifacts" + shell: bash + run: find ./dist -name '*.json' + - name: "Install buildx plugin" + # yamllint disable rule:line-length + run: | + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt install docker-buildx-plugin + - name: "Install regctl" + # yamllint disable rule:line-length + run: | + mkdir -p ~/bin + curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl + chmod 755 ${HOME}/bin/regctl + echo "${HOME}/bin" >>${GITHUB_PATH} + - name: "Merge regular images ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" + run: > + breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist + - name: "Merge slim images ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" run: > - breeze prod-image verify - --pull - --slim-image - --image-name - ${REPOSITORY}:slim-${AIRFLOW_VERSION}-python${PYTHON_VERSION} + breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images - name: "Docker logout" run: docker logout if: always() diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 939fa2b634179..435b49897da14 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -20,8 +20,12 @@ name: Unit tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining public AMD runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string test-group: @@ -37,7 +41,7 @@ on: # yamllint disable-line rule:truthy required: true type: string test-scope: - description: "The scope of the test to run: ('DB', 'Non-DB', 'All', 'ARM collection')" + description: "The scope of the test to run: ('DB', 'Non-DB', 'All')" required: true type: string test-name: @@ -125,7 +129,7 @@ jobs: ${{ inputs.test-scope }}-${{ inputs.test-group }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ ${{ matrix.python-version}}:${{ matrix.test-types.description }}" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -147,6 +151,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }} + PLATFORM: "${{ inputs.platform }}" # yamllint disable rule:line-length JOB_ID: "${{ inputs.test-group }}-${{ matrix.test-types.description }}-${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}" MOUNT_SOURCES: "skip" @@ -168,7 +173,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} - name: > diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index d47907f923f54..a41ab1693c830 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -20,8 +20,12 @@ name: Special tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string default-branch: @@ -90,7 +94,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -113,7 +118,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -136,7 +142,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -160,7 +167,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -185,7 +193,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -209,7 +218,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -233,7 +243,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" test-group: "core" @@ -256,7 +267,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" test-group: "providers" @@ -272,56 +284,6 @@ jobs: skip-providers-tests: ${{ inputs.skip-providers-tests }} use-uv: ${{ inputs.use-uv }} - - tests-arm-collection-core: - name: "ARM Collection test: core" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - test-name: "Postgres" - test-scope: "ARM collection" - test-group: "core" - backend: "postgres" - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} - excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ inputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - skip-providers-tests: ${{ inputs.skip-providers-tests }} - use-uv: ${{ inputs.use-uv }} - if: ${{ inputs.default-branch == 'main' }} - - tests-arm-collection-providers: - name: "ARM Collection test: providers" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} - test-name: "Postgres" - test-scope: "ARM collection" - test-group: "providers" - backend: "postgres" - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} - excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ inputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - skip-providers-tests: ${{ inputs.skip-providers-tests }} - use-uv: ${{ inputs.use-uv }} - if: ${{ inputs.default-branch == 'main' }} - - tests-system-core: name: "System test: ${{ matrix.test-group }}" uses: ./.github/workflows/run-unit-tests.yml @@ -329,7 +291,8 @@ jobs: contents: read packages: read with: - runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} test-name: "SystemTest" test-scope: "System" test-group: "core" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index 39f96b2c89afc..411b18daaa7b4 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -20,8 +20,12 @@ name: Provider tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on-as-json-default: - description: "The array of labels (in json form) determining default runner used for the build." + runners: + description: "The array of labels (in json form) determining public AMD runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string canary-run: @@ -68,7 +72,7 @@ jobs: prepare-install-verify-provider-distributions: timeout-minutes: 80 name: "Providers ${{ matrix.package-format }} tests" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -91,7 +95,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" @@ -104,14 +108,14 @@ jobs: - name: "Prepare provider distributions: ${{ matrix.package-format }}" run: > breeze release-management prepare-provider-distributions --include-not-ready-providers - --version-suffix-for-pypi dev0 --distribution-format ${{ matrix.package-format }} + --skip-tag-check --distribution-format ${{ matrix.package-format }} - name: "Prepare airflow package: ${{ matrix.package-format }}" run: > - breeze release-management prepare-airflow-distributions --version-suffix-for-pypi dev0 + breeze release-management prepare-airflow-distributions --distribution-format ${{ matrix.package-format }} - name: "Prepare task-sdk package: ${{ matrix.package-format }}" run: > - breeze release-management prepare-task-sdk-distributions --version-suffix-for-pypi dev0 + breeze release-management prepare-task-sdk-distributions --distribution-format ${{ matrix.package-format }} - name: "Verify ${{ matrix.package-format }} packages with twine" run: | @@ -163,7 +167,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.compat.test-types.description }} - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -175,7 +179,6 @@ jobs: GITHUB_USERNAME: ${{ github.actor }} INCLUDE_NOT_READY_PROVIDERS: "true" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.compat.python-version }}" - VERSION_SUFFIX_FOR_PYPI: "dev0" VERBOSE: "true" CLEAN_AIRFLOW_INSTALLATION: "true" if: inputs.skip-providers-tests != 'true' @@ -190,7 +193,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ matrix.compat.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ matrix.compat.python-version }} use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" @@ -198,7 +201,7 @@ jobs: - name: "Prepare provider distributions: wheel" run: > breeze release-management prepare-provider-distributions --include-not-ready-providers - --distribution-format wheel + --distribution-format wheel --skip-tag-check # yamllint disable rule:line-length - name: Remove incompatible Airflow ${{ matrix.compat.airflow-version }}:Python ${{ matrix.compat.python-version }} provider distributions env: diff --git a/.gitignore b/.gitignore index 1b0a579167847..026a52c50a6cf 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ airflow/git_version airflow/ui/coverage/ logs/ airflow-webserver.pid +airflow-api-server.pid standalone_admin_password.txt warnings.txt warn-summary-*.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 503814fbf5c77..1fc3e81f2ad48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ default_stages: [pre-commit, pre-push] default_language_version: python: python3 - node: 22.14.0 + node: 22.15.0 minimum_pre_commit_version: '3.2.0' exclude: ^.*/.*_vendor/ repos: @@ -511,7 +511,7 @@ repos: name: Update versions in docs entry: ./scripts/ci/pre_commit/update_versions.py language: python - files: ^docs|^airflow-core/src/airflow/__init__\.py$ + files: ^docs|^airflow-core/src/airflow/__init__\.py$|.*/pyproject\.toml$ pass_filenames: false additional_dependencies: ['rich>=12.4.4'] - id: check-pydevd-left-in-code @@ -886,7 +886,7 @@ repos: name: Update Airflow's meta-package pyproject.toml language: python entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py - files: ^pyproject\.toml$ + files: ^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$ pass_filenames: false require_serial: true additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1', 'packaging>=23.2' ] diff --git a/Dockerfile b/Dockerfile index 37ac245da5b20..f705e48d4f029 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ ARG AIRFLOW_UID="50000" ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here -ARG AIRFLOW_VERSION="2.10.5" +ARG AIRFLOW_VERSION="3.0.1" ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" @@ -54,10 +54,10 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm" # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` -ARG AIRFLOW_PIP_VERSION=25.0.1 +ARG AIRFLOW_PIP_VERSION=25.1.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_SETUPTOOLS_VERSION=78.1.0 -ARG AIRFLOW_UV_VERSION=0.6.13 +ARG AIRFLOW_SETUPTOOLS_VERSION=80.1.0 +ARG AIRFLOW_UV_VERSION=0.7.2 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" @@ -462,7 +462,7 @@ function common::get_packaging_tool() { UV_CONCURRENT_DOWNLOADS=$(nproc --all) export UV_CONCURRENT_DOWNLOADS if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then - EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow" + EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary" fi else echo @@ -491,7 +491,7 @@ function common::get_airflow_version_specification() { function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then - if [[ ${AIRFLOW_VERSION} =~ v?2.* && ! ${AIRFLOW_VERSION} =~ .*dev.* ]]; then + if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION} else AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH} @@ -1599,12 +1599,12 @@ COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} ARG ADDITIONAL_PYTHON_DEPS="" -ARG VERSION_SUFFIX_FOR_PYPI="" +ARG VERSION_SUFFIX="" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_DISTRIBUTIONS_FROM_CONTEXT=${INSTALL_DISTRIBUTIONS_FROM_CONTEXT} \ USE_CONSTRAINTS_FOR_CONTEXT_DISTRIBUTIONS=${USE_CONSTRAINTS_FOR_CONTEXT_DISTRIBUTIONS} \ - VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI} + VERSION_SUFFIX=${VERSION_SUFFIX} WORKDIR ${AIRFLOW_HOME} diff --git a/Dockerfile.ci b/Dockerfile.ci index b722ff0132935..39a5373ee1c83 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -401,7 +401,7 @@ function common::get_packaging_tool() { UV_CONCURRENT_DOWNLOADS=$(nproc --all) export UV_CONCURRENT_DOWNLOADS if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then - EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow" + EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary" fi else echo @@ -430,7 +430,7 @@ function common::get_airflow_version_specification() { function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then - if [[ ${AIRFLOW_VERSION} =~ v?2.* && ! ${AIRFLOW_VERSION} =~ .*dev.* ]]; then + if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION} else AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH} @@ -950,6 +950,12 @@ function environment_initialization() { cd "${AIRFLOW_SOURCES}" + # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags + # in the UI when testing in Breeze. This might be solved differently in the future + if [[ -d /opt/airflow/providers/standard/tests ]]; then + export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests + fi + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} wait_for_asset_compilation @@ -1038,12 +1044,12 @@ function check_downgrade_sqlalchemy() { return fi local min_sqlalchemy_version - min_sqlalchemy_version=$(grep "sqlalchemy>=" airflow-core/pyproject.toml | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs) + min_sqlalchemy_version=$(grep "sqlalchemy\[asyncio\]>=" airflow-core/pyproject.toml | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs) echo echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" echo # shellcheck disable=SC2086 - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy==${min_sqlalchemy_version}" + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy[asyncio]==${min_sqlalchemy_version}" pip check } @@ -1066,13 +1072,6 @@ function check_run_tests() { return fi - if [[ ${REMOVE_ARM_PACKAGES:="false"} == "true" ]]; then - # Test what happens if we do not have ARM packages installed. - # This is useful to see if pytest collection works without ARM packages which is important - # for the MacOS M1 users running tests in their ARM machines with `breeze testing *-tests` command - python "${IN_CONTAINER_DIR}/remove_arm_packages.py" - fi - if [[ ${TEST_GROUP:=""} == "system" ]]; then exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}" else @@ -1122,7 +1121,7 @@ function start_api_server_with_examples(){ return fi export AIRFLOW__CORE__LOAD_EXAMPLES=True - export AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True + export AIRFLOW__API__EXPOSE_CONFIG=True echo echo "${COLOR_BLUE}Initializing database${COLOR_RESET}" echo @@ -1353,10 +1352,10 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` -ARG AIRFLOW_PIP_VERSION=25.0.1 +ARG AIRFLOW_PIP_VERSION=25.1.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_SETUPTOOLS_VERSION=78.1.0 -ARG AIRFLOW_UV_VERSION=0.6.13 +ARG AIRFLOW_SETUPTOOLS_VERSION=80.1.0 +ARG AIRFLOW_UV_VERSION=0.7.2 # TODO(potiuk): automate with upgrade check (possibly) ARG AIRFLOW_PRE_COMMIT_VERSION="4.2.0" ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4" @@ -1391,10 +1390,10 @@ COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/ COPY . ${AIRFLOW_SOURCES}/ ARG UPGRADE_RANDOM_INDICATOR_STRING="" -ARG VERSION_SUFFIX_FOR_PYPI="" +ARG VERSION_SUFFIX="" ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \ - VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI} + VERSION_SUFFIX=${VERSION_SUFFIX} # The goal of this line is to install the dependencies from the most current pyproject.toml from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast diff --git a/README.md b/README.md index 95c5c3670509a..a0360260691e6 100644 --- a/README.md +++ b/README.md @@ -96,11 +96,11 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (3.0.0) | +| | Main version (dev) | Stable version (3.0.1) | |------------|------------------------|------------------------| | Python | 3.9, 3.10, 3.11, 3.12 | 3.9, 3.10, 3.11, 3.12 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.29, 1.30, 1.31, 1.32 | 1.29, 1.30, 1.31, 1.32 | +| Kubernetes | 1.30, 1.31, 1.32, 1.33 | 1.30, 1.31, 1.32, 1.33 | | PostgreSQL | 13, 14, 15, 16, 17 | 13, 14, 15, 16, 17 | | MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | | SQLite | 3.15.0+ | 3.15.0+ | @@ -174,15 +174,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==3.0.0' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.0/constraints-3.9.txt" +pip install 'apache-airflow==3.0.1' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.1/constraints-3.9.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==3.0.0' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.0/constraints-3.9.txt" +pip install 'apache-airflow[postgres,google]==3.0.1' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.1/constraints-3.9.txt" ``` For information on installing provider distributions, check @@ -289,14 +289,14 @@ Apache Airflow version life cycle: -| Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | -|-----------|-----------------------|-----------|-----------------|-------------------|------------------| -| 3 | 3.0.0 | Supported | Apr 22, 2025 | TBD | TBD | -| 2 | 2.10.5 | Supported | Dec 17, 2020 | TBD | TBD | -| 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | -| 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | -| 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | -| 1.7 | 1.7.1.2 | EOL | Mar 28, 2016 | Mar 19, 2017 | Mar 19, 2017 | +| Version | Current Patch/Minor | State | First Release | Limited Maintenance | EOL/Terminated | +|-----------|-----------------------|-----------|-----------------|-----------------------|------------------| +| 3 | 3.0.1 | Supported | Apr 22, 2025 | TBD | TBD | +| 2 | 2.10.5 | Supported | Dec 17, 2020 | TBD | TBD | +| 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | +| 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | +| 1.8 | 1.8.2 | EOL | Mar 19, 2017 | Jan 03, 2018 | Jan 03, 2018 | +| 1.7 | 1.7.1.2 | EOL | Mar 28, 2016 | Mar 19, 2017 | Mar 19, 2017 | diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 4df06ea010716..f4ff9d9323415 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,6 +21,137 @@ .. towncrier release notes start +Airflow 3.0.1 (2025-05-12) +-------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +No significant changes. + +Bug Fixes +""""""""" + +- Improves the handling of value masking when setting Airflow variables for enhanced security (#43123) +- Make entire task box clickable to select the task (#49299) +- Vertically align task log header components in full screen mode (#49569) +- Remove ``dag_code`` records with no serialized dag (#49478) +- Clear out the ``dag_code`` and ``serialized_dag`` tables on 3.0 upgrade (#49563) +- Remove extra slash so that the runs tab is selected (#49600) +- Null out the ``scheduler_interval`` field on downgrade (#49583) +- Logout functionality should respect ``base_url`` in api server (#49545) +- Fix bug with showing invalid credentials on Login UI (#49556) +- Fix Dag Code text selection when dark mode is enabled (#49649) +- Bugfix: ``max_active_tis_per_dag`` is not respected by dynamically mapped tasks (#49708) +- Fix infinite redirect caused by mistakenly setting token cookie as secure (#49721) +- Better handle safe url redirects in login form for ``SimpleAuthManager`` (#49697)(#49866) +- API: Add missing ``bundle_version`` to DagRun response (#49726) +- Display bundle version in Dag details tab (#49787) +- Fix gcp remote log module import in airflow local settings (#49788) +- Bugfix: Grid view stops loading when there is a pending task to be expanded (#49772) +- Treat single ``task_ids`` in ``xcom_pull`` the same as multiple when provided as part of a list (#49692) +- UI: Auto refresh Home page stats (#49830) +- UI: Error alert overflows out of the alert box (#49880) +- Show backfill banner after creating a new backfill (#49666) +- Mark ``DAGModel`` stale and associate bundle on import errors to aid migration from 2.10.5 (#49769) +- Improve detection and handling of timed out DAG processor processes (#49868) +- Fix editing port for connections (#50002) +- Improve & Fix grid endpoint response time (#49969) +- Update time duration format (#49914) +- Fix Dashboard overflow and handle no status tasks (#49964) +- Fix timezone setting for logical date input on Trigger Run form (#49662) +- Help ``pip`` with avoiding resolution too deep issues in Python 3.12 (#49853) +- Bugfix: backfill dry run does not use same timezone as create backfill (#49911) +- Fix Edit Connection when connection is imported (#49989) +- Bugfix: Filtering items from a mapped task is broken (#50011) +- Fix Dashboard for queued DagRuns (#49961) +- Fix backwards-compat import path for ``BashSensor`` (#49935) +- Apply task group sorting based on webserver config in grid structure response (#49418) +- Render custom ``map_index_template`` on task completion (#49809) +- Fix ``ContinuousTimetable`` false triggering when last run ends in future (#45175) +- Make Trigger Dag form warning more obvious (#49981) +- Restore task hover and selection indicators in the Grid view (#50050) +- Fix datetime validation for backfills (#50116) +- Fix duration charts (#50094) +- Fix DAG node selections (#50095) +- UI: Fix date range field alignment (#50086) +- Add auto-refresh for ``Stats`` (#50088) +- UI: Fixes validation error and adds error indicator for Params form (#50127) +- fix: wrap overflowing texts of asset events (#50173) +- Add audit log extra to table and improve UX (#50100) +- Handle map indexes for Mapped ``TaskGroup`` (#49996) +- Do not use introspection in migration to fix offline SQL generation (#49873) +- Fix operator extra links for mapped tasks (#50238) +- Fix backfill form (#50249)(#50243) +- UI: Fix operator overflow in graph (#50252) +- UI: Pass ``mapIndex`` to clear the relevant task instances. (#50256) +- Fix markdown rendering on dag docs (#50142) + +Miscellaneous +""""""""""""" + +- Add ``STRAIGHT_JOIN`` prefix for MySQL query optimization in ``get_sorted_triggers`` (#46303) +- Ensure ``sqlalchemy[asyncio]`` extra is in core deps (#49452) +- Remove unused constant ``HANDLER_SUPPORTS_TRIGGERER`` (#49370) +- Remove sort indicators on XCom table to avoid confusion (#49547) +- Remove ``gitpython`` as a core dependency (#49537) +- Bump ``@babel/runtime`` from ``7.26.0`` to ``7.27.0`` (#49479) +- Add backwards compatibility shim for ``get_current_context`` (#49630) +- AIP-38: enhance layout for ``RunBackfillForm`` (#49609) +- AIP-38: merge Backfill and Trigger Dag Run (#49490) +- Add count to Stats Cards in Dashboard (#49519) +- Add auto-refresh to health section for live updates. (#49645) +- Tweak Execution API OpenAPI spec to improve code Generation (#49700) +- Stricter validation for ``backfill_id`` (#49691)(#49716) +- Add ``SimpleAllAdminMiddleware`` to allow api usage without auth header in request (#49599) +- Bump ``react-router`` and ``react-router-dom`` from 7.4.0 to 7.5.2 (#49742) +- Remove reference to ``root_dag_id`` in dagbag and restore logic (#49668) +- Fix a few SqlAlchemy deprecation warnings (#49477) +- Make default execution server URL be relative to API Base URL (#49747)(#49782) +- Common ``airflow.cfg`` files across all containers in default ``docker-compose.yaml`` (#49681) +- Add redirects for old operators location to standard provider (#49776) +- Bump packaging from 24.2 to 25.0 in ``/airflow-core`` (#49512) +- Move some non-core dependencies to the ``apache-airflow`` meta package (#49846) +- Add more lower-bind limits to address resolution too deep (#49860) +- UI: Add counts to pool bar (#49894) +- Add type hints for ``@task.kuberenetes_cmd`` (#46913) +- Bump ``vite`` from ``5.4.17`` to ``5.4.19`` for Airflow UI (#49162)(#50074) +- Add ``map_index`` filter option to ``GetTICount`` and ``GetTaskStates`` (#49818) +- Add ``stats`` ui endpoint (#49985) +- Add link to tag to filter dags associated with the tag (#49680) +- Add keyboard shortcut for full screen and wrap in logs. (#50008) +- Update graph node styling to decrease border width on tasks in UI (#50047) (#50073) +- Allow non-string valid JSON values in Variable import. (#49844) +- Bump min versions of crucial providers (#50076) +- Add ``state`` attribute to ``RuntimeTaskInstance`` for easier ``ti.state`` access in Task Context (#50031) +- Move SQS message queue to Amazon provider (#50057) +- Execution API: Improve task instance logging with structlog context (#50120) +- Add ``dag_run_conf`` to ``RunBackfillForm`` (#49763) +- Refactor Dashboard to enhance layout (#50026) +- Add the download button on the assets page (#50045) +- Add ``dateInterval`` validation and error handling (#50072) +- Add ``Task Instances [{map_index}]`` tab to mapped task details (#50085) +- Add focus view on grid and graph on second click (#50125) +- Add formatted extra to asset events (#50124) +- Move webserver expose config to api section (#50209) + +Doc Only Changes +"""""""""""""""" + +- Remove flask application configuration from docs for AF3 (#49393) +- Docker compose: airflow-cli to depend on airflow common services (#49318) +- Better upgrade docs about flask/fab plugins in Airflow 3 (#49632)(#49614)(#49628) +- Various Airflow 3.0 Release notes & Updating guide docs updates (#49623)(#49401)(#49654)(#49663)(#49988)(#49954)(#49840)(#50195)(#50264) +- Add using the rest api by referring to ``security/api.rst`` (#49675) +- Add correct redirects for rest api and upgrade docs (#49764) +- Update ``max_consecutive_failed_dag_runs`` default value to zero in TaskSDK dag (#49795) (#49803) +- Fix spacing issues in params example dag (``example_params_ui_tutorial``) (#49905) +- Doc: Fix Kubernetes duplicated version in maintenance policy (#50030) +- Fix links to source examples in Airflow docs (#50082) +- Update ruff instructions for migration checks (#50232) +- Fix example of backfill command (#50222) +- Update docs for running behind proxy for Content-Security-Policy (#50236) + Airflow 3.0.0 (2025-04-22) -------------------------- We are proud to announce the General Availability of Apache Airflow 3.0 — the most significant release in the project's @@ -123,6 +254,19 @@ validations. The ``@asset`` decorator and related changes to the DAG parser enable clearer, asset-centric DAG definitions, allowing Airflow to more naturally support event-driven and data-aware scheduling patterns. +This renaming impacts modules, classes, functions, configuration keys, and internal models. Key changes include: + +- ``Dataset`` → ``Asset`` +- ``DatasetEvent`` → ``AssetEvent`` +- ``DatasetAlias`` → ``AssetAlias`` +- ``airflow.datasets.*`` → ``airflow.sdk.*`` +- ``airflow.timetables.simple.DatasetTriggeredTimetable`` → ``airflow.timetables.simple.AssetTriggeredTimetable`` +- ``airflow.timetables.datasets.DatasetOrTimeSchedule`` → ``airflow.timetables.assets.AssetOrTimeSchedule`` +- ``airflow.listeners.spec.dataset.on_dataset_created`` → ``airflow.listeners.spec.asset.on_asset_created`` +- ``airflow.listeners.spec.dataset.on_dataset_changed`` → ``airflow.listeners.spec.asset.on_asset_changed`` +- ``core.dataset_manager_class`` → ``core.asset_manager_class`` +- ``core.dataset_manager_kwargs`` → ``core.asset_manager_kwargs`` + Unified Scheduling Field """""""""""""""""""""""" @@ -137,6 +281,8 @@ Airflow 3.0 changes the default behavior for new DAGs by setting ``catchup_by_de file. This means DAGs that do not explicitly set ``catchup=...`` will no longer backfill missed intervals by default. This change reduces confusion for new users and better reflects the growing use of on-demand and event-driven workflows. +The default DAG schedule has been changed to ``None`` from ``@once``. + Restricted Metadata Database Access """"""""""""""""""""""""""""""""""" @@ -222,11 +368,85 @@ Airflow 3.0 fixes a bug that caused incorrect task statistics to be reported for accurately reflect the number of mapped task instances and their statuses, improving observability and debugging for dynamic workflows. +``SequentialExecutor`` has been removed +""""""""""""""""""""""""""""""""""""""" + +``SequentialExecutor`` was primarily used for local testing but is now redundant, as ``LocalExecutor`` +supports SQLite with WAL mode and provides better performance with parallel execution. +Users should switch to ``LocalExecutor`` or ``CeleryExecutor`` as alternatives. + DAG Authoring Enhancements ^^^^^^^^^^^^^^^^^^^^^^^^^^ Airflow 3.0 includes several changes that improve consistency, clarity, and long-term stability for DAG authors. +New Stable DAG Authoring Interface: ``airflow.sdk`` +""""""""""""""""""""""""""""""""""""""""""""""""""" + +Airflow 3.0 introduces a new, stable public API for DAG authoring under the ``airflow.sdk`` namespace, +available via the ``apache-airflow-task-sdk`` package. + +The goal of this change is to **decouple DAG authoring from Airflow internals** (Scheduler, API Server, etc.), +providing a **forward-compatible, stable interface** for writing and maintaining DAGs across Airflow versions. + +DAG authors should now import core constructs from ``airflow.sdk`` rather than internal modules. + +**Key Imports from** ``airflow.sdk``: + +- Classes: + + - ``Asset`` + - ``BaseNotifier`` + - ``BaseOperator`` + - ``BaseOperatorLink`` + - ``BaseSensorOperator`` + - ``Connection`` + - ``Context`` + - ``DAG`` + - ``EdgeModifier`` + - ``Label`` + - ``ObjectStoragePath`` + - ``Param`` + - ``TaskGroup`` + - ``Variable`` + +- Decorators and Functions: + + - ``@asset`` + - ``@dag`` + - ``@setup`` + - ``@task`` + - ``@task_group`` + - ``@teardown`` + - ``chain`` + - ``chain_linear`` + - ``cross_downstream`` + - ``get_current_context`` + - ``get_parsing_context`` + +For an exhaustive list of available classes, decorators, and functions, check ``airflow.sdk.__all__``. + +All DAGs should update imports to use ``airflow.sdk`` instead of referencing internal Airflow modules directly. +Legacy import paths (e.g., ``airflow.models.dag.DAG``, ``airflow.decorator.task``) are **deprecated** and +will be **removed** in a future Airflow version. Some additional utilities and helper functions +that DAGs sometimes use from ``airflow.utils.*`` and others will be progressively migrated to the Task SDK in future +minor releases. + +These future changes aim to **complete the decoupling** of DAG authoring constructs +from internal Airflow services. DAG authors should expect continued improvements +to ``airflow.sdk`` with no backwards-incompatible changes to existing constructs. + +For example, update: + +.. code-block:: python + + # Old (Airflow 2.x) + from airflow.models import DAG + from airflow.decorators import task + + # New (Airflow 3.x) + from airflow.sdk import DAG, task + Renamed Parameter: ``fail_stop`` → ``fail_fast`` """"""""""""""""""""""""""""""""""""""""""""""""" @@ -269,6 +489,60 @@ A new utility function, ``create_asset_aliases()``, allows DAG authors to define referenced Assets. This improves modularity and reuse across DAG files and is particularly helpful for teams adopting asset-centric DAGs. +Operator Links interface changed +"""""""""""""""""""""""""""""""" + +The Operator Extra links, which can be defined either via plugins or custom operators +now do not execute any user code in the Airflow UI, but instead push the "full" +links to XCom backend and the link is fetched from the XCom backend when viewing +task details, for example from grid view. + +Example for users with custom links class: + +.. code-block:: python + + @attr.s(auto_attribs=True) + class CustomBaseIndexOpLink(BaseOperatorLink): + """Custom Operator Link for Google BigQuery Console.""" + + index: int = attr.ib() + + @property + def name(self) -> str: + return f"BigQuery Console #{self.index + 1}" + + @property + def xcom_key(self) -> str: + return f"bigquery_{self.index + 1}" + + def get_link(self, operator, *, ti_key): + search_queries = XCom.get_one( + task_id=ti_key.task_id, dag_id=ti_key.dag_id, run_id=ti_key.run_id, key="search_query" + ) + return f"https://console.cloud.google.com/bigquery?j={search_query}" + +The link has an ``xcom_key`` defined, which is how it will be stored in the XCOM backend, with key as xcom_key and +value as the entire link, this case: ``https://console.cloud.google.com/bigquery?j=search`` + +Plugins no longer support adding executors, operators & hooks +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Operator (including Sensors), Executors & Hooks can no longer be registered or imported via Airflow's plugin mechanism. These types of classes +are just treated as plain Python classes by Airflow, so there is no need to register them with Airflow. They +can be imported directly from their respective provider packages. + +Before: + +.. code-block:: python + + from airflow.hooks.my_plugin import MyHook + +You should instead import it as: + +.. code-block:: python + + from my_plugin import MyHook + Support for ML & AI Use Cases (AIP-83) """"""""""""""""""""""""""""""""""""""" @@ -306,8 +580,8 @@ Refactored Internal Utilities Several core components have been moved to more intuitive or stable locations: -- The ``SecretsMasker`` class has been relocated to ``airflow.utils.secrets_masker``. -- The ``ObjectStoragePath`` utility previously located under ``airflow.io`` is now available via ``airflow.utils.object_storage_path``. +- The ``SecretsMasker`` class has been relocated to ``airflow.sdk.execution_time.secrets_masker``. +- The ``ObjectStoragePath`` utility previously located under ``airflow.io`` is now available via ``airflow.sdk``. These changes simplify imports and reflect broader efforts to stabilize utility interfaces across the Airflow codebase. @@ -331,13 +605,9 @@ The internal representation of asset event triggers now also includes an explici aligning with the broader asset-aware execution model introduced in Airflow 3.0. DAG authors interacting directly with ``inlet_events`` may need to update logic that assumes the previous structure. - - Behaviour change in ``xcom_pull`` """"""""""""""""""""""""""""""""" -**Pulling without setting ``task_ids``**: - In Airflow 2, the ``xcom_pull()`` method allowed pulling XComs by key without specifying task_ids, despite the fact that the underlying DB model defines task_id as part of the XCom primary key. This created ambiguity: if two tasks pushed XComs with the same key, ``xcom_pull()`` would pull whichever one happened to be first, leading to unpredictable behavior. @@ -354,42 +624,39 @@ Should be updated to:: kwargs["ti"].xcom_pull(task_ids="task1", key="key") -**Return Type Change for Single Task ID**: - -In Airflow 2, when using ``xcom_pull()`` with a single task ID in a list (e.g., ``task_ids=["task1"]``), it would return a ``LazyXComSelectSequence`` -object containing one value. In Airflow 3.0.0, this behavior was changed to return the value directly. - -So, if you previously used: - -.. code-block:: python - - xcom_values = kwargs["ti"].xcom_pull(task_ids=["task1"], key="key") - xcom_value = xcom_values[0] # Access the first value - -You would now get the value directly, rather than a sequence containing one value. - -.. code-block:: python - - xcom_value = kwargs["ti"].xcom_pull(task_ids=["task1"], key="key") - -The previous behaviour (returning list when passed a list) will be restored in Airflow 3.0.1 to maintain backward compatibility. - -However, it is recommended to be explicit about your intentions when using ``task_ids`` (after the fix in 3.0.1): - -- If you want a single value, use ``task_ids="task1"`` -- If you want a sequence, use ``task_ids=["task1"]`` - -This makes the code more explicit and easier to understand. - - Removed Configuration Keys """"""""""""""""""""""""""" As part of the deprecation cleanup, several legacy configuration options have been removed. These include: -- ``scheduler.allow_trigger_in_future`` -- ``scheduler.use_job_schedule`` -- ``scheduler.use_local_tz`` +- ``[scheduler] allow_trigger_in_future`` +- ``[scheduler] use_job_schedule`` +- ``[scheduler] use_local_tz`` +- ``[scheduler] processor_poll_interval`` +- ``[logging] dag_processor_manager_log_location`` +- ``[logging] dag_processor_manager_log_stdout`` +- ``[logging] log_processor_filename_template`` + +All the webserver configurations have also been removed since API server now replaces webserver, so +the configurations like below have no effect: + +- ``[webserver] allow_raw_html_descriptions`` +- ``[webserver] cookie_samesite`` +- ``[webserver] error_logfile`` +- ``[webserver] access_logformat`` +- ``[webserver] web_server_master_timeout`` +- etc + +Several configuration options previously located under the ``[webserver]`` section have +been **moved to the new ``[api]`` section**. The following configuration keys have been moved: + +- ``[webserver] web_server_host`` → ``[api] host`` +- ``[webserver] web_server_port`` → ``[api] port`` +- ``[webserver] workers`` → ``[api] workers`` +- ``[webserver] web_server_worker_timeout`` → ``[api] worker_timeout`` +- ``[webserver] web_server_ssl_cert`` → ``[api] ssl_cert`` +- ``[webserver] web_server_ssl_key`` → ``[api] ssl_key`` +- ``[webserver] access_logfile`` → ``[api] access_logfile`` Users should review their ``airflow.cfg`` files or use the ``airflow config lint`` command to identify outdated or removed options. @@ -401,7 +668,7 @@ Airflow 3.0 includes improved support for upgrade validation. Use the following configs or deprecated usage patterns: - ``airflow config lint``: Identifies removed or invalid config keys -- ``ruff check --select AIR30``: Flags removed interfaces and common migration issues +- ``ruff check --select AIR30 --preview``: Flags removed interfaces and common migration issues CLI & API Changes ^^^^^^^^^^^^^^^^^ @@ -419,6 +686,19 @@ The Airflow CLI has been split into two distinct interfaces: This change improves security and modularity for deployments that use Airflow in a distributed or API-first context. +REST API v2 replaces v1 +""""""""""""""""""""""" + +The legacy REST API v1, previously built with Connexion and Marshmallow, has been replaced by a modern FastAPI-based REST API v2. + +This new implementation improves performance, aligns more closely with web standards, and provides a consistent developer experience across the API and UI. + +Key changes include stricter validation (422 errors instead of 400), the removal of the ``execution_date`` parameter in favor of ``logical_date``, and more consistent query parameter handling. + +The v2 API is now the stable, fully supported interface for programmatic access to Airflow, and also powers the new UI - achieving full feature parity between the UI and API. + +For details, see the :doc:`Airflow REST API v2 ` documentation. + REST API: DAG Trigger Behavior Updated """""""""""""""""""""""""""""""""""""" @@ -432,8 +712,22 @@ Removed CLI Flags and Commands """""""""""""""""""""""""""""" Several deprecated CLI arguments and commands that were marked for removal in earlier versions have now been cleaned up -in Airflow 3.0. Refer to the Deprecations & Removals section or run ``airflow --help`` to review the current set of -available commands and arguments. +in Airflow 3.0. Run ``airflow --help`` to review the current set of available commands and arguments. + +- Deprecated ``--ignore-depends-on-past`` cli option is replaced by ``--depends-on-past ignore``. + +- ``--tree`` flag for ``airflow tasks list`` command is removed. The format of the output with that flag can be + expensive to generate and extremely large, depending on the DAG. ``airflow dag show`` is a better way to + visualize the relationship of tasks in a DAG. + +- Changing ``dag_id`` from flag (``-d``, ``--dag-id``) to a positional argument in the ``dags list-runs`` CLI command. + +- The ``airflow db init`` and ``airflow db upgrade`` commands have been removed. Use ``airflow db migrate`` instead + to initialize or migrate the metadata database. If you would like to create default connections use + ``airflow connections create-default-connections``. + +- ``airflow api-server`` has replaced ``airflow webserver`` cli command. + Provider Refactor & Standardization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -589,7 +883,7 @@ are called out individually above. +-------------------------------------------+----------------------------------------------------------+ | Deprecated core imports | Import from appropriate provider package | +-------------------------------------------+----------------------------------------------------------+ -| DebugExecutor | Use LocalExecutor for testing | +| ``SequentialExecutor`` & ``DebugExecutor``| Use LocalExecutor for testing | +-------------------------------------------+----------------------------------------------------------+ | ``.airflowignore`` regex | Uses glob syntax by default | +-------------------------------------------+----------------------------------------------------------+ @@ -625,8 +919,8 @@ compatible with Airflow 3.0. These checks are packaged under the ``AIR30x`` rule .. code-block:: bash - ruff check dags/ --select AIR301 - ruff check dags/ --select AIR301 --fix + ruff check dags/ --select AIR301 --preview + ruff check dags/ --select AIR301 --fix --preview These checks can automatically fix many common issues such as renamed arguments, removed imports, or legacy context variable usage. diff --git a/airflow-core/docs/administration-and-deployment/plugins.rst b/airflow-core/docs/administration-and-deployment/plugins.rst index 6d143c0960991..5e5dc46f40fbd 100644 --- a/airflow-core/docs/administration-and-deployment/plugins.rst +++ b/airflow-core/docs/administration-and-deployment/plugins.rst @@ -104,16 +104,10 @@ looks like: name = None # A list of references to inject into the macros namespace macros = [] - # A list of Blueprint object created from flask.Blueprint. For use with the flask_appbuilder based GUI - flask_blueprints = [] # A list of dictionaries containing FastAPI app objects and some metadata. See the example below. fastapi_apps = [] # A list of dictionaries containing FastAPI middleware factory objects and some metadata. See the example below. fastapi_root_middlewares = [] - # A list of dictionaries containing FlaskAppBuilder BaseView object and some metadata. See example below - appbuilder_views = [] - # A list of dictionaries containing kwargs for FlaskAppBuilder add_link. See example below - appbuilder_menu_items = [] # A callback to perform actions when Airflow starts and the plugin is loaded. # NOTE: Ensure your plugin has *args, and **kwargs in the method definition @@ -164,13 +158,9 @@ definitions in Airflow. # This is the class you derive to create a plugin from airflow.plugins_manager import AirflowPlugin - from airflow.security import permissions - from airflow.providers.fab.www.auth import has_access from fastapi import FastAPI from fastapi.middleware.trustedhost import TrustedHostMiddleware - from flask import Blueprint - from flask_appbuilder import expose, BaseView as AppBuilderBaseView # Importing base classes that we need to derive from airflow.hooks.base import BaseHook @@ -183,15 +173,6 @@ definitions in Airflow. pass - # Creating a flask blueprint to integrate the templates and static folder - bp = Blueprint( - "test_plugin", - __name__, - template_folder="templates", # registers airflow/plugins/templates as a Jinja template folder - static_folder="static", - static_url_path="/static/test_plugin", - ) - # Creating a FastAPI application to integrate in Airflow Rest API. app = FastAPI() @@ -213,65 +194,12 @@ definitions in Airflow. } - # Creating a flask appbuilder BaseView - class TestAppBuilderBaseView(AppBuilderBaseView): - default_view = "test" - - @expose("/") - @has_access( - [ - (permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE), - ] - ) - def test(self): - return self.render_template("test_plugin/test.html", content="Hello galaxy!") - - - # Creating a flask appbuilder BaseView - class TestAppBuilderBaseNoMenuView(AppBuilderBaseView): - default_view = "test" - - @expose("/") - @has_access( - [ - (permissions.ACTION_CAN_READ, permissions.RESOURCE_WEBSITE), - ] - ) - def test(self): - return self.render_template("test_plugin/test.html", content="Hello galaxy!") - - - v_appbuilder_view = TestAppBuilderBaseView() - v_appbuilder_package = { - "name": "Test View", - "category": "Test Plugin", - "view": v_appbuilder_view, - } - - v_appbuilder_nomenu_view = TestAppBuilderBaseNoMenuView() - v_appbuilder_nomenu_package = {"view": v_appbuilder_nomenu_view} - - # Creating flask appbuilder Menu Items - appbuilder_mitem = { - "name": "Google", - "href": "https://www.google.com", - "category": "Search", - } - appbuilder_mitem_toplevel = { - "name": "Apache", - "href": "https://www.apache.org/", - } - - # Defining the plugin class class AirflowTestPlugin(AirflowPlugin): name = "test_plugin" macros = [plugin_macro] - flask_blueprints = [bp] fastapi_apps = [app_with_metadata] fastapi_root_middlewares = [middleware_with_metadata] - appbuilder_views = [v_appbuilder_package, v_appbuilder_nomenu_package] - appbuilder_menu_items = [appbuilder_mitem, appbuilder_mitem_toplevel] .. seealso:: :doc:`/howto/define-extra-link` @@ -307,21 +235,10 @@ will automatically load the registered plugins from the entrypoint list. # my_package/my_plugin.py from airflow.plugins_manager import AirflowPlugin - from flask import Blueprint - - # Creating a flask blueprint to integrate the templates and static folder - bp = Blueprint( - "test_plugin", - __name__, - template_folder="templates", # registers airflow/plugins/templates as a Jinja template folder - static_folder="static", - static_url_path="/static/test_plugin", - ) class MyAirflowPlugin(AirflowPlugin): name = "my_namespace" - flask_blueprints = [bp] Then inside pyproject.toml: @@ -330,6 +247,15 @@ Then inside pyproject.toml: [project.entry-points."airflow.plugins"] my_plugin = "my_package.my_plugin:MyAirflowPlugin" +Flask Appbuilder and Flask Blueprints in Airflow 3 +-------------------------------------------------- + +Airflow 2 supported Flask Appbuilder views (``appbuilder_views``), Flask AppBuilder menu items (``appbuilder_menu_items``), +and Flask Blueprints (``flask_blueprints``) in plugins. These has been superseded by FastAPI apps in Airflow 3. All new plugins should use FastAPI apps (``fastapi_apps``) instead. + +However, a compatibility layer is provided for Flask and FAB plugins to ease the transition to Airflow 3 - simply install the FAB provider. +Ideally, you should convert your plugins to FastAPI apps (`fastapi_apps`) during the upgrade process, as this compatibility layer is deprecated. + Troubleshooting --------------- diff --git a/airflow-core/docs/authoring-and-scheduling/assets.rst b/airflow-core/docs/authoring-and-scheduling/assets.rst index f7f018c6d97db..dbbecd91ef882 100644 --- a/airflow-core/docs/authoring-and-scheduling/assets.rst +++ b/airflow-core/docs/authoring-and-scheduling/assets.rst @@ -280,7 +280,7 @@ The following example creates an asset event against the S3 URI ``f"s3://bucket/ .. code-block:: python - from airflow.sdk.definitions.asset import AssetAlias + from airflow.sdk import AssetAlias @task(outlets=[AssetAlias("my-task-outputs")]) @@ -292,7 +292,7 @@ The following example creates an asset event against the S3 URI ``f"s3://bucket/ .. code-block:: python - from airflow.sdk.definitions.asset.metadata import Metadata + from airflow.sdk import Metadata @task(outlets=[AssetAlias("my-task-outputs")]) @@ -304,7 +304,7 @@ Only one asset event is emitted for an added asset, even if it is added to the a .. code-block:: python - from airflow.sdk.definitions.asset import AssetAlias + from airflow.sdk import AssetAlias @task( diff --git a/airflow-core/docs/authoring-and-scheduling/connections.rst b/airflow-core/docs/authoring-and-scheduling/connections.rst index ba74826519eb2..7f9cbaa443ecc 100644 --- a/airflow-core/docs/authoring-and-scheduling/connections.rst +++ b/airflow-core/docs/authoring-and-scheduling/connections.rst @@ -47,5 +47,5 @@ Airflow allows to define custom connection types. This is what is described in d :doc:`apache-airflow-providers:index` - providers give you the capability of defining your own connections. The connection customization can be done by any provider, but also many of the providers managed by the community define custom connection types. -The full list of all providers delivered by ``Apache Airflow community managed providers`` can be found in +The full list of all connections delivered by ``Apache Airflow community managed providers`` can be found in :doc:`apache-airflow-providers:core-extensions/connections`. diff --git a/airflow-core/docs/authoring-and-scheduling/dynamic-task-mapping.rst b/airflow-core/docs/authoring-and-scheduling/dynamic-task-mapping.rst index 1ca973125c184..03119f411a6df 100644 --- a/airflow-core/docs/authoring-and-scheduling/dynamic-task-mapping.rst +++ b/airflow-core/docs/authoring-and-scheduling/dynamic-task-mapping.rst @@ -203,7 +203,7 @@ Since the template is rendered after the main execution block, it is possible to .. code-block:: python - from airflow.providers.standard.operators.python import get_current_context + from airflow.sdk import get_current_context @task(map_index_template="{{ my_variable }}") diff --git a/airflow-core/docs/best-practices.rst b/airflow-core/docs/best-practices.rst index bc4889b5f20ec..268f3e7150f63 100644 --- a/airflow-core/docs/best-practices.rst +++ b/airflow-core/docs/best-practices.rst @@ -303,11 +303,7 @@ Code Quality and Linting Maintaining high code quality is essential for the reliability and maintainability of your Airflow workflows. Utilizing linting tools can help identify potential issues and enforce coding standards. One such tool is ``ruff``, a fast Python linter that now includes specific rules for Airflow. -ruff assists in detecting deprecated features and patterns that may affect your migration to Airflow 3.0. For instance, it includes rules prefixed with ``AIR`` to flag potential issues: - -- **AIR301**: Flags DAGs without an explicit ``schedule`` argument. -- **AIR302**: Identifies usage of deprecated ``schedule_interval`` parameter. -- **AIR303**: Detects imports from modules that have been relocated or removed in Airflow 3.0. +ruff assists in detecting deprecated features and patterns that may affect your migration to Airflow 3.0. For instance, it includes rules prefixed with ``AIR`` to flag potential issues. The full list is detailed in `Airflow (AIR) `_. Installing and Using ruff ------------------------- @@ -316,13 +312,13 @@ Installing and Using ruff .. code-block:: bash - pip install "ruff>=0.9.5" + pip install "ruff>=0.11.6" 2. **Running ruff**: Execute ``ruff`` to check your dags for potential issues: .. code-block:: bash - ruff check dags/ --select AIR301,AIR302,AIR303 + ruff check dags/ --select AIR3 --preview This command will analyze your dags located in the ``dags/`` directory and report any issues related to the specified rules. diff --git a/airflow-core/docs/conf.py b/airflow-core/docs/conf.py index 43026bb39f258..cf6c0d8a80ba1 100644 --- a/airflow-core/docs/conf.py +++ b/airflow-core/docs/conf.py @@ -67,8 +67,6 @@ SYSTEM_TESTS_DIR: pathlib.Path | None SYSTEM_TESTS_DIR = AIRFLOW_REPO_ROOT_PATH / "airflow-core" / "tests" / "system" / "core" -conf_py_path = f"/docs/{PACKAGE_NAME}/" - os.environ["AIRFLOW_PACKAGE_NAME"] = PACKAGE_NAME # Hack to allow changing for piece of the code to behave differently while @@ -238,6 +236,7 @@ def add_airflow_core_exclude_patterns_to_sphinx(exclude_patterns: list[str]): # html theme options html_theme_options: dict[str, Any] = get_html_theme_options() +conf_py_path = "/airflow-core/docs/" # A dictionary of values to pass into the template engine's context for all pages. html_context = get_html_context(conf_py_path) @@ -263,6 +262,13 @@ def add_airflow_core_exclude_patterns_to_sphinx(exclude_patterns: list[str]): }, } +# Use for generate rst_epilog and other post-generation substitutions +global_substitutions = { + "version": PACKAGE_VERSION, + "airflow-version": airflow.__version__, + "experimental": "This is an :ref:`experimental feature `.", +} + # -- Options for sphinx.ext.autodoc -------------------------------------------- # See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html diff --git a/airflow-core/docs/core-concepts/auth-manager/index.rst b/airflow-core/docs/core-concepts/auth-manager/index.rst index 417bad407db1f..95cd05aae4f7a 100644 --- a/airflow-core/docs/core-concepts/auth-manager/index.rst +++ b/airflow-core/docs/core-concepts/auth-manager/index.rst @@ -148,7 +148,7 @@ delete the cookie. response = RedirectResponse(url="/") - secure = conf.has_option("api", "ssl_cert") + secure = bool(conf.get("api", "ssl_cert", fallback="")) response.set_cookie(COOKIE_NAME_JWT_TOKEN, token, secure=secure) return response diff --git a/airflow-core/docs/core-concepts/executor/index.rst b/airflow-core/docs/core-concepts/executor/index.rst index c83f10cb050bc..33427ff2bcf23 100644 --- a/airflow-core/docs/core-concepts/executor/index.rst +++ b/airflow-core/docs/core-concepts/executor/index.rst @@ -48,8 +48,6 @@ If you want to check which executor is currently set, you can use the ``airflow $ airflow config get-value core executor LocalExecutor - - Executor Types -------------- @@ -232,6 +230,40 @@ Some reasons you may want to write a custom executor include: * You'd like to use an executor that leverages a compute service from your preferred cloud provider. * You have a private tool/service for task execution that is only available to you or your organization. +Workloads +^^^^^^^^^ + +A workload in context of an Executor is the fundamental unit of execution for an executor. It represents a discrete +operation or job that the executor runs on a worker. For example, it can run user code encapsulated in an Airflow task +on a worker. + +Example: + +.. code-block:: python + + ExecuteTask( + token="mock", + ti=TaskInstance( + id=UUID("4d828a62-a417-4936-a7a6-2b3fabacecab"), + task_id="mock", + dag_id="mock", + run_id="mock", + try_number=1, + map_index=-1, + pool_slots=1, + queue="default", + priority_weight=1, + executor_config=None, + parent_context_carrier=None, + context_carrier=None, + queued_dttm=None, + ), + dag_rel_path=PurePosixPath("mock.py"), + bundle_info=BundleInfo(name="n/a", version="no matter"), + log_path="mock.log", + type="ExecuteTask", + ) + Important BaseExecutor Methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -239,7 +271,7 @@ Important BaseExecutor Methods These methods don't require overriding to implement your own executor, but are useful to be aware of: * ``heartbeat``: The Airflow scheduler Job loop will periodically call heartbeat on the executor. This is one of the main points of interaction between the Airflow scheduler and the executor. This method updates some metrics, triggers newly queued tasks to execute and updates state of running/completed tasks. -* ``queue_command``: The Airflow Executor will call this method of the BaseExecutor to provide tasks to be run by the executor. The BaseExecutor simply adds the TaskInstances to an internal list of queued tasks within the executor. +* ``queue_workload``: The Airflow Executor will call this method of the BaseExecutor to provide tasks to be run by the executor. The BaseExecutor simply adds the *workloads* (check section above to understand) to an internal list of queued workloads to run within the executor. All executors present in the repository use this method. * ``get_event_buffer``: The Airflow scheduler calls this method to retrieve the current state of the TaskInstances the executor is executing. * ``has_task``: The scheduler uses this BaseExecutor method to determine if an executor already has a specific task instance queued or running. * ``send_callback``: Sends any callbacks to the sink configured on the executor. @@ -251,7 +283,7 @@ Mandatory Methods to Implement The following methods must be overridden at minimum to have your executor supported by Airflow: * ``sync``: Sync will get called periodically during executor heartbeats. Implement this method to update the state of the tasks which the executor knows about. Optionally, attempting to execute queued tasks that have been received from the scheduler. -* ``execute_async``: Executes a command asynchronously. A command in this context is an Airflow CLI command to run an Airflow task. This method is called (after a few layers) during executor heartbeat which is run periodically by the scheduler. In practice, this method often just enqueues tasks into an internal or external queue of tasks to be run (e.g. ``KubernetesExecutor``). But can also execute the tasks directly as well (e.g. ``LocalExecutor``). This will depend on the executor. +* ``execute_async``: Executes a *workload* asynchronously. This method is called (after a few layers) during executor heartbeat which is run periodically by the scheduler. In practice, this method often just enqueues tasks into an internal or external queue of tasks to be run (e.g. ``KubernetesExecutor``). But can also execute the tasks directly as well (e.g. ``LocalExecutor``). This will depend on the executor. Optional Interface Methods to Implement diff --git a/airflow-core/docs/core-concepts/index.rst b/airflow-core/docs/core-concepts/index.rst index fdb9c2d146aaa..8ba314cd36778 100644 --- a/airflow-core/docs/core-concepts/index.rst +++ b/airflow-core/docs/core-concepts/index.rst @@ -43,6 +43,7 @@ Here you can find detailed documentation about each one of the core concepts of auth-manager/index objectstorage backfill + message-queues **Communication** diff --git a/airflow-core/docs/core-concepts/message-queues.rst b/airflow-core/docs/core-concepts/message-queues.rst new file mode 100644 index 0000000000000..573189a24f891 --- /dev/null +++ b/airflow-core/docs/core-concepts/message-queues.rst @@ -0,0 +1,41 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _concepts:message-queues: + +Message Queues +============== + +The Message Queues are a way to expose capability of external event-driven scheduling of Dags. + +Apache Airflow is primarily designed for time-based and dependency-based scheduling of workflows. However, +modern data architectures often require near real-time processing and the ability to react to +events from various sources, such as message queues. + +Airflow has native event-driven capability, allowing users to create workflows that can be +triggered by external events, thus enabling more responsive data pipelines. + +Airflow supports poll-based event-driven scheduling, where the Triggerer can poll +external message queues using built-in :class:`airflow.triggers.base.BaseTrigger` classes. This allows users +to create workflows that can be triggered by external events, such as messages arriving +in a queue or changes in a database efficiently. + +Airflow constantly monitors the state of an external resource and updates the asset whenever the external +resource reaches a given state (if it does reach it). To achieve this, we leverage Airflow Triggers. +Triggers are small, asynchronous pieces of Python code whose job is to poll an external resource state. + +The list of supported message queues is available in :doc:`apache-airflow-providers:core-extensions/message-queues`. diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index 9c360209ac22c..5aae73989e024 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -106,6 +106,8 @@ python dependencies for the provided package. +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | pandas | ``pip install 'apache-airflow[pandas]'`` | Install Pandas library compatible with Airflow | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ +| polars | ``pip install 'apache-airflow[polars]'`` | Polars hooks and operators | ++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend | +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+ | sentry | ``pip install 'apache-airflow[sentry]'`` | Sentry service for application logging and monitoring | diff --git a/airflow-core/docs/faq.rst b/airflow-core/docs/faq.rst index e552e024a4925..8727deac08e71 100644 --- a/airflow-core/docs/faq.rst +++ b/airflow-core/docs/faq.rst @@ -31,8 +31,8 @@ There are very many reasons why your task might not be getting scheduled. Here a - Does your script "compile", can the Airflow engine parse it and find your DAG object? To test this, you can run ``airflow dags list`` and confirm that your DAG shows up in the list. You can also run - ``airflow tasks list foo_dag_id --tree`` and confirm that your task - shows up in the list as expected. If you use the CeleryExecutor, you + ``airflow dags show foo_dag_id`` and confirm that your task + shows up in the graphviz format as expected. If you use the CeleryExecutor, you may want to confirm that this works both where the scheduler runs as well as where the worker runs. diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index cb99c6085db39..17f2e93bf144b 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -71,8 +71,7 @@ x-airflow-common: # for other purpose (development, test and especially production usage) build/extend Airflow image. _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} # The following line can be used to set a custom config file, stored in the local config folder - # If you want to use it, outcomment it and replace airflow.cfg with the name of your config file - # AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' + AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' volumes: - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs @@ -218,6 +217,7 @@ services: echo "For other operating systems you can get rid of the warning with manually created .env file:" echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" echo + export AIRFLOW_UID=$(id -u) fi one_meg=1048576 mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) @@ -252,9 +252,38 @@ services: echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" echo fi - mkdir -p /sources/logs /sources/dags /sources/plugins - chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} - exec /entrypoint airflow version + echo + echo "Creating missing opt dirs if missing:" + echo + mkdir -v -p /opt/airflow/{logs,dags,plugins,config} + echo + echo "Airflow version:" + /entrypoint airflow version + echo + echo "Files in shared volumes:" + echo + ls -la /opt/airflow/{logs,dags,plugins,config} + echo + echo "Running airflow config list to create default config file if missing." + echo + /entrypoint airflow config list >/dev/null + echo + echo "Files in shared volumes:" + echo + ls -la /opt/airflow/{logs,dags,plugins,config} + echo + echo "Change ownership of files in /opt/airflow to ${AIRFLOW_UID}:0" + echo + chown -R "${AIRFLOW_UID}:0" /opt/airflow/ + echo + echo "Change ownership of files in shared volumes to ${AIRFLOW_UID}:0" + echo + chown -v -R "${AIRFLOW_UID}:0" /opt/airflow/{logs,dags,plugins,config} + echo + echo "Files in shared volumes:" + echo + ls -la /opt/airflow/{logs,dags,plugins,config} + # yamllint enable rule:line-length environment: <<: *airflow-common-env @@ -264,8 +293,6 @@ services: _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} _PIP_ADDITIONAL_REQUIREMENTS: '' user: "0:0" - volumes: - - ${AIRFLOW_PROJ_DIR:-.}:/sources airflow-cli: <<: *airflow-common diff --git a/airflow-core/docs/howto/docker-compose/index.rst b/airflow-core/docs/howto/docker-compose/index.rst index 1e60262da14a0..df46066a0bc0a 100644 --- a/airflow-core/docs/howto/docker-compose/index.rst +++ b/airflow-core/docs/howto/docker-compose/index.rst @@ -89,7 +89,7 @@ This file contains several service definitions: - ``airflow-scheduler`` - The :doc:`scheduler ` monitors all tasks and dags, then triggers the task instances once their dependencies are complete. - ``airflow-dag-processor`` - The DAG processor parses DAG files. -- ``airflow-webserver`` - The webserver is available at ``http://localhost:8080``. +- ``airflow-api-server`` - The api server is available at ``http://localhost:8080``. - ``airflow-worker`` - The worker that executes the tasks given by the scheduler. - ``airflow-triggerer`` - The triggerer runs an event loop for deferrable tasks. - ``airflow-init`` - The initialization service. @@ -125,8 +125,8 @@ Setting the right Airflow user ------------------------------ On **Linux**, the quick-start needs to know your host user id and needs to have group id set to ``0``. -Otherwise the files created in ``dags``, ``logs`` and ``plugins`` will be created with ``root`` user ownership. -You have to make sure to configure them for the docker-compose: +Otherwise the files created in ``dags``, ``logs``, ``config`` and ``plugins`` will be created with +``root`` user ownership. You have to make sure to configure them for the docker-compose: .. code-block:: bash @@ -143,6 +143,17 @@ safely ignore it. You can also manually create an ``.env`` file in the same fold AIRFLOW_UID=50000 +Initialize airflow.cfg (Optional) +--------------------------------- + +If you want to initialize ``airflow.cfg`` with default values before launching the airflow service, run. + +.. code-block:: bash + + docker compose run airflow-cli airflow config list + +This will seed ``airflow.cfg`` with default values in ``config`` folder. + Initialize the database ----------------------- @@ -199,7 +210,7 @@ In a second terminal you can check the condition of the containers and make sure CONTAINER ID IMAGE |version-spacepad| COMMAND CREATED STATUS PORTS NAMES 247ebe6cf87a apache/airflow:|version| "/usr/bin/dumb-init …" 3 minutes ago Up 3 minutes (healthy) 8080/tcp compose_airflow-worker_1 ed9b09fc84b1 apache/airflow:|version| "/usr/bin/dumb-init …" 3 minutes ago Up 3 minutes (healthy) 8080/tcp compose_airflow-scheduler_1 - 7cb1fb603a98 apache/airflow:|version| "/usr/bin/dumb-init …" 3 minutes ago Up 3 minutes (healthy) 0.0.0.0:8080->8080/tcp compose_airflow-webserver_1 + 7cb1fb603a98 apache/airflow:|version| "/usr/bin/dumb-init …" 3 minutes ago Up 3 minutes (healthy) 0.0.0.0:8080->8080/tcp compose_airflow-api_server_1 74f3bbe506eb postgres:13 |version-spacepad| "docker-entrypoint.s…" 18 minutes ago Up 17 minutes (healthy) 5432/tcp compose_postgres_1 0bd6576d23cb redis:latest |version-spacepad| "docker-entrypoint.s…" 10 hours ago Up 17 minutes (healthy) 0.0.0.0:6379->6379/tcp compose_redis_1 @@ -346,12 +357,9 @@ Special case - Adding a custom config file If you have a custom config file and wish to use it in your Airflow instance, you need to perform the following steps: -1) Remove comment from the ``AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'`` line - in the ``docker-compose.yaml`` file. - -2) Place your custom ``airflow.cfg`` file in the local config folder. +1) Replace the auto-generated ``airflow.cfg`` file in the local config folder with your custom config file. -3) If your config file has a different name than ``airflow.cfg``, adjust the filename in +2) If your config file has a different name than ``airflow.cfg``, adjust the filename in ``AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'`` Networking diff --git a/airflow-core/docs/howto/index.rst b/airflow-core/docs/howto/index.rst index aa8372dd9195a..a4bd791200839 100644 --- a/airflow-core/docs/howto/index.rst +++ b/airflow-core/docs/howto/index.rst @@ -30,6 +30,7 @@ configuring an Airflow environment. :maxdepth: 2 Using the CLI + Using the REST API <../security/api> add-dag-tags notifications set-config diff --git a/airflow-core/docs/howto/run-behind-proxy.rst b/airflow-core/docs/howto/run-behind-proxy.rst index 294823753a585..c9eb3295bd29c 100644 --- a/airflow-core/docs/howto/run-behind-proxy.rst +++ b/airflow-core/docs/howto/run-behind-proxy.rst @@ -51,6 +51,11 @@ To do so, you need to set the following setting in your ``airflow.cfg``:: } } +- Some parts of the UI are rendered inside iframes (Auth managers security links for instance), you need to make sure that you are not setting a restricted CSP for iframe rendering + such as ``frame-ancestors 'none'``. You can set the CSP header in your reverse proxy configuration, for example:: + + add_header Content-Security-Policy "frame-ancestors 'self';"; + - Use ``--proxy-headers`` CLI flag to tell Uvicorn to respect these headers: ``airflow api-server --proxy-headers`` - If your proxy server is not on the same host (or in the same docker container) as Airflow, then you will need to diff --git a/airflow-core/docs/img/airflow_erd.sha256 b/airflow-core/docs/img/airflow_erd.sha256 index 33fe09fd943e4..f92146f77a88f 100644 --- a/airflow-core/docs/img/airflow_erd.sha256 +++ b/airflow-core/docs/img/airflow_erd.sha256 @@ -1 +1 @@ -bc93e7288a7a8355b15dc721accaf80260f370f3afa0d478248f9fe4692a1f1d \ No newline at end of file +1b5221fa589cfc8652242b95f24d218c632168238933b82b533321a217c2447f \ No newline at end of file diff --git a/airflow-core/docs/img/ui-dark/task_group.gif b/airflow-core/docs/img/ui-dark/task_group.gif index 32ed29f8b2ce3..88e74965ddede 100644 Binary files a/airflow-core/docs/img/ui-dark/task_group.gif and b/airflow-core/docs/img/ui-dark/task_group.gif differ diff --git a/airflow-core/docs/img/ui-light/task_group.gif b/airflow-core/docs/img/ui-light/task_group.gif index 39b7f94da1259..743e86667d3cb 100644 Binary files a/airflow-core/docs/img/ui-light/task_group.gif and b/airflow-core/docs/img/ui-light/task_group.gif differ diff --git a/airflow-core/docs/installation/supported-versions.rst b/airflow-core/docs/installation/supported-versions.rst index b1063453cc2f2..ec663422222af 100644 --- a/airflow-core/docs/installation/supported-versions.rst +++ b/airflow-core/docs/installation/supported-versions.rst @@ -26,16 +26,16 @@ Apache Airflow® version life cycle: .. This table is automatically updated by pre-commit scripts/ci/pre_commit/supported_versions.py .. Beginning of auto-generated table -========= ===================== ========= =============== ================= ================ -Version Current Patch/Minor State First Release Limited Support EOL/Terminated -========= ===================== ========= =============== ================= ================ -3 3.0.0 Supported Apr 22, 2025 TBD TBD -2 2.10.5 Supported Dec 17, 2020 TBD TBD -1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 -1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 -1.8 1.8.2 EOL Mar 19, 2017 Jan 03, 2018 Jan 03, 2018 -1.7 1.7.1.2 EOL Mar 28, 2016 Mar 19, 2017 Mar 19, 2017 -========= ===================== ========= =============== ================= ================ +========= ===================== ========= =============== ===================== ================ +Version Current Patch/Minor State First Release Limited Maintenance EOL/Terminated +========= ===================== ========= =============== ===================== ================ +3 3.0.1 Supported Apr 22, 2025 TBD TBD +2 2.10.5 Supported Dec 17, 2020 TBD TBD +1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 +1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 +1.8 1.8.2 EOL Mar 19, 2017 Jan 03, 2018 Jan 03, 2018 +1.7 1.7.1.2 EOL Mar 28, 2016 Mar 19, 2017 Mar 19, 2017 +========= ===================== ========= =============== ===================== ================ .. End of auto-generated table diff --git a/airflow-core/docs/installation/upgrading_to_airflow3.rst b/airflow-core/docs/installation/upgrading_to_airflow3.rst index 04c9bda2fa0e1..d44f46bbcc5a8 100644 --- a/airflow-core/docs/installation/upgrading_to_airflow3.rst +++ b/airflow-core/docs/installation/upgrading_to_airflow3.rst @@ -41,6 +41,10 @@ Step 2: Clean and back up your existing Airflow Instance upgrade process. These schema changes can take a long time if the database is large. For a faster, safer migration, we recommend that you clean up your Airflow meta-database before the upgrade. You can use the ``airflow db clean`` :ref:`Airflow CLI command` to trim your Airflow database. +- Ensure that there are no errors related to dag processing, such as ``AirflowDagDuplicatedIdException``. You should + be able to run ``airflow dags reserialize`` with no errors. If you have have to resolve errors from dag processing, + ensure you deploy your changes to your old instance prior to upgrade, and wait until your dags have all been reprocessed + (and all errors gone) before you proceed with upgrade. Step 3: DAG Authors - Check your Airflow DAGs for compatibility ---------------------------------------------------------------- @@ -52,19 +56,22 @@ for dag incompatibilities that will need to be fixed before they will work as ex .. code-block:: bash - ruff check dag/ --select AIR301 + ruff check dag/ --select AIR301 --preview To preview the recommended fixes, run the following command: .. code-block:: bash - ruff check dag/ --select AIR301 --show-fixes + ruff check dag/ --select AIR301 --show-fixes --preview Some changes can be automatically fixed. To do so, run the following command: .. code-block:: bash - ruff check dag/ --select AIR301 --fix + ruff check dag/ --select AIR301 --fix --preview + + +You can also configure these flags through configuration files. See `Configuring Ruff `_ for details. Step 4: Install the Standard Providers -------------------------------------- @@ -102,8 +109,9 @@ The biggest part of an Airflow upgrade is the database upgrade. The database upg airflow db migrate -You should now be able to start up your Airflow 3 instance. - +If you have plugins that use Flask-AppBuilder views ( ``appbuilder_views`` ), Flask-AppBuilder menu items ( ``appbuilder_menu_items`` ), or Flask blueprints ( ``flask_blueprints`` ), you will either need to convert +them to FastAPI apps or ensure you install the FAB provider which provides a backwards compatibility layer for Airflow 3. +Ideally, you should convert your plugins to FastAPI apps ( ``fastapi_apps`` ), as the compatibility layer in the FAB provider is deprecated. Step 6: Changes to your startup scripts --------------------------------------- @@ -120,6 +128,7 @@ The dag processor must now be started independently, even for local or developme airflow dag-processor +You should now be able to start up your Airflow 3 instance. .. _breaking-changes: @@ -131,8 +140,10 @@ These include: - **SubDAGs**: Replaced by TaskGroups, Assets, and Data Aware Scheduling. - **Sequential Executor**: Replaced by LocalExecutor, which can be used with SQLite for local development use cases. +- **CeleryKubernetesExecutor and LocalKubernetesExecutor**: Replaced by `Multiple Executor Configuration `_ - **SLAs**: Deprecated and removed; Will be replaced by forthcoming `Deadline Alerts `_. - **Subdir**: Used as an argument on many CLI commands, ``--subdir`` or ``-S`` has been superseded by :doc:`DAG bundles `. +- **REST API** (``/api/v1``) replaced: Use the modern FastAPI-based stable ``/api/v2`` instead; see :doc:`Airflow API v2 ` for details. - **Some Airflow context variables**: The following keys are no longer available in a :ref:`task instance's context `. If not replaced, will cause dag errors: - ``tomorrow_ds`` - ``tomorrow_ds_nodash`` diff --git a/airflow-core/docs/operators-and-hooks-ref.rst b/airflow-core/docs/operators-and-hooks-ref.rst index 511775d117eb4..fad6b04724f6c 100644 --- a/airflow-core/docs/operators-and-hooks-ref.rst +++ b/airflow-core/docs/operators-and-hooks-ref.rst @@ -1,3 +1,4 @@ + .. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -50,20 +51,26 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Operators - Guides - * - :mod:`airflow.operators.branch` - - + * - :mod:`airflow.providers.standard.operators.bash` + - :doc:`How to use ` + + * - :mod:`airflow.providers.standard.operators.python` + - :doc:`How to use ` + + * - :mod:`airflow.providers.standard.operators.datetime` + - :doc:`How to use ` * - :mod:`airflow.providers.standard.operators.empty` - - * - :mod:`airflow.operators.generic_transfer` - - + * - :mod:`airflow.providers.common.sql.operators.generic_transfer.GenericTransfer` + - :doc:`How to use ` * - :mod:`airflow.providers.standard.operators.latest_only` - - + - :doc:`How to use ` * - :mod:`airflow.providers.standard.operators.trigger_dagrun` - - + - :doc:`How to use ` **Sensors:** @@ -73,8 +80,23 @@ For details see: :doc:`apache-airflow-providers:operators-and-hooks-ref/index`. * - Sensors - Guides - * - :mod:`airflow.sensors.base` - - + * - :mod:`airflow.providers.standard.sensors.bash` + - :doc:`How to use ` + + * - :mod:`airflow.providers.standard.sensors.python` + - :doc:`How to use ` + + * - :mod:`airflow.providers.standard.sensors.filesystem` + - :doc:`How to use ` + + * - :mod:`airflow.providers.standard.sensors.date_time` + - :doc:`How to use ` + + * - :mod:`airflow.providers.standard.sensors.external_task` + - :doc:`How to use ` + + + **Hooks:** diff --git a/airflow-core/docs/redirects.txt b/airflow-core/docs/redirects.txt index a8f31ee0eb4f0..aadf5e09f1a61 100644 --- a/airflow-core/docs/redirects.txt +++ b/airflow-core/docs/redirects.txt @@ -75,7 +75,7 @@ start/index.rst start.rst # References cli-ref.rst cli-and-env-variables-ref.rst _api/index.rst public-airflow-interface.rst -rest-api-ref.rst deprecated-rest-api-ref.rst +deprecated-rest-api-ref.rst rest-api-ref.rst macros-ref.rst templates-ref.rst # Concepts @@ -85,7 +85,6 @@ scheduler.rst administration-and-deployment/scheduler.rst # Installation installation.rst installation/index.rst upgrade-check.rst installation/upgrade-check.rst -upgrading-to-2.rst howto/upgrading-from-1-10/index.rst # Release Notes changelog.rst release_notes.rst @@ -174,3 +173,13 @@ howto/define_extra_link.rst howto/define-extra-link.rst # Use test config (it's not a howto for users but a howto for developers so we redirect it back to index) howto/use-test-config.rst index.rst + +# Operators/Sensors moved to standard providers + +howto/operator/bash.rst ../apache-airflow-providers-standard/stable/operators/bash.rst +howto/operator/datetime.rst ../apache-airflow-providers-standard/stable/operators/datetime.rst +howto/operator/external_task_sensor.rst ../apache-airflow-providers-standard/stable/sensors/external_task_sensor.rst +howto/operator/file.rst ../apache-airflow-providers-standard/stable/sensors/file.rst +howto/operator/python.rst ../apache-airflow-providers-standard/stable/operators/python.rst +howto/operator/time.rst ../apache-airflow-providers-standard/stable/sensors/datetime.rst +howto/operator/weekday.rst ../apache-airflow-providers-standard/stable/operators/datetime.rst#branchdayofweekoperator diff --git a/airflow-core/docs/start.rst b/airflow-core/docs/start.rst index 81b57ea3b387a..bf66a34a27ed2 100644 --- a/airflow-core/docs/start.rst +++ b/airflow-core/docs/start.rst @@ -66,7 +66,7 @@ This quick start guide will help you bootstrap an Airflow standalone instance on :substitutions: - AIRFLOW_VERSION=3.0.0 + AIRFLOW_VERSION=3.0.1 # Extract the version of Python you have installed. If you're currently using a Python version that is not supported by Airflow, you may want to set this manually. # See above for supported versions. @@ -94,7 +94,7 @@ and create the "airflow.cfg" file with defaults that will get you going fast. You can override defaults using environment variables, see :doc:`/configurations-ref`. You can inspect the file either in ``$AIRFLOW_HOME/airflow.cfg``, or through the UI in the ``Admin->Configuration`` menu. The PID file for the webserver will be stored -in ``$AIRFLOW_HOME/airflow-webserver.pid`` or in ``/run/airflow/webserver.pid`` +in ``$AIRFLOW_HOME/airflow-api-server.pid`` or in ``/run/airflow/airflow-webserver.pid`` if started by systemd. As you grow and deploy Airflow to production, you will also want to move away @@ -111,8 +111,8 @@ run the commands below. airflow tasks test example_bash_operator runme_0 2015-01-01 # run a backfill over 2 days airflow backfill create --dag-id example_bash_operator \ - --start-date 2015-01-01 \ - --end-date 2015-01-02 + --from-date 2015-01-01 \ + --to-date 2015-01-02 If you want to run the individual parts of Airflow manually rather than using the all-in-one ``standalone`` command, you can instead run: diff --git a/airflow-core/docs/tutorial/fundamentals.rst b/airflow-core/docs/tutorial/fundamentals.rst index e1fa0f3684b56..09929b0d888ec 100644 --- a/airflow-core/docs/tutorial/fundamentals.rst +++ b/airflow-core/docs/tutorial/fundamentals.rst @@ -210,7 +210,7 @@ times. Working with Time Zones ----------------------- -Creating a time zone aware DAG straightforward. Just ensure you use time zone aware dates +Creating a time zone aware DAG is straightforward. Just ensure you use time zone aware dates with ``pendulum``. Avoid using the standard library `timezone `_ as they have known limitations. @@ -252,8 +252,8 @@ Let's validate your script further by running a few commands: # prints the list of tasks in the "tutorial" DAG airflow tasks list tutorial - # prints the hierarchy of tasks in the "tutorial" DAG - airflow tasks list tutorial --tree + # prints the graphviz representation of "tutorial" DAG + airflow dags show tutorial Testing Task Instances and DAG Runs diff --git a/airflow-core/docs/tutorial/taskflow.rst b/airflow-core/docs/tutorial/taskflow.rst index c673d1f182b30..b5804dea94c18 100644 --- a/airflow-core/docs/tutorial/taskflow.rst +++ b/airflow-core/docs/tutorial/taskflow.rst @@ -269,7 +269,7 @@ system-level packages. TaskFlow supports multiple execution environments to isol Creates a temporary virtualenv at task runtime. Great for experimental or dynamic tasks, but may have cold start overhead. -.. exampleinclude:: /../src/airflow/example_dags/example_python_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_python_venv] @@ -283,7 +283,7 @@ overhead. Executes the task using a pre-installed Python interpreter — ideal for consistent environments or shared virtualenvs. -.. exampleinclude:: /../src/airflow/example_dags/example_python_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_python_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_external_python] @@ -333,7 +333,7 @@ Using Sensors Use ``@task.sensor`` to build lightweight, reusable sensors using Python functions. These support both poke and reschedule modes. -.. exampleinclude:: /../src/airflow/example_dags/example_sensor_decorator.py +.. exampleinclude:: /../../providers/standard/tests/system/standard/example_sensor_decorator.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] @@ -388,7 +388,7 @@ method. .. code-block:: python - from airflow.providers.standard.operators.python import get_current_context + from airflow.sdk import get_current_context def some_function_in_your_library(): diff --git a/airflow-core/newsfragments/24842.significant.rst b/airflow-core/newsfragments/24842.significant.rst deleted file mode 100644 index f1b5e57cba6d5..0000000000000 --- a/airflow-core/newsfragments/24842.significant.rst +++ /dev/null @@ -1,17 +0,0 @@ -Default DAG schedule changed to *None* - -When a *schedule* parameter is not passed to the ``DAG`` constructor, Airflow -now defaults to never automatically schedule the DAG at all. The created DAG -can still be manually triggered, either by the user directly, or from another -DAG with ``TriggerDagRunOperator``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/40029.significant.rst b/airflow-core/newsfragments/40029.significant.rst deleted file mode 100644 index 1d9bc26ef858a..0000000000000 --- a/airflow-core/newsfragments/40029.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Removed deprecated airflow configuration ``webserver.allow_raw_html_descriptions`` from UI Trigger forms. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``webserver.allow_raw_html_descriptions`` diff --git a/airflow-core/newsfragments/40931.significant.rst b/airflow-core/newsfragments/40931.significant.rst deleted file mode 100644 index 7893e6dc0a014..0000000000000 --- a/airflow-core/newsfragments/40931.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Removed dagbag deprecated ``store_serialized_dags`` parameter. Please use ``read_dags_from_db`` parameter. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/40936.bugfix.rst b/airflow-core/newsfragments/40936.bugfix.rst deleted file mode 100644 index 207aeb97521af..0000000000000 --- a/airflow-core/newsfragments/40936.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix: ``on_success_callback`` will no longer execute if a task is skipped. Previously, this callback was triggered even when the task was skipped, which could lead to unintended behavior or inconsistencies in downstream processes. This is a breaking change because workflows that rely on ``on_success_callback`` running for skipped tasks will need to be updated. Consider updating your DAGs to handle cases where the callback is not invoked due to task skipping. diff --git a/airflow-core/newsfragments/41096.significant.rst b/airflow-core/newsfragments/41096.significant.rst deleted file mode 100644 index e08520b14255d..0000000000000 --- a/airflow-core/newsfragments/41096.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Removed deprecated ``processor_poll_interval`` configuration parameter from ``scheduler`` section. Please use ``scheduler_idle_sleep_time`` configuration parameter. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``scheduler.processor_poll_interval`` → ``scheduler.scheduler_idle_sleep_time`` diff --git a/airflow-core/newsfragments/41348.significant.rst b/airflow-core/newsfragments/41348.significant.rst deleted file mode 100644 index b212a5f18e35f..0000000000000 --- a/airflow-core/newsfragments/41348.significant.rst +++ /dev/null @@ -1,324 +0,0 @@ -Rename ``Dataset`` as ``Asset`` - -* list of renamed objects - - * Rename module ``airflow.api_connexion.schemas.dataset_schema`` as ``airflow.api_connexion.schemas.asset_schema`` - - * Rename variable ``create_dataset_event_schema`` as ``create_asset_event_schema`` - * Rename variable ``dataset_collection_schema`` as ``asset_collection_schema`` - * Rename variable ``dataset_event_collection_schema`` as ``asset_event_collection_schema`` - * Rename variable ``dataset_event_schema`` as ``asset_event_schema`` - * Rename variable ``dataset_schema`` as ``asset_schema`` - * Rename class ``TaskOutletDatasetReferenceSchema`` as ``TaskOutletAssetReferenceSchema`` - * Rename class ``DagScheduleDatasetReferenceSchema`` as ``DagScheduleAssetReferenceSchema`` - * Rename class ``DatasetAliasSchema`` as ``AssetAliasSchema`` - * Rename class ``DatasetSchema`` as ``AssetSchema`` - * Rename class ``DatasetCollection`` as ``AssetCollection`` - * Rename class ``DatasetEventSchema`` as ``AssetEventSchema`` - * Rename class ``DatasetEventCollection`` as ``AssetEventCollection`` - * Rename class ``DatasetEventCollectionSchema`` as ``AssetEventCollectionSchema`` - * Rename class ``CreateDatasetEventSchema`` as ``CreateAssetEventSchema`` - - * Move module ``airflow.datasets`` to ``airflow.sdk.definitions.asset`` - - * Rename class ``DatasetAlias`` as ``AssetAlias`` - * Rename class ``DatasetAll`` as ``AssetAll`` - * Rename class ``DatasetAny`` as ``AssetAny`` - * Rename function ``expand_alias_to_datasets`` as ``expand_alias_to_assets`` - * Rename class ``DatasetAliasEvent`` as ``AssetAliasEvent`` - - * Rename attribute ``dest_dataset_uri`` as ``dest_asset_uri`` - - * Rename class ``BaseDataset`` as ``BaseAsset`` - - * Rename method ``iter_datasets`` as ``iter_assets`` - * Rename method ``iter_dataset_aliases`` as ``iter_asset_aliases`` - - * Rename class ``Dataset`` as ``Asset`` - - * Rename method ``iter_datasets`` as ``iter_assets`` - * Rename method ``iter_dataset_aliases`` as ``iter_asset_aliases`` - - * Rename class ``_DatasetBooleanCondition`` as ``_AssetBooleanCondition`` - - * Rename method ``iter_datasets`` as ``iter_assets`` - * Rename method ``iter_dataset_aliases`` as ``iter_asset_aliases`` - - * Rename module ``airflow.datasets.manager`` as ``airflow.assets.manager`` - - * Rename variable ``dataset_manager`` as ``asset_manager`` - * Rename function ``resolve_dataset_manager`` as ``resolve_asset_manager`` - * Rename class ``DatasetManager`` as ``AssetManager`` - - * Rename method ``register_dataset_change`` as ``register_asset_change`` - * Rename method ``create_datasets`` as ``create_assets`` - * Rename method ``register_dataset_change`` as ``notify_asset_created`` - * Rename method ``notify_dataset_changed`` as ``notify_asset_changed`` - * Rename method ``notify_dataset_alias_created`` as ``notify_asset_alias_created`` - - * Rename module ``airflow.models.dataset`` as ``airflow.models.asset`` - - * Rename class ``DatasetDagRunQueue`` as ``AssetDagRunQueue`` - * Rename class ``DatasetEvent`` as ``AssetEvent`` - * Rename class ``DatasetModel`` as ``AssetModel`` - * Rename class ``DatasetAliasModel`` as ``AssetAliasModel`` - * Rename class ``DagScheduleDatasetReference`` as ``DagScheduleAssetReference`` - * Rename class ``TaskOutletDatasetReference`` as ``TaskOutletAssetReference`` - * Rename class ``DagScheduleDatasetAliasReference`` as ``DagScheduleAssetAliasReference`` - - * Rename module ``airflow.api_ui.views.datasets`` as ``airflow.api_ui.views.assets`` - - * Rename variable ``dataset_router`` as ``asset_rounter`` - - * Rename module ``airflow.listeners.spec.dataset`` as ``airflow.listeners.spec.asset`` - - * Rename function ``on_dataset_created`` as ``on_asset_created`` - * Rename function ``on_dataset_changed`` as ``on_asset_changed`` - - * Rename module ``airflow.timetables.datasets`` as ``airflow.timetables.assets`` - - * Rename class ``DatasetOrTimeSchedule`` as ``AssetOrTimeSchedule`` - - * Rename module ``airflow.serialization.pydantic.dataset`` as ``airflow.serialization.pydantic.asset`` - - * Rename class ``DagScheduleDatasetReferencePydantic`` as ``DagScheduleAssetReferencePydantic`` - * Rename class ``TaskOutletDatasetReferencePydantic`` as ``TaskOutletAssetReferencePydantic`` - * Rename class ``DatasetPydantic`` as ``AssetPydantic`` - * Rename class ``DatasetEventPydantic`` as ``AssetEventPydantic`` - - * Rename module ``airflow.datasets.metadata`` as ``airflow.sdk.definitions.asset.metadata`` - - * In module ``airflow.jobs.scheduler_job_runner`` - - * and its class ``SchedulerJobRunner`` - - * Rename method ``_create_dag_runs_dataset_triggered`` as ``_create_dag_runs_asset_triggered`` - * Rename method ``_orphan_unreferenced_datasets`` as ``_orphan_unreferenced_datasets`` - - * In module ``airflow.api_connexion.security`` - - * Rename decorator ``requires_access_dataset`` as ``requires_access_asset`` - - * In module ``airflow.api_fastapi.auth.managers.models.resource_details`` - - * Rename class ``DatasetDetails`` as ``AssetDetails`` - - * In module ``airflow.api_fastapi.auth.managers.base_auth_manager`` - - * Rename function ``is_authorized_dataset`` as ``is_authorized_asset`` - - * In module ``airflow.timetables.simple`` - - * Rename class ``DatasetTriggeredTimetable`` as ``AssetTriggeredTimetable`` - - * In module ``airflow.lineage.hook`` - - * Rename class ``DatasetLineageInfo`` as ``AssetLineageInfo`` - - * Rename attribute ``dataset`` as ``asset`` - - * In its class ``HookLineageCollector`` - - * Rename method ``create_dataset`` as ``create_asset`` - * Rename method ``add_input_dataset`` as ``add_input_asset`` - * Rename method ``add_output_dataset`` as ``add_output_asset`` - * Rename method ``collected_datasets`` as ``collected_assets`` - - * In module ``airflow.models.dag`` - - * Rename function ``get_dataset_triggered_next_run_info`` as ``get_asset_triggered_next_run_info`` - - * In its class ``DagModel`` - - * Rename method ``get_dataset_triggered_next_run_info`` as ``get_asset_triggered_next_run_info`` - - * In module ``airflow.models.taskinstance`` - - * and its class ``TaskInstance`` - - * Rename method ``_register_dataset_changes`` as ``_register_asset_changes`` - - * In module ``airflow.providers_manager`` - - * and its class ``ProvidersManager`` - - * Rename method ``initialize_providers_dataset_uri_resources`` as ``initialize_providers_asset_uri_resources`` - * Rename attribute ``_discover_dataset_uri_resources`` as ``_discover_asset_uri_resources`` - * Rename property ``dataset_factories`` as ``asset_factories`` - * Rename property ``dataset_uri_handlers`` as ``asset_uri_handlers`` - * Rename property ``dataset_to_openlineage_converters`` as ``asset_to_openlineage_converters`` - - * In module ``airflow.security.permissions`` - - * Rename constant ``RESOURCE_DATASET`` as ``RESOURCE_ASSET`` - - * In module ``airflow.serialization.enums`` - - * and its class DagAttributeTypes - - * Rename attribute ``DATASET_EVENT_ACCESSORS`` as ``ASSET_EVENT_ACCESSORS`` - * Rename attribute ``DATASET_EVENT_ACCESSOR`` as ``ASSET_EVENT_ACCESSOR`` - * Rename attribute ``DATASET`` as ``ASSET`` - * Rename attribute ``DATASET_ALIAS`` as ``ASSET_ALIAS`` - * Rename attribute ``DATASET_ANY`` as ``ASSET_ANY`` - * Rename attribute ``DATASET_ALL`` as ``ASSET_ALL`` - - * In module ``airflow.serialization.pydantic.taskinstance`` - - * and its class ``TaskInstancePydantic`` - - * Rename method ``_register_dataset_changes`` as ``_register_dataset_changes`` - - * In module ``airflow.serialization.serialized_objects`` - - * Rename function ``encode_dataset_condition`` as ``encode_asset_condition`` - * Rename function ``decode_dataset_condition`` as ``decode_asset_condition`` - - * In module ``airflow.timetables.base`` - - * Rename class ```_NullDataset``` as ```_NullAsset``` - - * Rename method ``iter_datasets`` as ``iter_assets`` - * Rename method ``iter_dataset_aliases`` as ``iter_assets_aliases`` - - * In module ``airflow.utils.context`` - - * Rename class ``LazyDatasetEventSelectSequence`` as ``LazyAssetEventSelectSequence`` - - * In module ``airflow.www.auth`` - - * Rename function ``has_access_dataset`` as ``has_access_asset`` - - * Rename configuration ``core.dataset_manager_class`` as ``core.asset_manager_class`` and ``core.dataset_manager_kwargs`` as ``core.asset_manager_kwargs`` - * Rename example dags ``example_dataset_alias.py``, ``example_dataset_alias_with_no_taskflow.py``, ``example_datasets.py`` as ``example_asset_alias.py``, ``example_asset_alias_with_no_taskflow.py``, ``example_assets.py`` - * Rename DagDependency name ``dataset-alias``, ``dataset`` as ``asset-alias``, ``asset`` - * Rename context key ``triggering_dataset_events`` as ``triggering_asset_events`` - * Rename resource key ``dataset-uris`` as ``asset-uris`` for providers amazon, common.io, mysql, fab, postgres, trino - - * In provider ``airflow.providers.amazon.aws`` - - * Rename package ``datasets`` as ``assets`` - - * In its module ``s3`` - - * Rename method ``create_dataset`` as ``create_asset`` - * Rename method ``convert_dataset_to_openlineage`` as ``convert_asset_to_openlineage`` - - * and its module ``auth_manager.avp.entities`` - - * Rename attribute ``AvpEntities.DATASET`` as ``AvpEntities.ASSET`` - - * and its module ``auth_manager.auth_manager.aws_auth_manager`` - - * Rename function ``is_authorized_dataset`` as ``is_authorized_asset`` - - * In provider ``airflow.providers.common.io`` - - * Rename package ``datasets`` as ``assets`` - - * in its module ``file`` - - * Rename method ``create_dataset`` as ``create_asset`` - * Rename method ``convert_dataset_to_openlineage`` as ``convert_asset_to_openlineage`` - - * In provider ``airflow.providers.fab`` - - * in its module ``auth_manager.fab_auth_manager`` - - * Rename function ``is_authorized_dataset`` as ``is_authorized_asset`` - - * In provider ``airflow.providers.openlineage`` - - * in its module ``utils.utils`` - - * Rename class ``DatasetInfo`` as ``AssetInfo`` - * Rename function ``translate_airflow_dataset`` as ``translate_airflow_asset`` - - * Rename package ``airflow.providers.postgres.datasets`` as ``airflow.providers.postgres.assets`` - * Rename package ``airflow.providers.mysql.datasets`` as ``airflow.providers.mysql.assets`` - * Rename package ``airflow.providers.trino.datasets`` as ``airflow.providers.trino.assets`` - * Add module ``airflow.providers.common.compat.assets`` - * Add module ``airflow.providers.common.compat.openlineage.utils.utils`` - * Add module ``airflow.providers.common.compat.security.permissions`` - -* Types of change - - * [x] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.dataset_manager_class`` → ``core.asset_manager_class`` - * [x] ``core.dataset_manager_kwargs`` → ``core.asset_manager_kwargs`` - - * ruff - - * AIR302 - - * [ ] context key ``triggering_dataset_events`` → ``triggering_asset_events`` - * [x] ``airflow.api_connexion.security.requires_access_dataset`` → ``airflow.api_connexion.security.requires_access_asset`` - * [x] ``airflow.auth.managers.base_auth_manager.is_authorized_dataset`` → ``airflow.api_fastapi.auth.managers.base_auth_manager.is_authorized_asset`` - * [x] ``airflow.auth.managers.models.resource_details.DatasetDetails`` → ``airflow.api_fastapi.auth.managers.models.resource_details.AssetDetails`` - * [x] ``airflow.lineage.hook.DatasetLineageInfo`` → ``airflow.lineage.hook.AssetLineageInfo`` - * [x] ``airflow.security.permissions.RESOURCE_DATASET`` → ``airflow.security.permissions.RESOURCE_ASSET`` - * [x] ``airflow.www.auth.has_access_dataset`` → ``airflow.www.auth.has_access_dataset.has_access_asset`` - * [x] ``airflow.datasets.DatasetAliasEvent`` - * [x] ``airflow.datasets.Dataset`` → ``airflow.sdk.definitions.asset.Asset`` - * [x] ``airflow.Dataset`` → ``airflow.sdk.definitions.asset.Asset`` - * [x] ``airflow.datasets.DatasetAlias`` → ``airflow.sdk.definitions.asset.AssetAlias`` - * [x] ``airflow.datasets.DatasetAll`` → ``airflow.sdk.definitions.asset.AssetAll`` - * [x] ``airflow.datasets.DatasetAny`` → ``airflow.sdk.definitions.asset.AssetAny`` - * [x] ``airflow.datasets.metadata`` → ``airflow.sdk.definitions.asset.metadata`` - * [x] ``airflow.datasets.expand_alias_to_datasets`` → ``airflow.sdk.definitions.asset.expand_alias_to_assets`` - * [x] ``airflow.datasets.manager.dataset_manager`` → ``airflow.assets.manager`` - * [x] ``airflow.datasets.manager.resolve_dataset_manager`` → ``airflow.assets.resolve_asset_manager`` - * [x] ``airflow.datasets.manager.DatasetManager`` → ``airflow.assets.AssetManager`` - * [x] ``airflow.listeners.spec.dataset.on_dataset_created`` → ``airflow.listeners.spec.asset.on_asset_created`` - * [x] ``airflow.listeners.spec.dataset.on_dataset_changed`` → ``airflow.listeners.spec.asset.on_asset_changed`` - * [x] ``airflow.timetables.simple.DatasetTriggeredTimetable`` → ``airflow.timetables.simple.AssetTriggeredTimetable`` - * [x] ``airflow.timetables.datasets.DatasetOrTimeSchedule`` → ``airflow.timetables.assets.AssetOrTimeSchedule`` - * [x] ``airflow.providers.amazon.auth_manager.avp.entities.AvpEntities.DATASET`` → ``airflow.providers.amazon.auth_manager.avp.entities.AvpEntities.ASSET`` - * [x] ``airflow.providers.amazon.aws.datasets.s3.create_dataset`` → ``airflow.providers.amazon.aws.assets.s3.create_asset`` - * [x] ``airflow.providers.amazon.aws.datasets.s3.convert_dataset_to_openlineage`` → ``airflow.providers.amazon.aws.datasets.s3.convert_dataset_to_openlineage`` - * [x] ``airflow.providers.amazon.aws.datasets.s3.sanitize_uri`` → ``airflow.providers.amazon.aws.assets.s3.sanitize_uri`` - * [x] ``airflow.providers.common.io.datasets.file.convert_dataset_to_openlineage`` → ``airflow.providers.common.io.assets.file.convert_asset_to_openlineage`` - * [x] ``airflow.providers.common.io.datasets.file.sanitize_uri`` → ``airflow.providers.common.io.assets.file.sanitize_uri`` - * [x] ``airflow.providers.common.io.datasets.file.create_dataset`` → ``airflow.providers.common.io.assets.file.create_asset`` - * [x] ``airflow.providers.google.datasets.bigquery.sanitize_uri`` → ``airflow.providers.google.assets.bigquery.sanitize_uri`` - * [x] ``airflow.providers.google.datasets.gcs.create_dataset`` → ``airflow.providers.google.assets.gcs.create_asset`` - * [x] ``airflow.providers.google.datasets.gcs.sanitize_uri`` → ``airflow.providers.google.assets.gcs.sanitize_uri`` - * [x] ``airflow.providers.google.datasets.gcs.convert_dataset_to_openlineage`` → ``airflow.providers.google.assets.gcs.convert_asset_to_openlineage`` - * [x] ``airflow.providers.fab.auth_manager.fab_auth_manager.is_authorized_dataset`` → ``airflow.providers.fab.auth_manager.fab_auth_manager.is_authorized_asset`` - * [x] ``airflow.providers.openlineage.utils.utils.DatasetInfo`` → ``airflow.providers.openlineage.utils.utils.AssetInfo`` - * [x] ``airflow.providers.openlineage.utils.utils.translate_airflow_dataset`` → ``airflow.providers.openlineage.utils.utils.translate_airflow_asset`` - * [x] ``airflow.providers.postgres.datasets.postgres.sanitize_uri`` → ``airflow.providers.postgres.assets.postgres.sanitize_uri`` - * [x] ``airflow.providers.mysql.datasets.mysql.sanitize_uri`` → ``airflow.providers.mysql.assets.mysql.sanitize_uri`` - * [x] ``airflow.providers.trino.datasets.trino.sanitize_uri`` → ``airflow.providers.trino.assets.trino.sanitize_uri`` - * [x] property ``airflow.providers_manager.ProvidersManager.dataset_factories`` → ``airflow.providers_manager.ProvidersManager.asset_factories`` - * [x] property ``airflow.providers_manager.ProvidersManager.dataset_uri_handlers`` → ``airflow.providers_manager.ProvidersManager.asset_uri_handlers`` - * [x] property ``airflow.providers_manager.ProvidersManager.dataset_to_openlineage_converters`` → ``airflow.providers_manager.ProvidersManager.asset_to_openlineage_converters`` - * [x] class attribute ``airflow.lineage.hook.DatasetLineageInfo.dataset`` → ``airflow.lineage.hook.AssetLineageInfo.asset`` - * [x] method ``airflow.datasets.manager.DatasetManager.register_dataset_change`` → ``airflow.assets.manager.AssetManager.register_asset_change`` - * [x] method ``airflow.datasets.manager.DatasetManager.create_datasets`` → ``airflow.assets.manager.AssetManager.create_assets`` - * [x] method ``airflow.datasets.manager.DatasetManager.notify_dataset_created`` → ``airflow.assets.manager.AssetManager.notify_asset_created`` - * [x] method ``airflow.datasets.manager.DatasetManager.notify_dataset_changed`` → ``airflow.assets.manager.AssetManager.notify_asset_changed`` - * [x] method ``airflow.datasets.manager.DatasetManager.notify_dataset_alias_created`` → ``airflow.assets.manager.AssetManager.notify_asset_alias_created`` - * [x] method ``airflow.providers.amazon.auth_manager.aws_auth_manager.AwsAuthManager.is_authorized_dataset`` → ``airflow.providers.amazon.auth_manager.aws_auth_manager.AwsAuthManager.is_authorized_asset`` - * [x] method ``airflow.lineage.hook.HookLineageCollector.create_dataset`` → ``airflow.lineage.hook.HookLineageCollector.create_asset`` - * [x] method ``airflow.lineage.hook.HookLineageCollector.add_input_dataset`` → ``airflow.lineage.hook.HookLineageCollector.add_input_asset`` - * [x] method ``airflow.lineage.hook.HookLineageCollector.add_output_dataset`` → ``airflow.lineage.hook.HookLineageCollector.dd_output_asset`` - * [x] method ``airflow.lineage.hook.HookLineageCollector.collected_datasets`` → ``airflow.lineage.hook.HookLineageCollector.collected_assets`` - * [x] method ``airflow.providers_manager.ProvidersManager.initialize_providers_dataset_uri_resources`` → ``airflow.providers_manager.ProvidersManager.initialize_providers_asset_uri_resources`` - * [x] method ``airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_uri`` → ``airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_value`` - * [x] method ``airflow.secrets.base_secrets.BaseSecretsBackend.get_connections`` → ``airflow.secrets.base_secrets.BaseSecretsBackend.get_connection`` - * [x] method ``airflow.hooks.base.BaseHook.get_connections`` → ``airflow.hooks.base.BaseHook.get_connection`` - * [x] method ``airflow.datasets.BaseDataset.iter_datasets`` → ``airflow.sdk.definitions.asset.BaseAsset.iter_assets`` - * [x] method ``airflow.datasets.BaseDataset.iter_dataset_aliases`` → ``airflow.sdk.definitions.asset.BaseAsset.iter_asset_aliases`` diff --git a/airflow-core/newsfragments/41366.significant.rst b/airflow-core/newsfragments/41366.significant.rst deleted file mode 100644 index edd588d90bd75..0000000000000 --- a/airflow-core/newsfragments/41366.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -``airflow.contrib`` modules have been removed - -All modules from ``airflow.contrib``, which were deprecated in Airflow 2, have been removed. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.contrib.*`` diff --git a/airflow-core/newsfragments/41367.significant.rst b/airflow-core/newsfragments/41367.significant.rst deleted file mode 100644 index 1e7405bd8e2ee..0000000000000 --- a/airflow-core/newsfragments/41367.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -Deprecated ``ImportError`` removed from ``airflow.models`` - -The deprecated ``ImportError`` class can no longer be imported from ``airflow.models``. -It has been moved to ``airflow.models.errors.ParseImportError``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41368.significant.rst b/airflow-core/newsfragments/41368.significant.rst deleted file mode 100644 index 641696a4b3ffb..0000000000000 --- a/airflow-core/newsfragments/41368.significant.rst +++ /dev/null @@ -1,171 +0,0 @@ -Support for deprecated core imports removed - -Support for importing classes etc from the following locations was deprecated at various times during Airflow 2s lifecycle, and has been removed: - -- ``airflow.executors`` -- ``airflow.hooks`` -- ``airflow.macros`` -- ``airflow.operators`` -- ``airflow.sensors`` - -Instead, import from the right provider or more specific module instead. -For example, instead of ``from airflow.sensors import TimeDeltaSensor``, use ``from airflow.sensors.time_delta import TimeDeltaSensor``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.sensors.base_sensor_operator.BaseSensorOperator`` → ``airflow.sdk.bases.sensor.BaseSensorOperator`` - * [x] ``airflow.hooks.base_hook.BaseHook`` → ``airflow.hooks.base.BaseHook`` - - * AIR303 - - * [x] ``airflow.sensors.external_task_sensor.ExternalTaskMarker`` → ``airflow.providers.standard.sensors.external_task.ExternalTaskMarker`` - * [x] ``airflow.sensors.external_task_sensor.ExternalTaskSensor`` → ``airflow.providers.standard.sensors.external_task.ExternalTaskSensor`` - * [x] ``airflow.sensors.external_task_sensor.ExternalTaskSensorLink`` → ``airflow.providers.standard.sensors.external_task.ExternalTaskSensorLink`` - * [x] ``airflow.sensors.time_delta_sensor.TimeDeltaSensor`` → ``airflow.providers.standard.sensors.time_delta.TimeDeltaSensor`` - * [x] ``airflow.operators.dagrun_operator.TriggerDagRunLink`` → ``airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunLink`` - * [x] ``airflow.operators.dagrun_operator.TriggerDagRunOperator`` → ``airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunOperator`` - * [x] ``airflow.operators.python_operator.BranchPythonOperator`` → ``airflow.providers.standard.operators.python.BranchPythonOperator`` - * [x] ``airflow.operators.python_operator.PythonOperator`` → ``airflow.providers.standard.operators.python.PythonOperator`` - * [x] ``airflow.operators.python_operator.PythonVirtualenvOperator`` → ``airflow.providers.standard.operators.python.PythonVirtualenvOperator`` - * [x] ``airflow.operators.python_operator.ShortCircuitOperator`` → ``airflow.providers.standard.operators.python.ShortCircuitOperator`` - * [x] ``airflow.operators.latest_only_operator.LatestOnlyOperator`` → ``airflow.providers.standard.operators.latest_only.LatestOnlyOperator`` - * [x] ``airflow.operators.bash_operator.BashOperator`` → ``airflow.providers.standard.operators.bash.BashOperator`` - * [x] ``airflow.operators.branch_operator.BaseBranchOperator`` → ``airflow.providers.standard.operators.branch.BaseBranchOperator`` - * [x] ``airflow.sensors.date_time_sensor.DateTimeSensor`` → ``airflow.providers.standardi.sensors.DateTimeSensor`` - * [x] ``airflow.operators.dummy.EmptyOperator`` → ``airflow.providers.standard.operators.empty.EmptyOperator`` - * [x] ``airflow.operators.dummy.DummyOperator`` → ``airflow.providers.standard.operators.empty.EmptyOperator`` - * [x] ``airflow.operators.dummy_operator.EmptyOperator`` → ``airflow.providers.standard.operators.empty.EmptyOperator`` - * [x] ``airflow.operators.dummy_operator.DummyOperator`` → ``airflow.providers.standard.operators.empty.EmptyOperator`` - * [x] ``airflow.operators.email_operator.EmailOperator`` → ``airflow.providers.smtp.operators.smtp.EmailOperator`` - * [x] ``airflow.executors.celery_executor.CeleryExecutor`` → ``airflow.providers.celery.executors.celery_executor.CeleryExecutor`` - * [x] ``airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor`` → ``airflow.providers.celery.executors.celery_kubernetes_executor.CeleryKubernetesExecutor`` - * [x] ``airflow.executors.dask_executor.DaskExecutor`` → ``airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor`` - * [x] ``airflow.executors.kubernetes_executor.KubernetesExecutor`` → ``airflow.providers.cncf.kubernetes.executors.kubernetes_executor.KubernetesExecutor`` - * [x] ``airflow.executors.kubernetes_executor_utils.AirflowKubernetesScheduler`` → ``airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils.AirflowKubernetesScheduler`` - * [x] ``airflow.executors.kubernetes_executor_utils.KubernetesJobWatcher`` → ``airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils.KubernetesJobWatcher`` - * [x] ``airflow.executors.kubernetes_executor_utils.ResourceVersion`` → ``airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils.ResourceVersion`` - * [x] ``airflow.executors.local_kubernetes_executor.LocalKubernetesExecutor`` → ``airflow.providers.cncf.kubernetes.executors.LocalKubernetesExecutor`` - * [x] ``airflow.hooks.S3_hook.S3Hook`` → ``airflow.providers.amazon.aws.hooks.s3.S3Hook`` - * [x] ``airflow.hooks.S3_hook.provide_bucket_name`` → ``airflow.providers.amazon.aws.hooks.s3.provide_bucket_name`` - * [x] ``airflow.hooks.base_hook.BaseHook`` → ``airflow.hooks.base.BaseHook`` - * [x] ``airflow.hooks.dbapi_hook.DbApiHook`` → ``airflow.providers.common.sql.hooks.sql.DbApiHook`` - * [x] ``airflow.hooks.docker_hook.DockerHook`` → ``airflow.providers.docker.hooks.docker.DockerHook`` - * [x] ``airflow.hooks.druid_hook.DruidDbApiHook`` → ``airflow.providers.apache.druid.hooks.druid.DruidDbApiHook`` - * [x] ``airflow.hooks.druid_hook.DruidHook`` → ``airflow.providers.apache.druid.hooks.druid.DruidHook`` - * [x] ``airflow.hooks.hive_hooks.HIVE_QUEUE_PRIORITIES`` → ``airflow.providers.apache.hive.hooks.hive.HIVE_QUEUE_PRIORITIES`` - * [x] ``airflow.hooks.hive_hooks.HiveCliHook`` → ``airflow.providers.apache.hive.hooks.hive.HiveCliHook`` - * [x] ``airflow.hooks.hive_hooks.HiveMetastoreHook`` → ``airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook`` - * [x] ``airflow.hooks.hive_hooks.HiveServer2Hook`` → ``airflow.providers.apache.hive.hooks.hive.HiveServer2Hook`` - * [x] ``airflow.hooks.http_hook.HttpHook`` → ``airflow.providers.http.hooks.http.HttpHook`` - * [x] ``airflow.hooks.jdbc_hook.JdbcHook`` → ``airflow.providers.jdbc.hooks.jdbc.JdbcHook`` - * [x] ``airflow.hooks.jdbc_hook.jaydebeapi`` → ``airflow.providers.jdbc.hooks.jdbc.jaydebeapi`` - * [x] ``airflow.hooks.mssql_hook.MsSqlHook`` → ``airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook`` - * [x] ``airflow.hooks.mysql_hook.MySqlHook`` → ``airflow.providers.mysql.hooks.mysql.MySqlHook`` - * [x] ``airflow.hooks.oracle_hook.OracleHook`` → ``airflow.providers.oracle.hooks.oracle.OracleHook`` - * [x] ``airflow.hooks.pig_hook.PigCliHook`` → ``airflow.providers.apache.pig.hooks.pig.PigCliHook`` - * [x] ``airflow.hooks.postgres_hook.PostgresHook`` → ``airflow.providers.postgres.hooks.postgres.PostgresHook`` - * [x] ``airflow.hooks.presto_hook.PrestoHook`` → ``airflow.providers.presto.hooks.presto.PrestoHook`` - * [x] ``airflow.hooks.samba_hook.SambaHook`` → ``airflow.providers.samba.hooks.samba.SambaHook`` - * [x] ``airflow.hooks.slack_hook.SlackHook`` → ``airflow.providers.slack.hooks.slack.SlackHook`` - * [x] ``airflow.hooks.sqlite_hook.SqliteHook`` → ``airflow.providers.sqlite.hooks.sqlite.SqliteHook`` - * [x] ``airflow.hooks.webhdfs_hook.WebHDFSHook`` → ``airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook`` - * [x] ``airflow.hooks.zendesk_hook.ZendeskHook`` → ``airflow.providers.zendesk.hooks.zendesk.ZendeskHook`` - * [x] ``airflow.operators.check_operator.SQLCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLCheckOperator`` - * [x] ``airflow.operators.check_operator.SQLIntervalCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator`` - * [x] ``airflow.operators.check_operator.SQLThresholdCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLThresholdCheckOperator`` - * [x] ``airflow.operators.check_operator.SQLValueCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLValueCheckOperator`` - * [x] ``airflow.operators.check_operator.CheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLCheckOperator`` - * [x] ``airflow.operators.check_operator.IntervalCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator`` - * [x] ``airflow.operators.check_operator.ThresholdCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLThresholdCheckOperator`` - * [x] ``airflow.operators.check_operator.ValueCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLValueCheckOperator`` - * [x] ``airflow.operators.dagrun_operator.TriggerDagRunLink`` → ``airflow.operators.trigger_dagrun.TriggerDagRunLink`` - * [x] ``airflow.operators.dagrun_operator.TriggerDagRunOperator`` → ``airflow.operators.trigger_dagrun.TriggerDagRunOperator`` - * [x] ``airflow.operators.docker_operator.DockerOperator`` → ``airflow.providers.docker.operators.docker.DockerOperator`` - * [x] ``airflow.operators.druid_check_operator.DruidCheckOperator`` → ``airflow.providers.apache.druid.operators.druid_check.DruidCheckOperator`` - * [x] ``airflow.operators.gcs_to_s3.GCSToS3Operator`` → ``airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator`` - * [x] ``airflow.operators.google_api_to_s3_transfer.GoogleApiToS3Operator`` → ``airflow.providers.amazon.aws.transfers.google_api_to_s3.GoogleApiToS3Operator`` - * [x] ``airflow.operators.google_api_to_s3_transfer.GoogleApiToS3Transfer`` → ``airflow.providers.amazon.aws.transfers.google_api_to_s3.GoogleApiToS3Operator`` - * [x] ``airflow.operators.hive_operator.HiveOperator`` → ``airflow.providers.apache.hive.operators.hive.HiveOperator`` - * [x] ``airflow.operators.hive_stats_operator.HiveStatsCollectionOperator`` → ``airflow.providers.apache.hive.operators.hive_stats.HiveStatsCollectionOperator`` - * [x] ``airflow.operators.hive_to_druid.HiveToDruidOperator`` → ``airflow.providers.apache.druid.transfers.hive_to_druid.HiveToDruidOperator`` - * [x] ``airflow.operators.hive_to_druid.HiveToDruidTransfer`` → ``airflow.providers.apache.druid.transfers.hive_to_druid.HiveToDruidOperator`` - * [x] ``airflow.operators.hive_to_mysql.HiveToMySqlOperator`` → ``airflow.providers.apache.hive.transfers.hive_to_mysql.HiveToMySqlOperator`` - * [x] ``airflow.operators.hive_to_mysql.HiveToMySqlTransfer`` → ``airflow.providers.apache.hive.transfers.hive_to_mysql.HiveToMySqlOperator`` - * [x] ``airflow.operators.local_kubernetes_executor.HiveToSambaOperator`` → ``airflow.providers.apache.hive.transfers.hive_to_samba.HiveToSambaOperator`` - * [x] ``airflow.operators.hive_to_samba_operator.SimpleHttpOperator`` → ``airflow.providers.http.operators.http.SimpleHttpOperator`` - * [x] ``airflow.operators.jdbc_operator.JdbcOperator`` → ``airflow.providers.jdbc.operators.jdbc.JdbcOperator`` - * [x] ``airflow.operators.latest_only_operator.LatestOnlyOperator`` → ``airflow.operators.latest_only.LatestOnlyOperator`` - * [x] ``airflow.operators.mssql_operator.MsSqlOperator`` → ``airflow.providers.microsoft.mssql.operators.mssql.MsSqlOperator`` - * [x] ``airflow.operators.mssql_to_hive.MsSqlToHiveOperator`` → ``airflow.providers.apache.hive.transfers.mssql_to_hive.MsSqlToHiveOperator`` - * [x] ``airflow.operators.mssql_to_hive.MsSqlToHiveTransfer`` → ``airflow.providers.apache.hive.transfers.mssql_to_hive.MsSqlToHiveOperator`` - * [x] ``airflow.operators.mysql_operator.MySqlOperator`` → ``airflow.providers.mysql.operators.mysql.MySqlOperator`` - * [x] ``airflow.operators.mysql_to_hive.MySqlToHiveOperator`` → ``airflow.providers.apache.hive.transfers.mysql_to_hive.MySqlToHiveOperator`` - * [x] ``airflow.operators.mysql_to_hive.MySqlToHiveTransfer`` → ``airflow.providers.apache.hive.transfers.mysql_to_hive.MySqlToHiveOperator`` - * [x] ``airflow.operators.oracle_operator.OracleOperator`` → ``airflow.providers.oracle.operators.oracle.OracleOperator`` - * [x] ``airflow.operators.papermill_operator.PapermillOperator`` → ``airflow.providers.papermill.operators.papermill.PapermillOperator`` - * [x] ``airflow.operators.pig_operator.PigOperator`` → ``airflow.providers.apache.pig.operators.pig.PigOperator`` - * [x] ``airflow.operators.postgres_operator.Mapping`` → ``airflow.providers.postgres.operators.postgres.Mapping`` - * [x] ``airflow.operators.postgres_operator.PostgresOperator`` → ``airflow.providers.postgres.operators.postgres.PostgresOperator`` - * [x] ``airflow.operators.presto_check_operator.SQLCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLCheckOperator`` - * [x] ``airflow.operators.presto_check_operator.SQLIntervalCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator`` - * [x] ``airflow.operators.presto_check_operator.SQLValueCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLValueCheckOperator`` - * [x] ``airflow.operators.presto_check_operator.PrestoCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLCheckOperator`` - * [x] ``airflow.operators.presto_check_operator.PrestoIntervalCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator`` - * [x] ``airflow.operators.presto_check_operator.PrestoValueCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLValueCheckOperator`` - * [x] ``airflow.operators.presto_to_mysql.PrestoToMySqlOperator`` → ``airflow.providers.mysql.transfers.presto_to_mysql.PrestoToMySqlOperator`` - * [x] ``airflow.operators.presto_to_mysql.PrestoToMySqlTransfer`` → ``airflow.providers.mysql.transfers.presto_to_mysql.PrestoToMySqlOperator`` - * [x] ``airflow.operators.python_operator.BranchPythonOperator`` → ``airflow.operators.python.BranchPythonOperator`` - * [x] ``airflow.operators.python_operator.PythonOperator`` → ``airflow.operators.python.PythonOperator`` - * [x] ``airflow.operators.python_operator.PythonVirtualenvOperator`` → ``airflow.operators.python.PythonVirtualenvOperator`` - * [x] ``airflow.operators.python_operator.ShortCircuitOperator`` → ``airflow.operators.python.ShortCircuitOperator`` - * [x] ``airflow.operators.redshift_to_s3_operator.RedshiftToS3Operator`` → ``airflow.providers.amazon.aws.transfers.redshift_to_s3.RedshiftToS3Operator`` - * [x] ``airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer`` → ``airflow.providers.amazon.aws.transfers.redshift_to_s3.RedshiftToS3Operator`` - * [x] ``airflow.operators.s3_file_transform_operator.S3FileTransformOperator`` → ``airflow.providers.amazon.aws.operators.s3_file_transform.S3FileTransformOperator`` - * [x] ``airflow.operators.s3_to_hive_operator.S3ToHiveOperator`` → ``airflow.providers.apache.hive.transfers.s3_to_hive.S3ToHiveOperator`` - * [x] ``airflow.operators.s3_to_hive_operator.S3ToHiveTransfer`` → ``airflow.providers.apache.hive.transfers.s3_to_hive.S3ToHiveOperator`` - * [x] ``airflow.operators.s3_to_redshift_operator.S3ToRedshiftOperator`` → ``airflow.providers.amazon.aws.transfers.s3_to_redshift.S3ToRedshiftOperator`` - * [x] ``airflow.operators.s3_to_redshift_operator.S3ToRedshiftTransfer`` → ``airflow.providers.amazon.aws.transfers.s3_to_redshift.S3ToRedshiftOperator`` - * [x] ``airflow.operators.slack_operator.SlackAPIOperator`` → ``airflow.providers.slack.operators.slack.SlackAPIOperator`` - * [x] ``airflow.operators.slack_operator.SlackAPIPostOperator`` → ``airflow.providers.slack.operators.slack.SlackAPIPostOperator`` - * [x] ``airflow.operators.sql.BaseSQLOperator`` → ``airflow.providers.common.sql.operators.sql.BaseSQLOperator`` - * [x] ``airflow.operators.sql.BranchSQLOperator`` → ``airflow.providers.common.sql.operators.sql.BranchSQLOperator`` - * [x] ``airflow.operators.sql.SQLCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLCheckOperator`` - * [x] ``airflow.operators.sql.SQLColumnCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLColumnCheckOperator`` - * [x] ``airflow.operators.sql.SQLIntervalCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLIntervalCheckOperator`` - * [x] ``airflow.operators.sql.SQLTableCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLTableCheckOperator`` - * [x] ``airflow.operators.sql.SQLThresholdCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLThresholdCheckOperator`` - * [x] ``airflow.operators.sql.SQLValueCheckOperator`` → ``airflow.providers.common.sql.operators.sql.SQLValueCheckOperator`` - * [x] ``airflow.operators.sql._convert_to_float_if_possible`` → ``airflow.providers.common.sql.operators.sql._convert_to_float_if_possible`` - * [x] ``airflow.operators.sql.parse_boolean`` → ``airflow.providers.common.sql.operators.sql.parse_boolean`` - * [x] ``airflow.operators.sql_branch_operator.BranchSQLOperator`` → ``airflow.providers.common.sql.operators.sql.BranchSQLOperator`` - * [x] ``airflow.operators.sql_branch_operator.BranchSqlOperator`` → ``airflow.providers.common.sql.operators.sql.BranchSQLOperator`` - * [x] ``airflow.operators.sqlite_operator.SqliteOperator`` → ``airflow.providers.sqlite.operators.sqlite.SqliteOperator`` - * [x] ``airflow.sensors.hive_partition_sensor.HivePartitionSensor`` → ``airflow.providers.apache.hive.sensors.hive_partition.HivePartitionSensor`` - * [x] ``airflow.sensors.http_sensor.HttpSensor`` → ``airflow.providers.http.sensors.http.HttpSensor`` - * [x] ``airflow.sensors.metastore_partition_sensor.MetastorePartitionSensor`` → ``airflow.providers.apache.hive.sensors.metastore_partition.MetastorePartitionSensor`` - * [x] ``airflow.sensors.named_hive_partition_sensor.NamedHivePartitionSensor`` → ``airflow.providers.apache.hive.sensors.named_hive_partition.NamedHivePartitionSensor`` - * [x] ``airflow.sensors.s3_key_sensor.S3KeySensor`` → ``airflow.providers.amazon.aws.sensors.s3.S3KeySensor`` - * [x] ``airflow.sensors.sql.SqlSensor`` → ``airflow.providers.common.sql.sensors.sql.SqlSensor`` - * [x] ``airflow.sensors.sql_sensor.SqlSensor`` → ``airflow.providers.common.sql.sensors.sql.SqlSensor`` - * [x] ``airflow.sensors.web_hdfs_sensor.WebHdfsSensor`` → ``airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor`` - * [x] ``airflow.executors.kubernetes_executor_types.ALL_NAMESPACES`` → ``airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types.ALL_NAMESPACES`` - * [x] ``airflow.executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY`` → ``airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY`` - * [x] ``airflow.hooks.hive_hooks.HIVE_QUEUE_PRIORITIES`` → ``airflow.providers.apache.hive.hooks.hive.HIVE_QUEUE_PRIORITIES`` - * [x] ``airflow.executors.celery_executor.app`` → ``airflow.providers.celery.executors.celery_executor_utils.app`` - * [x] ``airflow.macros.hive.closest_ds_partition`` → ``airflow.providers.apache.hive.macros.hive.closest_ds_partition`` - * [x] ``airflow.macros.hive.max_partition`` → ``airflow.providers.apache.hive.macros.hive.max_partition`` diff --git a/airflow-core/newsfragments/41390.significant.rst b/airflow-core/newsfragments/41390.significant.rst deleted file mode 100644 index 5a88c547048fe..0000000000000 --- a/airflow-core/newsfragments/41390.significant.rst +++ /dev/null @@ -1,31 +0,0 @@ -Support for SubDags is removed - -Subdags have been removed from the following locations: - -- CLI -- API -- ``SubDagOperator`` - -This removal marks the end of Subdag support across all interfaces. Users -should transition to using TaskGroups as a more efficient and maintainable -alternative. Please ensure your DAGs are updated to -remove any usage of Subdags to maintain compatibility with future Airflow releases. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [x] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.operators.subdag.*`` diff --git a/airflow-core/newsfragments/41391.significant.rst b/airflow-core/newsfragments/41391.significant.rst deleted file mode 100644 index de169be0e4835..0000000000000 --- a/airflow-core/newsfragments/41391.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -The ``airflow.providers.standard.sensors.external_task.ExternalTaskSensorLink`` class has been removed. - -This class was deprecated and is no longer available. Users should now use -the ``airflow.providers.standard.sensors.external_task.ExternalDagLink`` class directly. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.sensors.external_task.ExternalTaskSensorLink`` → ``airflow.sensors.external_task.ExternalDagLink`` diff --git a/airflow-core/newsfragments/41393.significant.rst b/airflow-core/newsfragments/41393.significant.rst deleted file mode 100644 index 26b3724ca4aa0..0000000000000 --- a/airflow-core/newsfragments/41393.significant.rst +++ /dev/null @@ -1,34 +0,0 @@ -The ``use_task_execution_day`` parameter has been removed from the ``DayOfWeekSensor`` class. This parameter was previously deprecated in favor of ``use_task_logical_date``. - -If your code still uses ``use_task_execution_day``, you should update it to use ``use_task_logical_date`` -instead to ensure compatibility with future Airflow versions. - -Example update: - -.. code-block:: python - - sensor = DayOfWeekSensor( - task_id="example", - week_day="Tuesday", - use_task_logical_date=True, - dag=dag, - ) - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] arguments ``use_task_execution_day`` → ``use_task_logical_date`` in ``airflow.operators.weekday.DayOfWeekSensor`` diff --git a/airflow-core/newsfragments/41394.significant.rst b/airflow-core/newsfragments/41394.significant.rst deleted file mode 100644 index 49414a50b8e31..0000000000000 --- a/airflow-core/newsfragments/41394.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -The ``airflow.models.taskMixin.TaskMixin`` class has been removed. It was previously deprecated in favor of the ``airflow.models.taskMixin.DependencyMixin`` class. - -If your code relies on ``TaskMixin``, please update it to use ``DependencyMixin`` instead -to ensure compatibility with Airflow 3.0 and beyond. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41395.significant.rst b/airflow-core/newsfragments/41395.significant.rst deleted file mode 100644 index be9427b8b9d6f..0000000000000 --- a/airflow-core/newsfragments/41395.significant.rst +++ /dev/null @@ -1,33 +0,0 @@ -**Breaking Change** - -The following deprecated functions, constants, and classes have been removed as part of Airflow 3.0: - -- ``airflow.executors.executor_loader.UNPICKLEABLE_EXECUTORS``: No direct replacement; this constant is no longer needed. -- ``airflow.utils.dag_cycle_tester.test_cycle`` function: Use ``airflow.utils.dag_cycle_tester.check_cycle`` instead. -- ``airflow.utils.file.TemporaryDirectory`` function: Use ``tempfile.TemporaryDirectory`` instead. -- ``airflow.utils.file.mkdirs`` function: Use ``pathlib.Path.mkdir`` instead. -- ``airflow.utils.state.SHUTDOWN`` state: No action needed; this state is no longer used. -- ``airflow.utils.state.terminating_states`` constant: No action needed; this constant is no longer used. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.file.TemporaryDirectory`` → ``tempfile.TemporaryDirectory`` - * [x] ``airflow.utils.file.mkdirs`` → ``pathlib.Path({path}).mkdir`` - * [x] ``airflow.utils.dag_cycle_tester.test_cycle`` - * [x] ``airflow.utils.state.SHUTDOWN`` - * [x] ``airflow.utils.state.terminating_states`` diff --git a/airflow-core/newsfragments/41420.significant.rst b/airflow-core/newsfragments/41420.significant.rst deleted file mode 100644 index b2d05c97c60b4..0000000000000 --- a/airflow-core/newsfragments/41420.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Replaced Python's ``list`` with ``MutableSet`` for the property ``DAG.tags``. - -At the constractur you still can use list, -you actually can use any data structure that implements the -``Collection`` interface. - -The ``tags`` property of the ``DAG`` model would be of type -``MutableSet`` instead of ``list``, -as there are no actual duplicates at the tags. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41429.improvement.rst b/airflow-core/newsfragments/41429.improvement.rst deleted file mode 100644 index 6d04d5dfe61af..0000000000000 --- a/airflow-core/newsfragments/41429.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``run_with_db_retries`` when the scheduler updates the deferred Task as failed to tolerate database deadlock issues. diff --git a/airflow-core/newsfragments/41434.significant.rst b/airflow-core/newsfragments/41434.significant.rst deleted file mode 100644 index 7a4eed657d11d..0000000000000 --- a/airflow-core/newsfragments/41434.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -Experimental API is removed - -Experimental API is no longer available in Airflow. Users -should transition to using Rest API as an alternative. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41440.significant.rst b/airflow-core/newsfragments/41440.significant.rst deleted file mode 100644 index c23b6edfb5efc..0000000000000 --- a/airflow-core/newsfragments/41440.significant.rst +++ /dev/null @@ -1,28 +0,0 @@ -Removed unused methods / properties in ``airflow/models/dag.py`` - -* Methods removed - - * ``date_range`` - * ``is_fixed_time_schedule`` - * ``next_dagrun_after_date`` - * ``get_run_dates`` - * ``normalize_schedule`` - * ``full_filepath`` - * ``concurrency`` - * ``filepath`` - * ``concurrency_reached`` - * ``normalized_schedule_interval`` - * ``latest_execution_date`` - * ``set_dag_runs_state`` - * ``bulk_sync_to_db`` - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41453.significant.rst b/airflow-core/newsfragments/41453.significant.rst deleted file mode 100644 index 8ca0de3df3202..0000000000000 --- a/airflow-core/newsfragments/41453.significant.rst +++ /dev/null @@ -1,31 +0,0 @@ -Removed legacy scheduling arguments on DAG - -The ``schedule_interval`` and ``timetable`` arguments are removed from ``DAG``. - -The ``schedule_interval`` _attribute_ has also been removed. In the API, a new -``timetable_summary`` field has been added to replace ``schedule_interval`` for -presentation purposes. - -Since the DAG object no longer has the ``schedule_interval`` attribute, -OpenLineage facets that contain the ``dag`` key produced on Airflow 3.0 or -later will also no longer contain the field. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] arguments ``schedule_interval`` in ``DAG`` - * [x] arguments ``timetable`` in ``DAG`` diff --git a/airflow-core/newsfragments/41496.significant.rst b/airflow-core/newsfragments/41496.significant.rst deleted file mode 100644 index e19f1df46c5a7..0000000000000 --- a/airflow-core/newsfragments/41496.significant.rst +++ /dev/null @@ -1,26 +0,0 @@ -Removed deprecated methods in ``airflow/utils/dates.py`` - -* Methods removed - - * ``date_range`` - * ``days_ago`` (Use ``pendulum.today('UTC').add(days=-N, ...)``) - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.dates.date_range`` - * [x] ``airflow.utils.dates.days_ago`` → ``pendulum.today("UTC").add(days=-N, ...)`` diff --git a/airflow-core/newsfragments/41520.significant.rst b/airflow-core/newsfragments/41520.significant.rst deleted file mode 100644 index b2f4fe7351b0f..0000000000000 --- a/airflow-core/newsfragments/41520.significant.rst +++ /dev/null @@ -1,26 +0,0 @@ -Removed deprecated methods in ``airflow/utils/helpers.py`` - -* Methods removed: - - * ``chain`` (Use ``airflow.sdk.chain``) - * ``cross_downstream`` (Use ``airflow.sdk.cross_downstream``) - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.helpers.chain`` → ``airflow.sdk.chain`` - * [x] ``airflow.utils.helpers.cross_downstream`` → ``airflow.sdk.cross_downstream`` diff --git a/airflow-core/newsfragments/41533.significant.rst b/airflow-core/newsfragments/41533.significant.rst deleted file mode 100644 index 0f898f21ad332..0000000000000 --- a/airflow-core/newsfragments/41533.significant.rst +++ /dev/null @@ -1,45 +0,0 @@ -The ``load_connections`` function has been removed from the ``local_file_system``. - -This function was previously deprecated in favor of ``load_connections_dict``. - -If your code still uses ``load_connections``, you should update it to use ``load_connections_dict`` -instead to ensure compatibility with future Airflow versions. - -Example update: - -.. code-block:: python - - connection_by_conn_id = local_filesystem.load_connections_dict(file_path="a.json") - -The ``get_connections`` function has been removed from the ``LocalFilesystemBackend`` class. -This function was previously deprecated in favor of ``get_connection``. - -If your code still uses ``get_connections``, you should update it to use ``get_connection`` -instead to ensure compatibility with future Airflow versions. - - -Example update: - -.. code-block:: python - - connection_by_conn_id = LocalFilesystemBackend().get_connection(conn_id="conn_id") - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.secrets.local_filesystem.load_connections`` → ``airflow.secrets.local_filesystem.load_connections_dict`` - * [x] ``airflow.secrets.local_filesystem.LocalFilesystemBackend.get_connection`` → ``airflow.secrets.local_filesystem.LocalFilesystemBackend.load_connections_dict`` diff --git a/airflow-core/newsfragments/41539.significant.rst b/airflow-core/newsfragments/41539.significant.rst deleted file mode 100644 index 097dc6db75b05..0000000000000 --- a/airflow-core/newsfragments/41539.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -Removed deprecated ``smtp_user`` and ``smtp_password`` configuration parameters from ``smtp`` section. Please use smtp connection (``smtp_default``). - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``smtp.smtp_user`` - * [x] ``smtp.smtp_password`` diff --git a/airflow-core/newsfragments/41550.significant.rst b/airflow-core/newsfragments/41550.significant.rst deleted file mode 100644 index 352afcc9020ea..0000000000000 --- a/airflow-core/newsfragments/41550.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Removed deprecated ``session_lifetime_days`` and ``force_log_out_after`` configuration parameters from ``webserver`` section. Please use ``session_lifetime_minutes`` from ``fab`` section. - -Removed deprecated ``policy`` parameter from ``airflow_local_settings``. Please use ``task_policy``. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``webserver.session_lifetime_days`` → ``fab.session_lifetime_minutes`` - * [x] ``webserver.force_log_out_after`` → ``fab.session_lifetime_minutes`` - * [x] ``policy`` → ``task_policy`` diff --git a/airflow-core/newsfragments/41552.significant.rst b/airflow-core/newsfragments/41552.significant.rst deleted file mode 100644 index 99d6881e4032c..0000000000000 --- a/airflow-core/newsfragments/41552.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed deprecated ``filename_template`` argument from ``airflow.utils.log.file_task_handler.FileTaskHandler``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] argument ``filename_template`` in ``airflow.utils.log.file_task_handler.FileTaskHandler`` and its subclassses diff --git a/airflow-core/newsfragments/41564.significant.rst b/airflow-core/newsfragments/41564.significant.rst deleted file mode 100644 index eed922f86c1f1..0000000000000 --- a/airflow-core/newsfragments/41564.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -Move all time operators and sensors from airflow core to standard provider - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.operators.datetime.*`` → ``airflow.providers.standard.time.operators.datetime.*`` - * [x] ``airflow.operators.weekday.*`` → ``airflow.providers.standard.time.operators.weekday.*`` - * [x] ``airflow.sensors.date_time.*`` → ``airflow.providers.standard.time.sensors.date_time.*`` - * [x] ``airflow.sensors.time_sensor.*`` → ``airflow.providers.standard.time.sensors.time.*`` - * [x] ``airflow.sensors.time_delta.*`` → ``airflow.providers.standard.time.sensors.time_delta.*`` - * [x] ``airflow.sensors.weekday.*`` → ``airflow.providers.standard.time.sensors.weekday.*`` diff --git a/airflow-core/newsfragments/41579.significant.rst b/airflow-core/newsfragments/41579.significant.rst deleted file mode 100644 index d554b5b85303a..0000000000000 --- a/airflow-core/newsfragments/41579.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed deprecated ``apply_defaults`` function from ``airflow/utils/decorators.py``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.decorators.apply_defaults`` (auto applied) diff --git a/airflow-core/newsfragments/41609.significant.rst b/airflow-core/newsfragments/41609.significant.rst deleted file mode 100644 index b691aaea7d188..0000000000000 --- a/airflow-core/newsfragments/41609.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Removed deprecated ``dependency_detector`` parameter from ``scheduler``. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``scheduler.dependency_detector`` diff --git a/airflow-core/newsfragments/41635.significant.rst b/airflow-core/newsfragments/41635.significant.rst deleted file mode 100644 index da3a6e719f4de..0000000000000 --- a/airflow-core/newsfragments/41635.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Removed deprecated ``--ignore-depends-on-past`` cli option from task command. Please use ``--depends-on-past ignore``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41642.significant.rst b/airflow-core/newsfragments/41642.significant.rst deleted file mode 100644 index a0748da360d10..0000000000000 --- a/airflow-core/newsfragments/41642.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -Removed deprecated secrets backend methods ``get_conn_uri`` and ``get_connections``. - -Please use ``get_conn_value`` and ``get_connection`` instead. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_uri`` → ``airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_value`` - * [x] ``airflow.secrets.base_secrets.BaseSecretsBackend.get_connections`` → ``airflow.secrets.base_secrets.BaseSecretsBackend.get_connection`` diff --git a/airflow-core/newsfragments/41663.significant.rst b/airflow-core/newsfragments/41663.significant.rst deleted file mode 100644 index 2cfd9d3f2df27..0000000000000 --- a/airflow-core/newsfragments/41663.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed deprecated auth ``airflow.api.auth.backend.basic_auth`` from ``auth_backends``. Please use ``airflow.providers.fab.auth_manager.api.auth.backend.basic_auth`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.api.auth.backend.basic_auth`` → ``airflow.providers.fab.auth_manager.api.auth.backend.basic_auth`` diff --git a/airflow-core/newsfragments/41693.significant.rst b/airflow-core/newsfragments/41693.significant.rst deleted file mode 100644 index 89fbe92567725..0000000000000 --- a/airflow-core/newsfragments/41693.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ -Removed deprecated auth ``airflow.api.auth.backend.kerberos_auth`` and ``airflow.auth.managers.fab.api.auth.backend.kerberos_auth`` from ``auth_backends``. Please use ``airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.api.auth.backend.kerberos_auth`` → ``airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth`` - * [x] ``airflow.auth.managers.fab.api.auth.backend.kerberos_auth`` → ``airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth`` diff --git a/airflow-core/newsfragments/41708.significant.rst b/airflow-core/newsfragments/41708.significant.rst deleted file mode 100644 index 5f14dd77d1107..0000000000000 --- a/airflow-core/newsfragments/41708.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ -Removed deprecated auth manager ``airflow.auth.managers.fab.fab_auth_manager`` and ``airflow.auth.managers.fab.security_manager.override``. Please use ``airflow.providers.fab.auth_manager.security_manager.override`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.auth.managers.fab.fab_auth_manager`` → ``airflow.providers.fab.auth_manager.security_manager.override`` - * [x] ``airflow.auth.managers.fab.security_manager.override`` → ``airflow.providers.fab.auth_manager.security_manager.override`` diff --git a/airflow-core/newsfragments/41733.significant.rst b/airflow-core/newsfragments/41733.significant.rst deleted file mode 100644 index dda6856aad8b5..0000000000000 --- a/airflow-core/newsfragments/41733.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed deprecated function ``get_connections()`` function in ``airflow.hooks.base.BaseHook``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.hooks.base.BaseHook.get_connections`` → ``airflow.hooks.base.BaseHook.get_connection`` diff --git a/airflow-core/newsfragments/41735.significant.rst b/airflow-core/newsfragments/41735.significant.rst deleted file mode 100644 index 751ea773bf761..0000000000000 --- a/airflow-core/newsfragments/41735.significant.rst +++ /dev/null @@ -1,56 +0,0 @@ -Removed deprecated module ``airflow.kubernetes``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.kubernetes.kubernetes_helper_functions.add_pod_suffix`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.add_pod_suffix`` - * [x] ``airflow.kubernetes.kubernetes_helper_functions.annotations_for_logging_task_metadata`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.annotations_for_logging_task_metadata`` - * [x] ``airflow.kubernetes.kubernetes_helper_functions.annotations_to_key`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.annotations_to_key`` - * [x] ``airflow.kubernetes.kubernetes_helper_functions.create_pod_id`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.create_pod_id`` - * [x] ``airflow.kubernetes.kubernetes_helper_functions.get_logs_task_metadata`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.get_logs_task_metadata`` - * [x] ``airflow.kubernetes.kubernetes_helper_functions.rand_str`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.rand_str`` - * [x] ``airflow.kubernetes.pod.Port`` → ``kubernetes.client.models.V1ContainerPort`` - * [x] ``airflow.kubernetes.pod.Resources`` → ``kubernetes.client.models.V1ResourceRequirements`` - * [x] ``airflow.kubernetes.pod_launcher.PodLauncher`` → ``airflow.providers.cncf.kubernetes.pod_launcher.PodLauncher`` - * [x] ``airflow.kubernetes.pod_launcher.PodStatus`` → ``airflow.providers.cncf.kubernetes.pod_launcher.PodStatus`` - * [x] ``airflow.kubernetes.pod_launcher_deprecated.PodLauncher`` → ``airflow.providers.cncf.kubernetes.pod_launcher_deprecated.PodLauncher`` - * [x] ``airflow.kubernetes.pod_launcher_deprecated.PodStatus`` → ``airflow.providers.cncf.kubernetes.pod_launcher_deprecated.PodStatus`` - * [x] ``airflow.kubernetes.pod_launcher_deprecated.get_kube_client`` → ``airflow.providers.cncf.kubernetes.kube_client.get_kube_client`` - * [x] ``airflow.kubernetes.pod_launcher_deprecated.PodDefaults`` → ``airflow.providers.cncf.kubernetes.pod_generator_deprecated.PodDefaults`` - * [x] ``airflow.kubernetes.pod_runtime_info_env.PodRuntimeInfoEnv`` → ``kubernetes.client.models.V1EnvVar`` - * [x] ``airflow.kubernetes.volume.Volume`` → ``kubernetes.client.models.V1Volume`` - * [x] ``airflow.kubernetes.volume_mount.VolumeMount`` → ``kubernetes.client.models.V1VolumeMount`` - * [x] ``airflow.kubernetes.k8s_model.K8SModel`` → ``airflow.providers.cncf.kubernetes.k8s_model.K8SModel`` - * [x] ``airflow.kubernetes.k8s_model.append_to_pod`` → ``airflow.providers.cncf.kubernetes.k8s_model.append_to_pod`` - * [x] ``airflow.kubernetes.kube_client._disable_verify_ssl`` → ``airflow.kubernetes.airflow.providers.cncf.kubernetes.kube_client._disable_verify_ssl`` - * [x] ``airflow.kubernetes.kube_client._enable_tcp_keepalive`` → ``airflow.kubernetes.airflow.providers.cncf.kubernetes.kube_client._enable_tcp_keepalive`` - * [x] ``airflow.kubernetes.kube_client.get_kube_client`` → ``airflow.kubernetes.airflow.providers.cncf.kubernetes.kube_client.get_kube_client`` - * [x] ``airflow.kubernetes.pod_generator.datetime_to_label_safe_datestring`` → ``airflow.providers.cncf.kubernetes.pod_generator.datetime_to_label_safe_datestring`` - * [x] ``airflow.kubernetes.pod_generator.extend_object_field`` → ``airflow.kubernetes.airflow.providers.cncf.kubernetes.pod_generator.extend_object_field`` - * [x] ``airflow.kubernetes.pod_generator.label_safe_datestring_to_datetime`` → ``airflow.providers.cncf.kubernetes.pod_generator.label_safe_datestring_to_datetime`` - * [x] ``airflow.kubernetes.pod_generator.make_safe_label_value`` → ``airflow.providers.cncf.kubernetes.pod_generator.make_safe_label_value`` - * [x] ``airflow.kubernetes.pod_generator.merge_objects`` → ``airflow.providers.cncf.kubernetes.pod_generator.merge_objects`` - * [x] ``airflow.kubernetes.pod_generator.PodGenerator`` → ``airflow.providers.cncf.kubernetes.pod_generator.PodGenerator`` - * [x] ``airflow.kubernetes.pod_generator.PodGeneratorDeprecated`` → ``airflow.providers.cncf.kubernetes.pod_generator.PodGenerator`` - * [x] ``airflow.kubernetes.pod_generator.PodDefaults`` → ``airflow.providers.cncf.kubernetes.pod_generator_deprecated.PodDefaults`` - * [x] ``airflow.kubernetes.pod_generator.add_pod_suffix`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.add_pod_suffix`` - * [x] ``airflow.kubernetes.pod_generator.rand_str`` → ``airflow.providers.cncf.kubernetes.kubernetes_helper_functions.rand_str`` - * [x] ``airflow.kubernetes.pod_generator_deprecated.make_safe_label_value`` → ``airflow.providers.cncf.kubernetes.pod_generator_deprecated.make_safe_label_value`` - * [x] ``airflow.kubernetes.pod_generator_deprecated.PodDefaults`` → ``airflow.providers.cncf.kubernetes.pod_generator_deprecated.PodDefaults`` - * [x] ``airflow.kubernetes.pod_generator_deprecated.PodGenerator`` → ``airflow.providers.cncf.kubernetes.pod_generator_deprecated.PodGenerator`` - * [x] ``airflow.kubernetes.secret.Secret`` → ``airflow.providers.cncf.kubernetes.secret.Secret`` - * [x] ``airflow.kubernetes.secret.K8SModel`` → ``airflow.providers.cncf.kubernetes.k8s_model.K8SModel`` diff --git a/airflow-core/newsfragments/41736.significant.rst b/airflow-core/newsfragments/41736.significant.rst deleted file mode 100644 index 00f356217b59c..0000000000000 --- a/airflow-core/newsfragments/41736.significant.rst +++ /dev/null @@ -1,28 +0,0 @@ -Removed deprecated parameters from core-operators. - -Parameters removed: - -- ``airflow.operators.datetime.BranchDateTimeOperator``: ``use_task_execution_date`` -- ``airflow.operators.trigger_dagrun.TriggerDagRunOperator``: ``execution_date`` -- ``airflow.operators.weekday.BranchDayOfWeekOperator``: ``use_task_execution_day`` - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] argument ``execution_date`` in ``airflow.operators.trigger_dagrun.TriggerDagRunOperator`` - * [x] argument ``use_task_execution_day`` → ``use_task_logical_date`` in ``airflow.operators.datetime.BranchDateTimeOperator`` - * [x] argument ``use_task_execution_day`` → ``use_task_logical_date`` in ``airflow.operators.weekday.BranchDayOfWeekOperator`` diff --git a/airflow-core/newsfragments/41737.significant.rst b/airflow-core/newsfragments/41737.significant.rst deleted file mode 100644 index 1bf2ed6c13663..0000000000000 --- a/airflow-core/newsfragments/41737.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed deprecated ``TaskStateTrigger`` from ``airflow.triggers.external_task`` module. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.triggers.external_task.TaskStateTrigger`` diff --git a/airflow-core/newsfragments/41739.significant.rst b/airflow-core/newsfragments/41739.significant.rst deleted file mode 100644 index fc92582ee6d2a..0000000000000 --- a/airflow-core/newsfragments/41739.significant.rst +++ /dev/null @@ -1,14 +0,0 @@ -Removed backfill job command cli option ``ignore-first-depends-on-past``. Its value always set to True. No replcaement cli option. - -Removed backfill job command cli option ``treat-dag-as-regex``. Please use ``treat-dag-id-as-regex`` instead. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41748.significant.rst b/airflow-core/newsfragments/41748.significant.rst deleted file mode 100644 index 33546c3841995..0000000000000 --- a/airflow-core/newsfragments/41748.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Deprecated module ``airflow.hooks.dbapi`` removed. Please use ``airflow.providers.common.sql.hooks.sql`` instead. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.hooks.dbapi`` → ``airflow.providers.common.sql.hooks.sql`` diff --git a/airflow-core/newsfragments/41758.significant.rst b/airflow-core/newsfragments/41758.significant.rst deleted file mode 100644 index cf87332b94cfb..0000000000000 --- a/airflow-core/newsfragments/41758.significant.rst +++ /dev/null @@ -1,37 +0,0 @@ -Removed deprecated functions and modules from ``airflow.www`` module. - -- Config flag default warning ``cookie_samesite`` option in section ``[webserver]`` removed. -- Legacy decorator ``@has_access`` in ``airflow.www.auth``: Please use one of the decorator ``has_access_*`` - defined in airflow/www/auth.py instead. -- Removed legacy modules ``airflow.www.security``: Should be inherited from - ``airflow.providers.fab.auth_manager.security_manager.override.FabAirflowSecurityManagerOverride`` instead. - The constant value ``EXISTING_ROLES`` should be used from ``airflow.www.security_manager`` module. -- Removed the method ``get_sensitive_variables_fields()`` from ``airflow.www.utils``: Please use - ``airflow.utils.log.secrets_masker.get_sensitive_variables_fields`` instead. -- Removed the method ``should_hide_value_for_key()`` from ``airflow.www.utils``: Please use - ``airflow.utils.log.secrets_masker.should_hide_value_for_key`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.www.auth.has_access`` → ``airflow.www.auth.has_access_*`` - * [x] ``airflow.www.utils.get_sensitive_variables_fields`` → ``airflow.utils.log.secrets_masker.get_sensitive_variables_fields`` - * [x] ``airflow.www.utils.should_hide_value_for_key`` → ``airflow.utils.log.secrets_masker.should_hide_value_for_key`` - - * AIR303 - - * [x] ``airflow.www.security.FabAirflowSecurityManagerOverride`` → ``airflow.providers.fab.auth_manager.security_manager.override.FabAirflowSecurityManagerOverride`` diff --git a/airflow-core/newsfragments/41761.significant.rst b/airflow-core/newsfragments/41761.significant.rst deleted file mode 100644 index 20b472783e142..0000000000000 --- a/airflow-core/newsfragments/41761.significant.rst +++ /dev/null @@ -1,32 +0,0 @@ -Removed a set of deprecations in BaseOperator. - -- Parameter ``task_concurrency`` removed, please use ``max_active_tis_per_dag``. -- Support for additional (not defined) arguments removed. -- Support for trigger rule ``dummy`` removed. Please use ``always``. -- Support for trigger rule ``none_failed_or_skipped`` removed. Please use ``none_failed_min_one_success``. -- Support to load ``BaseOperatorLink`` via ``airflow.models.baseoperator`` module removed. -- Config ``operators.allow_illegal_arguments`` removed. - -* Types of change - - * [x] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``operators.allow_illegal_arguments`` - - * ruff - - * AIR302 - - * [x] ``airflow.utils.trigger_rule.TriggerRule.NONE_FAILED_OR_SKIPPED`` - * [x] argument ``task_concurrency`` → ``max_active_tis_per_dag`` in ``BaseOperator`` and its subclassses diff --git a/airflow-core/newsfragments/41762.significant.rst b/airflow-core/newsfragments/41762.significant.rst deleted file mode 100644 index a2d6271a247ed..0000000000000 --- a/airflow-core/newsfragments/41762.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Removed a set of deprecations in ``Connection`` from ``airflow.models``. - -- Validation of extra fields is now enforcing that JSON values are provided. If a non-JSON value is provided - a ValueError will be raised. -- Removed utility method ``parse_netloc_to_hostname()`` -- Removed utility method ``parse_from_uri()``. -- Removed utility method ``log_info()`` and ``debug_info()``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41774.significant.rst b/airflow-core/newsfragments/41774.significant.rst deleted file mode 100644 index 56ed3a8633a01..0000000000000 --- a/airflow-core/newsfragments/41774.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Removed a set of deprecations in ``DAG`` from ``airflow.models``. - -- Removed deprecated parameters ``full_filepath`` and ``concurrency`` (Replaced by ``max_active_tasks``) from DAG and ``@dag`` decorator. -- Removed legacy support for permissions named ``can_dag_read`` and ``can_dag_edit``. The permissions need to be named ``can_read`` and ``can_edit``. -- Removed legacy deprecated functions ``following_schedule()`` and ``previous_schedule``. -- Removed deprecated support for ``datetime`` in ``next_dagrun_info()``. Use ``DataInterval``. -- Removed legacy DAG property ``is_paused``. Please use ``get_is_paused`` instead. -- Removed legacy parameters ``get_tis``, ``recursion_depth`` and ``max_recursion_depth`` from ``DAG.clear()``. -- Removed implicit support to call ``create_dagrun()`` without data interval. -- Removed support for deprecated parameter ``concurrency`` in ``DagModel``. -- Removed support for ``datetime`` in ``DagModel.calculate_dagrun_date_fields``. Use ``DataInterval``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes diff --git a/airflow-core/newsfragments/41776.significant.rst b/airflow-core/newsfragments/41776.significant.rst deleted file mode 100644 index 0f8f53648da11..0000000000000 --- a/airflow-core/newsfragments/41776.significant.rst +++ /dev/null @@ -1,16 +0,0 @@ -Removed a set of deprecations in from ``airflow.models.param``. - -- Removed deprecated direct access to DagParam as module. Please import from ``airflow.models.param``. -- Ensure all param values are JSON serialiazable and raise a ``ParamValidationError`` if not. -- Ensure parsed date and time values are RFC3339 compliant. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41778.significant.rst b/airflow-core/newsfragments/41778.significant.rst deleted file mode 100644 index 017ec59d3a2c8..0000000000000 --- a/airflow-core/newsfragments/41778.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -Removed a set of deprecations in from ``airflow.models.dagrun``. - -- Removed deprecated method ``DagRun.get_run()``. Instead you should use standard Sqlalchemy DagRun model retrieval. -- Removed deprecated method ``DagRun.get_log_filename_template()``. Please use ``get_log_template()`` instead. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41779.significant.rst b/airflow-core/newsfragments/41779.significant.rst deleted file mode 100644 index 0c2861a856891..0000000000000 --- a/airflow-core/newsfragments/41779.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Remove deprecated support for ``airflow.models.errors.ImportError`` which has been renamed to ``ParseImportError``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41780.significant.rst b/airflow-core/newsfragments/41780.significant.rst deleted file mode 100644 index 453c961d5af8e..0000000000000 --- a/airflow-core/newsfragments/41780.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Remove deprecated support for passing ``execution_date`` to ``airflow.models.skipmixin.SkipMixin.skip()``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41784.significant.rst b/airflow-core/newsfragments/41784.significant.rst deleted file mode 100644 index 86669b5c011af..0000000000000 --- a/airflow-core/newsfragments/41784.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -Removed a set of deprecations in from ``airflow.models.taskinstance``. - - - Removed deprecated arg ``activate_dag_runs`` from ``TaskInstance.clear_task_instances()``. Please use ``dag_run_state`` instead. - - Removed deprecated arg ``execution_date`` from ``TaskInstance.__init__()``. Please use ``run_id`` instead. - - Removed deprecated property ``_try_number`` from ``TaskInstance``. Please use ``try_number`` instead. - - Removed deprecated property ``prev_attempted_tries`` from ``TaskInstance``. Please use ``try_number`` instead. - - Removed deprecated property ``next_try_number`` from ``TaskInstance``. Please use ``try_number + 1`` instead. - - Removed deprecated property ``previous_ti`` from ``TaskInstance``. Please use ``get_previous_ti`` instead. - - Removed deprecated property ``previous_ti_success`` from ``TaskInstance``. Please use ``get_previous_ti`` instead. - - Removed deprecated property ``previous_start_date_success`` from ``TaskInstance``. Please use ``get_previous_start_date`` instead. - - Removed deprecated function ``as_dict`` from ``SimpleTaskInstance``. Please use ``BaseSerialization.serialize`` instead. - - Removed deprecated function ``from_dict`` from ``SimpleTaskInstance``. Please use ``BaseSerialization.deserialize`` instead. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41808.significant.rst b/airflow-core/newsfragments/41808.significant.rst deleted file mode 100644 index 38b66a9959588..0000000000000 --- a/airflow-core/newsfragments/41808.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -Removed deprecations in ``airflow.models.taskreschedule``. - -Removed methods: - -- ``query_for_task_instance()`` -- ``find_for_task_instance()`` - -Note: there are no replacements. Direct access to DB is discouraged and will not be possible in Airflow 3 for tasks. The public REST API is the future way to interact with Airflow - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes diff --git a/airflow-core/newsfragments/41857.significant.rst b/airflow-core/newsfragments/41857.significant.rst deleted file mode 100644 index fd88c97bfa522..0000000000000 --- a/airflow-core/newsfragments/41857.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Airflow core now depends on Pydantic v2. If you have Pydantic v1 installed, please upgrade. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41910.significant.rst b/airflow-core/newsfragments/41910.significant.rst deleted file mode 100644 index 313291486f2e4..0000000000000 --- a/airflow-core/newsfragments/41910.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed deprecated method ``requires_access`` from module ``airflow.api_connexion.security``. Please use ``requires_access_*`` instead. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.api_connexion.security.requires_access`` → ``airflow.api_connexion.security.requires_access_*`` diff --git a/airflow-core/newsfragments/41964.significant.rst b/airflow-core/newsfragments/41964.significant.rst deleted file mode 100644 index f004b61636733..0000000000000 --- a/airflow-core/newsfragments/41964.significant.rst +++ /dev/null @@ -1,17 +0,0 @@ -``--tree`` flag for ``airflow tasks list`` command removed - -The format of the output with that flag can be expensive to generate and extremely large, depending on the DAG. -``airflow dag show`` is a better way to visualize the relationship of tasks in a DAG. - -``DAG.tree_view`` and ``DAG.get_tree_view`` have also been removed. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/41975.significant.rst b/airflow-core/newsfragments/41975.significant.rst deleted file mode 100644 index 1569035d713ed..0000000000000 --- a/airflow-core/newsfragments/41975.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -Metrics basic deprecated validators (``AllowListValidator`` and ``BlockListValidator``) were removed in favor of pattern matching. Pattern matching validators (``PatternAllowListValidator`` and ``PatternBlockListValidator``) are enabled by default. Configuration parameter ``metrics_use_pattern_match``was removed from the ``metrics`` section. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config liint`` - - * [x] ``metrics.metrics_use_pattern_match`` - - * ruff - - * AIR302 - - * [x] ``airflow.metrics.validators.AllowListValidator`` → ``airflow.metrics.validators.PatternAllowListValidator`` - * [x] ``airflow.metrics.validators.BlockListValidator`` → ``airflow.metrics.validators.PatternBlockListValidator`` diff --git a/airflow-core/newsfragments/42023.significant.rst b/airflow-core/newsfragments/42023.significant.rst deleted file mode 100644 index 48e8f7344da53..0000000000000 --- a/airflow-core/newsfragments/42023.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -Rename ``Dataset`` as ``Asset`` in API endpoints - -* list of changes - - * Rename property run_type value ``dataset_triggered`` as ``asset_triggered`` in DAGRun endpoint - * Rename property ``dataset_expression`` as ``asset_expression`` in DAGDetail endpoint - * Change the string ``dataset_triggered`` in RUN_ID_REGEX as ``asset_triggered`` which affects the valid run id that an user can provide - * Rename ``dataset`` as ``asset`` in all the database tables - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42042.significant.rst b/airflow-core/newsfragments/42042.significant.rst deleted file mode 100644 index a743da5326c9d..0000000000000 --- a/airflow-core/newsfragments/42042.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Removed ``is_active`` property from ``BaseUser``. This property is longer used. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes diff --git a/airflow-core/newsfragments/42054.significant.rst b/airflow-core/newsfragments/42054.significant.rst deleted file mode 100644 index dbe243df3854e..0000000000000 --- a/airflow-core/newsfragments/42054.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -Dataset and DatasetAlias are no longer hashable - -This means they can no longer be used as dict keys or put into a set. Dataset's -equality logic is also tweaked slightly to consider the extra dict. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42060.significant.rst b/airflow-core/newsfragments/42060.significant.rst deleted file mode 100644 index fc806729e0ec0..0000000000000 --- a/airflow-core/newsfragments/42060.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ -Removed deprecated configuration ``stalled_task_timeout`` from ``celery``, ``task_adoption_timeout`` from ``celery`` and ``worker_pods_pending_timeout`` from ``kubernetes_executor``. Please use ``task_queued_timeout`` from ``scheduler`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``celery.stalled_task_timeout`` - * [x] ``kubernetes_executor.worker_pods_pending_timeout`` → ``scheduler.task_queued_timeout`` - * [x] ``celery.task_adoption_timeout`` → ``scheduler.task_queued_timeout`` - * [x] ``kubernetes_executor.worker_pods_pending_timeout`` → ``scheduler.task_queued_timeout`` diff --git a/airflow-core/newsfragments/42088.significant.rst b/airflow-core/newsfragments/42088.significant.rst deleted file mode 100644 index db2d65c3bd882..0000000000000 --- a/airflow-core/newsfragments/42088.significant.rst +++ /dev/null @@ -1,42 +0,0 @@ -Removed deprecated metrics configuration. - - * Removed deprecated configuration ``statsd_allow_list`` from ``metrics``. Please use ``metrics_allow_list`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_block_list`` from ``metrics``. Please use ``metrics_block_list`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_on`` from ``scheduler``. Please use ``statsd_on`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_host`` from ``scheduler``. Please use ``statsd_host`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_port`` from ``scheduler``. Please use ``statsd_port`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_prefix`` from ``scheduler``. Please use ``statsd_prefix`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_allow_list`` from ``scheduler``. Please use ``statsd_allow_list`` from ``metrics`` instead. - * Removed deprecated configuration ``stat_name_handler`` from ``scheduler``. Please use ``stat_name_handler`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_datadog_enabled`` from ``scheduler``. Please use ``statsd_datadog_enabled`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_datadog_tags`` from ``scheduler``. Please use ``statsd_datadog_tags`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_datadog_metrics_tags`` from ``scheduler``. Please use ``statsd_datadog_metrics_tags`` from ``metrics`` instead. - * Removed deprecated configuration ``statsd_custom_client_path`` from ``scheduler``. Please use ``statsd_custom_client_path`` from ``metrics`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``metrics.statsd_allow_list`` → ``metrics.metrics_allow_list`` - * [x] ``metrics.statsd_block_list`` → ``metrics.metrics_block_list`` - * [x] ``scheduler.statsd_on`` → ``metrics.statsd_on`` - * [x] ``scheduler.statsd_host`` → ``metrics.statsd_host`` - * [x] ``scheduler.statsd_port`` → ``metrics.statsd_port`` - * [x] ``scheduler.statsd_prefix`` → ``metrics.statsd_prefix`` - * [x] ``scheduler.statsd_allow_list`` → ``metrics.statsd_allow_list`` - * [x] ``scheduler.stat_name_handler`` → ``metrics.stat_name_handler`` - * [x] ``scheduler.statsd_datadog_enabled`` → ``metrics.statsd_datadog_enabled`` - * [x] ``scheduler.statsd_datadog_tags`` → ``metrics.statsd_datadog_tags`` - * [x] ``scheduler.statsd_datadog_metrics_tags`` → ``metrics.statsd_datadog_metrics_tags`` - * [x] ``scheduler.statsd_custom_client_path`` → ``metrics.statsd_custom_client_path`` diff --git a/airflow-core/newsfragments/42100.significant.rst b/airflow-core/newsfragments/42100.significant.rst deleted file mode 100644 index 801626a21c158..0000000000000 --- a/airflow-core/newsfragments/42100.significant.rst +++ /dev/null @@ -1,56 +0,0 @@ -Removed deprecated logging configuration. - - * Removed deprecated configuration ``interleave_timestamp_parser`` from ``core``. Please use ``interleave_timestamp_parser`` from ``logging`` instead. - * Removed deprecated configuration ``base_log_folder`` from ``core``. Please use ``base_log_folder`` from ``logging`` instead. - * Removed deprecated configuration ``remote_logging`` from ``core``. Please use ``remote_logging`` from ``logging`` instead. - * Removed deprecated configuration ``remote_log_conn_id`` from ``core``. Please use ``remote_log_conn_id`` from ``logging`` instead. - * Removed deprecated configuration ``remote_base_log_folder`` from ``core``. Please use ``remote_base_log_folder`` from ``logging`` instead. - * Removed deprecated configuration ``encrypt_s3_logs`` from ``core``. Please use ``encrypt_s3_logs`` from ``logging`` instead. - * Removed deprecated configuration ``logging_level`` from ``core``. Please use ``logging_level`` from ``logging`` instead. - * Removed deprecated configuration ``fab_logging_level`` from ``core``. Please use ``fab_logging_level`` from ``logging`` instead. - * Removed deprecated configuration ``logging_config_class`` from ``core``. Please use ``logging_config_class`` from ``logging`` instead. - * Removed deprecated configuration ``colored_console_log`` from ``core``. Please use ``colored_console_log`` from ``logging`` instead. - * Removed deprecated configuration ``colored_log_format`` from ``core``. Please use ``colored_log_format`` from ``logging`` instead. - * Removed deprecated configuration ``colored_formatter_class`` from ``core``. Please use ``colored_formatter_class`` from ``logging`` instead. - * Removed deprecated configuration ``log_format`` from ``core``. Please use ``log_format`` from ``logging`` instead. - * Removed deprecated configuration ``simple_log_format`` from ``core``. Please use ``simple_log_format`` from ``logging`` instead. - * Removed deprecated configuration ``task_log_prefix_template`` from ``core``. Please use ``task_log_prefix_template`` from ``logging`` instead. - * Removed deprecated configuration ``log_filename_template`` from ``core``. Please use ``log_filename_template`` from ``logging`` instead. - * Removed deprecated configuration ``log_processor_filename_template`` from ``core``. Please use ``log_processor_filename_template`` from ``logging`` instead. - * Removed deprecated configuration ``dag_processor_manager_log_location`` from ``core``. Please use ``dag_processor_manager_log_location`` from ``logging`` instead. - * Removed deprecated configuration ``task_log_reader`` from ``core``. Please use ``task_log_reader`` from ``logging`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.interleave_timestamp_parser`` → ``logging.interleave_timestamp_parser`` - * [x] ``core.base_log_folder`` → ``logging.base_log_folder`` - * [x] ``core.remote_logging`` → ``logging.remote_logging`` - * [x] ``core.remote_log_conn_id`` → ``logging.remote_log_conn_id`` - * [x] ``core.remote_base_log_folder`` → ``logging.remote_base_log_folder`` - * [x] ``core.encrypt_s3_logs`` → ``logging.encrypt_s3_logs`` - * [x] ``core.logging_level`` → ``logging.logging_level`` - * [x] ``core.fab_logging_level`` → ``logging.fab_logging_level`` - * [x] ``core.logging_config_class`` → ``logging.logging_config_class`` - * [x] ``core.colored_console_log`` → ``logging.colored_console_log`` - * [x] ``core.colored_log_format`` → ``logging.colored_log_format`` - * [x] ``core.colored_formatter_class`` → ``logging.colored_formatter_class`` - * [x] ``core.log_format`` → ``logging.log_format`` - * [x] ``core.simple_log_format`` → ``logging.simple_log_format`` - * [x] ``core.task_log_prefix_template`` → ``logging.task_log_prefix_template`` - * [x] ``core.log_filename_template`` → ``logging.log_filename_template`` - * [x] ``core.log_processor_filename_template`` → ``logging.log_processor_filename_template`` - * [x] ``core.dag_processor_manager_log_location`` → ``logging.dag_processor_manager_log_location`` - * [x] ``core.task_log_reader`` → ``logging.task_log_reader`` diff --git a/airflow-core/newsfragments/42126.significant.rst b/airflow-core/newsfragments/42126.significant.rst deleted file mode 100644 index e1b6f431ac4fb..0000000000000 --- a/airflow-core/newsfragments/42126.significant.rst +++ /dev/null @@ -1,43 +0,0 @@ -Removed deprecated database configuration. - - * Removed deprecated configuration ``sql_alchemy_conn`` from ``core``. Please use ``sql_alchemy_conn`` from ``database`` instead. - * Removed deprecated configuration ``sql_engine_encoding`` from ``core``. Please use ``sql_engine_encoding`` from ``database`` instead. - * Removed deprecated configuration ``sql_engine_collation_for_ids`` from ``core``. Please use ``sql_engine_collation_for_ids`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_pool_enabled`` from ``core``. Please use ``sql_alchemy_pool_enabled`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_pool_size`` from ``core``. Please use ``sql_alchemy_pool_size`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_max_overflow`` from ``core``. Please use ``sql_alchemy_max_overflow`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_pool_recycle`` from ``core``. Please use ``sql_alchemy_pool_recycle`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_pool_pre_ping`` from ``core``. Please use ``sql_alchemy_pool_pre_ping`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_schema`` from ``core``. Please use ``sql_alchemy_schema`` from ``database`` instead. - * Removed deprecated configuration ``sql_alchemy_connect_args`` from ``core``. Please use ``sql_alchemy_connect_args`` from ``database`` instead. - * Removed deprecated configuration ``load_default_connections`` from ``core``. Please use ``load_default_connections`` from ``database`` instead. - * Removed deprecated configuration ``max_db_retries`` from ``core``. Please use ``max_db_retries`` from ``database`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - - * ``airflow config lint`` - - * [x] ``core.sql_alchemy_conn`` → ``database.sql_alchemy_conn`` - * [x] ``core.sql_engine_encoding`` → ``database.sql_engine_encoding`` - * [x] ``core.sql_engine_collation_for_ids`` → ``database.sql_engine_collation_for_ids`` - * [x] ``core.sql_alchemy_pool_enabled`` → ``database.sql_alchemy_pool_enabled`` - * [x] ``core.sql_alchemy_pool_size`` → ``database.sql_alchemy_pool_size`` - * [x] ``core.sql_alchemy_max_overflow`` → ``database.sql_alchemy_max_overflow`` - * [x] ``core.sql_alchemy_pool_recycle`` → ``database.sql_alchemy_pool_recycle`` - * [x] ``core.sql_alchemy_pool_pre_ping`` → ``database.sql_alchemy_pool_pre_ping`` - * [x] ``core.sql_alchemy_schema`` → ``database.sql_alchemy_schema`` - * [x] ``core.sql_alchemy_connect_args`` → ``database.sql_alchemy_connect_args`` - * [x] ``core.load_default_connections`` → ``database.load_default_connections`` - * [x] ``core.max_db_retries`` → ``database.max_db_retries`` diff --git a/airflow-core/newsfragments/42129.significant.rst b/airflow-core/newsfragments/42129.significant.rst deleted file mode 100644 index 6f15f46fede7d..0000000000000 --- a/airflow-core/newsfragments/42129.significant.rst +++ /dev/null @@ -1,48 +0,0 @@ -Removed deprecated configuration. - - * Removed deprecated configuration ``worker_precheck`` from ``core``. Please use ``worker_precheck`` from ``celery`` instead. - * Removed deprecated configuration ``max_threads`` from ``scheduler``. Please use ``parsing_processes`` from ``dag_processor`` instead. - * Removed deprecated configuration ``default_queue`` from ``celery``. Please use ``default_queue`` from ``operators`` instead. - * Removed deprecated configuration ``hide_sensitive_variable_fields`` from ``admin``. Please use ``hide_sensitive_var_conn_fields`` from ``core`` instead. - * Removed deprecated configuration ``sensitive_variable_fields`` from ``admin``. Please use ``sensitive_var_conn_names`` from ``core`` instead. - * Removed deprecated configuration ``non_pooled_task_slot_count`` from ``core``. Please use ``default_pool_task_slot_count`` from ``core`` instead. - * Removed deprecated configuration ``dag_concurrency`` from ``core``. Please use ``max_active_tasks_per_dag`` from ``core`` instead. - * Removed deprecated configuration ``access_control_allow_origin`` from ``api``. Please use ``access_control_allow_origins`` from ``api`` instead. - * Removed deprecated configuration ``auth_backend`` from ``api``. Please use ``auth_backends`` from ``api`` instead. - * Removed deprecated configuration ``deactivate_stale_dags_interval`` from ``scheduler``. Please use ``parsing_cleanup_interval`` from ``scheduler`` instead. - * Removed deprecated configuration ``worker_pods_pending_timeout_check_interval`` from ``kubernetes_executor``. Please use ``task_queued_timeout_check_interval`` from ``scheduler`` instead. - * Removed deprecated configuration ``update_fab_perms`` from ``webserver``. Please use ``update_fab_perms`` from ``fab`` instead. - * Removed deprecated configuration ``auth_rate_limited`` from ``webserver``. Please use ``auth_rate_limited`` from ``fab`` instead. - * Removed deprecated configuration ``auth_rate_limit`` from ``webserver``. Please use ``auth_rate_limit`` from ``fab`` instead. - * Removed deprecated configuration section ``kubernetes``. Please use ``kubernetes_executor`` instead. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.worker_precheck`` → ``celery.worker_precheck`` - * [x] ``scheduler.max_threads`` → ``dag_processor.parsing_processes`` - * [x] ``celery.default_queue`` → ``operators.default_queue`` - * [x] ``admin.hide_sensitive_variable_fields`` → ``core.hide_sensitive_var_conn_fields`` - * [x] ``admin.sensitive_variable_fields`` → ``core.sensitive_var_conn_names`` - * [x] ``core.non_pooled_task_slot_count`` → ``core.default_pool_task_slot_count`` - * [x] ``core.dag_concurrency`` → ``core.max_active_tasks_per_dag`` - * [x] ``api.access_control_allow_origin`` → ``api.access_control_allow_origins`` - * [x] ``api.auth_backend`` → ``api.auth_backends`` - * [x] ``scheduler.deactivate_stale_dags_interval`` → ``scheduler.parsing_cleanup_interval`` - * [x] ``kubernetes_executor.worker_pods_pending_timeout_check_interval`` → ``scheduler.task_queued_timeout_check_interval`` - * [x] ``webserver.update_fab_perms`` → ``fab.update_fab_perms`` - * [x] ``webserver.auth_rate_limited`` → ``fab.auth_rate_limited`` - * [x] ``webserver.auth_rate_limit`` → ``fab.auth_rate_limit`` - * [x] ``kubernetes`` → ``kubernetes_executor`` diff --git a/airflow-core/newsfragments/42137.significant.rst b/airflow-core/newsfragments/42137.significant.rst deleted file mode 100644 index 1055032dc2a64..0000000000000 --- a/airflow-core/newsfragments/42137.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Optional ``[saml]`` extra has been removed from Airflow core - instead Amazon Provider gets saml as required dependency. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42252.significant.rst b/airflow-core/newsfragments/42252.significant.rst deleted file mode 100644 index 2533fb451b498..0000000000000 --- a/airflow-core/newsfragments/42252.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Move bash operators from airflow core to standard provider - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.operators.bash.BashOperator`` → ``airflow.providers.standard.operators.bash.BashOperator`` diff --git a/airflow-core/newsfragments/42280.significant.rst b/airflow-core/newsfragments/42280.significant.rst deleted file mode 100644 index 35e37727cda6a..0000000000000 --- a/airflow-core/newsfragments/42280.significant.rst +++ /dev/null @@ -1,16 +0,0 @@ -Removed deprecated Rest API endpoints: - -* /api/v1/roles. Use /auth/fab/v1/roles instead -* /api/v1/permissions. Use /auth/fab/v1/permissions instead -* /api/v1/users. Use /auth/fab/v1/users instead - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42285.significant.rst b/airflow-core/newsfragments/42285.significant.rst deleted file mode 100644 index c7a545664b484..0000000000000 --- a/airflow-core/newsfragments/42285.significant.rst +++ /dev/null @@ -1,27 +0,0 @@ -The SLA feature is removed in Airflow 3.0, to be replaced with Airflow Alerts in 3.1 - -* Types of change - - * [x] Dag changes - * [x] Config changes - * [x] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641) - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.check_slas`` - - * ruff - - * AIR302 - - * [x] argument ``sla`` in ``BaseOperator`` and its subclassses - * [x] argument ``sla_miss_callback`` in ``DAG`` diff --git a/airflow-core/newsfragments/42343.feature.rst b/airflow-core/newsfragments/42343.feature.rst deleted file mode 100644 index 8a7cdf335a06e..0000000000000 --- a/airflow-core/newsfragments/42343.feature.rst +++ /dev/null @@ -1 +0,0 @@ -New function ``create_dataset_aliases`` added to DatasetManager for DatasetAlias creation. diff --git a/airflow-core/newsfragments/42343.significant.rst b/airflow-core/newsfragments/42343.significant.rst deleted file mode 100644 index 7af4954e265cc..0000000000000 --- a/airflow-core/newsfragments/42343.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -``DatasetManager.create_datasets`` now takes ``Dataset`` objects - -This function previously accepts a list of ``DatasetModel`` objects. it now -receives ``Dataset`` objects instead. A list of ``DatasetModel`` objects are -created inside, and returned by the function. - -Also, the ``session`` argument is now keyword-only. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42404.significant.rst b/airflow-core/newsfragments/42404.significant.rst deleted file mode 100644 index da57b01bde197..0000000000000 --- a/airflow-core/newsfragments/42404.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -Removed ``logical_date`` arguments from functions and APIs for DAG run lookups to align with Airflow 3.0. - -The shift towards using ``run_id`` as the sole identifier for DAG runs eliminates the limitations of ``execution_date`` and ``logical_date``, particularly for dynamic DAG runs and cases where multiple runs occur at the same logical time. This change impacts database models, templates, and functions: - -- Removed ``logical_date`` arguments from public APIs and Python functions related to DAG run lookups. -- ``run_id`` is now the exclusive identifier for DAG runs in these contexts. -- ``ds``, ``ds_nodash``, ``ts``, ``ts_nodash``, ``ts_nodash_with_tz`` (and ``logical_date``) will no longer exist for non-scheduled DAG runs (i.e. manually triggered runs) -- ``task_instance_key_str`` template variable has changed to use ``run_id``, not the logical_date. This means the value of it will change compared to 2.x, even for old runs - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42436.significant.rst b/airflow-core/newsfragments/42436.significant.rst deleted file mode 100644 index f3c0117e641a5..0000000000000 --- a/airflow-core/newsfragments/42436.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Default ``.airflowignore`` syntax changed to ``glob`` - -The default value to the configuration ``[core] dag_ignore_file_syntax`` has -been changed to ``glob``, which better matches the ignore file behavior of many -popular tools. - -To revert to the previous behavior, set the configuration to ``regexp``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42548.significant.rst b/airflow-core/newsfragments/42548.significant.rst deleted file mode 100644 index 93a5a7db417e4..0000000000000 --- a/airflow-core/newsfragments/42548.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Remove is_backfill attribute from DagRun object - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42579.significant.rst b/airflow-core/newsfragments/42579.significant.rst deleted file mode 100644 index fd386f83385ea..0000000000000 --- a/airflow-core/newsfragments/42579.significant.rst +++ /dev/null @@ -1,32 +0,0 @@ -Rename ``Dataset`` endpoints as ``Asset`` endpoints - -* list of changes - - * Rename dataset endpoints as asset endpoints - - * Rename ``/datasets`` as ``/assets`` - * Rename ``/datasets/{uri}`` as ``/assets/{uri}`` - * Rename ``/datasets/events`` as ``/assets/events`` - * Rename ``/datasets/queuedEvent/{uri}`` as ``/ui/next_run_assets/upstream`` - * Rename ``/dags/{dag_id}/dagRuns/{dag_run_id}/upstreamDatasetEvents`` as ``/ui/next_run_assets/upstream`` - * Rename ``/dags/{dag_id}/datasets/queuedEvent/{uri}`` as ``/ui/next_run_assets/upstream`` - * Rename ``/dags/{dag_id}/datasets/queuedEvent`` as ``/ui/next_run_assets/upstream`` - * Rename ``/ui/next_run_datasets/upstream`` as ``/ui/next_run_assets/upstream`` - - * Rename dataset schema as asset endpoints - - * Rename ``AssetCollection.datasets`` as ``AssetCollection.assets`` - * Rename ``AssetEventCollection.dataset_events`` as ``AssetEventCollection.asset_events`` - * Rename ``AssetEventCollectionSchema.dataset_events`` as ``AssetEventCollectionSchema.asset_events`` - * Rename ``CreateAssetEventSchema.dataset_uri`` as ``CreateAssetEventSchema.asset_uri`` - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42640.significant.rst b/airflow-core/newsfragments/42640.significant.rst deleted file mode 100644 index e5d0e81f3da11..0000000000000 --- a/airflow-core/newsfragments/42640.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Removed deprecated custom dag dependency detector. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42647.significant.rst b/airflow-core/newsfragments/42647.significant.rst deleted file mode 100644 index 71bee3bb3fb9b..0000000000000 --- a/airflow-core/newsfragments/42647.significant.rst +++ /dev/null @@ -1,30 +0,0 @@ -Removed deprecated aliases support for providers. - -* Removed deprecated ``atlas`` alias support. Please use ``apache-atlas`` instead. -* Removed deprecated ``aws`` alias support. Please use ``amazon`` instead. -* Removed deprecated ``azure`` alias support. Please use ``microsoft-azure`` instead. -* Removed deprecated ``cassandra`` alias support. Please use ``apache-cassandra`` instead. -* Removed deprecated ``crypto`` alias support. -* Removed deprecated ``druid`` alias support. Please use ``apache-druid`` instead. -* Removed deprecated ``gcp`` alias support. Please use ``google`` instead. -* Removed deprecated ``gcp-api`` alias support. Please use ``google`` instead. -* Removed deprecated ``hdfs`` alias support. Please use ``apache-hdfs`` instead. -* Removed deprecated ``hive`` alias support. Please use ``apache-hive`` instead. -* Removed deprecated ``kubernetes`` alias support. Please use ``cncf-kubernetes`` instead. -* Removed deprecated ``mssql`` alias support. Please use ``microsoft-mssql`` instead. -* Removed deprecated ``pinot`` alias support. Please use ``apache-pinot`` instead. -* Removed deprecated ``s3`` alias support. Please use ``amazon`` instead. -* Removed deprecated ``spark`` alias support. Please use ``apache-spark`` instead. -* Removed deprecated ``webhdfs`` alias support. Please use ``apache-webhdfs`` instead. -* Removed deprecated ``winrm`` alias support. Please use ``microsoft-winrm`` instead. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42658.significant.rst b/airflow-core/newsfragments/42658.significant.rst deleted file mode 100644 index 84381da3f02a5..0000000000000 --- a/airflow-core/newsfragments/42658.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -* Changing dag_id from flag (-d, --dag-id) to a positional argument in the 'dags list-runs' CLI command. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42660.significant.rst b/airflow-core/newsfragments/42660.significant.rst deleted file mode 100644 index 924259e11151d..0000000000000 --- a/airflow-core/newsfragments/42660.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Deprecated field ``concurrency`` from ``DAGDetailSchema`` has been removed. Please use ``max_active_tasks`` from ``DAGDetailSchema`` instead. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42739.significant.rst b/airflow-core/newsfragments/42739.significant.rst deleted file mode 100644 index eb565c37e494a..0000000000000 --- a/airflow-core/newsfragments/42739.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Remove support for Python 3.8 as this version is not maintained within Python release schedule, see https://peps.python.org/pep-0596/. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42776.significant.rst b/airflow-core/newsfragments/42776.significant.rst deleted file mode 100644 index 188bcd8315442..0000000000000 --- a/airflow-core/newsfragments/42776.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Removed deprecated ``Chainable`` type from ``BaseOperator``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/42783.improvement.rst b/airflow-core/newsfragments/42783.improvement.rst deleted file mode 100644 index eb6a2181bd06f..0000000000000 --- a/airflow-core/newsfragments/42783.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -Bash script files (``.sh`` and ``.bash``) with Jinja templating enabled (without the space after the file extension) are now rendered into a temporary file, and then executed. Instead of being directly executed as inline command. diff --git a/airflow-core/newsfragments/42794.significant.rst b/airflow-core/newsfragments/42794.significant.rst deleted file mode 100644 index 7113d0ced5094..0000000000000 --- a/airflow-core/newsfragments/42794.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Move filesystem, package_index, subprocess hooks to standard provider - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.hooks.filesystem.*`` → ``airflow.providers.standard.hooks.filesystem.*`` - * [x] ``airflow.hooks.package_index.*`` → ``airflow.providers.standard.hooks.package_index.*`` - * [x] ``airflow.hooks.subprocess.*`` → ``airflow.providers.standard.hooks.subprocess.*`` diff --git a/airflow-core/newsfragments/42953.significant.rst b/airflow-core/newsfragments/42953.significant.rst deleted file mode 100644 index eb301b1197e7a..0000000000000 --- a/airflow-core/newsfragments/42953.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -``DAG.max_active_tasks`` is now evaluated per-run - -Previously, this was evaluated across all runs of the dag. This behavior change was passed by lazy consensus. -Vote thread: https://lists.apache.org/thread/9o84d3yn934m32gtlpokpwtbbmtxj47l. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43067.significant.rst b/airflow-core/newsfragments/43067.significant.rst deleted file mode 100644 index 65501c54a6bf5..0000000000000 --- a/airflow-core/newsfragments/43067.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -Remove DAG.get_num_active_runs - -We don't need this function. There's already an almost-identical function on DagRun that we can use, namely DagRun.active_runs_of_dags. -Also, make DagRun.active_runs_of_dags private. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43073.significant.rst b/airflow-core/newsfragments/43073.significant.rst deleted file mode 100644 index 9dd26a3bf175f..0000000000000 --- a/airflow-core/newsfragments/43073.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Rename ``DagRunTriggeredByType.DATASET`` as ``DagRunTriggeredByType.ASSET`` and all the name ``dataset`` in all the UI component to ``asset``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43096.significant.rst b/airflow-core/newsfragments/43096.significant.rst deleted file mode 100644 index 1803b9ac041b7..0000000000000 --- a/airflow-core/newsfragments/43096.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed auth backend ``airflow.api.auth.backend.default`` - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.api.auth.backend.default`` → ``airflow.providers.fab.auth_manager.api.auth.backend.session`` diff --git a/airflow-core/newsfragments/43102.significant.rst b/airflow-core/newsfragments/43102.significant.rst deleted file mode 100644 index 18e59e5ac22ac..0000000000000 --- a/airflow-core/newsfragments/43102.significant.rst +++ /dev/null @@ -1,29 +0,0 @@ -Change in query parameter handling for list parameters - -The handling of list-type query parameters in the API has been updated. -FastAPI defaults the ``explode`` behavior to ``true`` for list parameters, -which affects how these parameters are passed in requests. -This adjustment applies to all list-type query parameters across the API. - -Before: - -.. code-block:: - - http://:/?param=item1,item2 - -After: - -.. code-block:: - - http://:/?param=item1¶m=item2 - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43183.significant.rst b/airflow-core/newsfragments/43183.significant.rst deleted file mode 100644 index 7f3e28aa0c5a0..0000000000000 --- a/airflow-core/newsfragments/43183.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Remove TaskContextLogger - -We introduced this as a way to inject messages into task logs from places -other than the task execution context. We later realized that we were better off -just using the Log table. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config list`` - - * [x] ``logging.enable_task_context_logger`` diff --git a/airflow-core/newsfragments/43289.significant.rst b/airflow-core/newsfragments/43289.significant.rst deleted file mode 100644 index aa6a51d89907f..0000000000000 --- a/airflow-core/newsfragments/43289.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -Support for adding executors via Airflow Plugins is removed - -Executors should no longer be registered or imported via Airflow's plugin mechanism -- these types of classes -are just treated as plain Python classes by Airflow, so there is no need to register them with Airflow. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [x] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] extension ``executors`` in ``airflow.plugins_manager.AirflowPlugin`` - * [x] extension ``operators`` in ``airflow.plugins_manager.AirflowPlugin`` - * [x] extension ``sensors`` in ``airflow.plugins_manager.AirflowPlugin`` diff --git a/airflow-core/newsfragments/43291.significant.rst b/airflow-core/newsfragments/43291.significant.rst deleted file mode 100644 index 227ccda5fdd14..0000000000000 --- a/airflow-core/newsfragments/43291.significant.rst +++ /dev/null @@ -1,35 +0,0 @@ -Support for adding Hooks via Airflow Plugins is removed - -Hooks should no longer be registered or imported via Airflow's plugin mechanism -- these types of classes -are just treated as plain Python classes by Airflow, so there is no need to register them with Airflow. - -Before: - -.. code-block:: python - - from airflow.hooks.my_plugin import MyHook - -You should instead import it as: - -.. code-block:: python - - from my_plugin import MyHook - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [x] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] extension ``hooks`` in ``airflow.plugins_manager.AirflowPlugin`` diff --git a/airflow-core/newsfragments/43349.significant.rst b/airflow-core/newsfragments/43349.significant.rst deleted file mode 100644 index 23af4ec5697f8..0000000000000 --- a/airflow-core/newsfragments/43349.significant.rst +++ /dev/null @@ -1,24 +0,0 @@ -Deprecated trigger rule ``TriggerRule.DUMMY`` removed - -**Breaking Change** - -The trigger rule ``TriggerRule.DUMMY`` was removed. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.trigger_rule.TriggerRule.DUMMY`` diff --git a/airflow-core/newsfragments/43490.significant.rst b/airflow-core/newsfragments/43490.significant.rst deleted file mode 100644 index 7d17cbec439a1..0000000000000 --- a/airflow-core/newsfragments/43490.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -The ``task_fail`` table has been removed from the Airflow database. - -This table was used to store task failures, but it was not used by any Airflow components. -Use the REST API to get task failures instead (which gets it from the ``task_instance`` table) - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43530.significant.rst b/airflow-core/newsfragments/43530.significant.rst deleted file mode 100644 index 0876bd92ef9d8..0000000000000 --- a/airflow-core/newsfragments/43530.significant.rst +++ /dev/null @@ -1,47 +0,0 @@ -Direct Access to Deprecated ``airflow.configuration`` Module Functions Removed - -Functions previously accessible directly via the ``airflow.configuration`` module, -such as ``get``, ``getboolean``, ``getfloat``, ``getint``, ``has_option``, ``remove_option``, ``as_dict``, and ``set``, -have been removed. These functions should now be accessed through ``airflow.configuration.conf``. - -Before: - -.. code-block:: python - - from airflow.configuration import get - - value = get("section", "key") - -After: - -.. code-block:: python - - from airflow.configuration import conf - - value = conf.get("section", "key") - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.configuration.getboolean`` → ``airflow.configuration.conf.getboolean`` - * [x] ``airflow.configuration.getfloat`` → ``airflow.configuration.conf.getfloat`` - * [x] ``airflow.configuration.getint`` → ``airflow.configuration.conf.getint`` - * [x] ``airflow.configuration.has_option`` → ``airflow.configuration.conf.has_option`` - * [x] ``airflow.configuration.remove_option`` → ``airflow.configuration.conf.remove_option`` - * [x] ``airflow.configuration.as_dict`` → ``airflow.configuration.conf.as_dict`` - * [x] ``airflow.configuration.set`` → ``airflow.configuration.conf.set`` - * [x] ``airflow.configuration.get`` → ``airflow.configuration.conf.get`` diff --git a/airflow-core/newsfragments/43533.significant.rst b/airflow-core/newsfragments/43533.significant.rst deleted file mode 100644 index c59f931005300..0000000000000 --- a/airflow-core/newsfragments/43533.significant.rst +++ /dev/null @@ -1,30 +0,0 @@ -Unused and redundant functions from ``airflow.utils.dates`` module have been removed. - -Following functions are removed: - -- ``parse_execution_date`` -- ``round_time`` -- ``scale_time_units`` -- ``infer_time_unit`` - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.dates.parse_execution_date`` - * [x] ``airflow.utils.dates.round_time`` - * [x] ``airflow.utils.dates.scale_time_units`` - * [x] ``airflow.utils.dates.infer_time_unit`` diff --git a/airflow-core/newsfragments/43562.significant.rst b/airflow-core/newsfragments/43562.significant.rst deleted file mode 100644 index 98b232213e05d..0000000000000 --- a/airflow-core/newsfragments/43562.significant.rst +++ /dev/null @@ -1,44 +0,0 @@ -Removed Deprecated Python Version Identifiers from the ``airflow`` Module - -Python version check constants, such as ``PY36``, ``PY37``, and others, have been removed from the ``airflow`` -module. To check Python versions, please use the ``sys.version_info`` attribute directly instead. - -Before: - -.. code-block:: python - - from airflow import PY36 - - if PY36: - # perform some action - ... - -After: - -.. code-block:: python - - import sys - - if sys.version_info >= (3, 6): - # perform some action - ... - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.PY\d\d`` diff --git a/airflow-core/newsfragments/43568.significant.rst b/airflow-core/newsfragments/43568.significant.rst deleted file mode 100644 index 801e7739f661e..0000000000000 --- a/airflow-core/newsfragments/43568.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -Remove ``virtualenv`` extra as PythonVirtualenvOperator has been moved to standard provider and switched to use built-in venv package. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43608.significant.rst b/airflow-core/newsfragments/43608.significant.rst deleted file mode 100644 index 8082b0725d3f4..0000000000000 --- a/airflow-core/newsfragments/43608.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Move Airflow core triggers to standard provider - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.triggers.external_task.*`` → ``airflow.providers.standard.triggers.external_task.*`` - * [x] ``airflow.triggers.file.*`` → ``airflow.providers.standard.triggers.file.*`` - * [x] ``airflow.triggers.temporal.*`` → ``airflow.providers.standard.triggers.temporal.*`` diff --git a/airflow-core/newsfragments/43611.significant.rst b/airflow-core/newsfragments/43611.significant.rst deleted file mode 100644 index 2f6e51fd78de4..0000000000000 --- a/airflow-core/newsfragments/43611.significant.rst +++ /dev/null @@ -1,17 +0,0 @@ -TaskInstance ``priority_weight`` is capped in 32-bit signed integer ranges. - -Some database engines are limited to 32-bit integer values. As some users reported errors in -weight rolled-over to negative values, we decided to cap the value to the 32-bit integer. Even -if internally in python smaller or larger values to 64 bit are supported, ``priority_weight`` is -capped and only storing values from -2147483648 to 2147483647. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43612.significant.rst b/airflow-core/newsfragments/43612.significant.rst deleted file mode 100644 index 1b37228e11510..0000000000000 --- a/airflow-core/newsfragments/43612.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -Virtualenv installation uses ``uv`` now per default if ``uv`` is available. - -If you want to control how the virtualenv is created, you can use the -AIRFLOW__STANDARD__VENV_INSTALL_METHOD option. The possible values are: - -- ``auto``: Automatically select, use ``uv`` if available, otherwise use ``pip``. -- ``pip``: Use pip to install the virtual environment. -- ``uv``: Use uv to install the virtual environment. Must be available in environment PATH. - -* Types of change - - * [x] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/43679.misc.rst b/airflow-core/newsfragments/43679.misc.rst deleted file mode 100644 index 792087dc63a75..0000000000000 --- a/airflow-core/newsfragments/43679.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Renamed ``airflow.security.kerberos.get_kerberos_principle`` to ``airflow.security.kerberos.get_kerberos_principal`` due to misspelling. diff --git a/airflow-core/newsfragments/43774.significant.rst b/airflow-core/newsfragments/43774.significant.rst deleted file mode 100644 index 30ba2885b1416..0000000000000 --- a/airflow-core/newsfragments/43774.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -``HookLineageCollector.create_asset`` now accept only keyword arguments - -To provider AIP-74 support, new arguments "name" and "group" are added to ``HookLineageCollector.create_asset``. -For easier change in the future, this function now takes only keyword arguments. - -.. Check the type of change that applies to this change - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] Calling ``HookLineageCollector.create_asset`` with positional argument should raise an error diff --git a/airflow-core/newsfragments/43890.significant.rst b/airflow-core/newsfragments/43890.significant.rst deleted file mode 100644 index 9bf17e35a0890..0000000000000 --- a/airflow-core/newsfragments/43890.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Move filesystem sensor to standard provider - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.sensors.filesystem.FileSensor`` → ``airflow.providers.standard.sensors.filesystem.FileSensor`` diff --git a/airflow-core/newsfragments/43902.significant.rst b/airflow-core/newsfragments/43902.significant.rst deleted file mode 100644 index 262e23586ee28..0000000000000 --- a/airflow-core/newsfragments/43902.significant.rst +++ /dev/null @@ -1,35 +0,0 @@ -Renamed ``execution_date`` to ``logical_date`` across the codebase to align with Airflow 3.0. - -The shift towards ``logical_date`` helps move away from the limitations of ``execution_date``, particularly with dynamic DAG runs and cases where multiple runs occur at the same time. This change impacts database models, templates, and functions: - -- Renamed columns and function references to ``logical_date``. -- Removed ``execution_date``, ``next_ds``, ``next_ds_nodash``, ``next_execution_date``, ``prev_ds``, ``prev_ds_nodash``, ``prev_execution_date``, ``prev_execution_date_success``, ``tomorrow_ds``, ``yesterday_ds`` and ``yesterday_ds_nodash`` from Airflow ``context``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] context key ``execution_date`` - * [ ] context key ``next_ds`` - * [ ] context key ``next_ds_nodash`` - * [ ] context key ``next_execution_date`` - * [ ] context key ``prev_ds`` - * [ ] context key ``prev_ds_nodash`` - * [ ] context key ``prev_execution_date`` - * [ ] context key ``prev_execution_date_success`` - * [ ] context key ``tomorrow_ds`` - * [ ] context key ``yesterday_ds`` - * [ ] context key ``yesterday_ds_nodash`` diff --git a/airflow-core/newsfragments/43915.significant.rst b/airflow-core/newsfragments/43915.significant.rst deleted file mode 100644 index a3ac8e96881a0..0000000000000 --- a/airflow-core/newsfragments/43915.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ -Configuration ``[core] strict_dataset_uri_validation`` is removed - -Asset URI with a defined scheme will now always be validated strictly, raising -a hard error on validation failure. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.strict_dataset_uri_validation`` diff --git a/airflow-core/newsfragments/43943.significant.rst b/airflow-core/newsfragments/43943.significant.rst deleted file mode 100644 index 36d81af1647ee..0000000000000 --- a/airflow-core/newsfragments/43943.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Remove the ``traces`` ``otel_task_log_event`` event config option and feature - -This was sending the task logs form the scheduler, and would be a huge -scheduling performance hit (blocking the entire all scheduling while it was -fetching logs to attach to the trace) - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``traces.otel_task_log_event`` diff --git a/airflow-core/newsfragments/43949.significant.rst b/airflow-core/newsfragments/43949.significant.rst deleted file mode 100644 index 745c10d455def..0000000000000 --- a/airflow-core/newsfragments/43949.significant.rst +++ /dev/null @@ -1,16 +0,0 @@ -The ``--clear-only`` option of ``airflow dags reserialize`` command is now removed. - -The ``--clear-only`` option was added to clear the serialized DAGs without reserializing them. -This option has been removed as it is no longer needed. We have implemented DAG versioning and can -no longer delete serialized dag without going through ``airflow db-clean`` command. This command is now only for reserializing DAGs. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/44053.significant.rst b/airflow-core/newsfragments/44053.significant.rst deleted file mode 100644 index 7b3054394d87d..0000000000000 --- a/airflow-core/newsfragments/44053.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Move ``TriggerDagRunOperator`` to standard provider - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.operators.trigger_dagrun import TriggerDagRunOperator`` → ``airflow.providers.standard.operators.trigger_dagrun.TriggerDagRunOperator`` diff --git a/airflow-core/newsfragments/44080.significant.rst b/airflow-core/newsfragments/44080.significant.rst deleted file mode 100644 index 3aeca5191e4b6..0000000000000 --- a/airflow-core/newsfragments/44080.significant.rst +++ /dev/null @@ -1,14 +0,0 @@ -PostgreSQL 12 is no longer supported - -PostgreSQL 12 is no longer being supported by the PostgreSQL community. You must upgrade to PostgreSQL 13+ to use this version of Airflow. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/44288.significant.rst b/airflow-core/newsfragments/44288.significant.rst deleted file mode 100644 index e1a38ac27cd39..0000000000000 --- a/airflow-core/newsfragments/44288.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ - Move external task sensor to standard provider #44288 - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.sensors.external_task.ExternalTaskMarker`` → ``airflow.providers.standard.sensors.external_task.ExternalTaskMarker`` - * [x] ``airflow.sensors.external_task.ExternalTaskSensor`` → ``airflow.providers.standard.sensors.external_task.ExternalTaskSensor`` diff --git a/airflow-core/newsfragments/44300.bugfix.rst b/airflow-core/newsfragments/44300.bugfix.rst deleted file mode 100644 index ffd4b07b2ab0d..0000000000000 --- a/airflow-core/newsfragments/44300.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix stats of dynamic mapped tasks after automatic retries of failed tasks diff --git a/airflow-core/newsfragments/44475.significant.rst b/airflow-core/newsfragments/44475.significant.rst deleted file mode 100644 index 4b55765e75fa1..0000000000000 --- a/airflow-core/newsfragments/44475.significant.rst +++ /dev/null @@ -1,26 +0,0 @@ -Remove ``TriggerRule.NONE_FAILED_OR_SKIPPED`` - -.. Provide additional contextual information - -.. Check the type of change that applies to this change - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641) - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``TriggerRule.NONE_FAILED_OR_SKIPPED`` diff --git a/airflow-core/newsfragments/44533.significant.rst b/airflow-core/newsfragments/44533.significant.rst deleted file mode 100644 index 5943741428d7f..0000000000000 --- a/airflow-core/newsfragments/44533.significant.rst +++ /dev/null @@ -1,16 +0,0 @@ -During offline migration, ``DagRun.conf`` is cleared - -.. Provide additional contextual information - -The ``conf`` column is changing from pickle to json, thus, the values in that column cannot be migrated during offline migrations. If you want to retain ``conf`` values for existing DagRuns, you must do a normal, non-offline, migration. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/44706.significant.rst b/airflow-core/newsfragments/44706.significant.rst deleted file mode 100644 index 6391000d4caf7..0000000000000 --- a/airflow-core/newsfragments/44706.significant.rst +++ /dev/null @@ -1,16 +0,0 @@ -Deprecated cli commands under ``db`` group removed - -The ``db init`` and ``db upgrade`` commands have been removed. Use ``db migrate`` instead to initialize or migrate the metadata database. - -If you would like to create default connections use ``airflow connections create-default-connections``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/44712.feature.rst b/airflow-core/newsfragments/44712.feature.rst deleted file mode 100644 index 9a26758c84368..0000000000000 --- a/airflow-core/newsfragments/44712.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new table to the db for the Deadline Alerts feature. diff --git a/airflow-core/newsfragments/44751.bugfix.rst b/airflow-core/newsfragments/44751.bugfix.rst deleted file mode 100644 index 1ca32178be1c5..0000000000000 --- a/airflow-core/newsfragments/44751.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -``TriggerRule.ALWAYS`` cannot be utilized within a task-generated mapping, either in bare tasks (fixed in this PR) or mapped task groups (fixed in PR #44368). The issue with doing so, is that the task is immediately executed without waiting for the upstreams's mapping results, which certainly leads to failure of the task. This fix avoids it by raising an exception when it is detected during DAG parsing. diff --git a/airflow-core/newsfragments/44820.significant.rst b/airflow-core/newsfragments/44820.significant.rst deleted file mode 100644 index c37be748c780c..0000000000000 --- a/airflow-core/newsfragments/44820.significant.rst +++ /dev/null @@ -1,43 +0,0 @@ -Removed ``conf`` from the Task template context - -The ``conf`` variable, which provided access to the full Airflow configuration (``airflow.cfg``), has been -removed from the Task (Jinja2) template context for security and simplicity. If you -need specific configuration values in your tasks, retrieve them explicitly in your DAG or task code -using the ``airflow.configuration.conf`` module. - -For users retrieving the webserver URL (e.g., to include log links in task or callbacks), one of the -most common use-case, use the ``ti.log_url`` property available in the ``TaskInstance`` context instead. - -Example: - -.. code-block:: python - - PythonOperator( - task_id="my_task", - python_callable=my_task_callable, - on_failure_callback=SmtpNotifier( - from_email="example@example.com", - to="example@example.com", - subject="Task {{ ti.task_id }} failed", - html_content="Task {{ ti.task_id }} failed. Log URL: {{ ti.log_url }}", - ), - ) - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] context key ``conf`` diff --git a/airflow-core/newsfragments/44908.feature.rst b/airflow-core/newsfragments/44908.feature.rst deleted file mode 100644 index 2a711a0149322..0000000000000 --- a/airflow-core/newsfragments/44908.feature.rst +++ /dev/null @@ -1 +0,0 @@ -The ``airflow config lint`` command has been introduced to help users migrate from Airflow 2.x to 3.0 by identifying removed or renamed configuration parameters in airflow.cfg. diff --git a/airflow-core/newsfragments/45017.significant.rst b/airflow-core/newsfragments/45017.significant.rst deleted file mode 100644 index b1140fcf8d434..0000000000000 --- a/airflow-core/newsfragments/45017.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -Remove deprecated ``DEFAULT_CELERY_CONFIG`` from config templates - -``DEFAULT_CELERY_CONFIG`` has been moved into the celery provider and -should be imported from ``airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG`` → ``airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG`` diff --git a/airflow-core/newsfragments/45134.bugfix.rst b/airflow-core/newsfragments/45134.bugfix.rst deleted file mode 100644 index 09aaae23a3487..0000000000000 --- a/airflow-core/newsfragments/45134.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -(v2 API & UI) Allow fetching XCom with forward slash from the API and escape it in the UI diff --git a/airflow-core/newsfragments/45327.significant.rst b/airflow-core/newsfragments/45327.significant.rst deleted file mode 100644 index 1e7423cd2e6ea..0000000000000 --- a/airflow-core/newsfragments/45327.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ -Renamed DAG argument ``fail_stop`` to ``fail_fast`` across the codebase to align with Airflow 3.0. - - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] arguments ``fail_stop`` → ``fail_fast`` in ``DAG`` diff --git a/airflow-core/newsfragments/45425.significant.rst b/airflow-core/newsfragments/45425.significant.rst deleted file mode 100644 index 8fb3c3d477083..0000000000000 --- a/airflow-core/newsfragments/45425.significant.rst +++ /dev/null @@ -1,21 +0,0 @@ -The ``airflow.io`` class ``ObjectStoragePath`` and function ``attach`` are moved to ``airflow.sdk``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] ``airflow.io.path.ObjectStoragePath`` → ``airflow.sdk.ObjectStoragePath`` - * [ ] ``airflow.io.attach`` → ``airflow.sdk.io.attach`` diff --git a/airflow-core/newsfragments/45530.significant.rst b/airflow-core/newsfragments/45530.significant.rst deleted file mode 100644 index 5805dc3789e2b..0000000000000 --- a/airflow-core/newsfragments/45530.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -Ensure teardown tasks are executed when DAG run is set to failed - -Previously when a DAG run was manually set to "failed" or to "success" state the terminal state was set to all tasks. -But this was a gap for cases when setup- and teardown tasks were defined: If teardown was used to clean-up infrastructure -or other resources, they were also skipped and thus resources could stay allocated. - -As of now when setup tasks had been executed before and the DAG is manually set to "failed" or "success" then teardown -tasks are executed. Teardown tasks are skipped if the setup was also skipped. - -As a side effect this means if the DAG contains teardown tasks, then the manual marking of DAG as "failed" or "success" -will need to keep the DAG in running state to ensure that teardown tasks will be scheduled. They would not be scheduled -if the DAG is diorectly set to "failed" or "success". - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/45694.significant.rst b/airflow-core/newsfragments/45694.significant.rst deleted file mode 100644 index 222083bdf2e7e..0000000000000 --- a/airflow-core/newsfragments/45694.significant.rst +++ /dev/null @@ -1,52 +0,0 @@ -``get_parsing_context`` have been moved to Task SDK - -As part of AIP-72: Task SDK, the function ``get_parsing_context`` has been moved to ``airflow.sdk`` module. -Previously, it was located in ``airflow.utils.dag_parsing_context`` module. - -This function is used to optimize DAG parsing during execution when DAGs are generated dynamically. - -Before: - -.. code-block:: python - - from airflow.models.dag import DAG - from airflow.utils.dag_parsing_context import get_parsing_context - - current_dag_id = get_parsing_context().dag_id - - for thing in list_of_things: - dag_id = f"generated_dag_{thing}" - if current_dag_id is not None and current_dag_id != dag_id: - continue # skip generation of non-selected DAG - - with DAG(dag_id=dag_id, ...): - ... - -After: - -.. code-block:: python - - from airflow.sdk import get_parsing_context - - current_dag_id = get_parsing_context().dag_id - - # The rest of the code remains the same - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.dag_parsing_context.get_parsing_context`` -> ``airflow.sdk.get_parsing_context`` diff --git a/airflow-core/newsfragments/45729.significant.rst b/airflow-core/newsfragments/45729.significant.rst deleted file mode 100644 index 5a722119cdf7d..0000000000000 --- a/airflow-core/newsfragments/45729.significant.rst +++ /dev/null @@ -1,14 +0,0 @@ -Standalone DAG processor is now required - -The scheduler is no longer able to parse DAGs itself - it relies on the standalone DAG processor (introduced in Airflow 2.3) to do it instead. You can start one by running ``airflow dag-processor``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/45960.significant.rst b/airflow-core/newsfragments/45960.significant.rst deleted file mode 100644 index dcc581ca137f8..0000000000000 --- a/airflow-core/newsfragments/45960.significant.rst +++ /dev/null @@ -1,37 +0,0 @@ -Change how asset uris are accessed in inlet_events - -We used to be able to read asset uri through - -.. code-block:: python - - @task - def access_inlet_events(inlet_events): - print(inlet_events["uri"]) - -Now we'll need to do - -.. code-block:: python - - @task - def access_inlet_events(inlet_events): - print(inlet_events["asset"]["uri"]) - - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] context_key ``inlet_events.uri`` → ``inlet_events.asset.uri`` diff --git a/airflow-core/newsfragments/45961.significant.rst b/airflow-core/newsfragments/45961.significant.rst deleted file mode 100644 index a8c763e0dfb1f..0000000000000 --- a/airflow-core/newsfragments/45961.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Replace the ``external_trigger`` check with ``DagRunType``, and update any logic that relies on ``external_trigger`` to use ``run_type`` instead. - - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] argument ``external_trigger`` in ``airflow...DAG.create_dagrun`` - * [ ] context key ``dag_run.external_trigger`` diff --git a/airflow-core/newsfragments/46231.significant.rst b/airflow-core/newsfragments/46231.significant.rst deleted file mode 100644 index 18b20ec71a19b..0000000000000 --- a/airflow-core/newsfragments/46231.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -Moving EmptyOperator from Airflow core to the ``standard`` provider. - -EmptyOperator has been moved from Airflow core (``airflow.operators directory``) to the ``standard`` provider. -For new and existing DAGs, users must import ``EmptyOperator`` from ``airflow.providers.standard.operators.empty``. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.operators.empty.EmptyOperator`` → ``airflow.providers.standard.operators.empty.EmptyOperator`` diff --git a/airflow-core/newsfragments/46375.significant.rst b/airflow-core/newsfragments/46375.significant.rst deleted file mode 100644 index dafd2b9f3875d..0000000000000 --- a/airflow-core/newsfragments/46375.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -``SecretsMasker`` has now been moved into the task SDK to be consumed by DAG authors and users - -Any occurrences of the ``secrets_masker`` module will have to be updated from ``airflow.utils.log.secrets_masker`` to the new path: ``airflow.sdk.execution_time.secrets_masker`` - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.log.secrets_masker`` → ``airflow.sdk.execution_time.secrets_masker`` diff --git a/airflow-core/newsfragments/46408.significant.rst b/airflow-core/newsfragments/46408.significant.rst deleted file mode 100644 index ebfa4b9e614ad..0000000000000 --- a/airflow-core/newsfragments/46408.significant.rst +++ /dev/null @@ -1,30 +0,0 @@ -DAG processor related config options removed - -The follow configuration options have been removed: - -- ``[logging] dag_processor_manager_log_location`` -- ``[logging] dag_processor_manager_log_stdout`` -- ``[logging] log_processor_filename_template`` - -If these config options are still present, they will have no effect any longer. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641) - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``[logging] dag_processor_manager_log_location`` - * [x] ``[logging] dag_processor_manager_log_stdout`` - * [x] ``[logging] log_processor_filename_template`` diff --git a/airflow-core/newsfragments/46415.significant.rst b/airflow-core/newsfragments/46415.significant.rst deleted file mode 100644 index ac2cd6f84d2f6..0000000000000 --- a/airflow-core/newsfragments/46415.significant.rst +++ /dev/null @@ -1,26 +0,0 @@ -Legacy signature for operator link is removed. - -``BaseOperatorLink.get_link`` used to accept execution date as an argument. This -has been changed to accept ``ti_key`` to identify a task instance instead. The -old signature, supported at runtime for compatibility, has been removed. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [ ] Signature of ``airflow.models.baseoperatorlink.BaseOperatorLink.get_link`` changed - - .. detailed in https://github.com/apache/airflow/pull/46415#issuecomment-2636186625 diff --git a/airflow-core/newsfragments/46573.significant.rst b/airflow-core/newsfragments/46573.significant.rst deleted file mode 100644 index 20a5b7dd5fa26..0000000000000 --- a/airflow-core/newsfragments/46573.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -In Airflow 3.0, ``airflow.operators.email.EmailOperator`` is removed. - -Instead, users can install ``smtp`` provider and import ``EmailOperator`` from the the module ``airflow.providers.smtp.operators.smtp``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.operators.email.EmailOperator`` → ``airflow.providers.smtp.operators.smtp.EmailOperator`` diff --git a/airflow-core/newsfragments/46613.significant.rst b/airflow-core/newsfragments/46613.significant.rst deleted file mode 100644 index c0c0a5b04498e..0000000000000 --- a/airflow-core/newsfragments/46613.significant.rst +++ /dev/null @@ -1,45 +0,0 @@ -Operator Links interface changed to not run user code in Airflow Webserver - -The Operator Extra links, which can be defined either via plugins or custom operators -now do not execute any user code in the Airflow Webserver, but instead push the "full" -links to XCom backend and the link is fetched from the XCom backend when viewing -task details, for example from grid view. - -Example for users with custom links class: - -.. code-block:: python - - @attr.s(auto_attribs=True) - class CustomBaseIndexOpLink(BaseOperatorLink): - """Custom Operator Link for Google BigQuery Console.""" - - index: int = attr.ib() - - @property - def name(self) -> str: - return f"BigQuery Console #{self.index + 1}" - - @property - def xcom_key(self) -> str: - return f"bigquery_{self.index + 1}" - - def get_link(self, operator, *, ti_key): - search_queries = XCom.get_one( - task_id=ti_key.task_id, dag_id=ti_key.dag_id, run_id=ti_key.run_id, key="search_query" - ) - return f"https://console.cloud.google.com/bigquery?j={search_query}" - -The link has an xcom_key defined, which is how it will be stored in the XCOM backend, with key as xcom_key and -value as the entire link, this case: https://console.cloud.google.com/bigquery?j=search - - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/46663.significant.rst b/airflow-core/newsfragments/46663.significant.rst deleted file mode 100644 index b347d4dbd69df..0000000000000 --- a/airflow-core/newsfragments/46663.significant.rst +++ /dev/null @@ -1,31 +0,0 @@ -Removed configuration ``scheduler.allow_trigger_in_future``. - -A DAG run with logical date in the future can never be started now. This only affects ``schedule=None``. - -Instead of using a future date, you can trigger with ``logical_date=None``. A custom ``run_id`` can be supplied if desired. If a date is needed, it can be passed as a DAG param instead. - -Property ``allow_future_exec_dates`` on the DAG class has also been removed. - - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] property ``airflow...DAG.allow_future_exec_dates`` - - * ``airflow config lint`` - - * [x] ``scheduler.allow_trigger_in_future`` diff --git a/airflow-core/newsfragments/46916.significant.rst b/airflow-core/newsfragments/46916.significant.rst deleted file mode 100644 index 9f24753713468..0000000000000 --- a/airflow-core/newsfragments/46916.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -Public API authentication is migrated to JWT token based authentication for default (Simple Auth Manager) and FAB provider. - -The default setting is using API to create a token (JWT) to authenticate the requests to access the API. -The endpoints are populated under ``/auth`` path. -If none of the providers are installed such as FAB, the API will use the default use Simple Auth Manager in the core. - -To integrate the same functioning into API requests using FAB provider. Please install ``apache-airflow-providers-fab``. -For more information, please look at :doc:`apache-airflow-providers-fab:auth-manager/api-authentication`. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/47070.significant.rst b/airflow-core/newsfragments/47070.significant.rst deleted file mode 100644 index 2b212933b9b03..0000000000000 --- a/airflow-core/newsfragments/47070.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -Auto data interval calculation is disabled by default - -Configurations ``[scheduler] create_cron_data_intervals`` and ``create_delta_data_intervals`` are now *False* -by default. This means schedules specified using cron expressions or time deltas now have their logical date -set to *when a new run can start* instead of one data interval before. - -See :ref:`Differences between "trigger" and "data interval" timetables` for more information. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config update`` - - * [x] ``scheduler.create_cron_data_intervals`` - * [x] ``scheduler.create_delta_data_intervals`` diff --git a/airflow-core/newsfragments/47083.significant.rst b/airflow-core/newsfragments/47083.significant.rst deleted file mode 100644 index 6d8401935991b..0000000000000 --- a/airflow-core/newsfragments/47083.significant.rst +++ /dev/null @@ -1,45 +0,0 @@ -``airflow api-server`` has replaced ``airflow webserver`` cli command - -The new Airflow UI is now being served as part of the ``airflow api-server`` command and the ``airflow webserver`` command has been removed. - -The following configuration options have moved to the ``[api]`` section: - -- ``[webserver] web_server_host`` has been moved to ``[api] host`` -- ``[webserver] web_server_port`` has been moved to ``[api] port`` -- ``[webserver] workers`` has been moved to ``[api] workers`` -- ``[webserver] web_server_worker_timeout`` has been moved to ``[api] worker_timeout`` -- ``[webserver] web_server_ssl_cert`` has been moved to ``[api] ssl_cert`` -- ``[webserver] web_server_ssl_key`` has been moved to ``[api] ssl_key`` -- ``[webserver] access_logfile`` has been moved to ``[api] access_logfile`` - -The following configuration options have been removed: - -- ``[webserver] error_logfile`` -- ``[webserver] access_logformat`` - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641) - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``[webserver] web_server_host`` → ``[api] host`` - * [x] ``[webserver] web_server_port`` → ``[api] port`` - * [x] ``[webserver] workers`` → ``[api] workers`` - * [x] ``[webserver] web_server_worker_timeout`` → ``[api] worker_timeout`` - * [x] ``[webserver] web_server_ssl_cert`` → ``[api] ssl_cert`` - * [x] ``[webserver] web_server_ssl_key`` → ``[api] ssl_key`` - * [x] ``[webserver] access_logfile`` → ``[api] access_logfile`` - * [x] ``[webserver] error_logfile`` removed - * [x] ``[webserver] access_logformat`` removed diff --git a/airflow-core/newsfragments/47131.significant.rst b/airflow-core/newsfragments/47131.significant.rst deleted file mode 100644 index e8bf03ff3eae5..0000000000000 --- a/airflow-core/newsfragments/47131.significant.rst +++ /dev/null @@ -1,12 +0,0 @@ -The Airflow UI is now started with the ``airflow api-server`` command. The ``airflow webserver`` command has been removed. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/47136.significant.rst b/airflow-core/newsfragments/47136.significant.rst deleted file mode 100644 index d881cd789fa99..0000000000000 --- a/airflow-core/newsfragments/47136.significant.rst +++ /dev/null @@ -1,13 +0,0 @@ -``uri`` is replaced with ``id`` in ``AssetDetails`` . Hence, ``is_authorized_asset`` method needs to be updated in Auth Managers to use ``id`` instead of ``uri``. - - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [X] Code interface changes diff --git a/airflow-core/newsfragments/47264.significant.rst b/airflow-core/newsfragments/47264.significant.rst deleted file mode 100644 index d66c18dd12644..0000000000000 --- a/airflow-core/newsfragments/47264.significant.rst +++ /dev/null @@ -1,16 +0,0 @@ -Removed leftover deprecations prior to 3.0.0. - -* Removed the ``RemovedInAirflow3Warning`` warning class. -* Removed the deprecated module ``airflow.api.auth.backend.session``. Please use ``airflow.providers.fab.auth_manager.api.auth.backend.session`` instead. -* Removed the deprecated ``cleanup_stuck_queued_tasks`` method from the ``BaseExecutor`` interface. It is replaced by function ``revoke_task``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes diff --git a/airflow-core/newsfragments/47354.significant.rst b/airflow-core/newsfragments/47354.significant.rst deleted file mode 100644 index d967cb7242859..0000000000000 --- a/airflow-core/newsfragments/47354.significant.rst +++ /dev/null @@ -1,19 +0,0 @@ -The ``catchup_by_default`` configuration is now ``False`` by default. This means dags which do not explicitly define ``catchup`` will not display catchup behavior. - -* Types of change - - * [x] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``scheduler.catchup_by_default`` default change from ``True`` to ``False``. diff --git a/airflow-core/newsfragments/47399.significant.rst b/airflow-core/newsfragments/47399.significant.rst deleted file mode 100644 index bd84287acec61..0000000000000 --- a/airflow-core/newsfragments/47399.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -Removed auth backends. Auth backends are no longer used in Airflow 3. Please refer to documentation on how to use Airflow 3 public API. - -Moved the configuration ``[api] auth_backends`` to ``[fab] auth_backends``. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``api.auth_backends`` → ``fab.auth_backends`` diff --git a/airflow-core/newsfragments/47414.significant.rst b/airflow-core/newsfragments/47414.significant.rst deleted file mode 100644 index 6c7f9ac84b047..0000000000000 --- a/airflow-core/newsfragments/47414.significant.rst +++ /dev/null @@ -1,17 +0,0 @@ -Default connections no longer created by ``airflow db reset`` - -When default connection creation was removed from the ``airflow db migrate`` command in in 2.7, -``airflow db reset`` was missed and still used the deprecated configuration option -``[database] load_default_connections``. ``airflow db reset`` no longer does, so after a DB reset you must call -``airflow connections create-default-connections`` explicitly if you'd like the default connections to be created. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/47441.significant.rst b/airflow-core/newsfragments/47441.significant.rst deleted file mode 100644 index 05fe947efe0eb..0000000000000 --- a/airflow-core/newsfragments/47441.significant.rst +++ /dev/null @@ -1,15 +0,0 @@ -There are no more production bundle or devel extras - -There are no more production ``all*`` or ``devel*`` bundle extras available in ``wheel`` package of airflow. -If you want to install airflow with all extras you can use ``uv pip install --all-extras`` command. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/47599.significant.rst b/airflow-core/newsfragments/47599.significant.rst deleted file mode 100644 index 5e6c5c713a580..0000000000000 --- a/airflow-core/newsfragments/47599.significant.rst +++ /dev/null @@ -1,23 +0,0 @@ -Remove the option import create_session from db.py util - -The ability to create session from ``utils/db.py`` is removed. - - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.utils.db.create_session`` diff --git a/airflow-core/newsfragments/47761.significant.rst b/airflow-core/newsfragments/47761.significant.rst deleted file mode 100644 index 1da22d754bfa6..0000000000000 --- a/airflow-core/newsfragments/47761.significant.rst +++ /dev/null @@ -1,34 +0,0 @@ -Old default values no longer auto-updated - -There are a few configuration options that had old default values that no longer worked, so -Airflow 2 would automatically update them on-the-fly to the new default values. This is no longer -the case, and users need to update their configuration files to the new default values. - -The configuration options that were auto-updated in Airflow 2 are: - -- ``[core] hostname`` with value ``:`` -- ``[email] email_backend`` with value ``airflow.contrib.utils.sendgrid.send_email`` -- ``[logging] log_filename_template`` with value ``{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log`` -- ``[elasticsearch] log_id_template`` with value ``{dag_id}-{task_id}-{logical_date}-{try_number}`` - -If you have these configuration options in your ``airflow.cfg`` file, you need to update them. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config update`` - - * [x] Remove ``[core] hostname`` configuration option from config if value is ``:`` - * [x] Remove ``[email] email_backend`` configuration option from config if value is ``airflow.contrib.utils.sendgrid.send_email`` - * [x] Remove ``[logging] log_filename_template`` configuration option from config if value is ``{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log`` - * [x] Remove ``[elasticsearch] log_id_template`` configuration option from config if value is ``{dag_id}-{task_id}-{logical_date}-{try_number}`` diff --git a/airflow-core/newsfragments/47892.significant.rst b/airflow-core/newsfragments/47892.significant.rst deleted file mode 100644 index f6a1e0331b173..0000000000000 --- a/airflow-core/newsfragments/47892.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -Relocate utils.weekday from core to standard provider - -Removed module ``airflow.utils.weekday`` removed. Please use ``from airflow.providers.standard.utils.weekday`` instead. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ruff - - * AIR303 - - * [x] ``airflow.utils.weekday`` → ``airflow.providers.standard.utils.weekday`` diff --git a/airflow-core/newsfragments/47927.significant.rst b/airflow-core/newsfragments/47927.significant.rst deleted file mode 100644 index d9a017abf40d8..0000000000000 --- a/airflow-core/newsfragments/47927.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -Pre-installed providers are minimized per default with Airflow 3.0. - -Before Airflow 3.0 a set of providers were always pre-installed. We removed the pre-install of ftp, http and imap as they are not needed for all cases. - -The following providers are pre-installed by default: - -* common.compat -* common.io -* common.sql -* fab>=1.0.2 -* smtp -* sqlite -* standard - - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/48008.significant.rst b/airflow-core/newsfragments/48008.significant.rst deleted file mode 100644 index 58a76acc5c5b8..0000000000000 --- a/airflow-core/newsfragments/48008.significant.rst +++ /dev/null @@ -1,20 +0,0 @@ -The BaseNotifier class has been moved to ``airflow.sdk``. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ruff - - * AIR302 - - * [x] ``airflow.notifications.basenotifier.BaseNotifier`` → ``airflow.sdk.BaseNotifier`` diff --git a/airflow-core/newsfragments/48027.significant.rst b/airflow-core/newsfragments/48027.significant.rst deleted file mode 100644 index bdabd5d40e2e5..0000000000000 --- a/airflow-core/newsfragments/48027.significant.rst +++ /dev/null @@ -1,14 +0,0 @@ -``password`` extra has been removed - -Nothing is using the dependencies that are installed in the ``password`` extra (``bcrypt`` and ``flask-bcrypt``), so the extra has been removed. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/48066.significant.rst b/airflow-core/newsfragments/48066.significant.rst deleted file mode 100644 index c0767207321fa..0000000000000 --- a/airflow-core/newsfragments/48066.significant.rst +++ /dev/null @@ -1,78 +0,0 @@ -Unused webserver configuration options have been removed - -The following configuration options are now unused and have been removed: - -- ``[webserver] web_server_master_timeout`` -- ``[webserver] worker_refresh_batch_size`` -- ``[webserver] worker_refresh_interval`` -- ``[webserver] reload_on_plugin_change`` -- ``[webserver] worker_class`` -- ``[webserver] expose_stacktrace`` -- ``[webserver] log_fetch_delay_sec`` -- ``[webserver] log_auto_tailing_offset`` -- ``[webserver] log_animation_speed`` -- ``[webserver] default_dag_run_display_number`` -- ``[webserver] enable_proxy_fix`` -- ``[webserver] proxy_fix_x_for`` -- ``[webserver] proxy_fix_x_proto`` -- ``[webserver] proxy_fix_x_host`` -- ``[webserver] proxy_fix_x_port`` -- ``[webserver] proxy_fix_x_prefix`` -- ``[webserver] cookie_secure`` -- ``[webserver] analytics_tool`` -- ``[webserver] analytics_id`` -- ``[webserver] analytics_url`` -- ``[webserver] show_recent_stats_for_completed_runs`` -- ``[webserver] run_internal_api`` -- ``[webserver] caching_hash_method`` -- ``[webserver] show_trigger_form_if_no_params`` -- ``[webserver] num_recent_configurations_for_trigger`` -- ``[webserver] allowed_payload_size`` -- ``[webserver] max_form_memory_size`` -- ``[webserver] max_form_parts`` - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641) - -* Migration rules needed - - * ``airflow config lint`` - - * [x] Remove configuration option ``[webserver] web_server_master_timeout`` - * [x] Remove configuration option ``[webserver] worker_refresh_batch_size`` - * [x] Remove configuration option ``[webserver] worker_refresh_interval`` - * [x] Remove configuration option ``[webserver] reload_on_plugin_change`` - * [x] Remove configuration option ``[webserver] worker_class`` - * [x] Remove configuration option ``[webserver] expose_stacktrace`` - * [x] Remove configuration option ``[webserver] log_fetch_delay_sec`` - * [x] Remove configuration option ``[webserver] log_auto_tailing_offset`` - * [x] Remove configuration option ``[webserver] log_animation_speed`` - * [x] Remove configuration option ``[webserver] default_dag_run_display_number`` - * [x] Remove configuration option ``[webserver] enable_proxy_fix`` - * [x] Remove configuration option ``[webserver] proxy_fix_x_for`` - * [x] Remove configuration option ``[webserver] proxy_fix_x_proto`` - * [x] Remove configuration option ``[webserver] proxy_fix_x_host`` - * [x] Remove configuration option ``[webserver] proxy_fix_x_port`` - * [x] Remove configuration option ``[webserver] proxy_fix_x_prefix`` - * [x] Remove configuration option ``[webserver] cookie_secure`` - * [x] Remove configuration option ``[webserver] analytics_tool`` - * [x] Remove configuration option ``[webserver] analytics_id`` - * [x] Remove configuration option ``[webserver] analytics_url`` - * [x] Remove configuration option ``[webserver] show_recent_stats_for_completed_runs`` - * [x] Remove configuration option ``[webserver] run_internal_api`` - * [x] Remove configuration option ``[webserver] caching_hash_method`` - * [x] Remove configuration option ``[webserver] show_trigger_form_if_no_params`` - * [x] Remove configuration option ``[webserver] num_recent_configurations_for_trigger`` - * [x] Remove configuration option ``[webserver] allowed_payload_size`` - * [x] Remove configuration option ``[webserver] max_form_memory_size`` - * [x] Remove configuration option ``[webserver] max_form_parts`` diff --git a/airflow-core/newsfragments/48218.significant.rst b/airflow-core/newsfragments/48218.significant.rst deleted file mode 100644 index cfde19d2fbec4..0000000000000 --- a/airflow-core/newsfragments/48218.significant.rst +++ /dev/null @@ -1,14 +0,0 @@ -Remove option for unlimited parallelism. - -Before Airflow 3.0 it was possible to set unlimited parallelism by setting ``[core] parallelism`` to 0. This was removed in Airflow 3.0.0. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/48223.significant.rst b/airflow-core/newsfragments/48223.significant.rst deleted file mode 100644 index 5aee91c591f19..0000000000000 --- a/airflow-core/newsfragments/48223.significant.rst +++ /dev/null @@ -1,30 +0,0 @@ -The Airflow distribution is now split into multiple distributions. - -In Airflow 2, all the dependencies were kept in the ``apache-airflow`` package were dynamically generated at -build time and some of the optional dependencies in the package were excessive due to limitation of the build -system used. - -With Airflow 3, Airflow is now split into several independent and isolated distribution packages on top of -already existing ``providers`` and the dependencies are isolated and simplified across those distribution -packages. - -While the original installation methods via ``apache-airflow`` distribution package and extras still -work as previously and it installs complete airflow installation ready to serve as scheduler, webserver, triggerer -and worker, the ``apache-airflow`` package is now a meta-package that cab install all the other distribution -packages (mandatory of via optional extras), it's also possible to install only the distribution -packages that are needed for a specific component you want to run airflow with. - -One change vs. Airflow 2 is that neither ``apache-airflow`` nor ``apache-airflow-core`` distribution packages -have ``leveldb`` extra that is an optional feature of ``apache-airflow-providers-google`` distribution package. -The simplest way to install ``leveldb`` dependencies is to install ``apache-airflow-providers-google[leveldb]`` - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [x] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/48388.significant.rst b/airflow-core/newsfragments/48388.significant.rst deleted file mode 100644 index cc60bb7686210..0000000000000 --- a/airflow-core/newsfragments/48388.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Task-level auto lineage collection is removed - -The ``prepare_lineage``, ``apply_lineage`` mechanism, along with the custom -lineage backend type that supports it, has been removed. This has been an -experimental feature that never caught on. - -The ``airflow.lineage.hook`` submodule is not affected. - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes diff --git a/airflow-core/newsfragments/48458.significant.rst b/airflow-core/newsfragments/48458.significant.rst deleted file mode 100644 index 1f834d4395bd0..0000000000000 --- a/airflow-core/newsfragments/48458.significant.rst +++ /dev/null @@ -1,14 +0,0 @@ -For the Airflow CLI, we have changed the CLI parameter ``--exec-date`` to ``--logical-date`` for the command ``airflow dags trigger`` to maintain consistency with Airflow 3.0 and above. - -This is a breaking change for users using the ``airflow dags trigger`` command with the ``--exec-date`` parameter in their scripts or workflows. - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [x] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/48528.significant.rst b/airflow-core/newsfragments/48528.significant.rst deleted file mode 100644 index 252585a6eff3d..0000000000000 --- a/airflow-core/newsfragments/48528.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -``SequentialExecutor`` has been removed from the in-tree executors. - -This executor was primarily used for local testing but is now redundant, as ``LocalExecutor`` -supports SQLite with WAL mode and provides better performance with parallel execution. -Users should switch to ``LocalExecutor`` or ``CeleryExecutor`` as alternatives. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] Convert all ``SequentialExecutor`` to ``LocalExecutor`` in ``[core] executor`` diff --git a/airflow-core/newsfragments/48579.significant.rst b/airflow-core/newsfragments/48579.significant.rst deleted file mode 100644 index a67384b8c10be..0000000000000 --- a/airflow-core/newsfragments/48579.significant.rst +++ /dev/null @@ -1,22 +0,0 @@ -``DebugExecutor`` has been removed from the in-tree executors. - -This executor was primarily used for local testing but is now redundant, as ``LocalExecutor`` -supports SQLite with WAL mode and provides better performance with parallel execution. -Users should switch to ``LocalExecutor`` or ``CeleryExecutor`` as alternatives. - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config update`` - - * [x] Convert all ``DebugExecutor`` to ``LocalExecutor`` when present in ``[core] executor`` diff --git a/airflow-core/newsfragments/49017.significant.rst b/airflow-core/newsfragments/49017.significant.rst deleted file mode 100644 index 56a9b08831c53..0000000000000 --- a/airflow-core/newsfragments/49017.significant.rst +++ /dev/null @@ -1,25 +0,0 @@ -Renamed FAB related configuration. - - * Rename configuration ``webserver.config_file`` as ``fab.config_file`` - * Rename configuration ``webserver.session_backend`` as ``fab.session_backend`` - * Rename configuration ``webserver.base_url`` as ``api.base_url`` - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - - * ``airflow config lint`` - - * [x] ``webserver.config_file`` → ``fab.config_file`` - * [x] ``webserver.session_backend`` → ``fab.session_backend`` - * [x] ``webserver.base_url`` → ``api.base_url`` diff --git a/airflow-core/newsfragments/49161.significant.rst b/airflow-core/newsfragments/49161.significant.rst deleted file mode 100644 index 549d27bdfe30e..0000000000000 --- a/airflow-core/newsfragments/49161.significant.rst +++ /dev/null @@ -1,18 +0,0 @@ -Removed airflow configuration ``navbar_logo_text_color`` - -* Types of change - - * [ ] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``webserver.navbar_logo_text_color`` diff --git a/airflow-core/newsfragments/49223.significant.rst b/airflow-core/newsfragments/49223.significant.rst deleted file mode 100644 index 090e079c2d309..0000000000000 --- a/airflow-core/newsfragments/49223.significant.rst +++ /dev/null @@ -1,36 +0,0 @@ -Update in ``airflow config update`` command to show breaking config changes by default. - - * By default, the ``airflow config update`` shows a dry run (i.e. it does not modify your ``airflow.cfg``) and displays only the breaking configuration changes. This helps users avoid being overwhelmed by non-critical recommendations. - * ``airflow config update --fix`` to applies only the breaking changes and updates airflow.cfg accordingly. - * ``airflow config update --fix --all-recommendations`` updates both breaking and non-breaking recommended changes in your configuration. - -* Breaking Migration rules created - - * ``airflow config update`` - - * ``core.executor``: default value change from ``SequentialExecutor`` to ``LocalExecutor`` - * ``logging.log_filename_template``: remove configuration if value equals either - ``{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log`` or - ``dag_id={{ ti.dag_id }}/run_id={{ ti.run_id }}/task_id={{ ti.task_id }}/{% if ti.map_index >= 0 %}map_index={{ ti.map_index }}/{% endif %}attempt={{ try_number }}.log`` - * ``webserver.web_server_host`` → ``api.host`` - * ``webserver.web_server_port`` → ``api.port`` - * ``webserver.workers`` → ``api.workers`` - * ``webserver.web_server_ssl_cert`` → ``api.ssl_cert`` - * ``webserver.web_server_ssl_key`` → ``api.ssl_key`` - * ``webserver.access_logfile`` → ``api.access_logfile`` - * ``scheduler.catchup_by_default``: default value change from ``True`` to ``False`` - * ``scheduler.dag_dir_list_interval`` → ``dag_processor.refresh_interval`` - * ``triggerer.default_capacity`` → ``triggerer.capacity`` - * ``elasticsearch.log_id_template``: remove configuration if value equals ``{dag_id}-{task_id}-{logical_date}-{try_number}`` - - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/aip-72.significant.rst b/airflow-core/newsfragments/aip-72.significant.rst deleted file mode 100644 index 863e19c8caee5..0000000000000 --- a/airflow-core/newsfragments/aip-72.significant.rst +++ /dev/null @@ -1,135 +0,0 @@ -Create a TaskExecution interface and enforce DB isolation from Tasks - -As part of this change the following breaking changes have occurred: - -- Tasks and DAG Parsing code is not able to access the Airflow metadata DB - - Access via Variables and Connection is still allowed (though these will use an API, not direct DB access) -- it should be assumed that any use of the database models from within ``airflow.models`` inside of DAG files or tasks will break. - -- Remove the concept of pluggable TaskRunners. - - The ``task_runner`` config in ``[core]`` has been removed. - - There were two build in options for this, Standard (the default) which used Fork or a new process as appropriate, and CGroupRunner to launch tasks in a new CGroup (not usable inside docker or Kubernetes). - - With the move of the execution time code into the TaskSDK we are using this opportunity to reduce complexity for seldom used features. - -- Shipping DAGs via pickle is no longer supported - - This was a feature that was not widely used and was a security risk. It has been removed. - -- Pickling is no longer supported for XCom serialization. - - XCom data will no longer support pickling. This change is intended to improve security and simplify data - handling by supporting JSON-only serialization. DAGs that depend on XCom pickling must update to use JSON-serializable data. - - As part of that change, ``[core] enable_xcom_pickling`` configuration option has been removed. - - If you still need to use pickling, you can use a custom XCom backend that stores references in the metadata DB and - the pickled data can be stored in a separate storage like S3. - - The ``value`` field in the XCom table has been changed to a ``JSON`` type via DB migration. The XCom records that - contains pickled data are archived in the ``_xcom_archive`` table. You can safely drop this table if you don't need - the data anymore. To drop the table, you can use the following command or manually drop the table from the database. - - .. code-block:: bash - - airflow db drop-archived -t "_xcom_archive" - -- The ability to specify scheduling conditions for an operator via the ``deps`` class attribute has been removed. - - If you were defining custom scheduling conditions on an operator class (usually by subclassing BaseTIDep) this ability has been removed. - - It is recommended that you replace such a custom operator with a deferrable sensor, a condition or another triggering mechanism. - -- ``BaseOperatorLink`` has now been moved into the task SDK to be consumed by DAG authors to write custom operator links. - - Any occurrences of imports from ``airflow.models.baseoperatorlink`` will need to be updated to ``airflow.sdk.definitions.baseoperatorlink`` - -- ``chain``, ``chain_linear`` and ``cross_downstream`` have been moved to the task SDK. - - Any occurrences of imports from ``airflow.models.baseoperator`` will need to be updated to ``airflow.sdk`` - - Old imports: - - .. code-block:: python - - from airflow.models.baseoperator import chain, chain_linear, cross_downstream - - New imports: - - .. code-block:: python - - from airflow.sdk import chain, chain_linear, cross_downstream - -- The ``Label`` class has been moved to the task SDK. - - Old imports: - - .. code-block:: python - - from airflow.utils.edgemodifier import Label - - New imports: - - .. code-block:: python - - from airflow.sdk import Label - -- We have removed DAG level settings that control the UI behaviour. - These are now as per-user settings controlled by the UI - - - ``default_view`` - -- The ``SkipMixin` class has been removed as a parent class from ``BaseSensorOperator``. - -- A new config ``[workers] secrets_backend[kwargs]`` & ``[workers] secrets_backend_kwargs`` has been introduced to configure secrets backend on the - workers directly to allow reducing the round trip to the API server and also to allow configuring a - different secrets backend. - Priority defined as workers backend > workers env > secrets backend on API server > API server env > metadata DB. - -- All the decorators have been moved to the task SDK. - - Old imports: - - .. code-block:: python - - from airflow.decorators import dag, task, task_group, setup, teardown - - New imports: - - .. code-block:: python - - from airflow.sdk import dag, task, task_group, setup, teardown - - -* Types of change - - * [x] Dag changes - * [x] Config changes - * [ ] API changes - * [ ] CLI changes - * [x] Behaviour changes - * [x] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.task_runner`` - * [x] ``core.enable_xcom_pickling`` - - * ruff - - * AIR302 - - * [x] ``airflow.models.baseoperatorlink.BaseOperatorLink`` → ``airflow.sdk.BaseOperatorLink`` - * [ ] ``airflow.models.dag.DAG`` → ``airflow.sdk.DAG`` - * [ ] ``airflow.models.DAG`` → ``airflow.sdk.DAG`` - * [ ] ``airflow.decorators.dag`` → ``airflow.sdk.dag`` - * [ ] ``airflow.decorators.task`` → ``airflow.sdk.task`` - * [ ] ``airflow.decorators.task_group`` → ``airflow.sdk.task_group`` - * [ ] ``airflow.decorators.setup`` → ``airflow.sdk.setup`` - * [ ] ``airflow.decorators.teardown`` → ``airflow.sdk.teardown`` diff --git a/airflow-core/newsfragments/aip-79.significant.rst b/airflow-core/newsfragments/aip-79.significant.rst deleted file mode 100644 index 5d7dba3057e65..0000000000000 --- a/airflow-core/newsfragments/aip-79.significant.rst +++ /dev/null @@ -1,65 +0,0 @@ -Remove Flask App Builder from core Airflow dependencies. - -As part of this change the following breaking changes have occurred: - -- The auth manager interface ``base_auth_manager`` have been updated with some breaking changes: - - - The constructor no longer take ``appbuilder`` as parameter. The constructor takes no parameter - - - A new abstract method ``deserialize_user`` needs to be implemented - - - A new abstract method ``serialize_user`` needs to be implemented - - - A new abstract method ``filter_authorized_menu_items`` needs to be implemented - - - The property ``security_manager`` has been removed from the interface - - - The method ``get_url_logout`` is now optional - - - The method ``get_permitted_dag_ids`` has been renamed ``get_authorized_dag_ids`` - - - The method ``filter_permitted_dag_ids`` has been renamed ``filter_authorized_dag_ids`` - - - All these methods have been removed from the interface: - - - ``filter_permitted_menu_items`` - - ``get_user_name`` - - ``get_user_display_name`` - - ``get_user`` - - ``get_user_id`` - - ``is_logged_in`` - - ``get_api_endpoints`` - - ``register_views`` - - - All the following method signatures changed to make the parameter ``user`` required (it was optional) - - - ``is_authorized_configuration`` - - ``is_authorized_connection`` - - ``is_authorized_dag`` - - ``is_authorized_asset`` - - ``is_authorized_pool`` - - ``is_authorized_variable`` - - ``is_authorized_view`` - - ``is_authorized_custom_view`` - - ``get_authorized_dag_ids`` (previously ``get_permitted_dag_ids``) - - ``filter_authorized_dag_ids`` (previously ``filter_permitted_dag_ids``) - - - All the following method signatures changed to add the parameter ``user`` - - - ``batch_is_authorized_connection`` - - ``batch_is_authorized_dag`` - - ``batch_is_authorized_pool`` - - ``batch_is_authorized_variable`` - -- The module ``airflow.www.auth`` has been moved to ``airflow.providers.fab.www.auth`` - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [x] Code interface changes diff --git a/airflow-core/newsfragments/aip-84.significant.rst b/airflow-core/newsfragments/aip-84.significant.rst deleted file mode 100644 index a783f607ff477..0000000000000 --- a/airflow-core/newsfragments/aip-84.significant.rst +++ /dev/null @@ -1,50 +0,0 @@ -Modernize and improve the REST API. - -As part of this change the following breaking changes have occurred: - -- The API returns 422 status code instead of 400 for validation errors. - - For instance when the request payload, path params, or query params are invalid. - -- When listing a resource for instance on GET ``/dags``, ``fields`` parameter is not supported anymore to obtain a partial response. - - The full objects will be returned by the endpoint. This feature might be added back in upcoming 3.x versions. - -- Passing list in query parameters switched from ``form, non exploded`` to ``form, exploded`` - i.e before ``?my_list=item1,item2`` now ``?my_list=item1&my_list=item2`` - -- ``execution_date`` was deprecated and has been removed. Any payload or parameter mentioning this field has been removed. - -- Datetime format are RFC3339-compliant in FastAPI, more permissive than ISO8601, - meaning that the API returns zulu datetime for responses, more info here https://github.com/fastapi/fastapi/discussions/7693#discussioncomment-5143311. - Both ``Z`` and ``00+xx`` are supported for payload and params. - - This is due FastAPI and pydantic v2 default behavior. - -- PATCH on ``DagRun`` and ``TaskInstance`` are more generic and allow in addition to update the resource state and the note content. - - Therefore the two legacy dedicated endpoints to update a ``DagRun`` note and ``TaskInstance`` note have been removed. - - Same for the set task instance state, it is now handled by the broader PATCH on task instances. - -- ``assets/queuedEvent`` endpoints have moved to ``assets/queuedEvents`` for consistency. - -- dag_parsing endpoint now returns a 409 when the DagPriorityParsingRequest already exists. It was returning 201 before. - -- ``clearTaskInstances`` endpoint default value for ``reset_dag_runs`` field has been updated from ``False`` to ``True``. - -- Pool name can't be modified in the PATCH pool endpoint anymore. Pool name shouldn't be updated via pool PATCH API call. - -- Logical date is now a nullable. In addition it is a nullable required payload field for Triggering a DagRun endpoint. - - -* Types of change - - * [ ] Dag changes - * [ ] Config changes - * [x] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes diff --git a/airflow-core/newsfragments/api-38.significant.rst b/airflow-core/newsfragments/api-38.significant.rst deleted file mode 100644 index 3167475a9aeaa..0000000000000 --- a/airflow-core/newsfragments/api-38.significant.rst +++ /dev/null @@ -1,31 +0,0 @@ -- With the new UI and we have removed DAG and config level settings that control some of the the UI behaviour. - - These are now as per-user settings controlled by the UI - - - ``default_view`` - - ``orientation`` - -* Types of change - - * [x] Dag changes - * [ ] Config changes - * [ ] API changes - * [ ] CLI changes - * [ ] Behaviour changes - * [ ] Plugin changes - * [ ] Dependency changes - * [ ] Code interface changes - -* Migration rules needed - - * ``airflow config lint`` - - * [x] ``core.dag_default_view`` - * [x] ``core.dag_orientation`` - - * ruff - - * AIR302 - - * [x] ``default_view`` argument to DAG removed - * [x] ``orientation`` argument to DAG removed diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index f84568833060b..85710ae6b13b6 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -20,7 +20,7 @@ requires = [ "GitPython==3.1.44", "gitdb==4.0.12", "hatchling==1.27.0", - "packaging==24.2", + "packaging==25.0", "pathspec==0.12.1", "pluggy==1.5.0", "smmap==5.0.2", @@ -59,7 +59,7 @@ classifiers = [ ] # Version is defined in src/airflow/__init__.py and it is automatically synchronized by pre-commit -version = "3.0.0" +version = "3.0.1" dependencies = [ "a2wsgi>=1.10.8", @@ -86,11 +86,11 @@ dependencies = [ "fastapi[standard]>=0.115.0,!=0.115.10", # We could get rid of flask and gunicorn if we replace serve_logs with a starlette + unicorn "flask>=2.1.1", - "gitpython>=3.1.40", # We could get rid of flask and gunicorn if we replace serve_logs with a starlette + unicorn "gunicorn>=20.1.0", "httpx>=0.25.0", 'importlib_metadata>=6.5;python_version<"3.12"', + 'importlib_metadata>=7.0;python_version>="3.12"', "itsdangerous>=2.0", "jinja2>=3.1.5", "jsonschema>=4.19.1", @@ -99,8 +99,8 @@ dependencies = [ "linkify-it-py>=2.0.0", "lockfile>=0.12.2", "methodtools>=0.4.7", - "opentelemetry-api>=1.24.0", - "opentelemetry-exporter-otlp>=1.24.0", + "opentelemetry-api>=1.26.0", + "opentelemetry-exporter-otlp>=1.26.0", "packaging>=23.2", "pathspec>=0.9.0", 'pendulum>=2.1.2,<4.0;python_version<"3.12"', @@ -124,7 +124,7 @@ dependencies = [ # See https://sqlalche.me/e/b8d9 for details of deprecated features # you can set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. # The issue tracking it is https://github.com/apache/airflow/issues/28723 - "sqlalchemy>=1.4.49,<2.0", + "sqlalchemy[asyncio]>=1.4.49,<2.0", "sqlalchemy-jsonfield>=1.0", "sqlalchemy-utils>=0.41.2", "svcs>=25.1.0", @@ -135,46 +135,22 @@ dependencies = [ # Does not work with it Tracked in https://github.com/fsspec/universal_pathlib/issues/276 "universal-pathlib>=0.2.2,!=0.2.4", "uuid6>=2024.7.10", - "apache-airflow-task-sdk<1.1.0,>=1.0.0", + "apache-airflow-task-sdk<1.1.0,>=1.0.1", # pre-installed providers "apache-airflow-providers-common-compat>=1.6.0", "apache-airflow-providers-common-io>=1.5.3", - "apache-airflow-providers-common-sql>=1.25.0", + "apache-airflow-providers-common-sql>=1.26.0", "apache-airflow-providers-smtp>=2.0.2", "apache-airflow-providers-standard>=0.4.0", ] [project.optional-dependencies] -# Aiobotocore required for AWS deferrable operators. -# There is conflict between boto3 and aiobotocore dependency botocore. -# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or -# boto3 have native aync support and we move away from aio aiobotocore -"aiobotocore" = [ - "apache-airflow-providers-amazon[aiobotocore]>=9.6.0", -] "async" = [ "eventlet>=0.37.0", "gevent>=24.2.1", "greenlet>=0.4.9", ] -"apache-atlas" = [ - "atlasclient>=0.1.2", -] -"apache-webhdfs" = [ - "apache-airflow-providers-apache-hdfs", -] -"cloudpickle" = [ - "cloudpickle>=2.2.1", -] -"github-enterprise" = [ - "apache-airflow-providers-fab", - "authlib>=1.0.0", -] -"google-auth" = [ - "apache-airflow-providers-fab", - "authlib>=1.0.0", -] "graphviz" = [ # The graphviz package creates friction when installing on MacOS as it needs graphviz system package to # be installed, and it's really only used for very obscure features of Airflow, so we can skip it on MacOS @@ -186,27 +162,9 @@ dependencies = [ "requests-kerberos>=0.14.0", "thrift-sasl>=0.4.2", ] -"ldap" = [ - "python-ldap>=3.4.4", -] "otel" = [ "opentelemetry-exporter-prometheus>=0.47b0", ] - -"pandas" = [ - # In pandas 2.2 minimal version of the sqlalchemy is 2.0 - # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies - # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723 - "pandas>=2.1.2,<2.3", -] -"rabbitmq" = [ - "amqp>=5.2.0", -] -"s3fs" = [ - # This is required for support of S3 file system which uses aiobotocore - # which can have a conflict with boto3 as mentioned in aiobotocore extra - "apache-airflow-providers-amazon[s3fs]", -] "sentry" = [ "blinker>=1.1", # Sentry SDK 1.33 is broken when greenlets are installed and fails to import @@ -216,11 +174,8 @@ dependencies = [ "statsd" = [ "statsd>=3.3.0", ] -"uv" = [ - "uv>=0.6.13", -] "all" = [ - "apache-airflow-core[aiobotocore,apache-atlas,apache-webhdfs,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]" + "apache-airflow-core[graphviz,kerberos,otel,sentry,statsd]" ] [project.scripts] @@ -281,11 +236,14 @@ dev = [ "apache-airflow-ctl", "apache-airflow-devel-common", "apache-airflow-task-sdk", + # TODO(potiuk): eventually we do not want any providers nor apache-airflow extras to be needed for + # airflow-core tests + "apache-airflow[pandas,polars]", "apache-airflow-providers-amazon", "apache-airflow-providers-celery", "apache-airflow-providers-cncf-kubernetes", + "apache-airflow-providers-fab", "apache-airflow-providers-git", - # TODO(potiuk): check if this is really needed "apache-airflow-providers-ftp", ] diff --git a/airflow-core/src/airflow/__init__.py b/airflow-core/src/airflow/__init__.py index 6a997cecbdfef..8f58e4367ad9e 100644 --- a/airflow-core/src/airflow/__init__.py +++ b/airflow-core/src/airflow/__init__.py @@ -25,7 +25,8 @@ # lib.) This is required by some IDEs to resolve the import paths. __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore -__version__ = "3.0.0" +__version__ = "3.0.1" + import os import sys diff --git a/airflow-core/src/airflow/api/common/mark_tasks.py b/airflow-core/src/airflow/api/common/mark_tasks.py index c957a5cd53ab3..ad8c7fe4928cd 100644 --- a/airflow-core/src/airflow/api/common/mark_tasks.py +++ b/airflow-core/src/airflow/api/common/mark_tasks.py @@ -215,17 +215,31 @@ def set_dag_run_state_to_success( if not run_id: raise ValueError(f"Invalid dag_run_id: {run_id}") - # Mark all task instances of the dag run to success - except for teardown as they need to complete work. + # Mark all task instances of the dag run to success - except for unfinished teardown as they need to complete work. normal_tasks = [task for task in dag.tasks if not task.is_teardown] + teardown_tasks = [task for task in dag.tasks if task.is_teardown] + unfinished_teardown_task_ids = set( + session.scalars( + select(TaskInstance.task_id).where( + TaskInstance.dag_id == dag.dag_id, + TaskInstance.run_id == run_id, + TaskInstance.task_id.in_([task.task_id for task in teardown_tasks]), + or_(TaskInstance.state.is_(None), TaskInstance.state.in_(State.unfinished)), + ) + ).all() + ) - # Mark the dag run to success. - if commit and len(normal_tasks) == len(dag.tasks): + # Mark the dag run to success if there are no unfinished teardown tasks. + if commit and len(unfinished_teardown_task_ids) == 0: _set_dag_run_state(dag.dag_id, run_id, DagRunState.SUCCESS, session) - for task in normal_tasks: + tasks_to_mark_success = normal_tasks + [ + task for task in teardown_tasks if task.task_id not in unfinished_teardown_task_ids + ] + for task in tasks_to_mark_success: task.dag = dag return set_state( - tasks=normal_tasks, + tasks=tasks_to_mark_success, run_id=run_id, state=TaskInstanceState.SUCCESS, commit=commit, diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/models/resource_details.py b/airflow-core/src/airflow/api_fastapi/auth/managers/models/resource_details.py index 7aefccab703bb..7be79aff058ca 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/managers/models/resource_details.py +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/models/resource_details.py @@ -20,6 +20,8 @@ from dataclasses import dataclass from enum import Enum +from pydantic import NonNegativeInt + @dataclass class ConfigurationDetails: @@ -46,7 +48,7 @@ class DagDetails: class BackfillDetails: """Represents the details of a backfill.""" - id: str | None = None + id: NonNegativeInt | None = None @dataclass diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/middleware.py b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/middleware.py new file mode 100644 index 0000000000000..6c73cd015fa9f --- /dev/null +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/middleware.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from fastapi import Request +from starlette.middleware.base import BaseHTTPMiddleware + +from airflow.api_fastapi.auth.managers.simple.services.login import SimpleAuthManagerLogin + + +class SimpleAllAdminMiddleware(BaseHTTPMiddleware): + """Middleware that automatically generates and includes auth header for simple auth manager.""" + + async def dispatch(self, request: Request, call_next): + # Starlette Request is expected to be immutable, but we modify it to add the auth header + # https://github.com/fastapi/fastapi/issues/2727#issuecomment-770202019 + token = SimpleAuthManagerLogin.create_token_all_admins() + request.scope["headers"].append((b"authorization", f"Bearer {token}".encode())) + return await call_next(request) diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/routes/login.py b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/routes/login.py index c901692d8f9c8..82875ceb1239c 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/routes/login.py +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/routes/login.py @@ -17,7 +17,7 @@ from __future__ import annotations -from fastapi import status +from fastapi import Request, status from starlette.responses import RedirectResponse from airflow.api_fastapi.auth.managers.base_auth_manager import COOKIE_NAME_JWT_TOKEN @@ -57,10 +57,14 @@ def create_token_all_admins() -> LoginResponse: status_code=status.HTTP_307_TEMPORARY_REDIRECT, responses=create_openapi_http_exception_doc([status.HTTP_403_FORBIDDEN]), ) -def login_all_admins() -> RedirectResponse: +def login_all_admins(request: Request) -> RedirectResponse: """Login the user with no credentials.""" response = RedirectResponse(url=conf.get("api", "base_url", fallback="/")) - secure = conf.has_option("api", "ssl_cert") + + # The default config has this as an empty string, so we can't use `has_option`. + # And look at the request info (needs `--proxy-headers` flag to api-server) + secure = request.base_url.scheme == "https" or bool(conf.get("api", "ssl_cert", fallback="")) + response.set_cookie( COOKIE_NAME_JWT_TOKEN, SimpleAuthManagerLogin.create_token_all_admins(), diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package-lock.json b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package-lock.json index 300d4544dce0e..78ba79adf398f 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package-lock.json +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package-lock.json @@ -16,7 +16,7 @@ "react-cookie": "^8.0.1", "react-dom": "^19.0.0", "react-hook-form": "^7.54.2", - "react-router-dom": "^7.4.0" + "react-router-dom": "^7.5.2" }, "devDependencies": { "@7nohe/openapi-react-query-codegen": "^1.6.2", @@ -289,9 +289,10 @@ } }, "node_modules/@babel/runtime": { - "version": "7.26.0", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", - "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz", + "integrity": "sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==", + "license": "MIT", "dependencies": { "regenerator-runtime": "^0.14.0" }, @@ -2151,12 +2152,6 @@ "dev": true, "peer": true }, - "node_modules/@types/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", - "license": "MIT" - }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -7364,12 +7359,11 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" }, "node_modules/react-router": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.4.0.tgz", - "integrity": "sha512-Y2g5ObjkvX3VFeVt+0CIPuYd9PpgqCslG7ASSIdN73LwA1nNWzcMLaoMRJfP3prZFI92svxFwbn7XkLJ+UPQ6A==", + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.5.2.tgz", + "integrity": "sha512-9Rw8r199klMnlGZ8VAsV/I8WrIF6IyJ90JQUdboupx1cdkgYqwnrYjH+I/nY/7cA1X5zia4mDJqH36npP7sxGQ==", "license": "MIT", "dependencies": { - "@types/cookie": "^0.6.0", "cookie": "^1.0.1", "set-cookie-parser": "^2.6.0", "turbo-stream": "2.4.0" @@ -7388,12 +7382,12 @@ } }, "node_modules/react-router-dom": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.4.0.tgz", - "integrity": "sha512-VlksBPf3n2bijPvnA7nkTsXxMAKOj+bWp4R9c3i+bnwlSOFAGOkJkKhzy/OsRkWaBMICqcAl1JDzh9ZSOze9CA==", + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.5.2.tgz", + "integrity": "sha512-yk1XW8Fj7gK7flpYBXF3yzd2NbX6P7Kxjvs2b5nu1M04rb5pg/Zc4fGdBNTeT4eDYL2bvzWNyKaIMJX/RKHTTg==", "license": "MIT", "dependencies": { - "react-router": "7.4.0" + "react-router": "7.5.2" }, "engines": { "node": ">=20.0.0" diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package.json b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package.json index 6c3a49a18a635..61e99e7773143 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package.json +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/package.json @@ -23,7 +23,7 @@ "react-cookie": "^8.0.1", "react-dom": "^19.0.0", "react-hook-form": "^7.54.2", - "react-router-dom": "^7.4.0" + "react-router-dom": "^7.5.2" }, "devDependencies": { "@7nohe/openapi-react-query-codegen": "^1.6.2", diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock.yaml b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock.yaml index 87f498791a11d..3087c9e38cdd9 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock.yaml +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock.yaml @@ -33,8 +33,8 @@ importers: specifier: ^7.54.2 version: 7.54.2(react@19.0.0) react-router-dom: - specifier: ^7.4.0 - version: 7.4.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + specifier: ^7.5.2 + version: 7.5.2(react-dom@19.0.0(react@19.0.0))(react@19.0.0) devDependencies: '@7nohe/openapi-react-query-codegen': specifier: ^1.6.2 @@ -781,9 +781,6 @@ packages: '@types/aria-query@5.0.4': resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} - '@types/cookie@0.6.0': - resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} - '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} @@ -2314,15 +2311,15 @@ packages: react-is@17.0.2: resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} - react-router-dom@7.4.0: - resolution: {integrity: sha512-VlksBPf3n2bijPvnA7nkTsXxMAKOj+bWp4R9c3i+bnwlSOFAGOkJkKhzy/OsRkWaBMICqcAl1JDzh9ZSOze9CA==} + react-router-dom@7.5.2: + resolution: {integrity: sha512-yk1XW8Fj7gK7flpYBXF3yzd2NbX6P7Kxjvs2b5nu1M04rb5pg/Zc4fGdBNTeT4eDYL2bvzWNyKaIMJX/RKHTTg==} engines: {node: '>=20.0.0'} peerDependencies: react: '>=18' react-dom: '>=18' - react-router@7.4.0: - resolution: {integrity: sha512-Y2g5ObjkvX3VFeVt+0CIPuYd9PpgqCslG7ASSIdN73LwA1nNWzcMLaoMRJfP3prZFI92svxFwbn7XkLJ+UPQ6A==} + react-router@7.5.2: + resolution: {integrity: sha512-9Rw8r199klMnlGZ8VAsV/I8WrIF6IyJ90JQUdboupx1cdkgYqwnrYjH+I/nY/7cA1X5zia4mDJqH36npP7sxGQ==} engines: {node: '>=20.0.0'} peerDependencies: react: '>=18' @@ -3479,8 +3476,6 @@ snapshots: '@types/aria-query@5.0.4': {} - '@types/cookie@0.6.0': {} - '@types/estree@1.0.6': {} '@types/estree@1.0.7': {} @@ -5461,15 +5456,14 @@ snapshots: react-is@17.0.2: {} - react-router-dom@7.4.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0): + react-router-dom@7.5.2(react-dom@19.0.0(react@19.0.0))(react@19.0.0): dependencies: react: 19.0.0 react-dom: 19.0.0(react@19.0.0) - react-router: 7.4.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + react-router: 7.5.2(react-dom@19.0.0(react@19.0.0))(react@19.0.0) - react-router@7.4.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0): + react-router@7.5.2(react-dom@19.0.0(react@19.0.0))(react@19.0.0): dependencies: - '@types/cookie': 0.6.0 cookie: 1.0.2 react: 19.0.0 set-cookie-parser: 2.7.1 diff --git a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/src/login/Login.tsx b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/src/login/Login.tsx index b1963a02484c5..6aa65345361fa 100644 --- a/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/src/login/Login.tsx +++ b/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/src/login/Login.tsx @@ -32,6 +32,17 @@ export type LoginBody = { username: string; }; +const isSafeUrl = (targetUrl: string): boolean => { + try { + const base = new URL(globalThis.location.origin); + const target = new URL(targetUrl, base); + + return (target.protocol === "http:" || target.protocol === "https:") && target.origin === base.origin; + } catch { + return false; + } +}; + const LOCAL_STORAGE_DISABLE_BANNER_KEY = "disable-sam-banner"; export const Login = () => { @@ -42,15 +53,20 @@ export const Login = () => { ); const onSuccess = (data: LoginResponse) => { + // Fallback similar to FabAuthManager, strip off the next + const fallback = "/"; + // Redirect to appropriate page with the token - const next = searchParams.get("next"); + const next = searchParams.get("next") ?? fallback; setCookie("_token", data.access_token, { path: "/", secure: globalThis.location.protocol !== "http:", }); - globalThis.location.replace(next ?? ""); + const redirectTarget = isSafeUrl(next) ? next : fallback; + + globalThis.location.replace(redirectTarget); }; const { createToken, error, isPending, setError } = useCreateToken({ onSuccess, @@ -79,7 +95,7 @@ export const Login = () => { - {error === null && } + {Boolean(error) && } Enter your username and password below: diff --git a/airflow-core/src/airflow/api_fastapi/common/router.py b/airflow-core/src/airflow/api_fastapi/common/router.py index aeb1fb22452b5..12d9fca072459 100644 --- a/airflow-core/src/airflow/api_fastapi/common/router.py +++ b/airflow-core/src/airflow/api_fastapi/common/router.py @@ -17,17 +17,10 @@ from __future__ import annotations -from collections.abc import Sequence -from enum import Enum from typing import Any, Callable -from fastapi import APIRouter, params -from fastapi.datastructures import Default -from fastapi.routing import APIRoute -from fastapi.types import DecoratedCallable, IncEx -from fastapi.utils import generate_unique_id -from starlette.responses import JSONResponse, Response -from starlette.routing import BaseRoute +from fastapi import APIRouter +from fastapi.types import DecoratedCallable class AirflowRouter(APIRouter): @@ -36,58 +29,15 @@ class AirflowRouter(APIRouter): def api_route( self, path: str, - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] = Default(JSONResponse), - name: str | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[APIRoute], str] = Default(generate_unique_id), + **kwargs: Any, ) -> Callable[[DecoratedCallable], DecoratedCallable]: def decorator(func: DecoratedCallable) -> DecoratedCallable: self.add_api_route( path, func, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, operation_id=operation_id or func.__name__, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, + **kwargs, ) return func diff --git a/airflow-core/src/airflow/api_fastapi/core_api/app.py b/airflow-core/src/airflow/api_fastapi/core_api/app.py index 49f522994313a..90327ecc26677 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/app.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/app.py @@ -167,4 +167,10 @@ def init_error_handlers(app: FastAPI) -> None: def init_middlewares(app: FastAPI) -> None: + from airflow.configuration import conf + app.add_middleware(FlaskExceptionsMiddleware) + if conf.getboolean("core", "simple_auth_manager_all_admins"): + from airflow.api_fastapi.auth.managers.simple.middleware import SimpleAllAdminMiddleware + + app.add_middleware(SimpleAllAdminMiddleware) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py index c74a7e2020313..59561e462dc32 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/backfills.py @@ -19,6 +19,8 @@ from datetime import datetime +from pydantic import NonNegativeInt + from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel from airflow.models.backfill import ReprocessBehavior @@ -38,7 +40,7 @@ class BackfillPostBody(StrictBaseModel): class BackfillResponse(BaseModel): """Base serializer for Backfill.""" - id: int + id: NonNegativeInt dag_id: str from_date: datetime to_date: datetime diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py index a6bcf3361adf4..49661dfd5db0a 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_run.py @@ -76,6 +76,7 @@ class DAGRunResponse(BaseModel): conf: dict note: str | None dag_versions: list[DagVersionResponse] + bundle_version: str | None class DAGRunCollectionResponse(BaseModel): diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py index 7a1fae594c62b..a6df211746475 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py @@ -17,6 +17,7 @@ from __future__ import annotations +import inspect from collections import abc from collections.abc import Iterable from datetime import datetime, timedelta @@ -163,6 +164,14 @@ def get_timezone(cls, tz: Timezone | FixedTimezone) -> str | None: return None return str(tz) + @field_validator("doc_md", mode="before") + @classmethod + def get_doc_md(cls, doc_md: str | None) -> str | None: + """Clean indentation in doc md.""" + if doc_md is None: + return None + return inspect.cleandoc(doc_md) + @field_validator("params", mode="before") @classmethod def get_params(cls, params: abc.MutableMapping | None) -> dict | None: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dashboard.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dashboard.py index ad80685882829..bf2afa9fcd3c8 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dashboard.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dashboard.py @@ -61,3 +61,12 @@ class HistoricalMetricDataResponse(BaseModel): dag_run_types: DAGRunTypes dag_run_states: DAGRunStates task_instance_states: TaskInstanceStateCount + + +class DashboardDagStatsResponse(BaseModel): + """Dashboard DAG Stats serializer for responses.""" + + active_dag_count: int + failed_dag_count: int + running_dag_count: int + queued_dag_count: int diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py index 2317d8a168b82..2905e752650cd 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py @@ -19,7 +19,7 @@ import json -from pydantic import Field, model_validator +from pydantic import Field, JsonValue, model_validator from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel from airflow.models.base import ID_LEN @@ -54,7 +54,7 @@ class VariableBody(StrictBaseModel): """Variable serializer for bodies.""" key: str = Field(max_length=ID_LEN) - value: str = Field(serialization_alias="val") + value: JsonValue = Field(serialization_alias="val") description: str | None = Field(default=None) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index 9c586c694b14c..5eead52583dd4 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -296,6 +296,22 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /ui/dashboard/dag_stats: + get: + tags: + - Dashboard + summary: Dag Stats + description: Return basic DAG stats with counts of DAGs in various states. + operationId: dag_stats + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/DashboardDagStatsResponse' + security: + - OAuth2PasswordBearer: [] /ui/structure/structure_data: get: tags: @@ -595,6 +611,7 @@ components: properties: id: type: integer + minimum: 0.0 title: Id dag_id: type: string @@ -930,6 +947,11 @@ components: $ref: '#/components/schemas/DagVersionResponse' type: array title: Dag Versions + bundle_version: + anyOf: + - type: string + - type: 'null' + title: Bundle Version type: object required: - dag_run_id @@ -948,6 +970,7 @@ components: - conf - note - dag_versions + - bundle_version title: DAGRunResponse description: DAG Run serializer for responses. DAGRunStates: @@ -1262,6 +1285,28 @@ components: - bundle_url title: DagVersionResponse description: Dag Version serializer for responses. + DashboardDagStatsResponse: + properties: + active_dag_count: + type: integer + title: Active Dag Count + failed_dag_count: + type: integer + title: Failed Dag Count + running_dag_count: + type: integer + title: Running Dag Count + queued_dag_count: + type: integer + title: Queued Dag Count + type: object + required: + - active_dag_count + - failed_dag_count + - running_dag_count + - queued_dag_count + title: DashboardDagStatsResponse + description: Dashboard DAG Stats serializer for responses. EdgeResponse: properties: source_id: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v1-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v1-rest-api-generated.yaml index 8e54659005ba9..4ea3c207c3f06 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v1-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v1-rest-api-generated.yaml @@ -951,7 +951,8 @@ paths: in: path required: true schema: - type: string + type: integer + minimum: 0 title: Backfill Id responses: '200': @@ -997,6 +998,8 @@ paths: in: path required: true schema: + type: integer + minimum: 0 title: Backfill Id responses: '200': @@ -1048,6 +1051,8 @@ paths: in: path required: true schema: + type: integer + minimum: 0 title: Backfill Id responses: '200': @@ -1099,6 +1104,8 @@ paths: in: path required: true schema: + type: integer + minimum: 0 title: Backfill Id responses: '200': @@ -7039,6 +7046,7 @@ components: properties: id: type: integer + minimum: 0.0 title: Id dag_id: type: string @@ -8223,6 +8231,11 @@ components: $ref: '#/components/schemas/DagVersionResponse' type: array title: Dag Versions + bundle_version: + anyOf: + - type: string + - type: 'null' + title: Bundle Version type: object required: - dag_run_id @@ -8241,6 +8254,7 @@ components: - conf - note - dag_versions + - bundle_version title: DAGRunResponse description: DAG Run serializer for responses. DAGRunsBatchBody: @@ -8995,6 +9009,7 @@ components: - unixname title: JobResponse description: Job serializer for responses. + JsonValue: {} PatchTaskInstanceBody: properties: new_state: @@ -10235,8 +10250,7 @@ components: maxLength: 250 title: Key value: - type: string - title: Value + $ref: '#/components/schemas/JsonValue' description: anyOf: - type: string diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py index 7b2e30ba2ab6a..48b6c46d2e221 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/backfills.py @@ -20,6 +20,7 @@ from fastapi import Depends, HTTPException, status from fastapi.exceptions import RequestValidationError +from pydantic import NonNegativeInt from sqlalchemy import select, update from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity @@ -98,7 +99,7 @@ def list_backfills( ], ) def get_backfill( - backfill_id: str, + backfill_id: NonNegativeInt, session: SessionDep, ) -> BackfillResponse: backfill = session.get(Backfill, backfill_id) @@ -121,7 +122,7 @@ def get_backfill( Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.RUN)), ], ) -def pause_backfill(backfill_id, session: SessionDep) -> BackfillResponse: +def pause_backfill(backfill_id: NonNegativeInt, session: SessionDep) -> BackfillResponse: b = session.get(Backfill, backfill_id) if not b: raise HTTPException(status.HTTP_404_NOT_FOUND, f"Could not find backfill with id {backfill_id}") @@ -147,7 +148,7 @@ def pause_backfill(backfill_id, session: SessionDep) -> BackfillResponse: Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.RUN)), ], ) -def unpause_backfill(backfill_id, session: SessionDep) -> BackfillResponse: +def unpause_backfill(backfill_id: NonNegativeInt, session: SessionDep) -> BackfillResponse: b = session.get(Backfill, backfill_id) if not b: raise HTTPException(status.HTTP_404_NOT_FOUND, f"Could not find backfill with id {backfill_id}") @@ -172,7 +173,7 @@ def unpause_backfill(backfill_id, session: SessionDep) -> BackfillResponse: Depends(requires_access_backfill(method="PUT")), ], ) -def cancel_backfill(backfill_id, session: SessionDep) -> BackfillResponse: +def cancel_backfill(backfill_id: NonNegativeInt, session: SessionDep) -> BackfillResponse: b: Backfill = session.get(Backfill, backfill_id) if not b: raise HTTPException(status.HTTP_404_NOT_FOUND, f"Could not find backfill with id {backfill_id}") diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/config.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/config.py index 1df1582591581..1509dd25e6fc9 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/config.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/config.py @@ -70,11 +70,11 @@ def _check_expose_config() -> bool: display_sensitive: bool | None = None - if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": + if conf.get("api", "expose_config").lower() == "non-sensitive-only": expose_config = True display_sensitive = False else: - expose_config = conf.getboolean("webserver", "expose_config") + expose_config = conf.getboolean("api", "expose_config") display_sensitive = True if not expose_config: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dashboard.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dashboard.py index e7f6d42c9d4c2..f3c0198364137 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dashboard.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dashboard.py @@ -18,14 +18,19 @@ from fastapi import Depends, status from sqlalchemy import func, select +from sqlalchemy.sql.expression import case, false from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity from airflow.api_fastapi.common.db.common import SessionDep from airflow.api_fastapi.common.parameters import DateTimeQuery, OptionalDateTimeQuery from airflow.api_fastapi.common.router import AirflowRouter -from airflow.api_fastapi.core_api.datamodels.ui.dashboard import HistoricalMetricDataResponse +from airflow.api_fastapi.core_api.datamodels.ui.dashboard import ( + DashboardDagStatsResponse, + HistoricalMetricDataResponse, +) from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.api_fastapi.core_api.security import requires_access_dag +from airflow.models.dag import DagModel from airflow.models.dagrun import DagRun, DagRunType from airflow.models.taskinstance import TaskInstance from airflow.utils import timezone @@ -53,7 +58,7 @@ def historical_metrics( dag_run_types = session.execute( select(DagRun.run_type, func.count(DagRun.run_id)) .where( - DagRun.start_date >= start_date, + func.coalesce(DagRun.start_date, current_time) >= start_date, func.coalesce(DagRun.end_date, current_time) <= func.coalesce(end_date, current_time), ) .group_by(DagRun.run_type) @@ -62,7 +67,7 @@ def historical_metrics( dag_run_states = session.execute( select(DagRun.state, func.count(DagRun.run_id)) .where( - DagRun.start_date >= start_date, + func.coalesce(DagRun.start_date, current_time) >= start_date, func.coalesce(DagRun.end_date, current_time) <= func.coalesce(end_date, current_time), ) .group_by(DagRun.state) @@ -73,7 +78,7 @@ def historical_metrics( select(TaskInstance.state, func.count(TaskInstance.run_id)) .join(TaskInstance.dag_run) .where( - DagRun.start_date >= start_date, + func.coalesce(DagRun.start_date, current_time) >= start_date, func.coalesce(DagRun.end_date, current_time) <= func.coalesce(end_date, current_time), ) .group_by(TaskInstance.state) @@ -97,3 +102,50 @@ def historical_metrics( } return HistoricalMetricDataResponse.model_validate(historical_metrics_response) + + +@dashboard_router.get( + "/dag_stats", + dependencies=[Depends(requires_access_dag(method="GET"))], +) +def dag_stats( + session: SessionDep, +) -> DashboardDagStatsResponse: + """Return basic DAG stats with counts of DAGs in various states.""" + latest_dates_subq = ( + select(DagRun.dag_id, func.max(DagRun.logical_date).label("max_logical_date")) + .where(DagRun.logical_date.is_not(None)) + .group_by(DagRun.dag_id) + .subquery() + ) + + latest_runs = ( + select( + DagModel.dag_id, + DagModel.is_paused, + DagRun.state, + ) + .join(DagModel, DagRun.dag_id == DagModel.dag_id) + .join( + latest_dates_subq, + (DagRun.dag_id == latest_dates_subq.c.dag_id) + & (DagRun.logical_date == latest_dates_subq.c.max_logical_date), + ) + .cte() + ) + + combined_query = select( + func.coalesce(func.sum(case((latest_runs.c.is_paused == false(), 1))), 0).label("active"), + func.coalesce(func.sum(case((latest_runs.c.state == DagRunState.FAILED, 1))), 0).label("failed"), + func.coalesce(func.sum(case((latest_runs.c.state == DagRunState.RUNNING, 1))), 0).label("running"), + func.coalesce(func.sum(case((latest_runs.c.state == DagRunState.QUEUED, 1))), 0).label("queued"), + ).select_from(latest_runs) + + counts = session.execute(combined_query).first() + + return DashboardDagStatsResponse( + active_dag_count=counts.active, + failed_dag_count=counts.failed, + running_dag_count=counts.running, + queued_dag_count=counts.queued, + ) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py index 4ae459b42bb25..d1fe586c61fa3 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py @@ -133,7 +133,8 @@ def grid_data( tis_of_dag_runs, _ = paginated_select( statement=select(TaskInstance) .join(TaskInstance.task_instance_note, isouter=True) - .where(TaskInstance.dag_id == dag.dag_id), + .where(TaskInstance.dag_id == dag.dag_id) + .where(TaskInstance.run_id.in_([dag_run.run_id for dag_run in dag_runs])), filters=[], order_by=SortParam(allowed_attrs=["task_id", "run_id"], model=TaskInstance).set_value("task_id"), offset=offset, diff --git a/airflow-core/src/airflow/api_fastapi/core_api/security.py b/airflow-core/src/airflow/api_fastapi/core_api/security.py index e57a6543faf46..adc6cf2e01433 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/security.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/security.py @@ -23,6 +23,7 @@ from fastapi import Depends, HTTPException, Request, status from fastapi.security import OAuth2PasswordBearer from jwt import ExpiredSignatureError, InvalidTokenError +from pydantic import NonNegativeInt from airflow.api_fastapi.app import get_auth_manager from airflow.api_fastapi.auth.managers.models.base_user import BaseUser @@ -198,7 +199,7 @@ def inner( request: Request, user: Annotated[BaseUser | None, Depends(get_user)] = None, ) -> None: - backfill_id: str | None = request.path_params.get("backfill_id") + backfill_id: NonNegativeInt | None = request.path_params.get("backfill_id") _requires_access( is_authorized_callback=lambda: get_auth_manager().is_authorized_backfill( diff --git a/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py b/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py index 45ba4e42837d0..346676e14cd48 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/services/ui/grid.py @@ -17,9 +17,7 @@ from __future__ import annotations -from functools import cache -from operator import methodcaller -from typing import Callable +import contextlib from uuid import UUID import structlog @@ -37,29 +35,21 @@ from airflow.api_fastapi.core_api.datamodels.ui.structure import ( StructureDataResponse, ) -from airflow.configuration import conf from airflow.models.baseoperator import BaseOperator as DBBaseOperator from airflow.models.dag_version import DagVersion from airflow.models.taskmap import TaskMap from airflow.sdk import BaseOperator +from airflow.sdk.definitions._internal.abstractoperator import NotMapped +from airflow.sdk.definitions._internal.expandinput import NotFullyPopulated from airflow.sdk.definitions.mappedoperator import MappedOperator from airflow.sdk.definitions.taskgroup import MappedTaskGroup, TaskGroup from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils.state import TaskInstanceState -from airflow.utils.task_group import task_group_to_dict +from airflow.utils.task_group import get_task_group_children_getter, task_group_to_dict log = structlog.get_logger(logger_name=__name__) -@cache -def get_task_group_children_getter() -> Callable: - """Get the Task Group Children Getter for the DAG.""" - sort_order = conf.get("webserver", "grid_view_sorting_order") - if sort_order == "topological": - return methodcaller("topological_sort") - return methodcaller("hierarchical_alphabetical_sort") - - def get_task_group_map(dag: DAG) -> dict[str, dict[str, Any]]: """ Get the Task Group Map for the DAG. @@ -140,19 +130,14 @@ def get_child_task_map(parent_task_id: str, task_node_map: dict[str, dict[str, A return [task_id for task_id, task_map in task_node_map.items() if task_map["parent_id"] == parent_task_id] -def _get_total_task_count( - run_id: str, task_count: list[int | MappedTaskGroup | MappedOperator], session: SessionDep -) -> int: - return sum( - node - if isinstance(node, int) - else ( - DBBaseOperator.get_mapped_ti_count(node, run_id=run_id, session=session) or 0 - if isinstance(node, (MappedTaskGroup, MappedOperator)) - else node - ) - for node in task_count - ) +def _count_tis(node: int | MappedTaskGroup | MappedOperator, run_id: str, session: SessionDep) -> int: + if not isinstance(node, (MappedTaskGroup, MappedOperator)): + return node + with contextlib.suppress(NotFullyPopulated, NotMapped): + return DBBaseOperator.get_mapped_ti_count(node, run_id=run_id, session=session) + # If the downstream is not actually mapped, or we don't have information to + # determine the length yet, simply return 1 to represent the stand-in ti. + return 1 def fill_task_instance_summaries( @@ -253,7 +238,7 @@ def fill_task_instance_summaries( end_date=ti_end_date, queued_dttm=ti_queued_dttm, child_states=child_states, - task_count=_get_total_task_count(run_id, task_node_map[task_id]["task_count"], session), + task_count=sum(_count_tis(n, run_id, session) for n in task_node_map[task_id]["task_count"]), state=TaskInstanceState[overall_ti_state.upper()] if overall_ti_state != "no_status" else None, @@ -264,7 +249,7 @@ def fill_task_instance_summaries( def get_structure_from_dag(dag: DAG) -> StructureDataResponse: """If we do not have TIs, we just get the structure from the DAG.""" - nodes = [task_group_to_dict(child) for child in dag.task_group.topological_sort()] + nodes = [task_group_to_dict(child) for child in get_task_group_children_getter()(dag.task_group)] return StructureDataResponse(nodes=nodes, edges=[]) @@ -301,7 +286,7 @@ def get_combined_structure(task_instances, session): if serdag: dags.append(serdag.dag) for dag in dags: - nodes = [task_group_to_dict(child) for child in dag.task_group.topological_sort()] + nodes = [task_group_to_dict(child) for child in get_task_group_children_getter()(dag.task_group)] _merge_node_dicts(merged_nodes, nodes) return StructureDataResponse(nodes=merged_nodes, edges=[]) diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/app.py b/airflow-core/src/airflow/api_fastapi/execution_api/app.py index c4c924180bb20..ef51da9827943 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/app.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/app.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: import httpx + from fastapi.routing import APIRoute import structlog @@ -113,6 +114,10 @@ def customize_openapi(self, openapi_schema: dict[str, Any]) -> dict[str, Any]: This is particularly useful for client SDKs that require models for types not directly exposed in any endpoint's request or response schema. + We also replace ``anyOf`` with ``oneOf`` in the API spec as this produces better results for the code + generators. This is because anyOf can technically be more than of the given schemas, but 99.9% of the + time (perhaps 100% in this API) the types are mutually exclusive, so oneOf is more correct + References: - https://fastapi.tiangolo.com/how-to/extending-openapi/#modify-the-openapi-schema """ @@ -124,11 +129,23 @@ def customize_openapi(self, openapi_schema: dict[str, Any]) -> dict[str, Any]: # The `JsonValue` component is missing any info. causes issues when generating models openapi_schema["components"]["schemas"]["JsonValue"] = { "title": "Any valid JSON value", - "anyOf": [ + "oneOf": [ {"type": t} for t in ("string", "number", "integer", "object", "array", "boolean", "null") ], } + def replace_any_of_with_one_of(spec): + if isinstance(spec, dict): + return { + ("oneOf" if key == "anyOf" else key): replace_any_of_with_one_of(value) + for key, value in spec.items() + } + if isinstance(spec, list): + return [replace_any_of_with_one_of(item) for item in spec] + return spec + + openapi_schema = replace_any_of_with_one_of(openapi_schema) + for comp in openapi_schema["components"]["schemas"].values(): for prop in comp.get("properties", {}).values(): # {"type": "string", "const": "deferred"} @@ -147,11 +164,16 @@ def create_task_execution_api_app() -> FastAPI: from airflow.api_fastapi.execution_api.routes import execution_api_router from airflow.api_fastapi.execution_api.versions import bundle + def custom_generate_unique_id(route: APIRoute): + # This is called only if the route doesn't provide an explicit operation ID + return route.name + # See https://docs.cadwyn.dev/concepts/version_changes/ for info about API versions app = CadwynWithOpenAPICustomization( title="Airflow Task Execution API", description="The private Airflow Task Execution API.", lifespan=lifespan, + generate_unique_id_function=custom_generate_unique_id, api_version_parameter_name="Airflow-API-Version", api_version_default_value=bundle.versions[0].value, versions=bundle, @@ -175,7 +197,7 @@ def get_extra_schemas() -> dict[str, dict]: """Get all the extra schemas that are not part of the main FastAPI app.""" from airflow.api_fastapi.execution_api.datamodels.taskinstance import TaskInstance from airflow.executors.workloads import BundleInfo - from airflow.utils.state import TerminalTIState + from airflow.utils.state import TaskInstanceState, TerminalTIState return { "TaskInstance": TaskInstance.model_json_schema(), @@ -183,6 +205,7 @@ def get_extra_schemas() -> dict[str, dict]: # Include the combined state enum too. In the datamodels we separate out SUCCESS from the other states # as that has different payload requirements "TerminalTIState": {"type": "string", "enum": list(TerminalTIState)}, + "TaskInstanceState": {"type": "string", "enum": list(TaskInstanceState)}, } diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py b/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py index cd8287be97b73..b83d731a54e35 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py @@ -75,6 +75,7 @@ class TITerminalStatePayload(StrictBaseModel): end_date: UtcDateTime """When the task completed executing""" + rendered_map_index: str | None = None class TISuccessStatePayload(StrictBaseModel): @@ -97,6 +98,7 @@ class TISuccessStatePayload(StrictBaseModel): task_outlets: Annotated[list[AssetProfile], Field(default_factory=list)] outlet_events: Annotated[list[dict[str, Any]], Field(default_factory=list)] + rendered_map_index: str | None = None class TITargetStatePayload(StrictBaseModel): @@ -136,6 +138,7 @@ class TIDeferredStatePayload(StrictBaseModel): Both forms will be passed along to the TaskSDK upon resume, the server will not handle either. """ + rendered_map_index: str | None = None class TIRescheduleStatePayload(StrictBaseModel): @@ -171,6 +174,7 @@ class TIRetryStatePayload(StrictBaseModel): ), ] end_date: UtcDateTime + rendered_map_index: str | None = None class TISkippedDownstreamTasksStatePayload(StrictBaseModel): @@ -310,7 +314,7 @@ class TIRunContext(BaseModel): connections: Annotated[list[ConnectionResponse], Field(default_factory=list)] """Connections that can be accessed by the task instance.""" - upstream_map_indexes: dict[str, int] | None = None + upstream_map_indexes: dict[str, int | list[int] | None] | None = None next_method: str | None = None """Method to call. Set when task resumes from a trigger.""" diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/asset_events.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/asset_events.py index ec7a50705f215..09fb0fd5e679f 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/asset_events.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/asset_events.py @@ -19,11 +19,10 @@ from typing import Annotated -from fastapi import HTTPException, Query, status +from fastapi import APIRouter, HTTPException, Query, status from sqlalchemy import and_, select from airflow.api_fastapi.common.db.common import SessionDep -from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.execution_api.datamodels.asset import AssetResponse from airflow.api_fastapi.execution_api.datamodels.asset_event import ( AssetEventResponse, @@ -32,7 +31,7 @@ from airflow.models.asset import AssetAliasModel, AssetEvent, AssetModel # TODO: Add dependency on JWT token -router = AirflowRouter( +router = APIRouter( responses={ status.HTTP_404_NOT_FOUND: {"description": "Asset not found"}, status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/assets.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/assets.py index 213c599befb3e..316d4fab4770d 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/assets.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/assets.py @@ -19,16 +19,15 @@ from typing import Annotated -from fastapi import HTTPException, Query, status +from fastapi import APIRouter, HTTPException, Query, status from sqlalchemy import select from airflow.api_fastapi.common.db.common import SessionDep -from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.execution_api.datamodels.asset import AssetResponse from airflow.models.asset import AssetModel # TODO: Add dependency on JWT token -router = AirflowRouter( +router = APIRouter( responses={ status.HTTP_404_NOT_FOUND: {"description": "Asset not found"}, status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/dag_runs.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/dag_runs.py index 9044f9bb65d13..db27f7bd93ed8 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/dag_runs.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/dag_runs.py @@ -20,12 +20,11 @@ import logging from typing import Annotated -from fastapi import HTTPException, Query, status +from fastapi import APIRouter, HTTPException, Query, status from sqlalchemy import func, select from airflow.api.common.trigger_dag import trigger_dag from airflow.api_fastapi.common.db.common import SessionDep -from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.common.types import UtcDateTime from airflow.api_fastapi.execution_api.datamodels.dagrun import DagRunStateResponse, TriggerDAGRunPayload from airflow.exceptions import DagRunAlreadyExists @@ -34,7 +33,7 @@ from airflow.models.dagrun import DagRun from airflow.utils.types import DagRunTriggeredByType -router = AirflowRouter() +router = APIRouter() log = logging.getLogger(__name__) diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/health.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/health.py index bed519fdb2d44..d808f51e1db6a 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/health.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/health.py @@ -17,12 +17,12 @@ from __future__ import annotations +from fastapi import APIRouter from fastapi.responses import JSONResponse -from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.execution_api.deps import DepContainer -router = AirflowRouter() +router = APIRouter() @router.get("") diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py index ca8058bfd3aa1..00a5ea9a5e627 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py @@ -18,11 +18,12 @@ from __future__ import annotations import json -import logging from collections import defaultdict -from typing import Annotated, Any +from collections.abc import Iterator +from typing import TYPE_CHECKING, Annotated, Any from uuid import UUID +import structlog from cadwyn import VersionedAPIRouter from fastapi import Body, Depends, HTTPException, Query, Request, status from pydantic import JsonValue @@ -30,6 +31,7 @@ from sqlalchemy.exc import NoResultFound, SQLAlchemyError from sqlalchemy.orm import joinedload from sqlalchemy.sql import select +from structlog.contextvars import bind_contextvars from airflow.api_fastapi.common.db.common import SessionDep from airflow.api_fastapi.common.types import UtcDateTime @@ -54,9 +56,14 @@ from airflow.models.taskreschedule import TaskReschedule from airflow.models.trigger import Trigger from airflow.models.xcom import XComModel +from airflow.sdk.definitions.taskgroup import MappedTaskGroup from airflow.utils import timezone from airflow.utils.state import DagRunState, TaskInstanceState, TerminalTIState +if TYPE_CHECKING: + from airflow.sdk.types import Operator + + router = VersionedAPIRouter() ti_id_router = VersionedAPIRouter( @@ -67,7 +74,7 @@ ) -log = logging.getLogger(__name__) +log = structlog.get_logger(__name__) @ti_id_router.patch( @@ -81,7 +88,10 @@ response_model_exclude_unset=True, ) def ti_run( - task_instance_id: UUID, ti_run_payload: Annotated[TIEnterRunningPayload, Body()], session: SessionDep + task_instance_id: UUID, + ti_run_payload: Annotated[TIEnterRunningPayload, Body()], + session: SessionDep, + request: Request, ) -> TIRunContext: """ Run a TaskInstance. @@ -90,6 +100,13 @@ def ti_run( """ # We only use UUID above for validation purposes ti_id_str = str(task_instance_id) + bind_contextvars(ti_id=ti_id_str) + log.debug( + "Starting task instance run", + hostname=ti_run_payload.hostname, + unixname=ti_run_payload.unixname, + pid=ti_run_payload.pid, + ) from sqlalchemy.sql import column from sqlalchemy.types import JSON @@ -118,8 +135,9 @@ def ti_run( ) try: ti = session.execute(old).one() + log.debug("Retrieved task instance details", state=ti.state, dag_id=ti.dag_id, task_id=ti.task_id) except NoResultFound: - log.error("Task Instance %s not found", ti_id_str) + log.error("Task Instance not found") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={ @@ -134,6 +152,7 @@ def ti_run( # don't update start date when resuming from deferral if ti.next_kwargs: data.pop("start_date") + log.debug("Removed start_date from update as task is resuming from deferral") query = update(TI).where(TI.id == ti_id_str).values(data) @@ -141,17 +160,16 @@ def ti_run( # If we are already running, but this is a duplicate request from the same client return the same OK # -- it's possible there was a network glitch and they never got the response - if previous_state == TaskInstanceState.RUNNING and (ti["hostname"], ti["unixname"], ti["pid"]) == ( + if previous_state == TaskInstanceState.RUNNING and (ti.hostname, ti.unixname, ti.pid) == ( ti_run_payload.hostname, ti_run_payload.unixname, ti_run_payload.pid, ): - log.info("Duplicate start request received from %s ", ti_run_payload.hostname) + log.info("Duplicate start request received", hostname=ti_run_payload.hostname) elif previous_state not in (TaskInstanceState.QUEUED, TaskInstanceState.RESTARTING): log.warning( - "Can not start Task Instance ('%s') in invalid state: %s", - ti_id_str, - previous_state, + "Cannot start Task Instance in invalid state", + previous_state=previous_state, ) # TODO: Pass a RFC 9457 compliant error message in "detail" field @@ -168,7 +186,7 @@ def ti_run( }, ) else: - log.info("Task with %s state started on %s ", previous_state, ti_run_payload.hostname) + log.info("Task started", previous_state=previous_state, hostname=ti_run_payload.hostname) # Ensure there is no end date set. query = query.values( end_date=None, @@ -181,7 +199,7 @@ def ti_run( try: result = session.execute(query) - log.info("TI %s state updated: %s row(s) affected", ti_id_str, result.rowcount) + log.info("Task instance state updated", rows_affected=result.rowcount) dr = ( session.scalars( @@ -194,6 +212,7 @@ def ti_run( ) if not dr: + log.error("DagRun not found", dag_id=ti.dag_id, run_id=ti.run_id) raise ValueError(f"DagRun with dag_id={ti.dag_id} and run_id={ti.run_id} not found.") # Send the keys to the SDK so that the client requests to clear those XComs from the server. @@ -223,6 +242,11 @@ def ti_run( or 0 ) + if dag := request.app.state.dag_bag.get_dag(ti.dag_id): + upstream_map_indexes = dict(_get_upstream_map_indexes(dag.get_task(ti.task_id), ti.map_index)) + else: + upstream_map_indexes = None + context = TIRunContext( dag_run=dr, task_reschedule_count=task_reschedule_count, @@ -232,6 +256,7 @@ def ti_run( connections=[], xcom_keys_to_clear=xcom_keys, should_retry=_is_eligible_to_retry(previous_state, ti.try_number, ti.max_tries), + upstream_map_indexes=upstream_map_indexes, ) # Only set if they are non-null @@ -247,6 +272,27 @@ def ti_run( ) +def _get_upstream_map_indexes( + task: Operator, ti_map_index: int +) -> Iterator[tuple[str, int | list[int] | None]]: + for upstream_task in task.upstream_list: + map_indexes: int | list[int] | None + if not isinstance(upstream_task.task_group, MappedTaskGroup): + # regular tasks or non-mapped task groups + map_indexes = None + elif task.task_group == upstream_task.task_group: + # tasks in the same mapped task group + # the task should use the map_index as the previous task in the same mapped task group + map_indexes = ti_map_index + else: + # tasks not in the same mapped task group + # the upstream mapped task group should combine the xcom as a list and return it + mapped_ti_count: int = upstream_task.task_group.get_parse_time_mapped_ti_count() + map_indexes = list(range(mapped_ti_count)) if mapped_ti_count is not None else None + + yield upstream_task.task_id, map_indexes + + @ti_id_router.patch( "/{task_instance_id}/state", status_code=status.HTTP_204_NO_CONTENT, @@ -268,10 +314,12 @@ def ti_update_state( Not all state transitions are valid, and transitioning to some states requires extra information to be passed along. (Check out the datamodels for details, the rendered docs might not reflect this accurately) """ - updated_state: str = "" - # We only use UUID above for validation purposes ti_id_str = str(task_instance_id) + bind_contextvars(ti_id=ti_id_str) + log.debug("Updating task instance state", new_state=ti_patch_payload.state) + + updated_state: str = "" old = select(TI.state, TI.try_number, TI.max_tries, TI.dag_id).where(TI.id == ti_id_str).with_for_update() try: @@ -281,8 +329,14 @@ def ti_update_state( max_tries, dag_id, ) = session.execute(old).one() + log.debug( + "Retrieved current task instance state", + previous_state=previous_state, + try_number=try_number, + max_tries=max_tries, + ) except NoResultFound: - log.error("Task Instance %s not found", ti_id_str) + log.error("Task Instance not found") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={ @@ -293,9 +347,8 @@ def ti_update_state( if previous_state != TaskInstanceState.RUNNING: log.warning( - "Cannot update Task Instance ('%s') because it is in an invalid state: %s for an update", - ti_id_str, - previous_state, + "Cannot update Task Instance in invalid state", + previous_state=previous_state, ) raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -421,9 +474,9 @@ def ti_update_state( # https://fastapi.tiangolo.com/tutorial/handling-errors/#install-custom-exception-handlers try: result = session.execute(query) - log.info("TI %s state updated to %s: %s row(s) affected", ti_id_str, updated_state, result.rowcount) + log.info("Task instance state updated", new_state=updated_state, rows_affected=result.rowcount) except SQLAlchemyError as e: - log.error("Error updating Task Instance state: %s", e) + log.error("Error updating Task Instance state", error=str(e)) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Database error occurred" ) @@ -443,12 +496,17 @@ def ti_skip_downstream( session: SessionDep, ): ti_id_str = str(task_instance_id) + bind_contextvars(ti_id=ti_id_str) + log.info("Skipping downstream tasks", task_count=len(ti_patch_payload.tasks)) + now = timezone.utcnow() tasks = ti_patch_payload.tasks dag_id, run_id = session.execute(select(TI.dag_id, TI.run_id).where(TI.id == ti_id_str)).fetchone() + log.debug("Retrieved DAG and run info", dag_id=dag_id, run_id=run_id) task_ids = [task if isinstance(task, tuple) else (task, -1) for task in tasks] + log.debug("Prepared task IDs for skipping", task_ids=task_ids) query = ( update(TI) @@ -458,7 +516,7 @@ def ti_skip_downstream( ) result = session.execute(query) - log.info("TI %s updated the state of %s task(s) to skipped", ti_id_str, result.rowcount) + log.info("Downstream tasks skipped", tasks_skipped=result.rowcount) @ti_id_router.put( @@ -479,6 +537,8 @@ def ti_heartbeat( ): """Update the heartbeat of a TaskInstance to mark it as alive & still running.""" ti_id_str = str(task_instance_id) + bind_contextvars(ti_id=ti_id_str) + log.debug("Processing heartbeat", hostname=ti_payload.hostname, pid=ti_payload.pid) # Hot path: since heartbeating a task is a very common operation, we try to do minimize the number of queries # and DB round trips as much as possible. @@ -487,8 +547,11 @@ def ti_heartbeat( try: (previous_state, hostname, pid) = session.execute(old).one() + log.debug( + "Retrieved current task state", state=previous_state, current_hostname=hostname, current_pid=pid + ) except NoResultFound: - log.error("Task Instance %s not found", ti_id_str) + log.error("Task Instance not found") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={ @@ -498,6 +561,13 @@ def ti_heartbeat( ) if hostname != ti_payload.hostname or pid != ti_payload.pid: + log.warning( + "Task running elsewhere", + current_hostname=hostname, + current_pid=pid, + requested_hostname=ti_payload.hostname, + requested_pid=ti_payload.pid, + ) raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ @@ -509,6 +579,7 @@ def ti_heartbeat( ) if previous_state != TaskInstanceState.RUNNING: + log.warning("Task not in running state", current_state=previous_state) raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ @@ -520,7 +591,7 @@ def ti_heartbeat( # Update the last heartbeat time! session.execute(update(TI).where(TI.id == ti_id_str).values(last_heartbeat_at=timezone.utcnow())) - log.debug("Task with %s state heartbeated", previous_state) + log.debug("Heartbeat updated", state=previous_state) @ti_id_router.put( @@ -543,12 +614,17 @@ def ti_put_rtif( ): """Add an RTIF entry for a task instance, sent by the worker.""" ti_id_str = str(task_instance_id) + bind_contextvars(ti_id=ti_id_str) + log.info("Updating RenderedTaskInstanceFields", field_count=len(put_rtif_payload)) + task_instance = session.scalar(select(TI).where(TI.id == ti_id_str)) if not task_instance: + log.error("Task Instance not found") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, ) task_instance.update_rtif(put_rtif_payload, session) + log.debug("RenderedTaskInstanceFields updated successfully") return {"message": "Rendered task instance fields successfully set"} @@ -569,8 +645,12 @@ def get_previous_successful_dagrun( The data from this endpoint is used to get values for Task Context. """ ti_id_str = str(task_instance_id) + bind_contextvars(ti_id=ti_id_str) + log.debug("Retrieving previous successful DAG run") + task_instance = session.scalar(select(TI).where(TI.id == ti_id_str)) if not task_instance or not task_instance.logical_date: + log.debug("No task instance or logical date found") return PrevSuccessfulDagRunResponse() dag_run = session.scalar( @@ -584,15 +664,23 @@ def get_previous_successful_dagrun( .limit(1) ) if not dag_run: + log.debug("No previous successful DAG run found") return PrevSuccessfulDagRunResponse() + log.debug( + "Found previous successful DAG run", + dag_id=dag_run.dag_id, + run_id=dag_run.run_id, + logical_date=dag_run.logical_date, + ) return PrevSuccessfulDagRunResponse.model_validate(dag_run) @router.get("/count", status_code=status.HTTP_200_OK) -def get_count( +def get_task_instance_count( dag_id: str, session: SessionDep, + map_index: Annotated[int | None, Query()] = None, task_ids: Annotated[list[str] | None, Query()] = None, task_group_id: Annotated[str | None, Query()] = None, logical_dates: Annotated[list[UtcDateTime] | None, Query()] = None, @@ -605,6 +693,9 @@ def get_count( if task_ids: query = query.where(TI.task_id.in_(task_ids)) + if map_index is not None: + query = query.where(TI.map_index == map_index) + if logical_dates: query = query.where(TI.logical_date.in_(logical_dates)) @@ -615,7 +706,12 @@ def get_count( group_tasks = _get_group_tasks(dag_id, task_group_id, session, logical_dates, run_ids) # Get unique (task_id, map_index) pairs + task_map_pairs = [(ti.task_id, ti.map_index) for ti in group_tasks] + + if map_index is not None: + task_map_pairs = [(ti.task_id, ti.map_index) for ti in group_tasks if ti.map_index == map_index] + if not task_map_pairs: # If no task group tasks found, default to checking the task group ID itself # This matches the behavior in _get_external_task_group_task_ids @@ -640,15 +736,16 @@ def get_count( @router.get("/states", status_code=status.HTTP_200_OK) -def get_task_states( +def get_task_instance_states( dag_id: str, session: SessionDep, + map_index: Annotated[int | None, Query()] = None, task_ids: Annotated[list[str] | None, Query()] = None, task_group_id: Annotated[str | None, Query()] = None, logical_dates: Annotated[list[UtcDateTime] | None, Query()] = None, run_ids: Annotated[list[str] | None, Query()] = None, ) -> TaskStatesResponse: - """Get the task states for the given criteria.""" + """Get the states for Task Instances with the given criteria.""" run_id_task_state_map: dict[str, dict[str, Any]] = defaultdict(dict) query = select(TI).where(TI.dag_id == dag_id) @@ -664,12 +761,21 @@ def get_task_states( results = session.scalars(query).all() - [run_id_task_state_map[task.run_id].update({task.task_id: task.state}) for task in results] - if task_group_id: group_tasks = _get_group_tasks(dag_id, task_group_id, session, logical_dates, run_ids) - [run_id_task_state_map[task.run_id].update({task.task_id: task.state}) for task in group_tasks] + results = results + group_tasks if task_ids else group_tasks + + if map_index is not None: + results = [task for task in results if task.map_index == map_index] + [ + run_id_task_state_map[task.run_id].update( + {task.task_id: task.state} + if task.map_index < 0 + else {f"{task.task_id}_{task.map_index}": task.state} + ) + for task in results + ] return TaskStatesResponse(task_states=run_id_task_state_map) diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_reschedules.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_reschedules.py index d3e940f47a08f..f763858f9b9c1 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_reschedules.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_reschedules.py @@ -19,15 +19,14 @@ from uuid import UUID -from fastapi import status +from fastapi import APIRouter, status from sqlalchemy import select from airflow.api_fastapi.common.db.common import SessionDep -from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.common.types import UtcDateTime from airflow.models.taskreschedule import TaskReschedule -router = AirflowRouter( +router = APIRouter( responses={ status.HTTP_404_NOT_FOUND: {"description": "Task Instance not found"}, status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/xcoms.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/xcoms.py index 96483e4f08d9a..9f5e7d686a3cb 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/xcoms.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/xcoms.py @@ -21,13 +21,12 @@ import sys from typing import Annotated, Any -from fastapi import Body, Depends, HTTPException, Path, Query, Request, Response, status +from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request, Response, status from pydantic import BaseModel, JsonValue from sqlalchemy import delete from sqlalchemy.sql.selectable import Select from airflow.api_fastapi.common.db.common import SessionDep -from airflow.api_fastapi.common.router import AirflowRouter from airflow.api_fastapi.execution_api.datamodels.xcom import XComResponse from airflow.api_fastapi.execution_api.deps import JWTBearerDep from airflow.models.taskmap import TaskMap @@ -57,7 +56,7 @@ async def has_xcom_access( return True -router = AirflowRouter( +router = APIRouter( responses={ status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, status.HTTP_403_FORBIDDEN: {"description": "Task does not have access to the XCom"}, @@ -95,7 +94,7 @@ async def xcom_query( "description": "Metadata about the number of matching XCom values", "headers": { "Content-Range": { - "pattern": r"^map_indexes \d+$", + "schema": {"pattern": r"^map_indexes \d+$"}, "description": "The number of (mapped) XCom values found for this task.", }, }, @@ -127,6 +126,7 @@ class GetXcomFilterParams(BaseModel): map_index: int = -1 include_prior_dates: bool = False + offset: int | None = None @router.get( @@ -142,18 +142,23 @@ def get_xcom( params: Annotated[GetXcomFilterParams, Query()], ) -> XComResponse: """Get an Airflow XCom from database - not other XCom Backends.""" - # The xcom_query allows no map_index to be passed. This endpoint should always return just a single item, - # so we override that query value xcom_query = XComModel.get_many( run_id=run_id, key=key, task_ids=task_id, dag_ids=dag_id, - map_indexes=params.map_index, include_prior_dates=params.include_prior_dates, session=session, ) - xcom_query = xcom_query.filter(XComModel.map_index == params.map_index) + if params.offset is not None: + xcom_query = xcom_query.filter(XComModel.value.is_not(None)).order_by(None) + if params.offset >= 0: + xcom_query = xcom_query.order_by(XComModel.map_index.asc()).offset(params.offset) + else: + xcom_query = xcom_query.order_by(XComModel.map_index.desc()).offset(-1 - params.offset) + else: + xcom_query = xcom_query.filter(XComModel.map_index == params.map_index) + # We use `BaseXCom.get_many` to fetch XComs directly from the database, bypassing the XCom Backend. # This avoids deserialization via the backend (e.g., from a remote storage like S3) and instead # retrieves the raw serialized value from the database. By not relying on `XCom.get_many` or `XCom.get_one` @@ -161,13 +166,19 @@ def get_xcom( # performance hits from retrieving large data files into the API server. result = xcom_query.limit(1).first() if result is None: - map_index = params.map_index + if params.offset is None: + message = ( + f"XCom with {key=} map_index={params.map_index} not found for " + f"task {task_id!r} in DAG run {run_id!r} of {dag_id!r}" + ) + else: + message = ( + f"XCom with {key=} offset={params.offset} not found for " + f"task {task_id!r} in DAG run {run_id!r} of {dag_id!r}" + ) raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail={ - "reason": "not_found", - "message": f"XCom with {key=} {map_index=} not found for task {task_id!r} in DAG run {run_id!r} of {dag_id!r}", - }, + detail={"reason": "not_found", "message": message}, ) return XComResponse(key=key, value=result.value) diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py b/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py index 0d3a225305b67..54329054c1213 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py @@ -19,7 +19,10 @@ from cadwyn import HeadVersion, Version, VersionBundle +from airflow.api_fastapi.execution_api.versions.v2025_04_28 import AddRenderedMapIndexField + bundle = VersionBundle( HeadVersion(), + Version("2025-04-28", AddRenderedMapIndexField), Version("2025-04-11"), ) diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2025_04_28.py b/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2025_04_28.py new file mode 100644 index 0000000000000..e0916b4c93d67 --- /dev/null +++ b/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2025_04_28.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from cadwyn import VersionChange, schema + +from airflow.api_fastapi.execution_api.datamodels.taskinstance import ( + TIDeferredStatePayload, + TIRetryStatePayload, + TISuccessStatePayload, + TITerminalStatePayload, +) + + +class AddRenderedMapIndexField(VersionChange): + """Add the `rendered_map_index` field to payload models.""" + + description = __doc__ + + instructions_to_migrate_to_previous_version = ( + schema(TITerminalStatePayload).field("rendered_map_index").didnt_exist, + schema(TISuccessStatePayload).field("rendered_map_index").didnt_exist, + schema(TIDeferredStatePayload).field("rendered_map_index").didnt_exist, + schema(TIRetryStatePayload).field("rendered_map_index").didnt_exist, + ) diff --git a/airflow-core/src/airflow/cli/cli_config.py b/airflow-core/src/airflow/cli/cli_config.py index 1122033dc94cd..4300d8c446839 100644 --- a/airflow-core/src/airflow/cli/cli_config.py +++ b/airflow-core/src/airflow/cli/cli_config.py @@ -1590,6 +1590,12 @@ class GroupCommand(NamedTuple): func=lazy_load_command("airflow.cli.commands.provider_command.executors_list"), args=(ARG_OUTPUT, ARG_VERBOSE), ), + ActionCommand( + name="queues", + help="Get information about queues provided", + func=lazy_load_command("airflow.cli.commands.provider_command.queues_list"), + args=(ARG_OUTPUT, ARG_VERBOSE), + ), ActionCommand( name="notifications", help="Get information about notifications provided", diff --git a/airflow-core/src/airflow/cli/commands/config_command.py b/airflow-core/src/airflow/cli/commands/config_command.py index cdf07f9c2c20f..0f5c5df5ba157 100644 --- a/airflow-core/src/airflow/cli/commands/config_command.py +++ b/airflow-core/src/airflow/cli/commands/config_command.py @@ -467,27 +467,37 @@ def message(self) -> str | None: ), ConfigChange( config=ConfigParameter("webserver", "enable_proxy_fix"), - was_deprecated=False, + renamed_to=ConfigParameter("fab", "enable_proxy_fix"), + breaking=True, ), ConfigChange( config=ConfigParameter("webserver", "proxy_fix_x_for"), - was_deprecated=False, + renamed_to=ConfigParameter("fab", "proxy_fix_x_for"), + breaking=True, ), ConfigChange( config=ConfigParameter("webserver", "proxy_fix_x_proto"), - was_deprecated=False, + renamed_to=ConfigParameter("fab", "proxy_fix_x_proto"), + breaking=True, ), ConfigChange( config=ConfigParameter("webserver", "proxy_fix_x_host"), - was_deprecated=False, + renamed_to=ConfigParameter("fab", "proxy_fix_x_host"), + breaking=True, ), ConfigChange( config=ConfigParameter("webserver", "proxy_fix_x_port"), - was_deprecated=False, + renamed_to=ConfigParameter("fab", "proxy_fix_x_port"), + breaking=True, ), ConfigChange( config=ConfigParameter("webserver", "proxy_fix_x_prefix"), - was_deprecated=False, + renamed_to=ConfigParameter("fab", "proxy_fix_x_prefix"), + breaking=True, + ), + ConfigChange( + config=ConfigParameter("webserver", "expose_config"), + renamed_to=ConfigParameter("api", "expose_config"), ), ConfigChange( config=ConfigParameter("webserver", "cookie_secure"), diff --git a/airflow-core/src/airflow/cli/commands/provider_command.py b/airflow-core/src/airflow/cli/commands/provider_command.py index bd03d07ee45b2..81a96b2a9bc24 100644 --- a/airflow-core/src/airflow/cli/commands/provider_command.py +++ b/airflow-core/src/airflow/cli/commands/provider_command.py @@ -220,6 +220,19 @@ def executors_list(args): ) +@suppress_logs_and_warning +@providers_configuration_loaded +def queues_list(args): + """List all queues at the command line.""" + AirflowConsole().print_as( + data=list(ProvidersManager().queue_class_names), + output=args.output, + mapper=lambda x: { + "queue_class_names": x, + }, + ) + + @suppress_logs_and_warning @providers_configuration_loaded def config_list(args): diff --git a/airflow-core/src/airflow/config_templates/airflow_local_settings.py b/airflow-core/src/airflow/config_templates/airflow_local_settings.py index 319cdc972593c..3cade5678261c 100644 --- a/airflow-core/src/airflow/config_templates/airflow_local_settings.py +++ b/airflow-core/src/airflow/config_templates/airflow_local_settings.py @@ -180,7 +180,7 @@ ) remote_task_handler_kwargs = {} elif remote_base_log_folder.startswith("gs://"): - from airflow.providers.google.cloud.logs.gcs_task_handler import GCSRemoteLogIO + from airflow.providers.google.cloud.log.gcs_task_handler import GCSRemoteLogIO key_path = conf.get_mandatory_value("logging", "google_key_path", fallback=None) diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index 616416183936e..50a849a099d5f 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -536,11 +536,13 @@ core: default: "4096" execution_api_server_url: description: | - The url of the execution api server. + The url of the execution api server. Default is ``{BASE_URL}/execution/`` + where ``{BASE_URL}`` is the base url of the API Server. If ``{BASE_URL}`` is not set, + it will use ``http://localhost:8080`` as the default base url. version_added: 3.0.0 type: string example: ~ - default: "http://localhost:8080/execution/" + default: ~ database: description: ~ options: @@ -1316,6 +1318,15 @@ secrets: api: description: ~ options: + expose_config: + description: | + Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values + except those that have security implications. ``True`` shows all values. ``False`` hides the + configuration completely. + version_added: ~ + type: string + example: ~ + default: "False" base_url: description: | The base url of the API server. Airflow cannot guess what domain or CNAME you are using. @@ -1737,15 +1748,6 @@ webserver: sensitive: true example: ~ default: "{SECRET_KEY}" - expose_config: - description: | - Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values - except those that have security implications. ``True`` shows all values. ``False`` hides the - configuration completely. - version_added: ~ - type: string - example: ~ - default: "False" expose_hostname: description: | Expose hostname in the web server diff --git a/airflow-core/src/airflow/configuration.py b/airflow-core/src/airflow/configuration.py index c48377c333ccb..ff214b3138ad7 100644 --- a/airflow-core/src/airflow/configuration.py +++ b/airflow-core/src/airflow/configuration.py @@ -355,6 +355,7 @@ def sensitive_config_values(self) -> set[tuple[str, str]]: ("api", "ssl_key"): ("webserver", "web_server_ssl_key", "3.0"), ("api", "access_logfile"): ("webserver", "access_logfile", "3.0"), ("triggerer", "capacity"): ("triggerer", "default_capacity", "3.0"), + ("api", "expose_config"): ("webserver", "expose_config", "3.0.1"), } # A mapping of new section -> (old section, since_version). @@ -1294,7 +1295,7 @@ def gettimedelta( def read( self, - filenames: (str | bytes | os.PathLike | Iterable[str | bytes | os.PathLike]), + filenames: str | bytes | os.PathLike | Iterable[str | bytes | os.PathLike], encoding=None, ): super().read(filenames=filenames, encoding=encoding) diff --git a/airflow-core/src/airflow/dag_processing/bundles/manager.py b/airflow-core/src/airflow/dag_processing/bundles/manager.py index 18e855230bce0..1ace2896028c6 100644 --- a/airflow-core/src/airflow/dag_processing/bundles/manager.py +++ b/airflow-core/src/airflow/dag_processing/bundles/manager.py @@ -16,10 +16,12 @@ # under the License. from __future__ import annotations +import os from typing import TYPE_CHECKING from airflow.configuration import conf from airflow.exceptions import AirflowConfigException +from airflow.models.dag_version import DagVersion from airflow.models.dagbundle import DagBundleModel from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.module_loading import import_string @@ -33,6 +35,7 @@ from airflow.dag_processing.bundles.base import BaseDagBundle _example_dag_bundle_name = "example_dags" +_example_standard_dag_bundle_name = "example_standard_dags" def _bundle_item_exc(msg): @@ -79,6 +82,25 @@ def _add_example_dag_bundle(config_list): ) +def _add_example_standard_dag_bundle(config_list): + # TODO(potiuk): make it more generic - for now we only add standard example_dags if they are locally available + try: + from system import standard + except ImportError: + return + + example_dag_folder = next(iter(standard.__path__)) + config_list.append( + { + "name": _example_standard_dag_bundle_name, + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": { + "path": example_dag_folder, + }, + } + ) + + class DagBundlesManager(LoggingMixin): """Manager for DAG bundles.""" @@ -111,6 +133,11 @@ def parse_config(self) -> None: _validate_bundle_config(config_list) if conf.getboolean("core", "LOAD_EXAMPLES"): _add_example_dag_bundle(config_list) + if ( + os.environ.get("BREEZE", "").lower() == "true" + or os.environ.get("_IN_UNIT_TESTS", "").lower() == "true" + ): + _add_example_standard_dag_bundle(config_list) for cfg in config_list: name = cfg["name"] @@ -123,17 +150,36 @@ def parse_config(self) -> None: def sync_bundles_to_db(self, *, session: Session = NEW_SESSION) -> None: self.log.debug("Syncing DAG bundles to the database") stored = {b.name: b for b in session.query(DagBundleModel).all()} - for name in self._bundle_config.keys(): + active_bundle_names = set(self._bundle_config.keys()) + for name in active_bundle_names: if bundle := stored.pop(name, None): bundle.active = True else: session.add(DagBundleModel(name=name)) self.log.info("Added new DAG bundle %s to the database", name) - + inactive_bundle_names = [] for name, bundle in stored.items(): bundle.active = False + inactive_bundle_names.append(name) self.log.warning("DAG bundle %s is no longer found in config and has been disabled", name) + if inactive_bundle_names and active_bundle_names: + new_bundle_name = sorted(active_bundle_names)[0] + updated_rows = ( + session.query(DagVersion) + .filter(DagVersion.bundle_name.in_(inactive_bundle_names)) + .update( + {DagVersion.bundle_name: new_bundle_name}, + synchronize_session=False, + ) + ) + + self.log.info( + "Updated %d DAG versions from inactive bundles to active bundle %s", + updated_rows, + new_bundle_name, + ) + def get_bundle(self, name: str, version: str | None = None) -> BaseDagBundle: """ Get a DAG bundle by name. diff --git a/airflow-core/src/airflow/dag_processing/collection.py b/airflow-core/src/airflow/dag_processing/collection.py index 591ad687f9953..bfc2cb20662dd 100644 --- a/airflow-core/src/airflow/dag_processing/collection.py +++ b/airflow-core/src/airflow/dag_processing/collection.py @@ -31,7 +31,7 @@ import traceback from typing import TYPE_CHECKING, NamedTuple -from sqlalchemy import delete, func, insert, select, tuple_ +from sqlalchemy import delete, func, insert, select, tuple_, update from sqlalchemy.exc import OperationalError from sqlalchemy.orm import joinedload, load_only @@ -296,9 +296,16 @@ def _update_import_errors( get_listener_manager().hook.on_new_dag_import_error(filename=filename, stacktrace=stacktrace) except Exception: log.exception("error calling listener") - session.query(DagModel).filter( - DagModel.fileloc == filename, DagModel.bundle_name == bundle_name - ).update({"has_import_errors": True}) + session.execute( + update(DagModel) + .where(DagModel.fileloc == filename) + .values( + has_import_errors=True, + bundle_name=bundle_name, + is_stale=True, + ) + .execution_options(synchronize_session="fetch") + ) def update_dag_parsing_results_in_db( diff --git a/airflow-core/src/airflow/dag_processing/manager.py b/airflow-core/src/airflow/dag_processing/manager.py index b8ddf688dcb2a..324813cec3828 100644 --- a/airflow-core/src/airflow/dag_processing/manager.py +++ b/airflow-core/src/airflow/dag_processing/manager.py @@ -992,7 +992,7 @@ def prepare_file_queue(self, known_files: dict[str, set[DagFileInfo]]): def _kill_timed_out_processors(self): """Kill any file processors that timeout to defend against process hangs.""" - now = time.time() + now = time.monotonic() processors_to_remove = [] for file, processor in self._processors.items(): duration = now - processor.start_time diff --git a/airflow-core/src/airflow/dag_processing/processor.py b/airflow-core/src/airflow/dag_processing/processor.py index 25f9e2a73ed87..ef3e5cb69ca56 100644 --- a/airflow-core/src/airflow/dag_processing/processor.py +++ b/airflow-core/src/airflow/dag_processing/processor.py @@ -312,9 +312,5 @@ def is_ready(self) -> bool: return self._num_open_sockets == 0 - @property - def start_time(self) -> float: - return self._process.create_time() - def wait(self) -> int: raise NotImplementedError(f"Don't call wait on {type(self).__name__} objects") diff --git a/airflow-core/src/airflow/example_dags/example_params_ui_tutorial.py b/airflow-core/src/airflow/example_dags/example_params_ui_tutorial.py index da6b4c585b1d3..40e4667d35b6f 100644 --- a/airflow-core/src/airflow/example_dags/example_params_ui_tutorial.py +++ b/airflow-core/src/airflow/example_dags/example_params_ui_tutorial.py @@ -58,7 +58,7 @@ type="integer", title="Your favorite number", description_md="Everybody should have a **favorite** number. Not only _math teachers_. " - "If you can not think of any at the moment please think of the 42 which is very famous because" + "If you can not think of any at the moment please think of the 42 which is very famous because " "of the book [The Hitchhiker's Guide to the Galaxy]" "(https://en.wikipedia.org/wiki/Phrases_from_The_Hitchhiker%27s_Guide_to_the_Galaxy#" "The_Answer_to_the_Ultimate_Question_of_Life,_the_Universe,_and_Everything_is_42).", @@ -99,7 +99,7 @@ format="date", title="Date Picker", description="Please select a date, use the button on the left for a pop-up calendar. " - "See that here are no times!", + "See that there are no times!", section="Typed parameters with Param object", ), "time": Param( @@ -181,7 +181,7 @@ # A multiple options selection can also be combined with values_display "multi_select_with_label": Param( ["2", "3"], - "Select from the list of options. See that options can have nicer text and still technical values" + "Select from the list of options. See that options can have nicer text and still technical values " "are propagated as values during trigger to the DAG.", type="array", title="Multi Select with Labels", diff --git a/airflow-core/src/airflow/executors/local_executor.py b/airflow-core/src/airflow/executors/local_executor.py index 87a8e71d3589d..4c8ca1e73cb93 100644 --- a/airflow-core/src/airflow/executors/local_executor.py +++ b/airflow-core/src/airflow/executors/local_executor.py @@ -108,6 +108,13 @@ def _execute_work(log: logging.Logger, workload: workloads.ExecuteTask) -> None: from airflow.sdk.execution_time.supervisor import supervise setproctitle(f"airflow worker -- LocalExecutor: {workload.ti.id}") + + base_url = conf.get("api", "base_url", fallback="/") + # If it's a relative URL, use localhost:8080 as the default + if base_url.startswith("/"): + base_url = f"http://localhost:8080{base_url}" + default_execution_api_server = f"{base_url.rstrip('/')}/execution/" + # This will return the exit code of the task process, but we don't care about that, just if the # _supervisor_ had an error reporting the state back (which will result in an exception.) supervise( @@ -116,7 +123,7 @@ def _execute_work(log: logging.Logger, workload: workloads.ExecuteTask) -> None: dag_rel_path=workload.dag_rel_path, bundle_info=workload.bundle_info, token=workload.token, - server=conf.get("core", "execution_api_server_url"), + server=conf.get("core", "execution_api_server_url", fallback=default_execution_api_server), log_path=workload.log_path, ) diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py b/airflow-core/src/airflow/jobs/triggerer_job_runner.py index 05ba3dd28adcf..5df6a03261523 100644 --- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py +++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py @@ -81,14 +81,6 @@ from airflow.sdk.types import RuntimeTaskInstanceProtocol as RuntimeTI from airflow.triggers.base import BaseTrigger -HANDLER_SUPPORTS_TRIGGERER = False -""" -If this value is true, root handler is configured to log individual trigger messages -visible in task logs. - -:meta private: -""" - logger = logging.getLogger(__name__) __all__ = [ @@ -438,6 +430,7 @@ def _handle_request(self, msg: ToTriggerSupervisor, log: FilteringBoundLogger) - elif isinstance(msg, GetTICount): resp = self.client.task_instances.get_count( dag_id=msg.dag_id, + map_index=msg.map_index, task_ids=msg.task_ids, task_group_id=msg.task_group_id, logical_dates=msg.logical_dates, @@ -448,6 +441,7 @@ def _handle_request(self, msg: ToTriggerSupervisor, log: FilteringBoundLogger) - elif isinstance(msg, GetTaskStates): run_id_task_state_map = self.client.task_instances.get_task_states( dag_id=msg.dag_id, + map_index=msg.map_index, task_ids=msg.task_ids, task_group_id=msg.task_group_id, logical_dates=msg.logical_dates, diff --git a/airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py b/airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py index 383319cd6d9e7..4387728020e77 100644 --- a/airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py +++ b/airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py @@ -28,7 +28,6 @@ from __future__ import annotations from alembic import op -from sqlalchemy import inspect # revision identifiers, used by Alembic. revision = "044f740568ec" @@ -38,54 +37,28 @@ airflow_version = "3.0.0" -def table_exists(table_name): - """Check if a table exists in the database.""" - inspector = inspect(op.get_bind()) - return table_name in inspector.get_table_names() - - -def constraint_exists(table_name, constraint_name): - """Check if a foreign key constraint exists on a table.""" - inspector = inspect(op.get_bind()) - foreign_keys = inspector.get_foreign_keys(table_name) - return any(fk["name"] == constraint_name for fk in foreign_keys) - - -def index_exists(table_name, index_name): - """Check if an index exists on a table.""" - inspector = inspect(op.get_bind()) - indexes = inspector.get_indexes(table_name) - return any(idx["name"] == index_name for idx in indexes) - - def upgrade(): """Apply Drop ab_user.id foreign key.""" - if constraint_exists("dag_run_note", "dag_run_note_user_fkey"): - with op.batch_alter_table("dag_run_note", schema=None) as batch_op: - batch_op.drop_constraint("dag_run_note_user_fkey", type_="foreignkey") - - if constraint_exists("task_instance_note", "task_instance_note_user_fkey"): - with op.batch_alter_table("task_instance_note", schema=None) as batch_op: - batch_op.drop_constraint("task_instance_note_user_fkey", type_="foreignkey") + with op.batch_alter_table("dag_run_note", schema=None) as batch_op: + batch_op.drop_constraint("dag_run_note_user_fkey", type_="foreignkey") + with op.batch_alter_table("task_instance_note", schema=None) as batch_op: + batch_op.drop_constraint("task_instance_note_user_fkey", type_="foreignkey") if op.get_bind().dialect.name == "mysql": - if index_exists("dag_run_note", "dag_run_note_user_fkey"): - with op.batch_alter_table("dag_run_note", schema=None) as batch_op: - batch_op.drop_index("dag_run_note_user_fkey") + with op.batch_alter_table("dag_run_note", schema=None) as batch_op: + batch_op.drop_index("dag_run_note_user_fkey") - if index_exists("task_instance_note", "task_instance_note_user_fkey"): - with op.batch_alter_table("task_instance_note", schema=None) as batch_op: - batch_op.drop_index("task_instance_note_user_fkey") + with op.batch_alter_table("task_instance_note", schema=None) as batch_op: + batch_op.drop_index("task_instance_note_user_fkey") def downgrade(): """Unapply Drop ab_user.id foreign key.""" - if table_exists("ab_user"): - with op.batch_alter_table("task_instance_note", schema=None) as batch_op: - batch_op.create_foreign_key("task_instance_note_user_fkey", "ab_user", ["user_id"], ["id"]) + with op.batch_alter_table("task_instance_note", schema=None) as batch_op: + batch_op.create_foreign_key("task_instance_note_user_fkey", "ab_user", ["user_id"], ["id"]) - with op.batch_alter_table("dag_run_note", schema=None) as batch_op: - batch_op.create_foreign_key("dag_run_note_user_fkey", "ab_user", ["user_id"], ["id"]) + with op.batch_alter_table("dag_run_note", schema=None) as batch_op: + batch_op.create_foreign_key("dag_run_note_user_fkey", "ab_user", ["user_id"], ["id"]) if op.get_bind().dialect.name == "mysql": with op.batch_alter_table("task_instance_note", schema=None) as batch_op: diff --git a/airflow-core/src/airflow/migrations/versions/0030_3_0_0_rename_schedule_interval_to_timetable_.py b/airflow-core/src/airflow/migrations/versions/0030_3_0_0_rename_schedule_interval_to_timetable_.py index b4434a65f3f48..16f782933100d 100644 --- a/airflow-core/src/airflow/migrations/versions/0030_3_0_0_rename_schedule_interval_to_timetable_.py +++ b/airflow-core/src/airflow/migrations/versions/0030_3_0_0_rename_schedule_interval_to_timetable_.py @@ -58,3 +58,4 @@ def downgrade(): type_=sa.Text, nullable=True, ) + op.execute("UPDATE dag SET schedule_interval=NULL;") diff --git a/airflow-core/src/airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py b/airflow-core/src/airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py index bd0a8ea725f89..a4d4238816a2c 100644 --- a/airflow-core/src/airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py +++ b/airflow-core/src/airflow/migrations/versions/0047_3_0_0_add_dag_versioning.py @@ -30,11 +30,9 @@ import sqlalchemy as sa from alembic import op from sqlalchemy_utils import UUIDType -from uuid6 import uuid7 from airflow.migrations.db_types import TIMESTAMP, StringID from airflow.models.base import naming_convention -from airflow.models.dagcode import DagCode from airflow.utils import timezone # revision identifiers, used by Alembic. @@ -45,27 +43,11 @@ airflow_version = "3.0.0" -def _get_rows(sql, conn): - stmt = sa.text(sql) - rows = conn.execute(stmt) - if rows: - rows = rows.fetchall() - else: - rows = [] - return rows - - -def _airflow_2_fileloc_hash(fileloc): - import hashlib - import struct - - # Only 7 bytes because MySQL BigInteger can hold only 8 bytes (signed). - return struct.unpack(">Q", hashlib.sha1(fileloc.encode("utf-8")).digest()[-8:])[0] >> 8 - - def upgrade(): """Apply add dag versioning.""" - conn = op.get_bind() + op.execute("delete from dag_code;") + op.execute("delete from serialized_dag;") + op.create_table( "dag_version", sa.Column("id", UUIDType(binary=False), nullable=False), @@ -81,165 +63,15 @@ def upgrade(): sa.PrimaryKeyConstraint("id", name=op.f("dag_version_pkey")), sa.UniqueConstraint("dag_id", "version_number", name="dag_id_v_name_v_number_unique_constraint"), ) - with op.batch_alter_table( - "dag_code", - ) as batch_op: - batch_op.drop_constraint("dag_code_pkey", type_="primary") - batch_op.add_column(sa.Column("id", UUIDType(binary=False))) - batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False))) - batch_op.add_column(sa.Column("source_code_hash", sa.String(length=32))) - batch_op.add_column(sa.Column("dag_id", StringID())) - batch_op.add_column(sa.Column("created_at", TIMESTAMP(), default=timezone.utcnow)) - - with op.batch_alter_table( - "serialized_dag", - ) as batch_op: - batch_op.add_column(sa.Column("id", UUIDType(binary=False))) - batch_op.drop_index("idx_fileloc_hash") - batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False))) - batch_op.add_column(sa.Column("created_at", TIMESTAMP(), default=timezone.utcnow)) - - # Data migration - rows = _get_rows("SELECT dag_id FROM serialized_dag", conn) - - stmt = sa.text(""" - UPDATE serialized_dag - SET id = :_id - WHERE dag_id = :dag_id AND id IS NULL - """) - - for row in rows: - id = uuid7() - if conn.dialect.name != "postgresql": - id = id.hex - else: - id = str(id) - - conn.execute(stmt.bindparams(_id=id, dag_id=row.dag_id)) - id2 = uuid7() - if conn.dialect.name != "postgresql": - id2 = id2.hex - else: - id2 = str(id2) - # Update dagversion table - conn.execute( - sa.text(""" - INSERT INTO dag_version (id, version_number, dag_id, created_at, last_updated) - VALUES (:id, 1, :dag_id, :created_at, :last_updated) - """).bindparams( - id=id2, dag_id=row.dag_id, created_at=timezone.utcnow(), last_updated=timezone.utcnow() - ) - ) - - # Update serialized_dag table with dag_version_id where dag_id matches - if conn.dialect.name == "mysql": - conn.execute( - sa.text(""" - UPDATE serialized_dag sd - JOIN dag_version dv ON sd.dag_id = dv.dag_id - SET sd.dag_version_id = dv.id, - sd.created_at = dv.created_at - """) - ) - else: - conn.execute( - sa.text(""" - UPDATE serialized_dag - SET dag_version_id = dag_version.id, - created_at = dag_version.created_at - FROM dag_version - WHERE serialized_dag.dag_id = dag_version.dag_id - """) - ) - # Update dag_code table where fileloc_hash of serialized_dag matches - if conn.dialect.name == "mysql": - conn.execute( - sa.text(""" - UPDATE dag_code dc - JOIN serialized_dag sd ON dc.fileloc_hash = sd.fileloc_hash - SET dc.dag_version_id = sd.dag_version_id, - dc.created_at = sd.created_at, - dc.dag_id = sd.dag_id - """) - ) - else: - conn.execute( - sa.text(""" - UPDATE dag_code - SET dag_version_id = dag_version.id, - created_at = serialized_dag.created_at, - dag_id = serialized_dag.dag_id - FROM serialized_dag, dag_version - WHERE dag_code.fileloc_hash = serialized_dag.fileloc_hash - AND serialized_dag.dag_version_id = dag_version.id - """) - ) - - # select all rows in serialized_dag where the dag_id is not in dag_code - - stmt = """ - SELECT dag_id, fileloc, fileloc_hash, dag_version_id - FROM serialized_dag - WHERE dag_id NOT IN (SELECT dag_id FROM dag_code) - AND dag_id in (SELECT dag_id FROM dag) - """ - rows = _get_rows(stmt, conn) - # Insert the missing rows from serialized_dag to dag_code - stmt = sa.text(""" - INSERT INTO dag_code (dag_version_id, dag_id, fileloc, fileloc_hash, source_code, last_updated, created_at) - VALUES (:dag_version_id, :dag_id, :fileloc, :fileloc_hash, :source_code, :last_updated, :created_at) - """) - for row in rows: - try: - source_code = DagCode.get_code_from_file(row.fileloc) - except FileNotFoundError: - source_code = "source_code" - conn.execute( - stmt.bindparams( - dag_version_id=row.dag_version_id, - dag_id=row.dag_id, - fileloc=row.fileloc, - fileloc_hash=row.fileloc_hash, - source_code=source_code, - last_updated=timezone.utcnow(), - created_at=timezone.utcnow(), - ) - ) - - stmt = "SELECT dag_id, fileloc FROM dag_code" - rows = _get_rows(stmt, conn) - stmt = sa.text(""" - UPDATE dag_code - SET id = :_id, - dag_id = :dag_id, - source_code = :source_code, - source_code_hash = :source_code_hash - WHERE dag_id = :dag_id AND id IS NULL - """) - for row in rows: - id = uuid7() - if conn.dialect.name != "postgresql": - id = id.hex - else: - id = str(id) - try: - source_code = DagCode.get_code_from_file(row.fileloc) - except FileNotFoundError: - source_code = "source_code" - conn.execute( - stmt.bindparams( - _id=id, - source_code_hash=DagCode.dag_source_hash(source_code), - source_code=source_code, - dag_id=row.dag_id, - ) - ) - with op.batch_alter_table("dag_code") as batch_op: - batch_op.alter_column("dag_id", existing_type=StringID(), nullable=False) - batch_op.alter_column("id", existing_type=UUIDType(binary=False), nullable=False) + batch_op.drop_constraint("dag_code_pkey", type_="primary") + batch_op.drop_column("fileloc_hash") + batch_op.add_column(sa.Column("id", UUIDType(binary=False), nullable=False)) batch_op.create_primary_key("dag_code_pkey", ["id"]) - batch_op.alter_column("dag_version_id", existing_type=UUIDType(binary=False), nullable=False) + batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False), nullable=False)) + batch_op.add_column(sa.Column("source_code_hash", sa.String(length=32), nullable=False)) + batch_op.add_column(sa.Column("dag_id", StringID(), nullable=False)) + batch_op.add_column(sa.Column("created_at", TIMESTAMP(), default=timezone.utcnow, nullable=False)) batch_op.create_foreign_key( batch_op.f("dag_code_dag_version_id_fkey"), "dag_version", @@ -248,16 +80,15 @@ def upgrade(): ondelete="CASCADE", ) batch_op.create_unique_constraint("dag_code_dag_version_id_uq", ["dag_version_id"]) - batch_op.drop_column("fileloc_hash") - batch_op.alter_column("source_code_hash", existing_type=sa.String(length=32), nullable=False) - batch_op.alter_column("created_at", existing_type=TIMESTAMP(), nullable=False) with op.batch_alter_table("serialized_dag") as batch_op: batch_op.drop_constraint("serialized_dag_pkey", type_="primary") - batch_op.alter_column("id", existing_type=UUIDType(binary=False), nullable=False) - batch_op.alter_column("dag_version_id", existing_type=UUIDType(binary=False), nullable=False) + batch_op.drop_index("idx_fileloc_hash") batch_op.drop_column("fileloc_hash") batch_op.drop_column("fileloc") + batch_op.add_column(sa.Column("id", UUIDType(binary=False), nullable=False)) + batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False), nullable=False)) + batch_op.add_column(sa.Column("created_at", TIMESTAMP(), default=timezone.utcnow, nullable=False)) batch_op.create_primary_key("serialized_dag_pkey", ["id"]) batch_op.create_foreign_key( batch_op.f("serialized_dag_dag_version_id_fkey"), @@ -267,7 +98,6 @@ def upgrade(): ondelete="CASCADE", ) batch_op.create_unique_constraint("serialized_dag_dag_version_id_uq", ["dag_version_id"]) - batch_op.alter_column("created_at", existing_type=TIMESTAMP(), nullable=False) with op.batch_alter_table("task_instance", schema=None) as batch_op: batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False))) @@ -296,113 +126,40 @@ def upgrade(): def downgrade(): """Unapply add dag versioning.""" - conn = op.get_bind() + with op.batch_alter_table("task_instance", schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f("task_instance_dag_version_id_fkey"), type_="foreignkey") + batch_op.drop_column("dag_version_id") with op.batch_alter_table("task_instance_history", schema=None) as batch_op: batch_op.drop_column("dag_version_id") - with op.batch_alter_table("task_instance", schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f("task_instance_dag_version_id_fkey"), type_="foreignkey") - batch_op.drop_column("dag_version_id") + with op.batch_alter_table("dag_run", schema=None) as batch_op: + batch_op.add_column(sa.Column("dag_hash", sa.String(length=32), autoincrement=False, nullable=True)) + batch_op.drop_constraint("created_dag_version_id_fkey", type_="foreignkey") + batch_op.drop_column("created_dag_version_id") + + op.execute("delete from dag_code;") + op.execute("delete from serialized_dag;") with op.batch_alter_table("dag_code", schema=None) as batch_op: + batch_op.drop_constraint("dag_code_pkey", type_="primary") batch_op.drop_constraint(batch_op.f("dag_code_dag_version_id_fkey"), type_="foreignkey") batch_op.add_column(sa.Column("fileloc_hash", sa.BigInteger, nullable=True)) + batch_op.create_primary_key("dag_code_pkey", ["fileloc_hash"]) batch_op.drop_column("source_code_hash") batch_op.drop_column("created_at") - - # Update the added fileloc_hash with the hash of fileloc - stmt = "SELECT fileloc FROM dag_code" - rows = _get_rows(stmt, conn) - stmt = sa.text(""" - UPDATE dag_code - SET fileloc_hash = :_hash - where fileloc = :fileloc and fileloc_hash is null - """) - for row in rows: - hash = _airflow_2_fileloc_hash(row.fileloc) - conn.execute(stmt.bindparams(_hash=hash, fileloc=row.fileloc)) + batch_op.drop_column("id") + batch_op.drop_column("dag_version_id") + batch_op.drop_column("dag_id") with op.batch_alter_table("serialized_dag", schema=None, naming_convention=naming_convention) as batch_op: batch_op.drop_column("id") - batch_op.add_column(sa.Column("fileloc", sa.String(length=2000), nullable=True)) - batch_op.add_column(sa.Column("fileloc_hash", sa.BIGINT(), nullable=True)) - batch_op.drop_constraint(batch_op.f("serialized_dag_dag_version_id_fkey"), type_="foreignkey") batch_op.drop_column("created_at") - - # Update the serialized fileloc with fileloc from dag_code where dag_version_id matches - if conn.dialect.name == "mysql": - conn.execute( - sa.text(""" - UPDATE serialized_dag sd - JOIN dag_code dc ON sd.dag_version_id = dc.dag_version_id - SET sd.fileloc = dc.fileloc, - sd.fileloc_hash = dc.fileloc_hash - """) - ) - else: - conn.execute( - sa.text(""" - UPDATE serialized_dag - SET fileloc = dag_code.fileloc, - fileloc_hash = dag_code.fileloc_hash - FROM dag_code - WHERE serialized_dag.dag_version_id = dag_code.dag_version_id - """) - ) - # Deduplicate the rows in dag_code with the same fileloc_hash so we can make fileloc_hash the primary key - stmt = sa.text(""" - WITH ranked_rows AS ( - SELECT - fileloc_hash, - ROW_NUMBER() OVER (PARTITION BY fileloc_hash ORDER BY id) as row_num - FROM dag_code - ) - DELETE FROM dag_code - WHERE EXISTS ( - SELECT 1 - FROM ranked_rows - WHERE ranked_rows.fileloc_hash = dag_code.fileloc_hash - AND ranked_rows.row_num > 1 - ); - """) - conn.execute(stmt) - with op.batch_alter_table("serialized_dag") as batch_op: batch_op.drop_column("dag_version_id") + batch_op.add_column(sa.Column("fileloc", sa.String(length=2000), nullable=False)) + batch_op.add_column(sa.Column("fileloc_hash", sa.BIGINT(), nullable=False)) batch_op.create_index("idx_fileloc_hash", ["fileloc_hash"], unique=False) batch_op.create_primary_key("serialized_dag_pkey", ["dag_id"]) - batch_op.alter_column("fileloc", existing_type=sa.String(length=2000), nullable=False) - batch_op.alter_column("fileloc_hash", existing_type=sa.BIGINT(), nullable=False) - - with op.batch_alter_table("dag_code") as batch_op: - batch_op.drop_column("id") - batch_op.create_primary_key("dag_code_pkey", ["fileloc_hash"]) - batch_op.drop_column("dag_version_id") - batch_op.drop_column("dag_id") - - with op.batch_alter_table("dag_run", schema=None) as batch_op: - batch_op.add_column(sa.Column("dag_hash", sa.String(length=32), autoincrement=False, nullable=True)) - batch_op.drop_constraint("created_dag_version_id_fkey", type_="foreignkey") - batch_op.drop_column("created_dag_version_id") - - # Update dag_run dag_hash with dag_hash from serialized_dag where dag_id matches - if conn.dialect.name == "mysql": - conn.execute( - sa.text(""" - UPDATE dag_run dr - JOIN serialized_dag sd ON dr.dag_id = sd.dag_id - SET dr.dag_hash = sd.dag_hash - """) - ) - else: - conn.execute( - sa.text(""" - UPDATE dag_run - SET dag_hash = serialized_dag.dag_hash - FROM serialized_dag - WHERE dag_run.dag_id = serialized_dag.dag_id - """) - ) op.drop_table("dag_version") diff --git a/airflow-core/src/airflow/migrations/versions/0059_3_0_0_remove_external_trigger_field.py b/airflow-core/src/airflow/migrations/versions/0059_3_0_0_remove_external_trigger_field.py index f559cf9f8d8f8..46d3cd9ca57e4 100644 --- a/airflow-core/src/airflow/migrations/versions/0059_3_0_0_remove_external_trigger_field.py +++ b/airflow-core/src/airflow/migrations/versions/0059_3_0_0_remove_external_trigger_field.py @@ -53,6 +53,6 @@ def downgrade(): ) op.execute( dag_run_table.update().values( - external_trigger=sa.case([(dag_run_table.c.run_type == "manual", True)], else_=False) + external_trigger=sa.case((dag_run_table.c.run_type == "manual", True), else_=False) ) ) diff --git a/airflow-core/src/airflow/models/baseoperator.py b/airflow-core/src/airflow/models/baseoperator.py index 4c766007142c2..9b57d3d5251e1 100644 --- a/airflow-core/src/airflow/models/baseoperator.py +++ b/airflow-core/src/airflow/models/baseoperator.py @@ -621,7 +621,7 @@ def get_mapped_ti_count(cls, task: DAGNode, run_id: str, *, session: Session) -> raise NotImplementedError(f"Not implemented for {type(task)}") # https://github.com/python/cpython/issues/86153 - # WHile we support Python 3.9 we can't rely on the type hint, we need to pass the type explicitly to + # While we support Python 3.9 we can't rely on the type hint, we need to pass the type explicitly to # register. @get_mapped_ti_count.register(TaskSDKAbstractOperator) @classmethod diff --git a/airflow-core/src/airflow/models/dag.py b/airflow-core/src/airflow/models/dag.py index 13a93bd7de31f..676541e85785e 100644 --- a/airflow-core/src/airflow/models/dag.py +++ b/airflow-core/src/airflow/models/dag.py @@ -116,6 +116,7 @@ from airflow.utils.types import DagRunTriggeredByType, DagRunType if TYPE_CHECKING: + from pydantic import NonNegativeInt from sqlalchemy.orm.query import Query from sqlalchemy.orm.session import Session @@ -257,7 +258,7 @@ def _create_orm_dagrun( state: DagRunState | None, run_type: DagRunType, creating_job_id: int | None, - backfill_id: int | None, + backfill_id: NonNegativeInt | None, triggered_by: DagRunTriggeredByType, session: Session = NEW_SESSION, ) -> DagRun: @@ -1251,7 +1252,7 @@ def set_task_instance_state( # Clear downstream tasks that are in failed/upstream_failed state to resume them. # Flush the session so that the tasks marked success are reflected in the db. session.flush() - subdag = self.partial_subset( + subset = self.partial_subset( task_ids={task_id}, include_downstream=True, include_upstream=False, @@ -1273,9 +1274,9 @@ def set_task_instance_state( } if not future and not past: # Simple case 1: we're only dealing with exactly one run. clear_kwargs["run_id"] = run_id - subdag.clear(**clear_kwargs) + subset.clear(**clear_kwargs) elif future and past: # Simple case 2: we're clearing ALL runs. - subdag.clear(**clear_kwargs) + subset.clear(**clear_kwargs) else: # Complex cases: we may have more than one run, based on a date range. # Make 'future' and 'past' make some sense when multiple runs exist # for the same logical date. We order runs by their id and only @@ -1287,7 +1288,7 @@ def set_task_instance_state( else: clear_kwargs["end_date"] = logical_date exclude_run_id_stmt = exclude_run_id_stmt.where(DagRun.id < dr_id) - subdag.clear(exclude_run_ids=frozenset(session.scalars(exclude_run_id_stmt)), **clear_kwargs) + subset.clear(exclude_run_ids=frozenset(session.scalars(exclude_run_id_stmt)), **clear_kwargs) return altered @provide_session @@ -1363,13 +1364,13 @@ def get_logical_date() -> datetime: # Clear downstream tasks that are in failed/upstream_failed state to resume them. # Flush the session so that the tasks marked success are reflected in the db. session.flush() - task_subset = self.partial_subset( + subset = self.partial_subset( task_ids=task_ids, include_downstream=True, include_upstream=False, ) - task_subset.clear( + subset.clear( start_date=start_date, end_date=end_date, only_failed=True, @@ -1795,7 +1796,7 @@ def create_dagrun( state: DagRunState, start_date: datetime | None = None, creating_job_id: int | None = None, - backfill_id: int | None = None, + backfill_id: NonNegativeInt | None = None, session: Session = NEW_SESSION, ) -> DagRun: """ @@ -2041,16 +2042,7 @@ def from_sdk_dag(cls, dag: TaskSDKDag) -> DAG: if not field.init or field.name in ["edge_info"]: continue - value = getattr(dag, field.name) - - # Handle special cases where values need conversion - if field.name == "max_consecutive_failed_dag_runs": - # SchedulerDAG requires this to be >= 0, while TaskSDKDag allows -1 - if value == -1: - # If it is -1, we get the default value from the DAG - continue - - kwargs[field.name] = value + kwargs[field.name] = getattr(dag, field.name) new_dag = cls(**kwargs) diff --git a/airflow-core/src/airflow/models/dagbag.py b/airflow-core/src/airflow/models/dagbag.py index 393d01ce7a2c2..d0c8bf98f4152 100644 --- a/airflow-core/src/airflow/models/dagbag.py +++ b/airflow-core/src/airflow/models/dagbag.py @@ -235,23 +235,21 @@ def get_dag(self, dag_id, session: Session = None): # If asking for a known subdag, we want to refresh the parent dag = None - root_dag_id = dag_id if dag_id in self.dags: dag = self.dags[dag_id] # If DAG Model is absent, we can't check last_expired property. Is the DAG not yet synchronized? - orm_dag = DagModel.get_current(root_dag_id, session=session) + orm_dag = DagModel.get_current(dag_id, session=session) if not orm_dag: return self.dags.get(dag_id) - # If the dag corresponding to root_dag_id is absent or expired - is_missing = root_dag_id not in self.dags + is_missing = dag_id not in self.dags is_expired = ( orm_dag.last_expired and dag and dag.last_loaded and dag.last_loaded < orm_dag.last_expired ) if is_expired: # Remove associated dags so we can re-add them. - self.dags = {key: dag for key, dag in self.dags.items()} + self.dags.pop(dag_id, None) if is_missing or is_expired: # Reprocess source file. found_dags = self.process_file( @@ -505,8 +503,8 @@ def bag_dag(self, dag: DAG): """ Add the DAG into the bag. - :raises: AirflowDagCycleException if a cycle is detected in this dag or its subdags. - :raises: AirflowDagDuplicatedIdException if this dag or its subdags already exists in the bag. + :raises: AirflowDagCycleException if a cycle is detected. + :raises: AirflowDagDuplicatedIdException if this dag already exists in the bag. """ check_cycle(dag) # throws if a task cycle is found @@ -588,6 +586,14 @@ def collect_dags( example_dag_folder = next(iter(example_dags.__path__)) files_to_parse.extend(list_py_file_paths(example_dag_folder, safe_mode=safe_mode)) + try: + from system import standard + + example_dag_folder_standard = next(iter(standard.__path__)) + files_to_parse.extend(list_py_file_paths(example_dag_folder_standard, safe_mode=safe_mode)) + except ImportError: + # Nothing happens - this should only work during tests + pass for filepath in files_to_parse: try: diff --git a/airflow-core/src/airflow/models/dagrun.py b/airflow-core/src/airflow/models/dagrun.py index 929ad80f25429..138481643550e 100644 --- a/airflow-core/src/airflow/models/dagrun.py +++ b/airflow-core/src/airflow/models/dagrun.py @@ -93,6 +93,7 @@ from datetime import datetime from opentelemetry.sdk.trace import Span + from pydantic import NonNegativeInt from sqlalchemy.orm import Query, Session from airflow.models.baseoperator import BaseOperator @@ -290,7 +291,7 @@ def __init__( creating_job_id: int | None = None, data_interval: tuple[datetime, datetime] | None = None, triggered_by: DagRunTriggeredByType | None = None, - backfill_id: int | None = None, + backfill_id: NonNegativeInt | None = None, bundle_version: str | None = None, ): # For manual runs where logical_date is None, ensure no data_interval is set. diff --git a/airflow-core/src/airflow/models/mappedoperator.py b/airflow-core/src/airflow/models/mappedoperator.py index e6fb66962d1ee..5ecdf14f59051 100644 --- a/airflow-core/src/airflow/models/mappedoperator.py +++ b/airflow-core/src/airflow/models/mappedoperator.py @@ -17,11 +17,13 @@ # under the License. from __future__ import annotations -from typing import TYPE_CHECKING +from functools import cached_property +from typing import TYPE_CHECKING, Any import attrs import structlog +from airflow.exceptions import AirflowException from airflow.sdk.definitions.mappedoperator import MappedOperator as TaskSDKMappedOperator from airflow.triggers.base import StartTriggerArgs from airflow.utils.helpers import prevent_duplicates @@ -29,6 +31,8 @@ if TYPE_CHECKING: from sqlalchemy.orm.session import Session + from airflow.models import TaskInstance + from airflow.sdk import BaseOperatorLink from airflow.sdk.definitions.context import Context log = structlog.get_logger(__name__) @@ -118,3 +122,54 @@ def expand_start_trigger_args(self, *, context: Context, session: Session) -> St next_kwargs=next_kwargs, timeout=timeout, ) + + @cached_property + def operator_extra_link_dict(self) -> dict[str, BaseOperatorLink]: + """Returns dictionary of all extra links for the operator.""" + op_extra_links_from_plugin: dict[str, Any] = {} + from airflow import plugins_manager + + plugins_manager.initialize_extra_operators_links_plugins() + if plugins_manager.operator_extra_links is None: + raise AirflowException("Can't load operators") + operator_class_type = self.operator_class["task_type"] # type: ignore + for ope in plugins_manager.operator_extra_links: + if ope.operators and any(operator_class_type in cls.__name__ for cls in ope.operators): + op_extra_links_from_plugin.update({ope.name: ope}) + + operator_extra_links_all = {link.name: link for link in self.operator_extra_links} + # Extra links defined in Plugins overrides operator links defined in operator + operator_extra_links_all.update(op_extra_links_from_plugin) + + return operator_extra_links_all + + @cached_property + def global_operator_extra_link_dict(self) -> dict[str, Any]: + """Returns dictionary of all global extra links.""" + from airflow import plugins_manager + + plugins_manager.initialize_extra_operators_links_plugins() + if plugins_manager.global_operator_extra_links is None: + raise AirflowException("Can't load operators") + return {link.name: link for link in plugins_manager.global_operator_extra_links} + + @cached_property + def extra_links(self) -> list[str]: + return sorted(set(self.operator_extra_link_dict).union(self.global_operator_extra_link_dict)) + + def get_extra_links(self, ti: TaskInstance, name: str) -> str | None: + """ + For an operator, gets the URLs that the ``extra_links`` entry points to. + + :meta private: + + :raise ValueError: The error message of a ValueError will be passed on through to + the fronted to show up as a tooltip on the disabled link. + :param ti: The TaskInstance for the URL being searched for. + :param name: The name of the link we're looking for the URL for. Should be + one of the options specified in ``extra_links``. + """ + link = self.operator_extra_link_dict.get(name) or self.global_operator_extra_link_dict.get(name) + if not link: + return None + return link.get_link(self, ti_key=ti.key) # type: ignore[arg-type] diff --git a/airflow-core/src/airflow/models/taskinstance.py b/airflow-core/src/airflow/models/taskinstance.py index 5cf657a36f35f..52c1d7d5e8ea9 100644 --- a/airflow-core/src/airflow/models/taskinstance.py +++ b/airflow-core/src/airflow/models/taskinstance.py @@ -1052,8 +1052,7 @@ def refresh_from_db( """ query = select( # Select the columns, not the ORM object, to bypass any session/ORM caching layer - c - for c in TaskInstance.__table__.columns + *TaskInstance.__table__.columns ).filter_by( dag_id=self.dag_id, run_id=self.run_id, diff --git a/airflow-core/src/airflow/models/trigger.py b/airflow-core/src/airflow/models/trigger.py index 8bcff1d470e82..94d8360edf9e7 100644 --- a/airflow-core/src/airflow/models/trigger.py +++ b/airflow-core/src/airflow/models/trigger.py @@ -350,6 +350,7 @@ def get_sorted_triggers(cls, capacity: int, alive_triggerer_ids: list[int] | Sel """ query = with_row_locks( select(cls.id) + .prefix_with("STRAIGHT_JOIN", dialect="mysql") .join(TaskInstance, cls.id == TaskInstance.trigger_id, isouter=False) .where(or_(cls.triggerer_id.is_(None), cls.triggerer_id.not_in(alive_triggerer_ids))) .order_by(coalesce(TaskInstance.priority_weight, 0).desc(), cls.created_date) diff --git a/airflow-core/src/airflow/operators/__init__.py b/airflow-core/src/airflow/operators/__init__.py index df46ec18164aa..5e0b62cc344f3 100644 --- a/airflow-core/src/airflow/operators/__init__.py +++ b/airflow-core/src/airflow/operators/__init__.py @@ -35,6 +35,7 @@ "ExternalPythonOperator": "airflow.providers.standard.operators.python.ExternalPythonOperator", "BranchExternalPythonOperator": "airflow.providers.standard.operators.python.BranchExternalPythonOperator", "BranchPythonVirtualenvOperator": "airflow.providers.standard.operators.python.BranchPythonVirtualenvOperator", + "get_current_context": "airflow.sdk.get_current_context", }, "bash":{ "BashOperator": "airflow.providers.standard.operators.bash.BashOperator", diff --git a/airflow-core/src/airflow/provider.yaml.schema.json b/airflow-core/src/airflow/provider.yaml.schema.json index 75ba892569b4e..c35e0d9de25e7 100644 --- a/airflow-core/src/airflow/provider.yaml.schema.json +++ b/airflow-core/src/airflow/provider.yaml.schema.json @@ -467,6 +467,18 @@ } } }, + "queues": { + "type": "array", + "description": "Message Queues exposed by the provider", + "items": { + "name": { + "type": "string" + }, + "message-queue-class": { + "type": "string" + } + } + }, "source-date-epoch": { "type": "integer", "description": "Source date epoch - seconds since epoch (gmtime) when the release documentation was prepared. Used to generate reproducible package builds with flint.", diff --git a/airflow-core/src/airflow/provider_info.schema.json b/airflow-core/src/airflow/provider_info.schema.json index 1785ba02ed623..3ca9756dfb2f6 100644 --- a/airflow-core/src/airflow/provider_info.schema.json +++ b/airflow-core/src/airflow/provider_info.schema.json @@ -416,6 +416,18 @@ "description": "Class to instantiate the plugin" } } + }, + "queues": { + "type": "array", + "description": "Message Queues exposed by the provider", + "items": { + "name": { + "type": "string" + }, + "message-queue-class": { + "type": "string" + } + } } }, "definitions": { diff --git a/airflow-core/src/airflow/providers_manager.py b/airflow-core/src/airflow/providers_manager.py index 85062e9f75e63..81611c8205df7 100644 --- a/airflow-core/src/airflow/providers_manager.py +++ b/airflow-core/src/airflow/providers_manager.py @@ -416,6 +416,7 @@ def __init__(self): self._auth_manager_class_name_set: set[str] = set() self._secrets_backend_class_name_set: set[str] = set() self._executor_class_name_set: set[str] = set() + self._queue_class_name_set: set[str] = set() self._provider_configs: dict[str, dict[str, Any]] = {} self._trigger_info_set: set[TriggerInfo] = set() self._notification_info_set: set[NotificationInfo] = set() @@ -533,6 +534,12 @@ def initialize_providers_executors(self): self.initialize_providers_list() self._discover_executors() + @provider_info_cache("queues") + def initialize_providers_queues(self): + """Lazy initialization of providers queue information.""" + self.initialize_providers_list() + self._discover_queues() + @provider_info_cache("notifications") def initialize_providers_notifications(self): """Lazy initialization of providers notifications information.""" @@ -1091,6 +1098,14 @@ def _discover_executors(self) -> None: if _correctness_check(provider_package, executors_class_name, provider): self._executor_class_name_set.add(executors_class_name) + def _discover_queues(self) -> None: + """Retrieve all queues defined in the providers.""" + for provider_package, provider in self._provider_dict.items(): + if provider.data.get("queues"): + for queue_class_name in provider.data["queues"]: + if _correctness_check(provider_package, queue_class_name, provider): + self._queue_class_name_set.add(queue_class_name) + def _discover_config(self) -> None: """Retrieve all configs defined in the providers.""" for provider_package, provider in self._provider_dict.items(): @@ -1221,6 +1236,11 @@ def executor_class_names(self) -> list[str]: self.initialize_providers_executors() return sorted(self._executor_class_name_set) + @property + def queue_class_names(self) -> list[str]: + self.initialize_providers_queues() + return sorted(self._queue_class_name_set) + @property def filesystem_module_names(self) -> list[str]: self.initialize_providers_filesystems() @@ -1268,9 +1288,11 @@ def _cleanup(self): self._auth_manager_class_name_set.clear() self._secrets_backend_class_name_set.clear() self._executor_class_name_set.clear() + self._queue_class_name_set.clear() self._provider_configs.clear() self._trigger_info_set.clear() self._notification_info_set.clear() self._plugins_set.clear() + self._initialized = False self._initialization_stack_trace = None diff --git a/airflow-core/src/airflow/security/permissions.py b/airflow-core/src/airflow/security/permissions.py index 647bcf0b0c6e3..7545fe9d9305e 100644 --- a/airflow-core/src/airflow/security/permissions.py +++ b/airflow-core/src/airflow/security/permissions.py @@ -95,13 +95,7 @@ class ResourceDetails(TypedDict): def resource_name(root_dag_id: str, resource: str) -> str: - """ - Return the resource name for a DAG id. - - Note that since a sub-DAG should follow the permission of its - parent DAG, you should pass ``DagModel.root_dag_id`` to this function, - for a subdag. A normal dag should pass the ``DagModel.dag_id``. - """ + """Return the resource name for a DAG id.""" if root_dag_id in RESOURCE_DETAILS_MAP.keys(): return root_dag_id if root_dag_id.startswith(tuple(PREFIX_RESOURCES_MAP.keys())): @@ -113,10 +107,6 @@ def resource_name_for_dag(root_dag_id: str) -> str: """ Return the resource name for a DAG id. - Note that since a sub-DAG should follow the permission of its - parent DAG, you should pass ``DagModel.root_dag_id`` to this function, - for a subdag. A normal dag should pass the ``DagModel.dag_id``. - Note: This function is kept for backwards compatibility. """ if root_dag_id == RESOURCE_DAG: diff --git a/airflow-core/src/airflow/sensors/__init__.py b/airflow-core/src/airflow/sensors/__init__.py index f174fcb55ed36..62a4037df2f66 100644 --- a/airflow-core/src/airflow/sensors/__init__.py +++ b/airflow-core/src/airflow/sensors/__init__.py @@ -32,7 +32,7 @@ "PythonSensor": "airflow.providers.standard.sensors.python.PythonSensor", }, "bash":{ - "BashSensor": "airflow.providers.standard.sensor.bash.BashSensor", + "BashSensor": "airflow.providers.standard.sensors.bash.BashSensor", }, "date_time":{ "DateTimeSensor": "airflow.providers.standard.sensors.date_time.DateTimeSensor", diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py b/airflow-core/src/airflow/serialization/serialized_objects.py index 8428765e2be0c..948111b2c17e5 100644 --- a/airflow-core/src/airflow/serialization/serialized_objects.py +++ b/airflow-core/src/airflow/serialization/serialized_objects.py @@ -2098,7 +2098,11 @@ def timetable(self) -> Timetable: @property def has_task_concurrency_limits(self) -> bool: return any( - task[Encoding.VAR].get("max_active_tis_per_dag") is not None for task in self.data["dag"]["tasks"] + task[Encoding.VAR].get("max_active_tis_per_dag") is not None + or task[Encoding.VAR].get("max_active_tis_per_dagrun") is not None + or task[Encoding.VAR].get("partial_kwargs", {}).get("max_active_tis_per_dag") is not None + or task[Encoding.VAR].get("partial_kwargs", {}).get("max_active_tis_per_dagrun") is not None + for task in self.data["dag"]["tasks"] ) @property diff --git a/airflow-core/src/airflow/timetables/simple.py b/airflow-core/src/airflow/timetables/simple.py index 49dcda81a4f6f..0dfcd6060de51 100644 --- a/airflow-core/src/airflow/timetables/simple.py +++ b/airflow-core/src/airflow/timetables/simple.py @@ -135,14 +135,21 @@ def next_dagrun_info( ) -> DagRunInfo | None: if restriction.earliest is None: # No start date, won't run. return None + + current_time = timezone.coerce_datetime(timezone.utcnow()) + if last_automated_data_interval is not None: # has already run once - start = last_automated_data_interval.end - end = timezone.coerce_datetime(timezone.utcnow()) + if last_automated_data_interval.end > current_time: # start date is future + start = restriction.earliest + elapsed = last_automated_data_interval.end - last_automated_data_interval.start + + end = start + elapsed.as_timedelta() + else: + start = last_automated_data_interval.end + end = current_time else: # first run start = restriction.earliest - end = max( - restriction.earliest, timezone.coerce_datetime(timezone.utcnow()) - ) # won't run any earlier than start_date + end = max(restriction.earliest, current_time) if restriction.latest is not None and end > restriction.latest: return None diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index a3fcabb4c27f5..a59e98c888b5d 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -270,7 +270,7 @@ export const UseBackfillServiceGetBackfillKeyFn = ( { backfillId, }: { - backfillId: string; + backfillId: number; }, queryKey?: Array, ) => [useBackfillServiceGetBackfillKey, ...(queryKey ?? [{ backfillId }])]; @@ -1711,6 +1711,16 @@ export const UseDashboardServiceHistoricalMetricsKeyFn = ( }, queryKey?: Array, ) => [useDashboardServiceHistoricalMetricsKey, ...(queryKey ?? [{ endDate, startDate }])]; +export type DashboardServiceDagStatsDefaultResponse = Awaited>; +export type DashboardServiceDagStatsQueryResult< + TData = DashboardServiceDagStatsDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useDashboardServiceDagStatsKey = "DashboardServiceDagStats"; +export const UseDashboardServiceDagStatsKeyFn = (queryKey?: Array) => [ + useDashboardServiceDagStatsKey, + ...(queryKey ?? []), +]; export type StructureServiceStructureDataDefaultResponse = Awaited< ReturnType >; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index cd06c2bff30ea..c97c8ac5c66be 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -354,7 +354,7 @@ export const ensureUseBackfillServiceGetBackfillData = ( { backfillId, }: { - backfillId: string; + backfillId: number; }, ) => queryClient.ensureQueryData({ @@ -2381,6 +2381,17 @@ export const ensureUseDashboardServiceHistoricalMetricsData = ( queryKey: Common.UseDashboardServiceHistoricalMetricsKeyFn({ endDate, startDate }), queryFn: () => DashboardService.historicalMetrics({ endDate, startDate }), }); +/** + * Dag Stats + * Return basic DAG stats with counts of DAGs in various states. + * @returns DashboardDagStatsResponse Successful Response + * @throws ApiError + */ +export const ensureUseDashboardServiceDagStatsData = (queryClient: QueryClient) => + queryClient.ensureQueryData({ + queryKey: Common.UseDashboardServiceDagStatsKeyFn(), + queryFn: () => DashboardService.dagStats(), + }); /** * Structure Data * Get Structure Data. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index 14649a703b1d9..c9f715befa4e8 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -354,7 +354,7 @@ export const prefetchUseBackfillServiceGetBackfill = ( { backfillId, }: { - backfillId: string; + backfillId: number; }, ) => queryClient.prefetchQuery({ @@ -2381,6 +2381,17 @@ export const prefetchUseDashboardServiceHistoricalMetrics = ( queryKey: Common.UseDashboardServiceHistoricalMetricsKeyFn({ endDate, startDate }), queryFn: () => DashboardService.historicalMetrics({ endDate, startDate }), }); +/** + * Dag Stats + * Return basic DAG stats with counts of DAGs in various states. + * @returns DashboardDagStatsResponse Successful Response + * @throws ApiError + */ +export const prefetchUseDashboardServiceDagStats = (queryClient: QueryClient) => + queryClient.prefetchQuery({ + queryKey: Common.UseDashboardServiceDagStatsKeyFn(), + queryFn: () => DashboardService.dagStats(), + }); /** * Structure Data * Get Structure Data. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index ae20d5e3d762b..ae5dabdcd0d1b 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -446,7 +446,7 @@ export const useBackfillServiceGetBackfill = < { backfillId, }: { - backfillId: string; + backfillId: number; }, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">, @@ -2849,6 +2849,25 @@ export const useDashboardServiceHistoricalMetrics = < queryFn: () => DashboardService.historicalMetrics({ endDate, startDate }) as TData, ...options, }); +/** + * Dag Stats + * Return basic DAG stats with counts of DAGs in various states. + * @returns DashboardDagStatsResponse Successful Response + * @throws ApiError + */ +export const useDashboardServiceDagStats = < + TData = Common.DashboardServiceDagStatsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseDashboardServiceDagStatsKeyFn(queryKey), + queryFn: () => DashboardService.dagStats() as TData, + ...options, + }); /** * Structure Data * Get Structure Data. @@ -3580,7 +3599,7 @@ export const useBackfillServicePauseBackfill = < TData, TError, { - backfillId: unknown; + backfillId: number; }, TContext >, @@ -3591,7 +3610,7 @@ export const useBackfillServicePauseBackfill = < TData, TError, { - backfillId: unknown; + backfillId: number; }, TContext >({ @@ -3616,7 +3635,7 @@ export const useBackfillServiceUnpauseBackfill = < TData, TError, { - backfillId: unknown; + backfillId: number; }, TContext >, @@ -3627,7 +3646,7 @@ export const useBackfillServiceUnpauseBackfill = < TData, TError, { - backfillId: unknown; + backfillId: number; }, TContext >({ @@ -3652,7 +3671,7 @@ export const useBackfillServiceCancelBackfill = < TData, TError, { - backfillId: unknown; + backfillId: number; }, TContext >, @@ -3663,7 +3682,7 @@ export const useBackfillServiceCancelBackfill = < TData, TError, { - backfillId: unknown; + backfillId: number; }, TContext >({ diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index 0bb328cbb241f..00cd46c9d0e3b 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -423,7 +423,7 @@ export const useBackfillServiceGetBackfillSuspense = < { backfillId, }: { - backfillId: string; + backfillId: number; }, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">, @@ -2826,6 +2826,25 @@ export const useDashboardServiceHistoricalMetricsSuspense = < queryFn: () => DashboardService.historicalMetrics({ endDate, startDate }) as TData, ...options, }); +/** + * Dag Stats + * Return basic DAG stats with counts of DAGs in various states. + * @returns DashboardDagStatsResponse Successful Response + * @throws ApiError + */ +export const useDashboardServiceDagStatsSuspense = < + TData = Common.DashboardServiceDagStatsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseDashboardServiceDagStatsKeyFn(queryKey), + queryFn: () => DashboardService.dagStats() as TData, + ...options, + }); /** * Structure Data * Get Structure Data. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 7c9f8b2bf5fe2..179da51471303 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -427,6 +427,7 @@ export const $BackfillResponse = { properties: { id: { type: "integer", + minimum: 0, title: "Id", }, dag_id: { @@ -2254,6 +2255,17 @@ export const $DAGRunResponse = { type: "array", title: "Dag Versions", }, + bundle_version: { + anyOf: [ + { + type: "string", + }, + { + type: "null", + }, + ], + title: "Bundle Version", + }, }, type: "object", required: [ @@ -2273,6 +2285,7 @@ export const $DAGRunResponse = { "conf", "note", "dag_versions", + "bundle_version", ], title: "DAGRunResponse", description: "DAG Run serializer for responses.", @@ -3379,6 +3392,8 @@ export const $JobResponse = { description: "Job serializer for responses.", } as const; +export const $JsonValue = {} as const; + export const $PatchTaskInstanceBody = { properties: { new_state: { @@ -5373,8 +5388,7 @@ export const $VariableBody = { title: "Key", }, value: { - type: "string", - title: "Value", + $ref: "#/components/schemas/JsonValue", }, description: { anyOf: [ @@ -6302,6 +6316,31 @@ export const $DAGWithLatestDagRunsResponse = { description: "DAG with latest dag runs response serializer.", } as const; +export const $DashboardDagStatsResponse = { + properties: { + active_dag_count: { + type: "integer", + title: "Active Dag Count", + }, + failed_dag_count: { + type: "integer", + title: "Failed Dag Count", + }, + running_dag_count: { + type: "integer", + title: "Running Dag Count", + }, + queued_dag_count: { + type: "integer", + title: "Queued Dag Count", + }, + }, + type: "object", + required: ["active_dag_count", "failed_dag_count", "running_dag_count", "queued_dag_count"], + title: "DashboardDagStatsResponse", + description: "Dashboard DAG Stats serializer for responses.", +} as const; + export const $EdgeResponse = { properties: { source_id: { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index 385f8256e7fd9..bfd0077783ab7 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -211,6 +211,7 @@ import type { GetDependenciesResponse, HistoricalMetricsData, HistoricalMetricsResponse, + DagStatsResponse2, StructureDataData, StructureDataResponse2, GridDataData, @@ -3499,6 +3500,19 @@ export class DashboardService { }, }); } + + /** + * Dag Stats + * Return basic DAG stats with counts of DAGs in various states. + * @returns DashboardDagStatsResponse Successful Response + * @throws ApiError + */ + public static dagStats(): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/ui/dashboard/dag_stats", + }); + } } export class StructureService { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 18528b9da12d7..d34e86cdb55bb 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -577,6 +577,7 @@ export type DAGRunResponse = { }; note: string | null; dag_versions: Array; + bundle_version: string | null; }; /** @@ -889,6 +890,8 @@ export type JobResponse = { unixname: string | null; }; +export type JsonValue = unknown; + /** * Request body for Clear Task Instances endpoint. */ @@ -1296,7 +1299,7 @@ export type ValidationError = { */ export type VariableBody = { key: string; - value: string; + value: JsonValue; description?: string | null; }; @@ -1563,6 +1566,16 @@ export type DAGWithLatestDagRunsResponse = { readonly file_token: string; }; +/** + * Dashboard DAG Stats serializer for responses. + */ +export type DashboardDagStatsResponse = { + active_dag_count: number; + failed_dag_count: number; + running_dag_count: number; + queued_dag_count: number; +}; + /** * Edge serializer for responses. */ @@ -1857,25 +1870,25 @@ export type CreateBackfillData = { export type CreateBackfillResponse = BackfillResponse; export type GetBackfillData = { - backfillId: string; + backfillId: number; }; export type GetBackfillResponse = BackfillResponse; export type PauseBackfillData = { - backfillId: unknown; + backfillId: number; }; export type PauseBackfillResponse = BackfillResponse; export type UnpauseBackfillData = { - backfillId: unknown; + backfillId: number; }; export type UnpauseBackfillResponse = BackfillResponse; export type CancelBackfillData = { - backfillId: unknown; + backfillId: number; }; export type CancelBackfillResponse = BackfillResponse; @@ -2632,6 +2645,8 @@ export type HistoricalMetricsData = { export type HistoricalMetricsResponse = HistoricalMetricDataResponse; +export type DagStatsResponse2 = DashboardDagStatsResponse; + export type StructureDataData = { dagId: string; externalDependencies?: boolean; @@ -5445,6 +5460,16 @@ export type $OpenApiTs = { }; }; }; + "/ui/dashboard/dag_stats": { + get: { + res: { + /** + * Successful Response + */ + 200: DashboardDagStatsResponse; + }; + }; + }; "/ui/structure/structure_data": { get: { req: StructureDataData; diff --git a/airflow-core/src/airflow/ui/package.json b/airflow-core/src/airflow/ui/package.json index 7fe31b5842c41..ba22d9c745a5c 100644 --- a/airflow-core/src/airflow/ui/package.json +++ b/airflow-core/src/airflow/ui/package.json @@ -85,7 +85,7 @@ "prettier": "^3.5.3", "typescript": "~5.5.4", "typescript-eslint": "^8.27.0", - "vite": "^5.4.17", + "vite": "^5.4.19", "vite-plugin-css-injected-by-js": "^3.5.2", "vitest": "^2.1.9", "web-worker": "^1.5.0" diff --git a/airflow-core/src/airflow/ui/pnpm-lock.yaml b/airflow-core/src/airflow/ui/pnpm-lock.yaml index 6f37bae0fe70d..3b90be0585b1e 100644 --- a/airflow-core/src/airflow/ui/pnpm-lock.yaml +++ b/airflow-core/src/airflow/ui/pnpm-lock.yaml @@ -158,7 +158,7 @@ importers: version: 15.5.13 '@vitejs/plugin-react-swc': specifier: ^3.8.1 - version: 3.8.1(@swc/helpers@0.5.15)(vite@5.4.17(@types/node@22.13.11)) + version: 3.8.1(@swc/helpers@0.5.15)(vite@5.4.19(@types/node@22.13.11)) '@vitest/coverage-v8': specifier: ^2.1.9 version: 2.1.9(vitest@2.1.9(@types/node@22.13.11)(happy-dom@17.4.4)(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4))) @@ -211,11 +211,11 @@ importers: specifier: ^8.27.0 version: 8.27.0(eslint@9.23.0(jiti@1.21.7))(typescript@5.5.4) vite: - specifier: ^5.4.17 - version: 5.4.17(@types/node@22.13.11) + specifier: ^5.4.19 + version: 5.4.19(@types/node@22.13.11) vite-plugin-css-injected-by-js: specifier: ^3.5.2 - version: 3.5.2(vite@5.4.17(@types/node@22.13.11)) + version: 3.5.2(vite@5.4.19(@types/node@22.13.11)) vitest: specifier: ^2.1.9 version: 2.1.9(@types/node@22.13.11)(happy-dom@17.4.4)(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4)) @@ -256,6 +256,10 @@ packages: resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==} engines: {node: '>=6.9.0'} + '@babel/code-frame@7.27.1': + resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} + engines: {node: '>=6.9.0'} + '@babel/generator@7.17.7': resolution: {integrity: sha512-oLcVCTeIFadUoArDTwpluncplrYBmTCCZZgXCbgNGvOBBiSDDK3eWO4b/+eOTli5tKv1lg+a5/NAXg+nTcei1w==} engines: {node: '>=6.9.0'} @@ -292,6 +296,10 @@ packages: resolution: {integrity: sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==} engines: {node: '>=6.9.0'} + '@babel/helper-validator-identifier@7.27.1': + resolution: {integrity: sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==} + engines: {node: '>=6.9.0'} + '@babel/parser@7.26.10': resolution: {integrity: sha512-6aQR2zGE/QFi8JpDLjUZEPYOs7+mhKXm86VaKFiLP35JQwQb6bwUE+XbvkH0EptsYhbNBSUGaUBLKqxH1xSgsA==} engines: {node: '>=6.0.0'} @@ -767,103 +775,103 @@ packages: resolution: {integrity: sha512-O3rHJzAQKamUz1fvE0Qaw0xSFqsA/yafi2iqeE0pvdFtCO1viYx8QL6f3Ln/aCCTLxs68SLf0KPM9eSeM8yBnA==} engines: {node: '>=14.0.0'} - '@rollup/rollup-android-arm-eabi@4.39.0': - resolution: {integrity: sha512-lGVys55Qb00Wvh8DMAocp5kIcaNzEFTmGhfFd88LfaogYTRKrdxgtlO5H6S49v2Nd8R2C6wLOal0qv6/kCkOwA==} + '@rollup/rollup-android-arm-eabi@4.40.1': + resolution: {integrity: sha512-kxz0YeeCrRUHz3zyqvd7n+TVRlNyTifBsmnmNPtk3hQURUyG9eAB+usz6DAwagMusjx/zb3AjvDUvhFGDAexGw==} cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.39.0': - resolution: {integrity: sha512-It9+M1zE31KWfqh/0cJLrrsCPiF72PoJjIChLX+rEcujVRCb4NLQ5QzFkzIZW8Kn8FTbvGQBY5TkKBau3S8cCQ==} + '@rollup/rollup-android-arm64@4.40.1': + resolution: {integrity: sha512-PPkxTOisoNC6TpnDKatjKkjRMsdaWIhyuMkA4UsBXT9WEZY4uHezBTjs6Vl4PbqQQeu6oION1w2voYZv9yquCw==} cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.39.0': - resolution: {integrity: sha512-lXQnhpFDOKDXiGxsU9/l8UEGGM65comrQuZ+lDcGUx+9YQ9dKpF3rSEGepyeR5AHZ0b5RgiligsBhWZfSSQh8Q==} + '@rollup/rollup-darwin-arm64@4.40.1': + resolution: {integrity: sha512-VWXGISWFY18v/0JyNUy4A46KCFCb9NVsH+1100XP31lud+TzlezBbz24CYzbnA4x6w4hx+NYCXDfnvDVO6lcAA==} cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.39.0': - resolution: {integrity: sha512-mKXpNZLvtEbgu6WCkNij7CGycdw9cJi2k9v0noMb++Vab12GZjFgUXD69ilAbBh034Zwn95c2PNSz9xM7KYEAQ==} + '@rollup/rollup-darwin-x64@4.40.1': + resolution: {integrity: sha512-nIwkXafAI1/QCS7pxSpv/ZtFW6TXcNUEHAIA9EIyw5OzxJZQ1YDrX+CL6JAIQgZ33CInl1R6mHet9Y/UZTg2Bw==} cpu: [x64] os: [darwin] - '@rollup/rollup-freebsd-arm64@4.39.0': - resolution: {integrity: sha512-jivRRlh2Lod/KvDZx2zUR+I4iBfHcu2V/BA2vasUtdtTN2Uk3jfcZczLa81ESHZHPHy4ih3T/W5rPFZ/hX7RtQ==} + '@rollup/rollup-freebsd-arm64@4.40.1': + resolution: {integrity: sha512-BdrLJ2mHTrIYdaS2I99mriyJfGGenSaP+UwGi1kB9BLOCu9SR8ZpbkmmalKIALnRw24kM7qCN0IOm6L0S44iWw==} cpu: [arm64] os: [freebsd] - '@rollup/rollup-freebsd-x64@4.39.0': - resolution: {integrity: sha512-8RXIWvYIRK9nO+bhVz8DwLBepcptw633gv/QT4015CpJ0Ht8punmoHU/DuEd3iw9Hr8UwUV+t+VNNuZIWYeY7Q==} + '@rollup/rollup-freebsd-x64@4.40.1': + resolution: {integrity: sha512-VXeo/puqvCG8JBPNZXZf5Dqq7BzElNJzHRRw3vjBE27WujdzuOPecDPc/+1DcdcTptNBep3861jNq0mYkT8Z6Q==} cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.39.0': - resolution: {integrity: sha512-mz5POx5Zu58f2xAG5RaRRhp3IZDK7zXGk5sdEDj4o96HeaXhlUwmLFzNlc4hCQi5sGdR12VDgEUqVSHer0lI9g==} + '@rollup/rollup-linux-arm-gnueabihf@4.40.1': + resolution: {integrity: sha512-ehSKrewwsESPt1TgSE/na9nIhWCosfGSFqv7vwEtjyAqZcvbGIg4JAcV7ZEh2tfj/IlfBeZjgOXm35iOOjadcg==} cpu: [arm] os: [linux] - '@rollup/rollup-linux-arm-musleabihf@4.39.0': - resolution: {integrity: sha512-+YDwhM6gUAyakl0CD+bMFpdmwIoRDzZYaTWV3SDRBGkMU/VpIBYXXEvkEcTagw/7VVkL2vA29zU4UVy1mP0/Yw==} + '@rollup/rollup-linux-arm-musleabihf@4.40.1': + resolution: {integrity: sha512-m39iO/aaurh5FVIu/F4/Zsl8xppd76S4qoID8E+dSRQvTyZTOI2gVk3T4oqzfq1PtcvOfAVlwLMK3KRQMaR8lg==} cpu: [arm] os: [linux] - '@rollup/rollup-linux-arm64-gnu@4.39.0': - resolution: {integrity: sha512-EKf7iF7aK36eEChvlgxGnk7pdJfzfQbNvGV/+l98iiMwU23MwvmV0Ty3pJ0p5WQfm3JRHOytSIqD9LB7Bq7xdQ==} + '@rollup/rollup-linux-arm64-gnu@4.40.1': + resolution: {integrity: sha512-Y+GHnGaku4aVLSgrT0uWe2o2Rq8te9hi+MwqGF9r9ORgXhmHK5Q71N757u0F8yU1OIwUIFy6YiJtKjtyktk5hg==} cpu: [arm64] os: [linux] - '@rollup/rollup-linux-arm64-musl@4.39.0': - resolution: {integrity: sha512-vYanR6MtqC7Z2SNr8gzVnzUul09Wi1kZqJaek3KcIlI/wq5Xtq4ZPIZ0Mr/st/sv/NnaPwy/D4yXg5x0B3aUUA==} + '@rollup/rollup-linux-arm64-musl@4.40.1': + resolution: {integrity: sha512-jEwjn3jCA+tQGswK3aEWcD09/7M5wGwc6+flhva7dsQNRZZTe30vkalgIzV4tjkopsTS9Jd7Y1Bsj6a4lzz8gQ==} cpu: [arm64] os: [linux] - '@rollup/rollup-linux-loongarch64-gnu@4.39.0': - resolution: {integrity: sha512-NMRUT40+h0FBa5fb+cpxtZoGAggRem16ocVKIv5gDB5uLDgBIwrIsXlGqYbLwW8YyO3WVTk1FkFDjMETYlDqiw==} + '@rollup/rollup-linux-loongarch64-gnu@4.40.1': + resolution: {integrity: sha512-ySyWikVhNzv+BV/IDCsrraOAZ3UaC8SZB67FZlqVwXwnFhPihOso9rPOxzZbjp81suB1O2Topw+6Ug3JNegejQ==} cpu: [loong64] os: [linux] - '@rollup/rollup-linux-powerpc64le-gnu@4.39.0': - resolution: {integrity: sha512-0pCNnmxgduJ3YRt+D+kJ6Ai/r+TaePu9ZLENl+ZDV/CdVczXl95CbIiwwswu4L+K7uOIGf6tMo2vm8uadRaICQ==} + '@rollup/rollup-linux-powerpc64le-gnu@4.40.1': + resolution: {integrity: sha512-BvvA64QxZlh7WZWqDPPdt0GH4bznuL6uOO1pmgPnnv86rpUpc8ZxgZwcEgXvo02GRIZX1hQ0j0pAnhwkhwPqWg==} cpu: [ppc64] os: [linux] - '@rollup/rollup-linux-riscv64-gnu@4.39.0': - resolution: {integrity: sha512-t7j5Zhr7S4bBtksT73bO6c3Qa2AV/HqiGlj9+KB3gNF5upcVkx+HLgxTm8DK4OkzsOYqbdqbLKwvGMhylJCPhQ==} + '@rollup/rollup-linux-riscv64-gnu@4.40.1': + resolution: {integrity: sha512-EQSP+8+1VuSulm9RKSMKitTav89fKbHymTf25n5+Yr6gAPZxYWpj3DzAsQqoaHAk9YX2lwEyAf9S4W8F4l3VBQ==} cpu: [riscv64] os: [linux] - '@rollup/rollup-linux-riscv64-musl@4.39.0': - resolution: {integrity: sha512-m6cwI86IvQ7M93MQ2RF5SP8tUjD39Y7rjb1qjHgYh28uAPVU8+k/xYWvxRO3/tBN2pZkSMa5RjnPuUIbrwVxeA==} + '@rollup/rollup-linux-riscv64-musl@4.40.1': + resolution: {integrity: sha512-n/vQ4xRZXKuIpqukkMXZt9RWdl+2zgGNx7Uda8NtmLJ06NL8jiHxUawbwC+hdSq1rrw/9CghCpEONor+l1e2gA==} cpu: [riscv64] os: [linux] - '@rollup/rollup-linux-s390x-gnu@4.39.0': - resolution: {integrity: sha512-iRDJd2ebMunnk2rsSBYlsptCyuINvxUfGwOUldjv5M4tpa93K8tFMeYGpNk2+Nxl+OBJnBzy2/JCscGeO507kA==} + '@rollup/rollup-linux-s390x-gnu@4.40.1': + resolution: {integrity: sha512-h8d28xzYb98fMQKUz0w2fMc1XuGzLLjdyxVIbhbil4ELfk5/orZlSTpF/xdI9C8K0I8lCkq+1En2RJsawZekkg==} cpu: [s390x] os: [linux] - '@rollup/rollup-linux-x64-gnu@4.39.0': - resolution: {integrity: sha512-t9jqYw27R6Lx0XKfEFe5vUeEJ5pF3SGIM6gTfONSMb7DuG6z6wfj2yjcoZxHg129veTqU7+wOhY6GX8wmf90dA==} + '@rollup/rollup-linux-x64-gnu@4.40.1': + resolution: {integrity: sha512-XiK5z70PEFEFqcNj3/zRSz/qX4bp4QIraTy9QjwJAb/Z8GM7kVUsD0Uk8maIPeTyPCP03ChdI+VVmJriKYbRHQ==} cpu: [x64] os: [linux] - '@rollup/rollup-linux-x64-musl@4.39.0': - resolution: {integrity: sha512-ThFdkrFDP55AIsIZDKSBWEt/JcWlCzydbZHinZ0F/r1h83qbGeenCt/G/wG2O0reuENDD2tawfAj2s8VK7Bugg==} + '@rollup/rollup-linux-x64-musl@4.40.1': + resolution: {integrity: sha512-2BRORitq5rQ4Da9blVovzNCMaUlyKrzMSvkVR0D4qPuOy/+pMCrh1d7o01RATwVy+6Fa1WBw+da7QPeLWU/1mQ==} cpu: [x64] os: [linux] - '@rollup/rollup-win32-arm64-msvc@4.39.0': - resolution: {integrity: sha512-jDrLm6yUtbOg2TYB3sBF3acUnAwsIksEYjLeHL+TJv9jg+TmTwdyjnDex27jqEMakNKf3RwwPahDIt7QXCSqRQ==} + '@rollup/rollup-win32-arm64-msvc@4.40.1': + resolution: {integrity: sha512-b2bcNm9Kbde03H+q+Jjw9tSfhYkzrDUf2d5MAd1bOJuVplXvFhWz7tRtWvD8/ORZi7qSCy0idW6tf2HgxSXQSg==} cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.39.0': - resolution: {integrity: sha512-6w9uMuza+LbLCVoNKL5FSLE7yvYkq9laSd09bwS0tMjkwXrmib/4KmoJcrKhLWHvw19mwU+33ndC69T7weNNjQ==} + '@rollup/rollup-win32-ia32-msvc@4.40.1': + resolution: {integrity: sha512-DfcogW8N7Zg7llVEfpqWMZcaErKfsj9VvmfSyRjCyo4BI3wPEfrzTtJkZG6gKP/Z92wFm6rz2aDO7/JfiR/whA==} cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.39.0': - resolution: {integrity: sha512-yAkUOkIKZlK5dl7u6dg897doBgLXmUHhIINM2c+sND3DZwnrdQkkSiDh7N75Ll4mM4dxSkYfXqU9fW3lLkMFug==} + '@rollup/rollup-win32-x64-msvc@4.40.1': + resolution: {integrity: sha512-ECyOuDeH3C1I8jH2MK1RtBJW+YPMvSfT0a5NN0nHfQYnDSJ6tUiZH3gzwVP5/Kfh/+Tt7tpWVF9LXNTnhTJ3kA==} cpu: [x64] os: [win32] @@ -3583,8 +3591,8 @@ packages: robust-predicates@3.0.2: resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} - rollup@4.39.0: - resolution: {integrity: sha512-thI8kNc02yNvnmJp8dr3fNWJ9tCONDhp6TV35X6HkKGGs9E6q7YWCHbe5vKiTa7TAiNcFEmXKj3X/pG2b3ci0g==} + rollup@4.40.1: + resolution: {integrity: sha512-C5VvvgCCyfyotVITIAv+4efVytl5F7wt+/I2i9q9GZcEXW9BP52YYOXC58igUi+LFZVHukErIIqQSWwv/M3WRw==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true @@ -4017,8 +4025,8 @@ packages: peerDependencies: vite: '>2.0.0-0' - vite@5.4.17: - resolution: {integrity: sha512-5+VqZryDj4wgCs55o9Lp+p8GE78TLVg0lasCH5xFZ4jacZjtqZa6JUw9/p0WeAojaOfncSM6v77InkFPGnvPvg==} + vite@5.4.19: + resolution: {integrity: sha512-qO3aKv3HoQC8QKiNSTuUM1l9o/XX3+c+VTgLHbJWHZGeTPVAg2XwazI9UWzoxjIJCGCV2zU60uqMzjeLZuULqA==} engines: {node: ^18.0.0 || >=20.0.0} hasBin: true peerDependencies: @@ -4298,6 +4306,12 @@ snapshots: js-tokens: 4.0.0 picocolors: 1.1.1 + '@babel/code-frame@7.27.1': + dependencies: + '@babel/helper-validator-identifier': 7.27.1 + js-tokens: 4.0.0 + picocolors: 1.1.1 + '@babel/generator@7.17.7': dependencies: '@babel/types': 7.17.0 @@ -4340,6 +4354,8 @@ snapshots: '@babel/helper-validator-identifier@7.25.9': {} + '@babel/helper-validator-identifier@7.27.1': {} + '@babel/parser@7.26.10': dependencies: '@babel/types': 7.26.10 @@ -4829,64 +4845,64 @@ snapshots: '@remix-run/router@1.23.0': {} - '@rollup/rollup-android-arm-eabi@4.39.0': + '@rollup/rollup-android-arm-eabi@4.40.1': optional: true - '@rollup/rollup-android-arm64@4.39.0': + '@rollup/rollup-android-arm64@4.40.1': optional: true - '@rollup/rollup-darwin-arm64@4.39.0': + '@rollup/rollup-darwin-arm64@4.40.1': optional: true - '@rollup/rollup-darwin-x64@4.39.0': + '@rollup/rollup-darwin-x64@4.40.1': optional: true - '@rollup/rollup-freebsd-arm64@4.39.0': + '@rollup/rollup-freebsd-arm64@4.40.1': optional: true - '@rollup/rollup-freebsd-x64@4.39.0': + '@rollup/rollup-freebsd-x64@4.40.1': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.39.0': + '@rollup/rollup-linux-arm-gnueabihf@4.40.1': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.39.0': + '@rollup/rollup-linux-arm-musleabihf@4.40.1': optional: true - '@rollup/rollup-linux-arm64-gnu@4.39.0': + '@rollup/rollup-linux-arm64-gnu@4.40.1': optional: true - '@rollup/rollup-linux-arm64-musl@4.39.0': + '@rollup/rollup-linux-arm64-musl@4.40.1': optional: true - '@rollup/rollup-linux-loongarch64-gnu@4.39.0': + '@rollup/rollup-linux-loongarch64-gnu@4.40.1': optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.39.0': + '@rollup/rollup-linux-powerpc64le-gnu@4.40.1': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.39.0': + '@rollup/rollup-linux-riscv64-gnu@4.40.1': optional: true - '@rollup/rollup-linux-riscv64-musl@4.39.0': + '@rollup/rollup-linux-riscv64-musl@4.40.1': optional: true - '@rollup/rollup-linux-s390x-gnu@4.39.0': + '@rollup/rollup-linux-s390x-gnu@4.40.1': optional: true - '@rollup/rollup-linux-x64-gnu@4.39.0': + '@rollup/rollup-linux-x64-gnu@4.40.1': optional: true - '@rollup/rollup-linux-x64-musl@4.39.0': + '@rollup/rollup-linux-x64-musl@4.40.1': optional: true - '@rollup/rollup-win32-arm64-msvc@4.39.0': + '@rollup/rollup-win32-arm64-msvc@4.40.1': optional: true - '@rollup/rollup-win32-ia32-msvc@4.39.0': + '@rollup/rollup-win32-ia32-msvc@4.40.1': optional: true - '@rollup/rollup-win32-x64-msvc@4.39.0': + '@rollup/rollup-win32-x64-msvc@4.40.1': optional: true '@stylistic/eslint-plugin@2.13.0(eslint@9.23.0(jiti@1.21.7))(typescript@5.5.4)': @@ -4983,7 +4999,7 @@ snapshots: '@testing-library/dom@10.4.0': dependencies: - '@babel/code-frame': 7.26.2 + '@babel/code-frame': 7.27.1 '@babel/runtime': 7.26.10 '@types/aria-query': 5.0.4 aria-query: 5.3.0 @@ -5681,10 +5697,10 @@ snapshots: d3-time-format: 4.1.0 internmap: 2.0.3 - '@vitejs/plugin-react-swc@3.8.1(@swc/helpers@0.5.15)(vite@5.4.17(@types/node@22.13.11))': + '@vitejs/plugin-react-swc@3.8.1(@swc/helpers@0.5.15)(vite@5.4.19(@types/node@22.13.11))': dependencies: '@swc/core': 1.11.12(@swc/helpers@0.5.15) - vite: 5.4.17(@types/node@22.13.11) + vite: 5.4.19(@types/node@22.13.11) transitivePeerDependencies: - '@swc/helpers' @@ -5713,14 +5729,14 @@ snapshots: chai: 5.2.0 tinyrainbow: 1.2.0 - '@vitest/mocker@2.1.9(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4))(vite@5.4.17(@types/node@22.13.11))': + '@vitest/mocker@2.1.9(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4))(vite@5.4.19(@types/node@22.13.11))': dependencies: '@vitest/spy': 2.1.9 estree-walker: 3.0.3 magic-string: 0.30.17 optionalDependencies: msw: 2.7.3(@types/node@22.13.11)(typescript@5.5.4) - vite: 5.4.17(@types/node@22.13.11) + vite: 5.4.19(@types/node@22.13.11) '@vitest/pretty-format@2.1.9': dependencies: @@ -8662,30 +8678,30 @@ snapshots: robust-predicates@3.0.2: {} - rollup@4.39.0: + rollup@4.40.1: dependencies: '@types/estree': 1.0.7 optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.39.0 - '@rollup/rollup-android-arm64': 4.39.0 - '@rollup/rollup-darwin-arm64': 4.39.0 - '@rollup/rollup-darwin-x64': 4.39.0 - '@rollup/rollup-freebsd-arm64': 4.39.0 - '@rollup/rollup-freebsd-x64': 4.39.0 - '@rollup/rollup-linux-arm-gnueabihf': 4.39.0 - '@rollup/rollup-linux-arm-musleabihf': 4.39.0 - '@rollup/rollup-linux-arm64-gnu': 4.39.0 - '@rollup/rollup-linux-arm64-musl': 4.39.0 - '@rollup/rollup-linux-loongarch64-gnu': 4.39.0 - '@rollup/rollup-linux-powerpc64le-gnu': 4.39.0 - '@rollup/rollup-linux-riscv64-gnu': 4.39.0 - '@rollup/rollup-linux-riscv64-musl': 4.39.0 - '@rollup/rollup-linux-s390x-gnu': 4.39.0 - '@rollup/rollup-linux-x64-gnu': 4.39.0 - '@rollup/rollup-linux-x64-musl': 4.39.0 - '@rollup/rollup-win32-arm64-msvc': 4.39.0 - '@rollup/rollup-win32-ia32-msvc': 4.39.0 - '@rollup/rollup-win32-x64-msvc': 4.39.0 + '@rollup/rollup-android-arm-eabi': 4.40.1 + '@rollup/rollup-android-arm64': 4.40.1 + '@rollup/rollup-darwin-arm64': 4.40.1 + '@rollup/rollup-darwin-x64': 4.40.1 + '@rollup/rollup-freebsd-arm64': 4.40.1 + '@rollup/rollup-freebsd-x64': 4.40.1 + '@rollup/rollup-linux-arm-gnueabihf': 4.40.1 + '@rollup/rollup-linux-arm-musleabihf': 4.40.1 + '@rollup/rollup-linux-arm64-gnu': 4.40.1 + '@rollup/rollup-linux-arm64-musl': 4.40.1 + '@rollup/rollup-linux-loongarch64-gnu': 4.40.1 + '@rollup/rollup-linux-powerpc64le-gnu': 4.40.1 + '@rollup/rollup-linux-riscv64-gnu': 4.40.1 + '@rollup/rollup-linux-riscv64-musl': 4.40.1 + '@rollup/rollup-linux-s390x-gnu': 4.40.1 + '@rollup/rollup-linux-x64-gnu': 4.40.1 + '@rollup/rollup-linux-x64-musl': 4.40.1 + '@rollup/rollup-win32-arm64-msvc': 4.40.1 + '@rollup/rollup-win32-ia32-msvc': 4.40.1 + '@rollup/rollup-win32-x64-msvc': 4.40.1 fsevents: 2.3.3 run-parallel@1.2.0: @@ -9158,7 +9174,7 @@ snapshots: debug: 4.4.0 es-module-lexer: 1.6.0 pathe: 1.1.2 - vite: 5.4.17(@types/node@22.13.11) + vite: 5.4.19(@types/node@22.13.11) transitivePeerDependencies: - '@types/node' - less @@ -9170,15 +9186,15 @@ snapshots: - supports-color - terser - vite-plugin-css-injected-by-js@3.5.2(vite@5.4.17(@types/node@22.13.11)): + vite-plugin-css-injected-by-js@3.5.2(vite@5.4.19(@types/node@22.13.11)): dependencies: - vite: 5.4.17(@types/node@22.13.11) + vite: 5.4.19(@types/node@22.13.11) - vite@5.4.17(@types/node@22.13.11): + vite@5.4.19(@types/node@22.13.11): dependencies: esbuild: 0.21.5 postcss: 8.5.3 - rollup: 4.39.0 + rollup: 4.40.1 optionalDependencies: '@types/node': 22.13.11 fsevents: 2.3.3 @@ -9186,7 +9202,7 @@ snapshots: vitest@2.1.9(@types/node@22.13.11)(happy-dom@17.4.4)(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4)): dependencies: '@vitest/expect': 2.1.9 - '@vitest/mocker': 2.1.9(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4))(vite@5.4.17(@types/node@22.13.11)) + '@vitest/mocker': 2.1.9(msw@2.7.3(@types/node@22.13.11)(typescript@5.5.4))(vite@5.4.19(@types/node@22.13.11)) '@vitest/pretty-format': 2.1.9 '@vitest/runner': 2.1.9 '@vitest/snapshot': 2.1.9 @@ -9202,7 +9218,7 @@ snapshots: tinyexec: 0.3.2 tinypool: 1.0.2 tinyrainbow: 1.2.0 - vite: 5.4.17(@types/node@22.13.11) + vite: 5.4.19(@types/node@22.13.11) vite-node: 2.1.9(@types/node@22.13.11) why-is-node-running: 2.3.0 optionalDependencies: diff --git a/airflow-core/src/airflow/ui/src/components/Assets/AssetEvent.tsx b/airflow-core/src/airflow/ui/src/components/Assets/AssetEvent.tsx index 6c97c648de5a7..6d69ae9a1ac99 100644 --- a/airflow-core/src/airflow/ui/src/components/Assets/AssetEvent.tsx +++ b/airflow-core/src/airflow/ui/src/components/Assets/AssetEvent.tsx @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { Box, Text, HStack, Code } from "@chakra-ui/react"; +import { Box, Text, HStack } from "@chakra-ui/react"; import { FiDatabase } from "react-icons/fi"; import { Link } from "react-router-dom"; @@ -24,30 +24,26 @@ import type { AssetEventResponse } from "openapi/requests/types.gen"; import Time from "src/components/Time"; import { Tooltip } from "src/components/ui"; +import RenderedJsonField from "../RenderedJsonField"; import { TriggeredRuns } from "./TriggeredRuns"; export const AssetEvent = ({ assetId, event, - showExtra, }: { readonly assetId?: number; readonly event: AssetEventResponse; - readonly showExtra?: boolean; }) => { let source = ""; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { from_rest_api, from_trigger, ...extra } = event.extra ?? {}; + const { from_rest_api: fromRestAPI, from_trigger: fromTrigger, ...extra } = event.extra ?? {}; - if (from_rest_api === true) { + if (fromRestAPI === true) { source = "API"; - } else if (from_trigger === true) { + } else if (fromTrigger === true) { source = "Trigger"; } - const extraString = JSON.stringify(extra); - return ( @@ -55,7 +51,9 @@ export const AssetEvent = ({ {Boolean(assetId) ? undefined : ( - + + + @@ -66,18 +64,22 @@ export const AssetEvent = ({ showArrow > - {event.name ?? ""} + + {event.name ?? ""} + )} - Source: + Source: {source === "" ? ( -1 ? `/mapped/${event.source_map_index}` : ""}`} > - {event.source_dag_id} + + {event.source_dag_id} + ) : ( source @@ -86,7 +88,9 @@ export const AssetEvent = ({ - {showExtra && extraString !== "{}" ? {extraString} : undefined} + {Object.keys(extra).length >= 1 ? ( + + ) : undefined} ); }; diff --git a/airflow-core/src/airflow/ui/src/components/Assets/AssetEvents.tsx b/airflow-core/src/airflow/ui/src/components/Assets/AssetEvents.tsx index f06f55185c917..4becd16cc9544 100644 --- a/airflow-core/src/airflow/ui/src/components/Assets/AssetEvents.tsx +++ b/airflow-core/src/airflow/ui/src/components/Assets/AssetEvents.tsx @@ -29,8 +29,8 @@ import { DataTable } from "../DataTable"; import type { CardDef, TableState } from "../DataTable/types"; import { AssetEvent } from "./AssetEvent"; -const cardDef = (assetId?: number, showExtra?: boolean): CardDef => ({ - card: ({ row }) => , +const cardDef = (assetId?: number): CardDef => ({ + card: ({ row }) => , meta: { customSkeleton: , }, @@ -42,7 +42,6 @@ type AssetEventProps = { readonly isLoading?: boolean; readonly setOrderBy?: (order: string) => void; readonly setTableUrlState?: (state: TableState) => void; - readonly showExtra?: boolean; readonly tableUrlState?: TableState; readonly title?: string; }; @@ -53,7 +52,6 @@ export const AssetEvents = ({ isLoading, setOrderBy, setTableUrlState, - showExtra, tableUrlState, title, }: AssetEventProps) => { @@ -100,7 +98,7 @@ export const AssetEvents = ({ )} { - await queryClient.invalidateQueries({ - queryKey: [useBackfillServiceListBackfills1Key], - }); -}; - const BackfillBanner = ({ dagId }: Props) => { const { data, isLoading } = useBackfillServiceListBackfills1({ dagId, }); const [backfill] = data?.backfills.filter((bf) => bf.completed_at === null) ?? []; + const queryClient = useQueryClient(); + const onSuccess = async () => { + await queryClient.invalidateQueries({ + queryKey: [useBackfillServiceListBackfills1Key], + }); + }; + const { isPending: isPausePending, mutate: pauseMutate } = useBackfillServicePauseBackfill({ onSuccess }); const { isPending: isUnPausePending, mutate: unpauseMutate } = useBackfillServiceUnpauseBackfill({ onSuccess, @@ -65,15 +66,21 @@ const BackfillBanner = ({ dagId }: Props) => { const { isPending: isStopPending, mutate: stopPending } = useBackfillServiceCancelBackfill({ onSuccess }); const togglePause = () => { - if (backfill?.is_paused) { + if (backfill === undefined) { + return; + } + if (backfill.is_paused) { unpauseMutate({ backfillId: backfill.id }); } else { - pauseMutate({ backfillId: backfill?.id }); + pauseMutate({ backfillId: backfill.id }); } }; const cancel = () => { - stopPending({ backfillId: backfill?.id }); + if (backfill === undefined) { + return; + } + stopPending({ backfillId: backfill.id }); }; if (isLoading || backfill === undefined) { diff --git a/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearTaskInstanceDialog.tsx b/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearTaskInstanceDialog.tsx index 53126f9e88e2a..3d251138ff15b 100644 --- a/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearTaskInstanceDialog.tsx +++ b/airflow-core/src/airflow/ui/src/components/Clear/TaskInstance/ClearTaskInstanceDialog.tsx @@ -131,7 +131,7 @@ const ClearTaskInstanceDialog = ({ onClose, open, taskInstance }: Props) => { include_past: past, include_upstream: upstream, only_failed: onlyFailed, - task_ids: [taskId], + task_ids: [[taskId, mapIndex]], }, }); if (note !== taskInstance.note) { diff --git a/airflow-core/src/airflow/ui/src/components/ConfigForm.tsx b/airflow-core/src/airflow/ui/src/components/ConfigForm.tsx new file mode 100644 index 0000000000000..c8e75edfa816b --- /dev/null +++ b/airflow-core/src/airflow/ui/src/components/ConfigForm.tsx @@ -0,0 +1,121 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Accordion, Box, Field } from "@chakra-ui/react"; +import { type Control, type FieldValues, type Path, Controller } from "react-hook-form"; + +import type { ParamsSpec } from "src/queries/useDagParams"; +import { useParamStore } from "src/queries/useParamStore"; + +import { FlexibleForm, flexibleFormDefaultSection } from "./FlexibleForm"; +import { JsonEditor } from "./JsonEditor"; + +type ConfigFormProps = { + readonly children?: React.ReactNode; + readonly control: Control; + readonly errors: { + conf?: string; + date?: unknown; + }; + readonly initialParamsDict: { paramsDict: ParamsSpec }; + readonly setErrors: React.Dispatch< + React.SetStateAction<{ + conf?: string; + date?: unknown; + }> + >; + readonly setFormError: (error: boolean) => void; +}; + +const ConfigForm = ({ + children, + control, + errors, + initialParamsDict, + setErrors, + setFormError, +}: ConfigFormProps) => { + const { conf, setConf } = useParamStore(); + + const validateAndPrettifyJson = (value: string) => { + try { + const parsedJson = JSON.parse(value) as JSON; + + setErrors((prev) => ({ ...prev, conf: undefined })); + + const formattedJson = JSON.stringify(parsedJson, undefined, 2); + + if (formattedJson !== conf) { + setConf(formattedJson); // Update only if the value is different + } + + return formattedJson; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "Unknown error occurred."; + + setErrors((prev) => ({ + ...prev, + conf: `Invalid JSON format: ${errorMessage}`, + })); + + return value; + } + }; + + return ( + + + + Advanced Options + + + {children} + } + render={({ field }) => ( + + Configuration JSON + { + field.onChange(validateAndPrettifyJson(field.value as string)); + }} + /> + {Boolean(errors.conf) ? {errors.conf} : undefined} + + )} + /> + + + + + ); +}; + +export default ConfigForm; diff --git a/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx b/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx index 1bbacf6dcede2..4b455aa317740 100644 --- a/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx +++ b/airflow-core/src/airflow/ui/src/components/DagActions/RunBackfillForm.tsx @@ -16,18 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -import { Input, Box, Spacer, HStack, Field, VStack, Flex, Text } from "@chakra-ui/react"; +import { Input, Box, Spacer, HStack, Field, VStack, Flex, Text, Skeleton } from "@chakra-ui/react"; +import dayjs from "dayjs"; import { useEffect, useState } from "react"; import { useForm, Controller, useWatch } from "react-hook-form"; import type { DAGResponse, DAGWithLatestDagRunsResponse, BackfillPostBody } from "openapi/requests/types.gen"; -import { Alert, Button } from "src/components/ui"; +import { Button } from "src/components/ui"; import { reprocessBehaviors } from "src/constants/reprocessBehaviourParams"; import { useCreateBackfill } from "src/queries/useCreateBackfill"; import { useCreateBackfillDryRun } from "src/queries/useCreateBackfillDryRun"; +import { useDagParams } from "src/queries/useDagParams"; +import { useParamStore } from "src/queries/useParamStore"; import { useTogglePause } from "src/queries/useTogglePause"; +import { pluralize } from "src/utils"; +import ConfigForm from "../ConfigForm"; +import { DateTimeInput } from "../DateTimeInput"; import { ErrorAlert } from "../ErrorAlert"; +import type { DagRunTriggerParams } from "../TriggerDag/TriggerDAGForm"; import { Checkbox } from "../ui/Checkbox"; import { RadioCardItem, RadioCardLabel, RadioCardRoot } from "../ui/RadioCard"; @@ -37,23 +44,27 @@ type RunBackfillFormProps = { }; const today = new Date().toISOString().slice(0, 16); +type BackfillFormProps = DagRunTriggerParams & Omit; + const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { const [errors, setErrors] = useState<{ conf?: string; date?: unknown }>({}); const [unpause, setUnpause] = useState(true); - - const { control, handleSubmit, reset, watch } = useForm({ + const [formError, setFormError] = useState(false); + const initialParamsDict = useDagParams(dag.dag_id, true); + const { conf } = useParamStore(); + const { control, handleSubmit, reset, watch } = useForm({ defaultValues: { + conf, dag_id: dag.dag_id, - dag_run_conf: {}, from_date: "", max_active_runs: 1, - reprocess_behavior: "failed", + reprocess_behavior: "none", run_backwards: false, to_date: "", }, mode: "onBlur", }); - const values = useWatch({ + const values = useWatch({ control, }); @@ -83,10 +94,21 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { } }, [dateValidationError]); + useEffect(() => { + if (conf) { + reset((prevValues) => ({ + ...prevValues, + conf, + })); + } + }, [conf, reset]); + const dataIntervalStart = watch("from_date"); const dataIntervalEnd = watch("to_date"); + const noDataInterval = !Boolean(dataIntervalStart) || !Boolean(dataIntervalEnd); + const dataIntervalInvalid = dayjs(dataIntervalStart).isAfter(dayjs(dataIntervalEnd)); - const onSubmit = (fdata: BackfillPostBody) => { + const onSubmit = (fdata: BackfillFormProps) => { if (unpause && dag.is_paused) { togglePause({ dagId: dag.dag_id, @@ -96,11 +118,14 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { }); } createBackfill({ - requestBody: fdata, + requestBody: { + ...fdata, + dag_run_conf: JSON.parse(fdata.conf) as Record, + }, }); }; - const onCancel = (fdata: BackfillPostBody) => { + const onCancel = (fdata: BackfillFormProps) => { reset(fdata); onClose(); }; @@ -114,26 +139,35 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { total_entries: 0, }; + const inlineMessage = isPendingDryRun ? ( + + ) : affectedTasks.total_entries > 0 ? ( + + {pluralize("run", affectedTasks.total_entries)} will be triggered + + ) : ( + + No runs matching selected criteria. + + ); + return ( <> - + + - + Date Range - + ( - - + + From + + Start Date must be before the End Date )} /> @@ -141,20 +175,15 @@ const RunBackfillForm = ({ dag, onClose }: RunBackfillFormProps) => { control={control} name="to_date" render={({ field }) => ( - - + + To + )} /> + {noDataInterval || dataIntervalInvalid ? undefined : {inlineMessage}} { field.onChange(event); }} > - Reprocess Behaviour + + Reprocess Behaviour + {reprocessBehaviors.map((item) => ( { )} /> - ( - - Run Backwards - - )} - /> - { )} /> - {affectedTasks.total_entries > 0 ? ( - {affectedTasks.total_entries} runs will be triggered - ) : ( - No runs matching selected criteria. - )} + ( + + Run Backwards + + )} + /> + + {dag.is_paused ? ( + <> + setUnpause(!unpause)}> + Unpause {dag.dag_display_name} on trigger + + + + ) : undefined} + + - {dag.is_paused ? ( - setUnpause(!unpause)}> - Unpause {dag.dag_display_name} on trigger - - ) : undefined} - - ) : undefined} - - + + + {isGroup ? "Task Group" : operator} + + {taskInstance === undefined ? undefined : ( + + + {taskInstance.state} + + {taskInstance.try_number > 1 ? : undefined} + + )} + {isGroup ? ( + + ) : undefined} + {Boolean(isMapped) || Boolean(isGroup && !isOpen) ? ( <> diff --git a/airflow-core/src/airflow/ui/src/components/LimitedItemsList.tsx b/airflow-core/src/airflow/ui/src/components/LimitedItemsList.tsx index 3c696beef1606..7d993413c2dff 100644 --- a/airflow-core/src/airflow/ui/src/components/LimitedItemsList.tsx +++ b/airflow-core/src/airflow/ui/src/components/LimitedItemsList.tsx @@ -16,25 +16,34 @@ * specific language governing permissions and limitations * under the License. */ -import { Box, Text, HStack } from "@chakra-ui/react"; +import { Box, Text, HStack, StackSeparator } from "@chakra-ui/react"; import React, { type ReactNode } from "react"; import { Tooltip } from "./ui"; type ListProps = { readonly icon?: ReactNode; + readonly interactive?: boolean; readonly items: Array; readonly maxItems?: number; readonly separator?: string; }; -export const LimitedItemsList = ({ icon, items, maxItems, separator = ", " }: ListProps) => { +export const LimitedItemsList = ({ + icon, + interactive = false, + items, + maxItems, + separator = ", ", +}: ListProps) => { const shouldTruncate = maxItems !== undefined && items.length > maxItems; const displayItems = shouldTruncate ? items.slice(0, maxItems) : items; const remainingItems = shouldTruncate ? items.slice(maxItems) : []; - const remainingItemsList = remainingItems - .map((item) => (typeof item === "string" ? item : "item")) - .join(", "); + const remainingItemsList = interactive ? ( + }>{remainingItems} + ) : ( + `More items: ${remainingItems.map((item) => (typeof item === "string" ? item : "item")).join(", ")}` + ); if (!items.length) { return undefined; @@ -57,9 +66,9 @@ export const LimitedItemsList = ({ icon, items, maxItems, separator = ", " }: Li remainingItems.length === 1 ? ( {remainingItems[0]} ) : ( - + - , +{remainingItems.length} more + +{remainingItems.length} more ) diff --git a/airflow-core/src/airflow/ui/src/components/RenderedJsonField.tsx b/airflow-core/src/airflow/ui/src/components/RenderedJsonField.tsx index 132a57116a6ce..5e32d8c67d7a9 100644 --- a/airflow-core/src/airflow/ui/src/components/RenderedJsonField.tsx +++ b/airflow-core/src/airflow/ui/src/components/RenderedJsonField.tsx @@ -47,7 +47,7 @@ const RenderedJsonField = ({ content, jsonProps, ...rest }: Props) => { {...jsonProps} /> - + ); diff --git a/airflow-core/src/airflow/ui/src/components/TaskInstanceTooltip.tsx b/airflow-core/src/airflow/ui/src/components/TaskInstanceTooltip.tsx index ed09f9a9f93d3..f2152d468ef79 100644 --- a/airflow-core/src/airflow/ui/src/components/TaskInstanceTooltip.tsx +++ b/airflow-core/src/airflow/ui/src/components/TaskInstanceTooltip.tsx @@ -25,6 +25,7 @@ import type { } from "openapi/requests/types.gen"; import Time from "src/components/Time"; import { Tooltip, type TooltipProps } from "src/components/ui"; +import { getDuration } from "src/utils"; type Props = { readonly taskInstance?: GridTaskInstanceSummary | TaskInstanceHistoryResponse | TaskInstanceResponse; @@ -47,8 +48,8 @@ const TaskInstanceTooltip = ({ children, positioning, taskInstance, ...rest }: P End Date: