Sync v2-10-stable with v2-10-test(sync_v2_10_test) to release 2.10.4

.github/actions/checkout_target_commit/action.yml

@@ -65,13 +65,16 @@ runs:
         rm -rfv "dev"
         rm -rfv ".github/actions"
         rm -rfv ".github/workflows"
+        rm -v ".dockerignore" || true
         mv -v "target-airflow/scripts/ci" "scripts"
         mv -v "target-airflow/dev" "."
         mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github"
+        mv -v "target-airflow/.dockerignore" ".dockerignore" || true
       if: inputs.pull-request-target == 'true' && inputs.is-committer-build != 'true'
       ####################################################################################################
-      #  AFTER IT'S SAFE. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE
-      #  BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAFE TO RUN.
+      #  AFTER IT'S SAFE. THE `dev`, `scripts/ci` AND `.github/actions` and `.dockerignore` ARE NOW COMING
+      #  FROM THE BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE
+      #  SAFE TO RUN AND CODE AVAILABLE IN THE DOCKER BUILD PHASE IS CONTROLLED BY THE `.dockerignore`.
       #  ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN
       #  BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
       #  ISOLATED FROM THE RUNNER.

.github/actions/install-pre-commit/action.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Install pre-commit'
+description: 'Installs pre-commit and related packages'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    default: 3.9
+  uv-version:
+    description: 'uv version to use'
+    default: 0.4.29
+  pre-commit-version:
+    description: 'pre-commit version to use'
+    default: 4.0.1
+  pre-commit-uv-version:
+    description: 'pre-commit-uv version to use'
+    default: 4.1.4
+runs:
+  using: "composite"
+  steps:
+    - name: Install pre-commit, uv, and pre-commit-uv
+      shell: bash
+      run: >
+        pip install
+        pre-commit==${{inputs.pre-commit-version}}
+        uv==${{inputs.uv-version}}
+        pre-commit-uv==${{inputs.pre-commit-uv-version}}
+    - name: Cache pre-commit envs
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+        restore-keys: |
+          pre-commit-${{inputs.python-version}}-

.github/workflows/additional-ci-image-checks.yml

@@ -80,6 +80,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources (true/false)"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv to build the image (true/false)"
+        required: true
+        type: string
 jobs:
   # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
   # tests to complete - it is run very early in the build process for "main" merges in order to refresh
@@ -109,7 +113,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ inputs.use-uv}}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.branch == 'main'
@@ -165,6 +169,6 @@ jobs:
 #      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: "true"
+#      use-uv: ${{ inputs.use-uv}}
 #      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
 #      docker-cache: ${{ inputs.docker-cache }}

.github/workflows/basic-tests.yml

@@ -232,16 +232,11 @@ jobs:
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         id: breeze
-      - name: Cache pre-commit envs
-        uses: actions/cache@v4
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
         with:
-          path: ~/.cache/pre-commit
-          # yamllint disable-line rule:line-length
-          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-          restore-keys: "\
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-\
-            ${{ hashFiles('.pre-commit-config.yaml') }}\n
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-"
+          python-version: ${{steps.breeze.outputs.host-python-version}}
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:

.github/workflows/build-images.yml

@@ -16,7 +16,7 @@
 # under the License.
 #
 ---
-name: "Build Images"
+name: Build Images
 run-name: >
   Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
 on:  # yamllint disable-line rule:truthy
@@ -54,7 +54,7 @@ concurrency:
 jobs:
   build-info:
     timeout-minutes: 10
-    name: "Build Info"
+    name: Build Info
     # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners
     runs-on: ["ubuntu-22.04"]
     env:
@@ -71,6 +71,7 @@ jobs:
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
       default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
       runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
       runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
@@ -89,7 +90,7 @@ jobs:
         }}"
     if: github.repository == 'apache/airflow'
     steps:
-      - name: "Cleanup repo"
+      - name: Cleanup repo
         shell: bash
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - name: Discover PR merge commit
@@ -154,13 +155,13 @@ jobs:
       #  COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT
       #  CODE WILL BE RUN FROM THE PR.
       ####################################################################################################
-      - name: "Cleanup docker"
+      - name: Cleanup docker
         run: ./scripts/ci/cleanup_docker.sh
-      - name: "Setup python"
+      - name: Setup python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
-      - name: "Install Breeze"
+          python-version: "3.9"
+      - name: Install Breeze
         uses: ./.github/actions/breeze
       ####################################################################################################
       #  WE RUN SELECTIVE CHECKS HERE USING THE TARGET COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM
@@ -202,7 +203,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -212,7 +213,7 @@ jobs:
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
 
   generate-constraints:
-    name: "Generate constraints"
+    name: Generate constraints
     needs: [build-info, build-ci-images]
     uses: ./.github/workflows/generate-constraints.yml
     with:
@@ -245,9 +246,9 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      platform: "linux/amd64"
+      platform: linux/amd64
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}

.github/workflows/check-providers.yml

@@ -28,6 +28,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
+      canary-run:
+        description: "Whether this is a canary run"
+        required: true
+        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -209,6 +213,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       VERBOSE: "true"
+      CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
     if: inputs.skip-provider-tests != 'true'
     steps:
       - name: "Cleanup repo"

.github/workflows/ci.yml

@@ -75,6 +75,7 @@ jobs:
       default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
       default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
       default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
       parallel-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
@@ -95,7 +96,7 @@ jobs:
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docs-build: ${{ steps.selective-checks.outputs.docs-build }}
-      mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
+      mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
       needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
       needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
       needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
@@ -199,7 +200,7 @@ jobs:
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -263,6 +264,7 @@ jobs:
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
 
 
   generate-constraints:
@@ -290,7 +292,7 @@ jobs:
       runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
-      mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
+      mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -304,6 +306,7 @@ jobs:
       ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      docs-build: ${{ needs.build-info.outputs.docs-build }}
 
   providers:
     name: "Provider checks"
@@ -319,6 +322,7 @@ jobs:
     with:
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
+      canary-run: ${{ needs.build-info.outputs.canary-run }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }}
@@ -541,7 +545,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
@@ -622,6 +626,7 @@ jobs:
       kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }}
       kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||

.github/workflows/finalize-tests.yml

@@ -145,7 +145,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.canary-run == 'true'

.github/workflows/generate-constraints.yml

@@ -95,7 +95,7 @@ jobs:
         timeout-minutes: 25
         run: >
           breeze release-management generate-constraints --run-in-parallel
-          --airflow-constraints-mode constraints-no-providers --answer yes
+          --airflow-constraints-mode constraints-no-providers --answer yes --parallelism 3
         # The no providers constraints are only needed when we want to update constraints (in canary builds)
         # They slow down the start of PROD image builds so we want to only run them when needed.
         if: inputs.generate-no-providers-constraints == 'true'
@@ -115,7 +115,7 @@ jobs:
         run: >
           breeze release-management generate-constraints --run-in-parallel
           --airflow-constraints-mode constraints --answer yes
-          --chicken-egg-providers "${{ inputs.chicken-egg-providers }}"
+          --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" --parallelism 3
       - name: "Dependency upgrade summary"
         shell: bash
         run: |

.github/workflows/integration-tests.yml

@@ -59,6 +59,7 @@ on:  # yamllint disable-line rule:truthy
 jobs:
   tests-integration:
     timeout-minutes: 130
+    if: inputs.testable-integrations != '[]'
     name: "Integration Tests: ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:

.github/workflows/k8s-tests.yml

@@ -44,6 +44,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to include success outputs"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv"
+        required: true
+        type: string
       debug-resources:
         description: "Whether to debug resources"
         required: true
@@ -96,6 +100,9 @@ jobs:
           key: "\
             k8s-env-${{ steps.breeze.outputs.host-python-version }}-\
             ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}"
+      - name: "Switch breeze to use uv"
+        run: breeze setup config --use-uv
+        if: inputs.use-uv == 'true'
       - name: Run complete K8S tests ${{ inputs.kubernetes-combos-list-as-string }}
         run: breeze k8s run-complete-tests --run-in-parallel --upgrade --no-copy-local-sources
         env:

.github/workflows/release_dockerhub_image.yml

@@ -85,6 +85,7 @@ jobs:
       "kaxil",
       "pierrejeambrun",
       "potiuk",
+      "utkarsharma2"
       ]'), github.event.sender.login)
     steps:
       - name: "Cleanup repo"