[v3-0-test] Add possibility to apply a commit with patch to docs when publishing (apache#53668)

.github/actions/breeze/action.yml

@@ -22,9 +22,6 @@ inputs:
   python-version:
     description: 'Python version to use'
     default: "3.9"
-  use-uv:
-    description: 'Whether to use uv tool'
-    required: true
 outputs:
   host-python-version:
     description: Python version used in host

.github/actions/prepare_breeze_and_image/action.yml

@@ -43,8 +43,6 @@ runs:
       shell: bash
     - name: "Install Breeze"
       uses: ./.github/actions/breeze
-      with:
-        use-uv: ${{ inputs.use-uv }}
       id: breeze
     - name: "Check free space"
       shell: bash

.github/workflows/additional-ci-image-checks.yml

@@ -142,7 +142,5 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: "Check that image builds quickly"
         run: breeze shell --max-time 900 --platform "${PLATFORM}"

.github/workflows/airflow-distributions-tests.yml

@@ -105,8 +105,6 @@ jobs:
         if: ${{ inputs.use-local-venv == 'true' }}
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         if: ${{ inputs.use-local-venv == 'true' }}
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*

.github/workflows/backport-cli.yml

@@ -70,8 +70,8 @@ jobs:
           TARGET_BRANCH: ${{ inputs.target-branch }}
           COMMIT_SHA: ${{ inputs.commit-sha }}
         run: |
-          git config --global user.email "name@example.com"
-          git config --global user.name "Your Name"
+          git config --global user.email "bot@airflow.apache.org"
+          git config --global user.name "Your friendly bot"
           set +e
           {
           echo 'cherry_picker_output<<EOF'

.github/workflows/basic-tests.yml

@@ -80,8 +80,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
         working-directory: ./dev/breeze/
   tests-ui:
@@ -169,8 +167,6 @@ jobs:
           persist-credentials: false
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         id: breeze
       - name: "Install pre-commit"
         uses: ./.github/actions/install-pre-commit
@@ -200,8 +196,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         id: breeze
       - name: "Install pre-commit"
         uses: ./.github/actions/install-pre-commit
@@ -256,8 +250,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         id: breeze
       - name: "Install pre-commit"
         uses: ./.github/actions/install-pre-commit
@@ -338,14 +330,12 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*
       - name: Setup git for tagging
         run: |
-          git config --global user.email "name@example.com"
-          git config --global user.name "Your Name"
+          git config --global user.email "bot@airflow.apache.org"
+          git config --global user.name "Your friendly bot"
       - name: Install twine
         run: pip install twine
       - name: "Check Airflow create minor branch command"

.github/workflows/ci-amd.yml

@@ -148,8 +148,6 @@ jobs:
           persist-credentials: false
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         id: breeze
       - name: "Get information about the Workflow"
         id: source-run-info

.github/workflows/ci-arm.yml

@@ -141,8 +141,6 @@ jobs:
           persist-credentials: false
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         id: breeze
       - name: "Get information about the Workflow"
         id: source-run-info

.github/workflows/ci-image-build.yml

@@ -124,8 +124,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: "Restore ci-cache mount image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
         with:

.github/workflows/ci-image-checks.yml

@@ -133,8 +133,6 @@ jobs:
         if: inputs.canary-run == 'true'
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         id: breeze
         if: inputs.canary-run == 'true'
       - name: "Install pre-commit"

.github/workflows/helm-tests.yml

@@ -100,12 +100,10 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: Setup git for tagging
         run: |
-          git config --global user.email "name@example.com"
-          git config --global user.name "Your Name"
+          git config --global user.email "bot@airflow.apache.org"
+          git config --global user.name "Your friendly bot"
       - name: "Remove old artifacts"
         run: rm -rf dist/*
       - name: "Setup k8s/helm environment"

.github/workflows/prod-image-build.yml

@@ -136,8 +136,6 @@ jobs:
         if: inputs.upload-package-artifact == 'true'
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
         if: inputs.upload-package-artifact == 'true'
       - name: "Prepare providers packages - all providers built from sources"
         shell: bash
@@ -218,8 +216,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: "Cleanup dist and context file"
         shell: bash
         run: rm -fv ./dist/* ./docker-context-files/*

.github/workflows/publish-docs-to-s3.yml

@@ -52,7 +52,18 @@ on:  # yamllint disable-line rule:truthy
         required: false
         default: false
         type: boolean
-
+      airflow-base-version:
+        required: false
+        description: "Override the Airflow Base Version to use for the docs build"
+        type: string
+      airflow-version:
+        required: false
+        description: "Override the Airflow Version to use for the docs build"
+        type: string
+      apply-commits:
+        required: false
+        description: "Optionally apply commit hashes before building - to patch the docs (coma separated)"
+        type: string
 permissions:
   contents: read
 jobs:
@@ -69,6 +80,9 @@ jobs:
       DESTINATION: ${{ inputs.destination }}
       SKIP_WRITE_TO_STABLE_FOLDER: ${{ inputs.skip-write-to-stable-folder }}
       BUILD_SBOMS: ${{ inputs.build-sboms }}
+      AIRFLOW_BASE_VERSION: ${{ inputs.airflow-base-version || '' }}
+      AIRFLOW_VERSION: ${{ inputs.airflow-version || '' }}
+      APPLY_COMMITS: ${{ inputs.apply-commits || '' }}
     outputs:
       include-docs: ${{ inputs.include-docs == 'all' && '' || inputs.include-docs }}
       destination-location: ${{ steps.parameters.outputs.destination-location }}
@@ -102,6 +116,9 @@ jobs:
           echo "Destination: '${DESTINATION}'"
           echo "Skip write to stable folder: '${SKIP_WRITE_TO_STABLE_FOLDER}'"
           echo "Build SBOMs: '${BUILD_SBOMS}'"
+          echo "Airflow Base Version: '${AIRFLOW_BASE_VERSION}'"
+          echo "Airflow Version: '${AIRFLOW_VERSION}'"
+          echo "Apply commits: '${APPLY_COMMITS}'"
           if [[ "${DESTINATION}" == "auto" ]]; then
              if [[ "${REF}" =~ ^.*[0-9]*\.[0-9]*\.[0-9]*$ ]]; then
                 echo "${REF} looks like final release, using live destination"
@@ -118,8 +135,20 @@ jobs:
              echo "destination-location=s3://staging-docs-airflow-apache-org/docs/" >> ${GITHUB_OUTPUT}
           fi
           if [[ " ${INCLUDE_DOCS} " =~ " apache-airflow " ]]; then
-             AIRFLOW_BASE_VERSION=$(echo "${REF}" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
-             AIRFLOW_VERSION="${REF}"
+             if [[ ${AIRFLOW_BASE_VERSION=} == "" && ${AIRFLOW_VERSION=} == "" ]]; then
+                echo "No Airflow Versions provided, using ${REF} to determine it."
+                AIRFLOW_VERSION="${REF}"
+                set +e
+                AIRFLOW_BASE_VERSION=$(echo "${REF}" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
+                set -e
+                if [[ ${AIRFLOW_BASE_VERSION=} == "" ]]; then
+                   echo
+                   echo "No Airflow Base Version found in ${REF}"
+                   echo "You need to force airflow version and airflow base version in the workflow."
+                   echo
+                   exit 1
+                fi
+             fi
              echo "airflow-base-version=${AIRFLOW_BASE_VERSION}" >> ${GITHUB_OUTPUT}
              echo "airflow-version=${AIRFLOW_VERSION}" >> ${GITHUB_OUTPUT}
           else
@@ -139,6 +168,7 @@ jobs:
       INCLUDE_SUCCESS_OUTPUTS: false
       VERBOSE: "true"
       EXTRA_BUILD_OPTIONS: ${{ needs.build-info.outputs.extra-build-options }}
+      APPLY_COMMITS: ${{ inputs.apply-commits || '' }}
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -161,17 +191,43 @@ jobs:
           ref: ${{ inputs.ref }}
           fetch-depth: 0
           fetch-tags: true
+      - name: "Apply patch commits if provided"
+        run: |
+          if [[ "${APPLY_COMMITS}" != "" ]]; then
+            git config --global user.email "bot@airflow.apache.org"
+            git config --global user.name "Your friendly bot"
+            echo "Applying commits ${APPLY_COMMIT} to the docs"
+            # Split APPLY_COMMITS by comma and apply each commit
+            IFS=',' read -ra COMMIT_ARRAY <<< "${APPLY_COMMITS}"
+            for APPLY_COMMIT in "${COMMIT_ARRAY[@]}"; do
+              echo "Applying commit ${APPLY_COMMIT}"
+              git fetch origin "${APPLY_COMMIT}"
+              git cherry-pick "${APPLY_COMMIT}"
+            done
+          else
+            echo "No commits provided to apply, skipping."
+          fi
       - name: "Install Breeze from the ${{ inputs.ref }} reference"
         uses: ./.github/actions/breeze
         with:
-          use-uv: ${{ inputs.use-uv }}
           python-version: "${{ needs.build-info.outputs.default-python-version }}"
       - name: "Building image from the ${{ inputs.ref }} reference"
         env:
           INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }}
           INCLUDE_COMMITS: ${{ startsWith(inputs.ref, 'providers') && 'true' || 'false' }}
+        # if the regular breeze ci-image build fails, we will try to build the image using docker buildx
+        # This is needed for the case when we are building an old image which tries to use main as
+        # a cache and it fails because the main branch has changed and does not have the same pyproject.toml
+        # Structure as the one we are trying to build.
         run: >
-          breeze ci-image build
+          breeze ci-image build ||
+          docker buildx build --load --builder default --progress=auto --pull
+          --build-arg AIRFLOW_EXTRAS=devel-ci --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES=false
+          --build-arg AIRFLOW_USE_UV=true --build-arg UV_HTTP_TIMEOUT=300
+          --build-arg BUILD_PROGRESS=auto --build-arg INSTALL_MYSQL_CLIENT_TYPE=mariadb
+          --build-arg VERSION_SUFFIX_FOR_PYPI=dev0
+          -t ghcr.io/apache/airflow/main/ci/python3.9:latest --target main .
+          -f Dockerfile.ci --platform linux/amd64
       - name: "Building docs with --docs-only flag using ${{ inputs.ref }} reference breeze"
         env:
           INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }}
@@ -268,8 +324,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: "Download docs prepared as artifacts"
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
         with:

.github/workflows/push-image-cache.yml

@@ -120,8 +120,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: Login to ghcr.io
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -191,8 +189,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: ${{ inputs.use-uv }}
       - name: "Cleanup dist and context file"
         run: rm -fv ./dist/* ./docker-context-files/*
       - name: "Download packages prepared as artifacts"

.github/workflows/release_dockerhub_image.yml

@@ -96,8 +96,6 @@ jobs:
         run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}"
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: "true"
       - name: Selective checks
         id: selective-checks
         env:

.github/workflows/release_single_dockerhub_image.yml

@@ -83,8 +83,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: "false"
       - name: Free space
         run: breeze ci free-space --answer yes
       - name: "Cleanup dist and context file"
@@ -181,8 +179,6 @@ jobs:
         run: ./scripts/ci/prepare_and_cleanup_runner.sh
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          use-uv: "false"
       - name: Free space
         run: breeze ci free-space --answer yes
       - name: "Cleanup dist and context file"

dev/README_RELEASE_AIRFLOW.md

@@ -376,6 +376,25 @@ Other available parameters can be found with:
 breeze workflow-run publish-docs --help
 ```
 
+In case you publish the documentation from branch, you can specify `--airflow-version` and `--airflow-base-version`
+parameters to specify which version of airflow you want to build the documentation for - as it cannot
+be automatically derived from tag name. Normally both are automatically derived from the tag name.
+
+One of the interesting features of publishing this way is that you can also rebuild historical version of
+the documentation with patches applied to the documentation (if they can be applied cleanly).
+
+Yoy should specify the `--apply-commits` parameter with the list of commits you want to apply
+separated by commas and the workflow will apply those commits to the documentation before
+building it. (don't forget to add --skip-write-to-stable-folder if you are publishing
+previous version of the distribution). Example:
+
+```shell script
+breeze workflow-run publish-docs --ref 3.0.3 --site-env staging \
+  --apply-commits 4ae273cbedec66c87dc40218c7a94863390a380d,e61e9618bdd6be8213d277b1427f67079fcb1d9b \
+  --skip-write-to-stable-folder \
+  apache-airflow docker-stack task-sdk
+```
+
 ### Manually using GitHub Actions
 
 There are two steps to publish the documentation:

dev/README_RELEASE_PROVIDERS.md

@@ -516,6 +516,20 @@ The `--ref` parameter should be the tag of the release candidate you are publish
 The `--site-env` parameter should be set to `staging` for pre-release versions or `live` for final releases. the default option is `auto`
 if the tag is rc it publishes to `staging` bucket, otherwise it publishes to `live` bucket.
 
+One of the interesting features of publishing this way is that you can also rebuild historical version of
+the documentation with patches applied to the documentation (if they can be applied cleanly).
+
+Yoy should specify the `--apply-commits` parameter with the list of commits you want to apply
+separated by commas and the workflow will apply those commits to the documentation before
+building it (don't forget to add --skip-write-to-stable-folder if you are publishing
+previous version of the distribution). Example:
+
+```shell script
+breeze workflow-run publish-docs --ref providers-apache-hive/9.0.0 --site-env live \
+  --apply-commits 4ae273cbedec66c87dc40218c7a94863390a380d --skip-write-to-stable-folder \
+  apache.hive
+```
+
 Other available parameters can be found with:
 
 ```shell