diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index 40f683c5a6fd4..46438127df678 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -22,9 +22,6 @@ inputs: python-version: description: 'Python version to use' default: "3.9" - use-uv: - description: 'Whether to use uv tool' - required: true outputs: host-python-version: description: Python version used in host diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 4a2685cc4ebd8..0724167cda519 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -43,8 +43,6 @@ runs: shell: bash - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} id: breeze - name: "Check free space" shell: bash diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 86a75b1d31fb4..f3982389301e6 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -142,7 +142,5 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: "Check that image builds quickly" run: breeze shell --max-time 900 --platform "${PLATFORM}" diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index 62d4ee1f6b420..98203e47b3ebc 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -105,8 +105,6 @@ jobs: if: ${{ inputs.use-local-venv == 'true' }} - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} if: ${{ inputs.use-local-venv == 'true' }} - name: "Cleanup dist files" run: rm -fv ./dist/* diff --git a/.github/workflows/backport-cli.yml b/.github/workflows/backport-cli.yml index 0ecdfb8e63e04..42f8178868267 100644 --- a/.github/workflows/backport-cli.yml +++ b/.github/workflows/backport-cli.yml @@ -70,8 +70,8 @@ jobs: TARGET_BRANCH: ${{ inputs.target-branch }} COMMIT_SHA: ${{ inputs.commit-sha }} run: | - git config --global user.email "name@example.com" - git config --global user.name "Your Name" + git config --global user.email "bot@airflow.apache.org" + git config --global user.name "Your friendly bot" set +e { echo 'cherry_picker_output<> ${GITHUB_OUTPUT} fi if [[ " ${INCLUDE_DOCS} " =~ " apache-airflow " ]]; then - AIRFLOW_BASE_VERSION=$(echo "${REF}" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') - AIRFLOW_VERSION="${REF}" + if [[ ${AIRFLOW_BASE_VERSION=} == "" && ${AIRFLOW_VERSION=} == "" ]]; then + echo "No Airflow Versions provided, using ${REF} to determine it." + AIRFLOW_VERSION="${REF}" + set +e + AIRFLOW_BASE_VERSION=$(echo "${REF}" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') + set -e + if [[ ${AIRFLOW_BASE_VERSION=} == "" ]]; then + echo + echo "No Airflow Base Version found in ${REF}" + echo "You need to force airflow version and airflow base version in the workflow." + echo + exit 1 + fi + fi echo "airflow-base-version=${AIRFLOW_BASE_VERSION}" >> ${GITHUB_OUTPUT} echo "airflow-version=${AIRFLOW_VERSION}" >> ${GITHUB_OUTPUT} else @@ -139,6 +168,7 @@ jobs: INCLUDE_SUCCESS_OUTPUTS: false VERBOSE: "true" EXTRA_BUILD_OPTIONS: ${{ needs.build-info.outputs.extra-build-options }} + APPLY_COMMITS: ${{ inputs.apply-commits || '' }} steps: - name: "Cleanup repo" shell: bash @@ -161,17 +191,43 @@ jobs: ref: ${{ inputs.ref }} fetch-depth: 0 fetch-tags: true + - name: "Apply patch commits if provided" + run: | + if [[ "${APPLY_COMMITS}" != "" ]]; then + git config --global user.email "bot@airflow.apache.org" + git config --global user.name "Your friendly bot" + echo "Applying commits ${APPLY_COMMIT} to the docs" + # Split APPLY_COMMITS by comma and apply each commit + IFS=',' read -ra COMMIT_ARRAY <<< "${APPLY_COMMITS}" + for APPLY_COMMIT in "${COMMIT_ARRAY[@]}"; do + echo "Applying commit ${APPLY_COMMIT}" + git fetch origin "${APPLY_COMMIT}" + git cherry-pick "${APPLY_COMMIT}" + done + else + echo "No commits provided to apply, skipping." + fi - name: "Install Breeze from the ${{ inputs.ref }} reference" uses: ./.github/actions/breeze with: - use-uv: ${{ inputs.use-uv }} python-version: "${{ needs.build-info.outputs.default-python-version }}" - name: "Building image from the ${{ inputs.ref }} reference" env: INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} INCLUDE_COMMITS: ${{ startsWith(inputs.ref, 'providers') && 'true' || 'false' }} + # if the regular breeze ci-image build fails, we will try to build the image using docker buildx + # This is needed for the case when we are building an old image which tries to use main as + # a cache and it fails because the main branch has changed and does not have the same pyproject.toml + # Structure as the one we are trying to build. run: > - breeze ci-image build + breeze ci-image build || + docker buildx build --load --builder default --progress=auto --pull + --build-arg AIRFLOW_EXTRAS=devel-ci --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES=false + --build-arg AIRFLOW_USE_UV=true --build-arg UV_HTTP_TIMEOUT=300 + --build-arg BUILD_PROGRESS=auto --build-arg INSTALL_MYSQL_CLIENT_TYPE=mariadb + --build-arg VERSION_SUFFIX_FOR_PYPI=dev0 + -t ghcr.io/apache/airflow/main/ci/python3.9:latest --target main . + -f Dockerfile.ci --platform linux/amd64 - name: "Building docs with --docs-only flag using ${{ inputs.ref }} reference breeze" env: INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} @@ -268,8 +324,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: "Download docs prepared as artifacts" uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index b523577b46c99..eccd176096137 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -120,8 +120,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: Login to ghcr.io env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -191,8 +189,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Download packages prepared as artifacts" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 2499e521d74a3..4678ea78f9d53 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -96,8 +96,6 @@ jobs: run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}" - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: "true" - name: Selective checks id: selective-checks env: diff --git a/.github/workflows/release_single_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml index fd572adbabab1..8cd5acca6aa19 100644 --- a/.github/workflows/release_single_dockerhub_image.yml +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -83,8 +83,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: "false" - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" @@ -181,8 +179,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: "false" - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index a61ec4496a745..854582e93237a 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -376,6 +376,25 @@ Other available parameters can be found with: breeze workflow-run publish-docs --help ``` +In case you publish the documentation from branch, you can specify `--airflow-version` and `--airflow-base-version` +parameters to specify which version of airflow you want to build the documentation for - as it cannot +be automatically derived from tag name. Normally both are automatically derived from the tag name. + +One of the interesting features of publishing this way is that you can also rebuild historical version of +the documentation with patches applied to the documentation (if they can be applied cleanly). + +Yoy should specify the `--apply-commits` parameter with the list of commits you want to apply +separated by commas and the workflow will apply those commits to the documentation before +building it. (don't forget to add --skip-write-to-stable-folder if you are publishing +previous version of the distribution). Example: + +```shell script +breeze workflow-run publish-docs --ref 3.0.3 --site-env staging \ + --apply-commits 4ae273cbedec66c87dc40218c7a94863390a380d,e61e9618bdd6be8213d277b1427f67079fcb1d9b \ + --skip-write-to-stable-folder \ + apache-airflow docker-stack task-sdk +``` + ### Manually using GitHub Actions There are two steps to publish the documentation: diff --git a/dev/README_RELEASE_PROVIDERS.md b/dev/README_RELEASE_PROVIDERS.md index 08492c7f2dec8..ed3e91aaaa8fd 100644 --- a/dev/README_RELEASE_PROVIDERS.md +++ b/dev/README_RELEASE_PROVIDERS.md @@ -516,6 +516,20 @@ The `--ref` parameter should be the tag of the release candidate you are publish The `--site-env` parameter should be set to `staging` for pre-release versions or `live` for final releases. the default option is `auto` if the tag is rc it publishes to `staging` bucket, otherwise it publishes to `live` bucket. +One of the interesting features of publishing this way is that you can also rebuild historical version of +the documentation with patches applied to the documentation (if they can be applied cleanly). + +Yoy should specify the `--apply-commits` parameter with the list of commits you want to apply +separated by commas and the workflow will apply those commits to the documentation before +building it (don't forget to add --skip-write-to-stable-folder if you are publishing +previous version of the distribution). Example: + +```shell script +breeze workflow-run publish-docs --ref providers-apache-hive/9.0.0 --site-env live \ + --apply-commits 4ae273cbedec66c87dc40218c7a94863390a380d --skip-write-to-stable-folder \ + apache.hive +``` + Other available parameters can be found with: ```shell diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 377cf320ceeb5..d15db05f27901 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -1,4 +1,4 @@ - +