Skip to content
Tests

Tests #2238

Sign in to view logs

Tests

Tests #2238

Workflow file for this run

.github/workflows/ci.yml at 01e40ab
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Tests
on: # yamllint disable-line rule:truthy
schedule:
- cron: '28 0 * * *'
push:
branches: ['main', 'v[0-9]+-[0-9]+-test']
pull_request:
branches: ['main']
permissions:
# All other permissions are set to none
contents: read
# Technically read access while waiting for images should be more than enough. However,
# there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission
# denied error when attempting to just pull private image, changing the token permission to write solves the
# issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml
# changes. If it is for a fork, then the token is read-only anyway.
packages: write
env:
ANSWER: "yes"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
# Airflow one is going to be used
CONSTRAINTS_GITHUB_REPOSITORY: >-
${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
# In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
USE_SUDO: "true"
INCLUDE_SUCCESS_OUTPUTS: "true"
INCLUDE_NOT_READY_PROVIDERS: "true"
AIRFLOW_ENABLE_AIP_44: "true"
MOUNT_SOURCES: "skip"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-info:
name: "Build info"
# The runs-on cannot refer to env. or secrets. context, so we have no
# option but to specify a hard-coded list here. This is "safe", the
# runner checks if the user is an owner or collaborator of the repo
# before running the workflow.
runs-on: ["ubuntu-22.04"]
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
# Version of Python used for reproducibility of the packages built
# Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
# Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
# TODO: remove me when we switch to Python 3.9 as minimal version
breeze-python-version: "3.9"
image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
affected-providers-list-as-string: >-
${{ steps.selective-checks.outputs.affected-providers-list-as-string }}
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
kubernetes-combos-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
parallel-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
docs-build: ${{ steps.selective-checks.outputs.docs-build }}
mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }}
helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
runs-on: ${{steps.selective-checks.outputs.runs-on}}
is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
build-job-description: ${{ steps.source-run-info.outputs.build-job-description }}
canary-run: ${{ steps.source-run-info.outputs.canary-run }}
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Get information about the Workflow"
id: source-run-info
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- name: env
run: printenv
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
# Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
# tests to complete - it is run very early in the build process for "main" merges in order to refresh
# cache using the current constraints. This will speed up cache refresh in cases when pyproject.toml
# changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will
# delay cache refresh. It does not attempt to upgrade to newer dependencies.
# We only push CI cache as PROD cache usually does not gain as much from fresh cache because
# it uses prepared airflow and provider packages that invalidate the cache anyway most of the time
push-early-buildx-cache-to-github-registry:
timeout-minutes: 110
name: "Push Early Image Cache"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
strategy:
fail-fast: false
matrix:
platform: ["linux/amd64", "linux/arm64"]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
UPGRADE_TO_NEWER_DEPENDENCIES: false
VERSION_SUFFIX_FOR_PYPI: "dev0"
continue-on-error: true
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
if: >
matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
- name: "Push CI cache ${{ matrix.platform }}"
run: >
breeze ci-image build
--builder airflow_cache
--prepare-buildx-cache
--run-in-parallel
--platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Push CI latest image ${{ matrix.platform }}"
run: >
breeze ci-image build
--tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
# We only push "amd" image as it is really only needed for any kind of automated builds in CI
# and currently there is not an easy way to make multi-platform image from two separate builds
if: >
matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: >
always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
- name: "Clean docker cache for ${{ matrix.platform }}"
run: docker system prune --all --force
if: >
matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
# Check that after earlier cache push, breeze command will build quickly
check-that-image-builds-quickly:
timeout-minutes: 5
name: "Check that image builds quickly"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- push-early-buildx-cache-to-github-registry
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
UPGRADE_TO_NEWER_DEPENDENCIES: false
PLATFORM: "linux/amd64"
if: >
needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Check that image builds quickly"
run: breeze shell --max-time 120
build-ci-images:
strategy:
fail-fast: true
matrix:
python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
timeout-minutes: 80
name: ${{needs.build-info.outputs.build-job-description}} CI image ${{ matrix.python-version }}
runs-on: ["ubuntu-22.04"]
needs: [build-info]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
# Force more parallelism for build even on public images
PARALLELISM: 6
VERSION_SUFFIX_FOR_PYPI: "dev0"
USE_UV: "true"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'true'
- name: Build CI Images ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-ci-images
if: needs.build-info.outputs.in-workflow-build == 'true'
with:
python-version: ${{ matrix.python-version }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
BUILD_TIMEOUT_MINUTES: 70
- name: Verify CI images ${{ matrix.python-version }}:${{ needs.build-info.outputs.image-tag }}
run: breeze ci-image verify --python ${{ matrix.python-version }}
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'true'
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
# Need to fetch all history for selective checks tests
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: "${{needs.build-info.outputs.default-python-version}}"
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: pip install --editable ./dev/breeze/
- run: python -m pytest -n auto --color=yes
working-directory: ./dev/breeze/
tests-www:
timeout-minutes: 10
name: React WWW tests
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.run-www-tests == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Setup node"
uses: actions/setup-node@v4
with:
node-version: 21
- name: "Cache eslint"
uses: actions/cache@v4
with:
path: 'airflow/www/node_modules'
key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
- run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
- run: yarn --cwd airflow/www/ run test
env:
FORCE_COLOR: 2
test-openapi-client:
timeout-minutes: 10
name: "Test OpenAPI client"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.needs-api-codegen == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
repository: "apache/airflow-client-python"
fetch-depth: 1
persist-credentials: false
path: ./airflow-client-python
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Generate client with breeze"
run: >
breeze release-management prepare-python-client --package-format both
--version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python
- name: "Show diff"
run: git diff --color HEAD
working-directory: ./airflow-client-python
- name: Install hatch
run: |
python -m pip install --upgrade pipx
pipx ensurepath
pipx install hatch --force
- name: Run tests
run: hatch run run-coverage
env:
HATCH_ENV: "test"
working-directory: ./clients/python
- name: "Install Airflow in editable mode with fab for webserver tests"
run: pip install -e ".[fab]"
- name: "Install Python client"
run: pip install ./dist/apache_airflow_client-*.whl
- name: "Initialize Airflow DB and start webserver"
run: |
airflow db init
# Let scheduler runs a few loops and get all DAG files from example DAGs serialized to DB
airflow scheduler --num-runs 100
airflow users create --username admin --password admin --firstname Admin --lastname Admin \
--role Admin --email admin@example.org
killall python || true # just in case there is a webserver running in the background
nohup airflow webserver --port 8080 &
echo "Started webserver"
env:
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.session,airflow.api.auth.backend.basic_auth
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True"
AIRFLOW__CORE__LOAD_EXAMPLES: "True"
- name: "Waiting for the webserver to be available"
run: |
timeout 30 bash -c 'until nc -z $0 $1; do echo "sleeping"; sleep 1; done' localhost 8080
sleep 5
- name: "Run test python client"
run: python ./clients/python/test_python_client.py
env:
FORCE_COLOR: "standard"
- name: "Stop running webserver"
run: killall python || true # just in case there is a webserver running in the background
if: always()
- name: "Upload python client packages"
uses: actions/upload-artifact@v4
with:
name: python-client-packages
path: ./dist/apache_airflow_client-*
retention-days: 7
if-no-files-found: error
test-git-clone-on-windows:
timeout-minutes: 5
name: "Test git clone on Windows"
runs-on: "windows-latest"
needs: [build-info]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
wait-for-ci-images:
timeout-minutes: 120
name: "Wait for CI images"
runs-on: ["ubuntu-22.04"]
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.ci-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
# Force more parallelism for pull even on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
id: wait-for-images
run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
generate-constraints:
permissions:
contents: read
timeout-minutes: 70
name: >
Generate constraints with CI image verifying
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
VERSION_SUFFIX_FOR_PYPI: "dev0"
if: needs.build-info.outputs.ci-image-build == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze ci-image pull --run-in-parallel --tag-as-latest
- name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze ci-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- name: "Source constraints"
shell: bash
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints-source-providers
- name: "No providers constraints"
shell: bash
timeout-minutes: 25
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints-no-providers
- name: "Prepare chicken-eggs provider packages"
# In case of provider packages which use latest dev0 version of providers, we should prepare them
# from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
# And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
shell: bash
run: >
breeze release-management prepare-provider-packages --include-not-ready-providers
--package-format wheel --version-suffix-for-pypi dev0
${{ needs.build-info.outputs.chicken-egg-providers }}
if: needs.build-info.outputs.chicken-egg-providers != ''
- name: "PyPI constraints"
shell: bash
timeout-minutes: 25
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints
--chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}"
- name: "Dependency upgrade summary"
shell: bash
run: |
for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
echo "Summarizing Python $PYTHON_VERSION"
cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true
done
- name: "Upload constraint artifacts"
uses: actions/upload-artifact@v4
with:
name: constraints
path: ./files/constraints-*/constraints-*.txt
retention-days: 7
if-no-files-found: error
static-checks:
timeout-minutes: 45
name: "Static checks"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
id: breeze
- name: Cache pre-commit envs
uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
# yamllint disable-line rule:line-length
key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: |
pre-commit-${{steps.breeze.outputs.host-python-version}}-
- name: "Static checks"
run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment
env:
VERBOSE: "false"
SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
COLUMNS: "250"
SKIP_GROUP_OUTPUT: "true"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
RUFF_FORMAT: "github"
upgrade-check:
timeout-minutes: 45
name: "Upgrade checks"
runs-on: ["ubuntu-22.04"]
needs: [build-info]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
if: needs.build-info.outputs.canary-run == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
# Install python from scratch. No cache used. We always want to have fresh version of everything
- uses: actions/setup-python@v5
with:
python-version: "${{needs.build-info.outputs.default-python-version}}"
- name: "Install latest pre-commit"
run: pip install pre-commit
- name: "Autoupdate all pre-commits"
run: pre-commit autoupdate
- name: "Run automated upgrade for black"
run: >
pre-commit run
--all-files --show-diff-on-failure --color always --verbose
--hook-stage manual
update-black-version
if: always()
- name: "Run automated upgrade for build dependencies"
run: >
pre-commit run
--all-files --show-diff-on-failure --color always --verbose
--hook-stage manual
update-build-dependencies
if: always()
- name: "Run automated upgrade for installers"
run: >
pre-commit run
--all-files --show-diff-on-failure --color always --verbose
--hook-stage manual
update-installers
if: always()
- name: "Run automated upgrade for chart dependencies"
run: >
pre-commit run
--all-files --show-diff-on-failure --color always --verbose
--hook-stage manual
update-chart-dependencies
if: always()
# Runs static checks for groups of files in the repository in a single process without passing .
# List of files
mypy:
timeout-minutes: 45
name: "MyPy checks"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
matrix:
mypy-folder: ${{fromJson(needs.build-info.outputs.mypy-folders)}}
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.needs-mypy == 'true'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.needs-mypy == 'true'
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
id: breeze
if: needs.build-info.outputs.needs-mypy == 'true'
- name: Cache pre-commit envs
uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
# yamllint disable-line rule:line-length
key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: |
pre-commit-${{steps.breeze.outputs.host-python-version}}-
if: needs.build-info.outputs.needs-mypy == 'true'
- name: "MyPy checks for ${{ matrix.mypy-folder }}"
run: |
pip install pre-commit
pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files
env:
VERBOSE: "false"
COLUMNS: "250"
SKIP_GROUP_OUTPUT: "true"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
RUFF_FORMAT: "github"
if: needs.build-info.outputs.needs-mypy == 'true'
# Those checks are run if no image needs to be built for checks. This is for simple changes that
# Do not touch any of the python code or any of the important files that might require building
# The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Setup python"
uses: actions/setup-python@v5
with:
python-version: "${{needs.build-info.outputs.default-python-version}}"
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: "Install Breeze"
uses: ./.github/actions/breeze
id: breeze
- name: Cache pre-commit envs
uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
# yamllint disable-line rule:line-length
key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: "\
pre-commit-${{steps.breeze.outputs.host-python-version}}-\
${{ hashFiles('.pre-commit-config.yaml') }}\n
pre-commit-${{steps.breeze.outputs.host-python-version}}-"
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Static checks: basic checks only"
run: >
breeze static-checks --show-diff-on-failure --color always --initialize-environment
--skip-image-upgrade-check --commit-ref "${{ github.sha }}"
env:
VERBOSE: "false"
SKIP_BREEZE_PRE_COMMITS: "true"
SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
COLUMNS: "250"
build-docs:
timeout-minutes: 60
name: "Build docs"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- uses: actions/cache@v4
id: cache-doc-inventories
with:
path: ./docs/_inventory_cache/
key: docs-inventory-${{ hashFiles('pyproject.toml;') }}
restore-keys: |
docs-inventory-${{ hashFiles('pyproject.toml;') }}
docs-inventory-
- name: "Build docs"
run: >
breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --docs-only
- name: "Clone airflow-site"
run: >
git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
if: needs.build-info.outputs.canary-run == 'true'
- name: "Publish docs"
run: >
breeze release-management publish-docs
--override-versioned --run-in-parallel
${{ needs.build-info.outputs.docs-list-as-string }}
if: needs.build-info.outputs.canary-run == 'true'
- name: "Generate back references for providers"
run: breeze release-management add-back-references all-providers
if: needs.build-info.outputs.canary-run == 'true'
- name: "Generate back references for apache-airflow"
run: breeze release-management add-back-references apache-airflow
if: needs.build-info.outputs.canary-run == 'true'
- name: "Generate back references for docker-stack"
run: breeze release-management add-back-references docker-stack
if: needs.build-info.outputs.canary-run == 'true'
- name: "Generate back references for helm-chart"
run: breeze release-management add-back-references helm-chart
if: needs.build-info.outputs.canary-run == 'true'
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
with:
aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
aws-region: eu-central-1
- name: "Upload documentation to AWS S3"
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
spellcheck-docs:
timeout-minutes: 120
name: "Spellcheck docs"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- uses: actions/cache@v4
id: cache-doc-inventories
with:
path: ./docs/_inventory_cache/
key: docs-inventory-${{ hashFiles('pyproject.toml;') }}
restore-keys: |
docs-inventory-${{ hashFiles('pyproject.toml;') }}
docs-inventory-
- name: "Spellcheck docs"
run: >
breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --spellcheck-only
prepare-test-provider-packages-wheel:
timeout-minutes: 80
name: "Provider packages wheel (verify)"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
if: needs.build-info.outputs.skip-provider-tests != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider documentation"
run: >
breeze release-management prepare-provider-documentation --include-not-ready-providers
--non-interactive
- name: "Prepare provider packages: wheel"
run: >
breeze release-management prepare-provider-packages --include-not-ready-providers
--version-suffix-for-pypi dev0 --package-format wheel
- name: "Prepare airflow package: wheel"
run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
- name: "Verify wheel packages with twine"
run: |
pipx uninstall twine || true
pipx install twine && twine check dist/*.whl
- name: "Test providers issue generation automatically"
run: >
breeze release-management generate-issue-content-providers
--only-available-in-dist --disable-progress
- name: "Generate source constraints from CI image"
shell: bash
run: >
breeze release-management generate-constraints
--airflow-constraints-mode constraints-source-providers
- name: "Install and verify all provider packages and airflow via wheel files"
run: >
breeze release-management verify-provider-packages
--use-packages-from-dist
--package-format wheel
--use-airflow-version wheel
--airflow-constraints-reference default
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
env:
AIRFLOW_SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Prepare airflow package: wheel without suffix and skipping the tag check"
run: >
breeze release-management prepare-provider-packages --skip-tag-check --package-format wheel
prepare-install-provider-packages-sdist:
timeout-minutes: 80
name: "Provider packages sdist (install)"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
if: >
needs.build-info.outputs.canary-run == 'true' &&
needs.build-info.outputs.skip-provider-tests != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider packages: sdist"
run: >
breeze release-management prepare-provider-packages --include-not-ready-providers
--version-suffix-for-pypi dev0 --package-format sdist
${{ needs.build-info.outputs.affected-providers-list-as-string }}
- name: "Prepare airflow package: sdist"
run: >
breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0 --package-format sdist
- name: "Verify sdist packages with twine"
run: |
pipx uninstall twine || true
pipx install twine && twine check dist/*.tar.gz
- name: "Generate source constraints from CI image"
shell: bash
run: >
breeze release-management generate-constraints
--airflow-constraints-mode constraints-source-providers
- name: "Install all provider packages and airflow via sdist files"
run: >
breeze release-management install-provider-packages
--use-packages-from-dist
--package-format sdist
--use-airflow-version sdist
--airflow-constraints-reference default
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
--run-in-parallel
if: needs.build-info.outputs.affected-providers-list-as-string == ''
- name: "Install affected provider packages and airflow via sdist files"
run: >
breeze release-management install-provider-packages
--use-packages-from-dist
--package-format sdist
--use-airflow-version sdist
--airflow-constraints-reference default
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
--run-in-parallel
if: needs.build-info.outputs.affected-providers-list-as-string != ''
providers-compatibility-checks:
timeout-minutes: 80
name: >
Compat ${{matrix.airflow-version}}:P${{matrix.python-version}} provider check
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
matrix:
include: ${{fromJson(needs.build-info.outputs.providers-compatibility-checks)}}
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
VERSION_SUFFIX_FOR_PYPI: "dev0"
if: needs.build-info.outputs.skip-provider-tests != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: contains(fromJson(needs.build-info.outputs.python-versions),matrix.python-version)
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider packages: wheel"
run: >
breeze release-management prepare-provider-packages --include-not-ready-providers
--package-format wheel
- name: >
Remove incompatible Airflow
${{matrix.airflow-version}}:Python ${{matrix.python-version}} provider packages
run: |
rm -vf ${{ matrix.remove-providers }}
working-directory: ./dist
if: matrix.remove-providers != ''
- name: "Download airflow package: wheel"
run: |
pip download "apache-airflow==${{matrix.airflow-version}}" -d dist --no-deps
- name: >
Install and verify all provider packages and airflow on
Airflow ${{matrix.airflow-version}}:Python ${{matrix.python-version}}
run: >
breeze release-management verify-provider-packages
--use-packages-from-dist
--package-format wheel
--use-airflow-version wheel
--airflow-constraints-reference constraints-${{matrix.airflow-version}}
--providers-skip-constraints
--install-airflow-with-constraints
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: >
needs.build-info.outputs.is-self-hosted-runner == 'true'
&& needs.build-info.outputs.is-airflow-runner == 'false'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Install required packages"
run: python -m pip install --editable ./dev/breeze/
- name: "Check Airflow create minor branch command"
run: breeze release-management create-minor-branch --version-branch 2-4 -a y
- name: "Check Airflow RC process command"
run: |
breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y
- name: "Check Airflow release process command"
run: |
breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y
tests-helm:
timeout-minutes: 80
name: "Unit tests Helm: ${{matrix.helm-test-package}}"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
matrix:
helm-test-package: ${{fromJson(needs.build-info.outputs.helm-test-packages)}}
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "Helm"
BACKEND: "none"
DB_RESET: "false"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
JOB_ID: "helm-tests"
USE_XDIST: "true"
if: >
needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Helm Unit Tests: ${{ matrix.helm-test-package }}"
run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}"
tests-helm-release:
timeout-minutes: 80
name: "Release Helm"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
if: >
needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: Setup git for tagging
run: |
git config --global user.email "name@example.com"
git config --global user.name "Your Name"
- name: "Remove old artifacts"
run: rm -rf dist/*
- name: "Setup k8s/helm environment"
run: breeze k8s setup-env
- name: "Install helm gpg plugin"
run: >
breeze k8s shell -c "helm plugin install https://github.com/technosophos/helm-gpg
--version 6303407eb63deaeb1b2f24de611e3468a27ec05b" || true
- name: "Helm release tarball"
run: >
breeze release-management prepare-helm-chart-tarball --ignore-version-check --override-tag
--skip-tag-signing --version 0.0.0 --version-suffix dev0
- name: Generate GPG key for signing
# Sometimes the key will be already added to the keyring, so we ignore the error
run: gpg --batch --passphrase '' --quick-gen-key dev@airflow.apache.org default default || true
- name: "Helm release package"
run: >
breeze release-management prepare-helm-chart-package --sign-email dev@airflow.apache.org
- name: "Sign artifacts for ASF distribution"
run: ./dev/sign.sh dist/airflow-*.tgz dist/airflow-*-source.tar.gz
env:
SIGN_WITH: dev@airflow.apache.org
- name: "Upload Helm artifacts"
uses: actions/upload-artifact@v4
with:
name: Helm artifacts
path: ./dist/airflow-*
retention-days: 7
if-no-files-found: error
tests-postgres:
timeout-minutes: 130
name: >
DB:Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}"
exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}"
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
POSTGRES_VERSION: "${{matrix.postgres-version}}"
BACKEND_VERSION: "${{matrix.postgres-version}}"
JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
run: breeze testing db-tests --collect-only --remove-arm-packages
if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version
- name: >
Post Tests success: Postgres"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Postgres"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-min-sqlalchemy:
timeout-minutes: 130
name: >
DB:MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
DOWNGRADE_SQLALCHEMY: "true"
JOB_ID: >
min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:MinSQLAlchemy"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:MinSQLAlchemy"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-boto:
timeout-minutes: 130
name: >
LatestBoto-Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
UPGRADE_BOTO: "true"
JOB_ID: >
boto-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing tests --run-in-parallel
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:LatestBoto"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:LatestBoto"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-pydantic:
timeout-minutes: 130
name: >
Pydantic-${{ matrix.pydantic }}-Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
strategy:
matrix:
pydantic: ["v1", "none"]
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
PYDANTIC: ${{ matrix.pydantic }}
JOB_ID: >
pydantic-${{ matrix.pydantic }}-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing tests --run-in-parallel
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:NoPydantic"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:NoPydantic"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-pendulum-2:
timeout-minutes: 130
name: >
Pendulum2-Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
DOWNGRADE_PENDULUM: "true"
JOB_ID: >
pendulum-2-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing tests --run-in-parallel
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Pendulum2"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Pendulum2"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-in-progress-features-disabled:
timeout-minutes: 130
name: >
InProgressDisabled-Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
AIRFLOW_ENABLE_AIP_44: "false"
JOB_ID: >
in-progress-disabled-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing tests --run-in-parallel
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-mysql:
timeout-minutes: 130
name: >
DB:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}"
exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}"
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "mysql"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
MYSQL_VERSION: "${{matrix.mysql-version}}"
BACKEND_VERSION: "${{matrix.mysql-version}}"
JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: MySQL"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: MySQL"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-sqlite:
timeout-minutes: 130
name: >
DB:Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }}
fail-fast: false
if: needs.build-info.outputs.run-tests == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "sqlite"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
BACKEND_VERSION: ""
JOB_ID: "sqlite-${{matrix.python-version}}"
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: Sqlite"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Sqlite"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-integration-postgres:
timeout-minutes: 130
name: Integration Tests Postgres
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
JOB_ID: "integration-postgres"
SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: "Integration Tests Postgres: cassandra"
run: |
breeze testing integration-tests --integration cassandra
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: mongo"
run: |
breeze testing integration-tests --integration mongo
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: pinot"
run: |
breeze testing integration-tests --integration pinot
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: celery"
run: |
breeze testing integration-tests --integration celery
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: trino, kerberos"
run: |
breeze testing integration-tests --integration trino --integration kerberos
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: Kafka"
run: |
breeze testing integration-tests --integration kafka
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: Qdrant"
run: breeze testing integration-tests --integration qdrant
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Integration Tests Postgres: all-testable"
run: breeze testing integration-tests --integration all-testable
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests success: Integration Postgres"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Integration Postgres"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-integration-mysql:
timeout-minutes: 130
name: Integration Tests MySQL
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "mysql"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
JOB_ID: "integration-mysql"
SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Integration Tests MySQL: all-testable"
run: breeze testing integration-tests --integration all-testable
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests success: Integration MySQL"
uses: ./.github/actions/post_tests_success
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests failure: Integration MySQL"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-quarantined:
timeout-minutes: 60
name: "Quarantined tests"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
continue-on-error: true
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
TEST_TYPE: "All-Quarantined"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.is-airflow-runner == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "postgres"
BACKEND_VERSION: ${{needs.build-info.outputs.default-postgres-version}}
POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}}
- name: >
Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "mysql"
BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
- name: >
Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "sqlite"
BACKEND_VERSION: ""
- name: >
Post Tests success: Quarantined"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Quarantined"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-no-db:
timeout-minutes: 60
name: >
Non-DB: Py${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
JOB_ID: "no-db-${{matrix.python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Tests: ${{matrix.python-version}}:Non-DB"
run: >
breeze testing non-db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: "Post Tests success: Non-DB"
uses: ./.github/actions/post_tests_success
if: success()
- name: "Post Tests failure: Non-DB"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-no-db-pendulum-2:
timeout-minutes: 60
name: >
Non-DB: Pendulum2, Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
JOB_ID: "no-db-pendulum-2-${{needs.build-info.outputs.default-python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
DOWNGRADE_PENDULUM: "true"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Tests: ${{matrix.python-version}}:Non-DB-Pendulum2"
run: >
breeze testing non-db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: "Post Tests success: Non-DB-Pendulum2"
uses: ./.github/actions/post_tests_success
if: success()
- name: "Post Tests failure: Non-DB-Pendulum2"
uses: ./.github/actions/post_tests_failure
if: failure()
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- tests-postgres
- tests-sqlite
- tests-mysql
- tests-quarantined
- tests-integration-postgres
- tests-integration-mysql
- tests-no-db
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Download all artifacts from the current build"
uses: actions/download-artifact@v4
with:
path: ./artifacts
- name: "Summarize all warnings"
run: |
ls -R ./artifacts/
cat ./artifacts/test-warnings*/* | sort | uniq
echo
echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l)
build-prod-images:
name: Build PROD images (in-workflow)
needs: [build-info, build-ci-images]
uses: ./.github/workflows/prod-image-build.yml
permissions:
contents: read
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
# For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
# "in-workflow-build" condition
packages: write
secrets: inherit
with:
build-type: "Regular"
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
push-image: "true"
use-uv: "true"
in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }}
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.in-workflow-build == 'true'
prod-image-extra-checks-main:
name: PROD image extra checks (main)
# Here we just need to wait for CI images: we build provider packages and can use source constraints
needs: [build-info, build-ci-images]
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.default-branch == 'main' && needs.build-info.outputs.canary-runs == 'true'
prod-image-extra-checks-release-branch:
name: PROD image extra checks (release)
# Here we need to wait for generate-constraints to complete because we have to use PyPI constraints
needs: [build-info, generate-constraints]
uses: ./.github/workflows/prod-image-extra-checks.yml
with:
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.default-branch != 'main' && needs.build-info.outputs.canary-runs == 'true'
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for PROD images"
runs-on: ["ubuntu-22.04"]
needs: [build-info, wait-for-ci-images, build-prod-images]
# This strange condition below is equivalent to: "all success or skipped"
# The "build-;rod-images" step might be skipped, in case production building happens in the
# "build-images" workflow, and in this case we sill want to wait for PROD images and run depending tests
if: >
always() && !failure() && !cancelled() &&
needs.build-info.outputs.prod-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
# Force more parallelism for pull on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
# We wait for the images to be available either from "build-images.yml' run as pull_request_target
# or from build-prod-images (or build-prod-images-release-branch) above.
# We are utilising single job to wait for all images because this job merely waits
# For the images to be available.
run: breeze prod-image pull --wait-for-image --run-in-parallel
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of production image building"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
if: needs.build-info.outputs.prod-image-build == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: >
Pull PROD image
"${{needs.build-info.outputs.default-python-version}}":${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image pull --tag-as-latest
env:
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
- name: "Setup python"
uses: actions/setup-python@v5
with:
python-version: ${{needs.build-info.outputs.default-python-version}}
cache: 'pip'
cache-dependency-path: ./dev/requirements.txt
- name: "Test examples of PROD image building"
# yamllint disable-line rule:line-length
run: >
cd ./docker_tests &&
python -m pip install -r requirements.txt &&
TEST_IMAGE="ghcr.io/${{env.GITHUB_REPOSITORY}}/${{needs.build-info.outputs.default-branch}}/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
env:
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Docker-compose quick start with PROD image verifying"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
if: needs.build-info.outputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
# Force more parallelism for pull even on public images
PARALLELISM: 6
- name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- name: "Test docker-compose quick start"
run: breeze testing docker-compose-tests
tests-kubernetes:
timeout-minutes: 240
name: "\
K8S System:${{matrix.executor}} - ${{matrix.use-standard-naming}} - \
${{needs.build-info.outputs.kubernetes-versions-list-as-string}}"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
strategy:
matrix:
executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
use-standard-naming: [true, false]
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.needs-helm-tests == 'true')
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
id: breeze
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
# Force more parallelism for pull even on public images
PARALLELISM: 6
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v4
with:
path: ".build/.k8s-env"
key: "\
k8s-env-${{steps.breeze.outputs.host-python-version}}-\
${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','pyproject.toml') }}"
- name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
run: breeze k8s run-complete-tests --run-in-parallel --upgrade
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
EXECUTOR: ${{matrix.executor}}
USE_STANDARD_NAMING: ${{matrix.use-standard-naming}}
VERBOSE: "false"
- name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v4
if: failure() || cancelled()
with:
name: kind-logs-${{matrix.executor}}-${{matrix.use-standard-naming}}
path: /tmp/kind_logs_*
retention-days: 7
- name: >
Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v4
if: failure() || cancelled()
with:
name: k8s-test-resources-${{matrix.executor}}-${{matrix.use-standard-naming}}
path: /tmp/k8s_test_resources_*
retention-days: 7
- name: "Delete clusters just in case they are left"
run: breeze k8s delete-cluster --all
if: always()
update-constraints:
permissions:
contents: write
packages: write
timeout-minutes: 80
name: "Update constraints"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- build-docs
- spellcheck-docs
- wait-for-ci-images
- wait-for-prod-images
- static-checks
- mypy
- tests-sqlite
- tests-mysql
- tests-postgres
- tests-no-db
- tests-integration-postgres
- tests-integration-mysql
- generate-constraints
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
# Needed to perform push action
persist-credentials: false
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
uses: actions/checkout@v4
with:
path: "constraints"
ref: ${{ steps.constraints-branch.outputs.branch }}
persist-credentials: true
fetch-depth: 0
- name: "Download constraints from the constraints generated by build CI image"
uses: actions/download-artifact@v4
with:
name: constraints
path: ./files
- name: "Diff in constraints for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_diff_constraints.sh
# only commit and push constraints in canary runs (main)
- name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
if: needs.build-info.outputs.canary-run == 'true'
- name: "Push changes"
if: needs.build-info.outputs.canary-run == 'true'
working-directory: "constraints"
run:
git push
# Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build
# is executed as result of direct push to "main" or one of the "vX-Y-test" branches
# It rebuilds all images using just-pushed constraints using buildx and pushes them to registry
# It will automatically check if a new python image was released and will pull the latest one if needed
push-buildx-cache-to-github-registry:
timeout-minutes: 110
name: "Push Image Cache"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- update-constraints
if: needs.build-info.outputs.canary-run == 'true'
strategy:
fail-fast: false
matrix:
platform: ["linux/amd64", "linux/arm64"]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
python-version: ${{ needs.build-info.outputs.breeze-python-version }}
env:
# Always use default Python version of CI image for preparing packages
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
- name: "Prepare airflow package for PROD build"
run: breeze release-management prepare-airflow-package --package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
- name: "Prepare providers packages for PROD build"
run: >
breeze release-management prepare-provider-packages
--package-list-file ./prod_image_installed_providers.txt
--package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
if: needs.build-info.outputs.default-branch == 'main'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
if: matrix.platform == 'linux/arm64'
- name: "Push CI cache ${{ matrix.platform }}"
run: >
breeze ci-image build
--builder airflow_cache
--prepare-buildx-cache
--run-in-parallel
--platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
- name: "Push CI latest image ${{ matrix.platform }}"
run: >
breeze ci-image build
--tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
if: matrix.platform == 'linux/amd64'
- name: "Copy dist packages to docker-context files"
run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
- name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}"
run: |
# Do not run parallel builds here as they often fail due to github token expiry issues similar to
# those described in https://github.com/moby/buildkit/issues/2367
for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
breeze prod-image build --builder airflow_cache --install-packages-from-context \
--airflow-constraints-mode constraints-source-providers --prepare-buildx-cache \
--platform ${{ matrix.platform }} --python ${python}
done
env:
COMMIT_SHA: ${{ github.sha }}
- name: "Push PROD latest image ${{ matrix.platform }}"
run: >
breeze prod-image build --tag-as-latest --install-packages-from-context
--push --run-in-parallel --platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
if: matrix.platform == 'linux/amd64'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always() && matrix.platform == 'linux/arm64'
# This is only a check if ARM images are successfully building when committer runs PR from
# Apache repository. This is needed in case you want to fix failing cache job in "canary" run
# There is no point in running this one in "canary" run, because the above step is doing the
# same build anyway.
build-ci-arm-images:
timeout-minutes: 110
name: >
Build CI ARM images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- build-docs
- spellcheck-docs
- static-checks
- mypy
- tests-sqlite
- tests-mysql
- tests-postgres
- tests-no-db
- tests-integration-postgres
- tests-integration-mysql
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
# Force more parallelism for build even on small instances
PARALLELISM: 6
USE_UV: "true"
if: >
needs.build-info.outputs.in-workflow-build == 'true' &&
needs.build-info.outputs.canary-run != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: >
Build CI ARM images ${{ needs.build-info.outputs.image-tag }}
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
run: >
breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always()