diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index fa2ed1374d138..93b70b8ac0474 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -48,6 +48,10 @@ on: # yamllint disable-line rule:truthy description: "Which version of python should be used by default" required: true type: string + shared-distributions-as-json: + description: "Json array of shared distributions to run tests for" + required: true + type: string canary-run: description: "Whether to run canary tests (true/false)" required: true @@ -60,6 +64,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv in the image" required: true type: string + uv-version: + description: 'uv version to use' + default: "0.8.13" # Keep this comment to allow automatic replacement of uv version + type: string platform: description: 'Platform for the build - linux/amd64 or linux/arm64' required: true @@ -86,42 +94,27 @@ jobs: uses: ./.github/actions/breeze - run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes working-directory: ./dev/breeze/ - tests-shared-secrets-masker: + tests-shared-distributions: timeout-minutes: 10 - name: Shared secrets masker tests + name: Shared ${{ matrix.shared-distribution }} tests + strategy: + fail-fast: false + matrix: + shared-distribution: ${{ fromJSON(inputs.shared-distributions-as-json) }} runs-on: ${{ fromJSON(inputs.runners) }} + env: + UV_VERSION: ${{inputs.uv-version}} steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + fetch-depth: 1 persist-credentials: false - - name: "Prepare and cleanup runner" - run: ./scripts/ci/prepare_and_cleanup_runner.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze + - name: "Install uv" + run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh - name: "Run shared secrets masker tests" - run: uv run --group dev pytest --color=yes - working-directory: ./shared/secrets_masker/ - tests-shared-timezones: - timeout-minutes: 10 - name: Shared timezone tests - runs-on: ${{ fromJSON(inputs.runners) }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: "Prepare and cleanup runner" - run: ./scripts/ci/prepare_and_cleanup_runner.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: "Run shared timezone tests" - run: uv run --group dev pytest --color=yes - working-directory: ./shared/timezones/ + run: uv run --group dev pytest --color=yes -n auto + working-directory: shared/${{ matrix.shared-distribution }} tests-ui: timeout-minutes: 15 name: React UI tests diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index f27383a7bc255..c3be70d386139 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -121,6 +121,7 @@ jobs: run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} + shared-distributions-as-json: ${{ steps.selective-checks.outputs.shared-distributions-as-json }} skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} source-head-repo: ${{ steps.source-run-info.outputs.head-repo }} @@ -212,6 +213,7 @@ jobs: latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} platform: "linux/amd64" + shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}} build-ci-images: name: Build CI images diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 37bac6d7e6782..bd3300ce5a699 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -114,6 +114,7 @@ jobs: run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} selected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} + shared-distributions-as-json: ${{ steps.selective-checks.outputs.shared-distributions-as-json }} skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} @@ -175,6 +176,7 @@ jobs: latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} use-uv: ${{needs.build-info.outputs.use-uv}} platform: "linux/arm64" + shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}} build-ci-images: name: Build CI images diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index f44e121004984..34f4a767b99c9 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1512,3 +1512,7 @@ def is_log_mocked_in_the_tests(self) -> bool: @cached_property def force_pip(self): return FORCE_PIP_LABEL in self._pr_labels + + @cached_property + def shared_distributions_as_json(self): + return json.dumps([file.name for file in (AIRFLOW_ROOT_PATH / "shared").iterdir() if file.is_dir()]) diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 67b01debd4979..0183819c63221 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -50,6 +50,7 @@ False, ), (AIRFLOW_ROOT_PATH / ".github" / "actions" / "install-prek" / "action.yml", False), + (AIRFLOW_ROOT_PATH / ".github" / "workflows" / "basic-test.yml", False), (AIRFLOW_ROOT_PATH / "dev" / "breeze" / "doc" / "ci" / "02_images.md", True), (AIRFLOW_ROOT_PATH / "dev" / "breeze" / "pyproject.toml", False), (AIRFLOW_ROOT_PATH / ".pre-commit-config.yaml", False),