Don't run submission benchmarks by default (#1592) #3523
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "*" | |
| schedule: | |
| # Runs "At 00:01" (see https://crontab.guru) | |
| - cron: "1 0 * * 0" # every Sunday | |
| workflow_dispatch: | |
| concurrency: | |
| # Include `github.event_name` to avoid pushes to `main` and | |
| # scheduled jobs canceling one another | |
| group: tests-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| # Required shell entrypoint to have properly activated conda environments | |
| run: | |
| shell: bash -l {0} | |
| jobs: | |
| tests: | |
| name: ${{ matrix.name_prefix }} ${{ matrix.os }} py${{ matrix.python_version }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python_version: ["3.10"] | |
| pytest_args: [tests --ignore=tests/tpch --ignore=tests/geospatial] | |
| extra-env: [""] | |
| name_prefix: [tests] | |
| include: | |
| # Run stability tests on the lowest and highest versions of Python only | |
| # These are temporarily redundant with the current global python_version | |
| # - pytest_args: tests/stability | |
| # python_version: "3.10" | |
| # os: ubuntu-latest | |
| # name_prefix: stability | |
| - pytest_args: tests/stability | |
| python_version: "3.11" | |
| os: ubuntu-latest | |
| name_prefix: stability | |
| # Run stability tests on Python Windows and MacOS (latest py39 only) | |
| - pytest_args: tests/stability | |
| python_version: "3.10" | |
| os: windows-latest | |
| name_prefix: stability | |
| - pytest_args: tests/stability | |
| python_version: "3.10" | |
| os: macos-latest | |
| name_prefix: stability | |
| - pytest_args: tests/workflows/test_snowflake.py | |
| python_version: "3.10" | |
| os: ubuntu-latest | |
| name_prefix: snowflake | |
| extra-env: ci/environment-snowflake.yml | |
| - pytest_args: tests/tpch | |
| python_version: "3.10" | |
| os: ubuntu-latest | |
| extra-env: ci/environment-tpch-nondask.yml | |
| name_prefix: tpch | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up environment | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-version: latest | |
| use-mamba: true | |
| condarc-file: ci/condarc | |
| python-version: ${{ matrix.python_version }} | |
| environment-file: ci/environment.yml | |
| - name: Add extra packages to environment | |
| if: ${{ matrix.extra-env != '' }} | |
| run: mamba env update --file ${{ matrix.extra-env }} | |
| - name: Upgrade dask to git tip | |
| run: mamba env update --file ci/environment-git-tip.yml | |
| - name: Add test dependencies | |
| run: mamba env update --file ci/environment-test.yml | |
| - name: Reconfigure pytest-timeout | |
| shell: bash -l {0} | |
| # No SIGALRM available on Windows | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: sed -i.bak 's/timeout_method = signal/timeout_method = thread/' setup.cfg | |
| - name: Dump environment | |
| run: | | |
| # For debugging | |
| echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--" | |
| mamba env export | grep -E -v '^prefix:.*$' | |
| - name: Disable workflows on most PRs | |
| # Run workflows on PRs with `workflows` label and nightly cron job | |
| if: | | |
| github.event_name != 'schedule' | |
| && !(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'workflows')) | |
| run: | | |
| echo PYTEST_MARKERS=" and not workflows" >> $GITHUB_ENV | |
| - name: Disable non-Dask TPCH benchmarks | |
| run: | | |
| echo PYTEST_MARKERS="${{ env.PYTEST_MARKERS }} and not tpch_nondask" >> $GITHUB_ENV | |
| - name: Finalize PYTEST_MARKERS | |
| run: | | |
| if [ -n "$PYTEST_MARKERS" ]; then | |
| PYTEST_MARKERS=${PYTEST_MARKERS# and } | |
| PYTEST_MARKERS="-m '${PYTEST_MARKERS}'" | |
| echo PYTEST_MARKERS=${PYTEST_MARKERS} | |
| echo PYTEST_MARKERS=${PYTEST_MARKERS} >> $GITHUB_ENV | |
| fi | |
| - name: Google auth | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| credentials_json: "${{ secrets.GCP_CREDENTIALS }}" | |
| - name: Run Coiled Runtime Tests | |
| env: | |
| DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }} | |
| AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }} | |
| AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME}} | |
| AZURE_STORAGE_SAS_TOKEN: ${{ secrets.AZURE_STORAGE_SAS_TOKEN}} | |
| COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }} | |
| DB_NAME: ${{ matrix.name_prefix }}-${{ matrix.os }}-py${{ matrix.python_version }}.db | |
| MEMRAY_PROFILE: "none" | |
| run: | | |
| pytest --benchmark -n 4 --dist loadscope --memray $MEMRAY_PROFILE ${{ env.PYTEST_MARKERS }} ${{ matrix.pytest_args }} | |
| - name: Dump coiled.Cluster kwargs | |
| run: cat cluster_kwargs.merged.yaml || true | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{ matrix.name_prefix }}-${{ matrix.os }}-py${{ matrix.python_version }} | |
| path: | | |
| ${{ matrix.name_prefix }}-${{ matrix.os }}-py${{ matrix.python_version }}.db | |
| cluster_kwargs.*.* | |
| mamba_env_export.yml | |
| process-results: | |
| needs: tests | |
| name: Combine separate benchmark results | |
| if: always() && github.repository == 'coiled/benchmarks' | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| # Fairly strict concurrency rule to avoid stepping on benchmark db. | |
| # Could eventually replace with a real db in coiled, RDS, or litestream | |
| group: process-benchmarks | |
| cancel-in-progress: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: pip install alembic | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: benchmarks | |
| - name: Download benchmark db | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason | |
| DB_NAME: benchmark.db | |
| run: | | |
| aws s3 cp s3://coiled-runtime-ci/benchmarks/$DB_NAME . || true | |
| - name: Combine benchmarks | |
| run: | | |
| ls -lhR benchmarks | |
| bash ci/scripts/combine-dbs.sh | |
| - name: Upload benchmark db | |
| if: always() && github.ref == 'refs/heads/main' && github.repository == 'coiled/benchmarks' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason | |
| DB_NAME: benchmark.db | |
| run: | | |
| aws s3 cp $DB_NAME s3://coiled-runtime-ci/benchmarks/ | |
| - name: Upload benchmark results as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark | |
| path: benchmark.db | |
| regressions: | |
| needs: [tests, process-results] | |
| # Always check for regressions, as this can be skipped even if an indirect dependency fails (like a test run) | |
| # Not running regressions when tests are cancelled, and on PRs because of volatility of single runs | |
| if: always() && github.event_name != 'pull_request' && needs.tests.result != 'cancelled' | |
| name: Detect regressions | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark | |
| - name: Set up environment | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-version: latest | |
| use-mamba: true | |
| environment-file: ci/environment-dashboard.yml | |
| - name: Run detect regressions | |
| run: | | |
| if [[ ${{ github.event_name }} = 'pull_request' ]] | |
| then | |
| export IS_PR='true' | |
| fi | |
| echo "IS_PR=$IS_PR" | |
| python detect_regressions.py | |
| - name: Create regressions summary | |
| if: always() | |
| run: | | |
| echo "$(<regressions_summary.md)" >> $GITHUB_STEP_SUMMARY | |
| report: | |
| name: report | |
| needs: [tests, regressions] | |
| if: | | |
| always() | |
| && github.event_name != 'pull_request' | |
| && github.repository == 'coiled/benchmarks' | |
| && (needs.tests.result == 'failure' || needs.regressions.result == 'failure') | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Report failures | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` | |
| const issue_body = `[Workflow Run URL](${workflow_url})` | |
| github.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: issue_body, | |
| title: "⚠️ CI failed ⚠️", | |
| labels: ["ci-failure"], | |
| }) | |
| static-site: | |
| needs: process-results | |
| # Always generate the site, as this can be skipped even if an indirect dependency fails (like a test run) | |
| if: always() | |
| name: Build static dashboards | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download tests database | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark | |
| - name: Set up environment | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-version: latest | |
| use-mamba: true | |
| environment-file: ci/environment-dashboard.yml | |
| - name: Generate dashboards | |
| run: python dashboard.py -d benchmark.db -o static | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: static-dashboard | |
| path: static | |
| - name: Deploy 🚀 | |
| uses: JamesIves/github-pages-deploy-action@v4.5.0 | |
| if: github.ref == 'refs/heads/main' && github.repository == 'coiled/benchmarks' | |
| with: | |
| branch: gh-pages | |
| folder: static | |
| single-commit: true |