test-integrations-data-processing.yml

name: Test Data Processing
on:
  push:
    branches:
      - master
      - release/**
      - sentry-sdk-2.0
  pull_request:
# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true
permissions:
  contents: read
env:
  BUILD_CACHE_KEY: ${{ github.sha }}
  CACHED_BUILD_PATHS: |
    ${{ github.workspace }}/dist-serverless
jobs:
  test-data_processing-latest:
    name: Data Processing (latest)
    timeout-minutes: 30
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.6","3.7","3.8","3.10","3.11","3.12"]
        # python3.6 reached EOL and is no longer being supported on
        # new versions of hosted runners on Github Actions
        # ubuntu-20.04 is the last version that supported python3.6
        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
        os: [ubuntu-20.04]
    steps:
      - uses: actions/checkout@v4.1.7
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Start Redis
        uses: supercharge/redis-github-action@1.8.0
      - name: Setup Test Env
        run: |
          pip install coverage tox
      - name: Erase coverage
        run: |
          coverage erase
      - name: Test arq latest
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh "py${{ matrix.python-version }}-arq-latest"
      - name: Test beam latest
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh "py${{ matrix.python-version }}-beam-latest"
      - name: Test celery latest
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh "py${{ matrix.python-version }}-celery-latest"
      - name: Test huey latest
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh "py${{ matrix.python-version }}-huey-latest"
      - name: Test rq latest
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh "py${{ matrix.python-version }}-rq-latest"
      - name: Test spark latest
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh "py${{ matrix.python-version }}-spark-latest"
      - name: Generate coverage XML
        if: ${{ !cancelled() }}
        run: |
          coverage combine .coverage*
          coverage xml -i
      - name: Upload coverage to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@v4.5.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: coverage.xml
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
  test-data_processing-pinned:
    name: Data Processing (pinned)
    timeout-minutes: 30
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12"]
        # python3.6 reached EOL and is no longer being supported on
        # new versions of hosted runners on Github Actions
        # ubuntu-20.04 is the last version that supported python3.6
        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
        os: [ubuntu-20.04]
    steps:
      - uses: actions/checkout@v4.1.7
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Start Redis
        uses: supercharge/redis-github-action@1.8.0
      - name: Setup Test Env
        run: |
          pip install coverage tox
      - name: Erase coverage
        run: |
          coverage erase
      - name: Test arq pinned
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-arq"
      - name: Test beam pinned
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-beam"
      - name: Test celery pinned
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-celery"
      - name: Test huey pinned
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-huey"
      - name: Test rq pinned
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-rq"
      - name: Test spark pinned
        run: |
          set -x # print commands that are executed
          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-spark"
      - name: Generate coverage XML
        if: ${{ !cancelled() }}
        run: |
          coverage combine .coverage*
          coverage xml -i
      - name: Upload coverage to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@v4.5.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: coverage.xml
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
  check_required_tests:
    name: All Data Processing tests passed
    needs: test-data_processing-pinned
    # Always run this, even if a dependent job failed
    if: always()
    runs-on: ubuntu-20.04
    steps:
      - name: Check for failures
        if: contains(needs.test-data_processing-pinned.result, 'failure') || contains(needs.test-data_processing-pinned.result, 'skipped')
        run: |
          echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1