metadata: fix tests not running, fix failing non-running tests, fix validate base images exist #112717
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Airbyte CI - Repository Health Check | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| jobs: | |
| # In case of self-hosted EC2 errors, remove this block. | |
| start-check-runner: | |
| name: Start EC2 Runner | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Start AWS Runner | |
| id: start-ec2-runner | |
| uses: ./.github/actions/start-aws-runner | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| # Use a beefier instance type than the default c5.2xlarge, but with the same per-core cost. | |
| # When gradle runs on this instance, it will use up all the available cores anyway. | |
| # There should be little to no difference in total cost, however the job latency will be improved. | |
| # At the time of this writing, the latency doesn't improve much beyond this instance size (approx 5 minutes). | |
| # This is largely thanks to the gradle cache. | |
| ec2-instance-type: "c5.4xlarge" | |
| github-token: ${{ env.PAT }} | |
| run-check: | |
| # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. | |
| needs: start-check-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner | |
| name: Gradle Check | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| # IMPORTANT! This is nessesary to make sure that a status is reported on the PR | |
| # even if the workflow is skipped. If we used github actions filters, the workflow | |
| # would not be reported as skipped, but instead would be forever pending. | |
| # | |
| # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. | |
| # | |
| # Also it gets worse | |
| # | |
| # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. | |
| # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. | |
| - name: Get changed files | |
| uses: tj-actions/changed-files@v39 | |
| id: changes | |
| with: | |
| files_yaml: | | |
| gradlecheck: | |
| - '**/*' | |
| - '!**/*.md' | |
| - '!.github/*' | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: "17" | |
| - name: Install Pip | |
| if: steps.changes.outputs.gradlecheck_any_changed == 'true' | |
| run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3 | |
| - name: Install Pyenv | |
| if: steps.changes.outputs.gradlecheck_any_changed == 'true' | |
| run: python3 -m pip install virtualenv --user | |
| - name: Docker login | |
| # Some tests use testcontainers which pull images from DockerHub. | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| - name: Run Gradle Check | |
| if: steps.changes.outputs.gradlecheck_any_changed == 'true' | |
| uses: burrunan/gradle-cache-action@v1 | |
| env: | |
| CI: true | |
| with: | |
| read-only: ${{ github.ref != 'refs/heads/master' }} | |
| # TODO: be able to remove the skipSlowTests property | |
| arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true | |
| # In case of self-hosted EC2 errors, remove this block. | |
| stop-check-runner: | |
| name: Stop EC2 Runner | |
| timeout-minutes: 10 | |
| needs: | |
| - start-check-runner # required to get output from the start-runner job | |
| - run-check # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. | |
| # Thus, we check for skipped here. | |
| if: ${{ always() && needs.start-check-runner.result != 'skipped'}} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Stop EC2 runner | |
| uses: supertopher/ec2-github-runner@base64v1.0.10 | |
| with: | |
| mode: stop | |
| github-token: ${{ env.PAT }} | |
| label: ${{ needs.start-check-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-check-runner.outputs.ec2-instance-id }} | |
| set-instatus-incident-on-failure: | |
| name: Create Instatus Incident on Failure | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-check | |
| if: ${{ failure() && github.ref == 'refs/heads/master' }} | |
| steps: | |
| - name: Call Instatus Webhook | |
| uses: joelwmale/webhook-action@master | |
| with: | |
| url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} | |
| body: '{ "trigger": "down", "status": "HASISSUES" }' | |
| set-instatus-incident-on-success: | |
| name: Create Instatus Incident on Success | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-check | |
| if: ${{ success() && github.ref == 'refs/heads/master' }} | |
| steps: | |
| - name: Call Instatus Webhook | |
| uses: joelwmale/webhook-action@master | |
| with: | |
| url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} | |
| body: '{ "trigger": "up" }' | |
| notify-failure-slack-channel: | |
| name: "Notify Slack Channel on Build Failures" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-check | |
| if: ${{ failure() && github.ref == 'refs/heads/master' }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Match GitHub User to Slack User | |
| id: match-github-to-slack-user | |
| uses: ./.github/actions/match-github-to-slack-user | |
| env: | |
| AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish to OSS Build Failure Slack Channel | |
| uses: abinoda/slack-action@master | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| args: >- | |
| {\"channel\":\"C03BEADRPNY\", \"blocks\":[ | |
| {\"type\":\"divider\"}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}}, | |
| {\"type\":\"divider\"}]} | |
| notify-failure-slack-channel-fixed-broken-build: | |
| name: "Notify Slack Channel on Build Fixes" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-check | |
| if: success() | |
| steps: | |
| - name: Get Previous Workflow Status | |
| uses: Mercymeilya/last-workflow-status@v0.3 | |
| id: last_status | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. | |
| - name: Publish Build Fixed Message to OSS Build Failure Slack Channel | |
| if: ${{ steps.last_status.outputs.last_status == 'failure' }} | |
| uses: abinoda/slack-action@master | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| args: >- | |
| {\"channel\":\"C03BEADRPNY\", \"blocks\":[ | |
| {\"type\":\"divider\"}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}}, | |
| {\"type\":\"divider\"}]} |