metadata: fix tests not running, fix failing non-running tests, fix validate base images exist #6309
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector Ops CI - Pipeline Unit Test | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| airbyte_ci_binary_url: | |
| description: "URL to airbyte-ci binary" | |
| required: false | |
| default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| jobs: | |
| run-airbyte-ci-tests: | |
| name: Run Airbyte CI tests | |
| runs-on: "ci-runner-connector-test-large-dagger-0-6-4" | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| # IMPORTANT! This is nessesary to make sure that a status is reported on the PR | |
| # even if the workflow is skipped. If we used github actions filters, the workflow | |
| # would not be reported as skipped, but instead would be forever pending. | |
| # | |
| # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. | |
| # | |
| # Also it gets worse | |
| # | |
| # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. | |
| # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. | |
| - name: Get changed files | |
| uses: tj-actions/changed-files@v39 | |
| id: changes | |
| with: | |
| files_yaml: | | |
| ops: | |
| - 'airbyte-ci/connectors/connector_ops/**' | |
| - '!**/*.md' | |
| base_images: | |
| - 'airbyte-ci/connectors/connector_ops/**' | |
| - 'airbyte-ci/connectors/base_images/**' | |
| - '!**/*.md' | |
| pipelines: | |
| - 'airbyte-ci/connectors/connector_ops/**' | |
| - 'airbyte-ci/connectors/base_images/**' | |
| - 'airbyte-ci/connectors/pipelines/**' | |
| - '!**/*.md' | |
| metadata_lib: | |
| - 'airbyte-ci/connectors/metadata_service/lib/**' | |
| - '!**/*.md' | |
| metadata_orchestrator: | |
| - 'airbyte-ci/connectors/metadata_service/lib/**' | |
| - 'airbyte-ci/connectors/metadata_service/orchestrator/**' | |
| - '!**/*.md' | |
| - name: Run airbyte-ci/connectors/connector_ops tests | |
| if: steps.changes.outputs.ops_any_changed == 'true' | |
| id: run-airbyte-ci-connectors-connector-ops-tests | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-ci/connectors/connector_ops" | |
| airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} | |
| - name: Run airbyte-ci/connectors/pipelines tests | |
| id: run-airbyte-ci-connectors-pipelines-tests | |
| if: steps.changes.outputs.pipelines_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-ci/connectors/pipelines" | |
| airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} | |
| - name: Run airbyte-ci/connectors/base_images tests | |
| id: run-airbyte-ci-connectors-base-images-tests | |
| if: steps.changes.outputs.base_images_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| context: "pull_request" | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| subcommand: "test airbyte-ci/connectors/base_images" | |
| airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} | |
| - name: Run test pipeline for the metadata lib | |
| id: metadata_lib-test-pipeline | |
| if: steps.changes.outputs.metadata_lib_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| subcommand: "test airbyte-ci/connectors/metadata_service/lib/" | |
| context: "pull_request" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} | |
| - name: Run test for the metadata orchestrator | |
| id: metadata_orchestrator-test-pipeline | |
| if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true' | |
| uses: ./.github/actions/run-dagger-pipeline | |
| with: | |
| subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/" | |
| context: "pull_request" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} |