diff --git a/.github/workflows/connector_integration_test_single_dagger.yml b/.github/workflows/connector_integration_test_single_dagger.yml index 98e56fe17606..8eae8b0d7ec5 100644 --- a/.github/workflows/connector_integration_test_single_dagger.yml +++ b/.github/workflows/connector_integration_test_single_dagger.yml @@ -10,38 +10,10 @@ on: paths: - "airbyte-integrations/connectors/**" jobs: - start-runner: - name: "Connectors Integration Test: Start Build EC2 Runner" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - pipeline-start-timestamp: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - steps: - - name: Get start timestamp - id: get-start-timestamp - run: echo "::set-output name=start-timestamp::$(date +%s)" - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ env.PAT }} - ec2-instance-type: "c5.2xlarge" # https://aws.amazon.com/ec2/instance-types/ connectors_ci: name: Connectors CI timeout-minutes: 240 # 4 hours - needs: start-runner - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + runs-on: medium-runner steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -61,7 +33,16 @@ jobs: run: pip install ./tools/ci_connector_ops\[pipelines]\ - name: Run connectors-ci test-connectors [WORKFLOW DISPATCH] if: github.event_name == 'workflow_dispatch' - run: connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors ${{ github.event.inputs.test-connectors-options }} + run: | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" + DAGGER_CLI_COMMIT="67c7e7635cf4ea0e446e2fed522a3e314c960f6a" + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then + mkdir -p "$DAGGER_TMP_BINDIR" + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" + fi + connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors ${{ github.event.inputs.test-connectors-options }} env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} @@ -75,7 +56,16 @@ jobs: CI_PIPELINE_START_TIMESTAMP: ${{ needs.start-runner.outputs.pipeline-start-timestamp }} - name: Run connectors-ci test-connectors [PULL REQUESTS] if: github.event_name == 'pull_request' - run: connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --modified + run: | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" + DAGGER_CLI_COMMIT="67c7e7635cf4ea0e446e2fed522a3e314c960f6a" + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then + mkdir -p "$DAGGER_TMP_BINDIR" + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" + fi + connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --modified env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} @@ -87,35 +77,3 @@ jobs: CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }} CI_CONTEXT: "pull_request" CI_PIPELINE_START_TIMESTAMP: ${{ needs.start-runner.outputs.pipeline-start-timestamp }} - - stop-runner: - name: "Connectors Integration Test: Stop Build EC2 Runner" - timeout-minutes: 10 - needs: - - start-runner # required to get output from the start-runner job - - connectors_ci # required to wait when the main job is done - runs-on: ubuntu-latest - # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. - # Thus, we check for skipped here. - if: ${{ always() && needs.start-runner.result != 'skipped'}} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ env.PAT }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}