Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Eks runner branch #23943

Merged
merged 5 commits into from
Mar 21, 2023
Merged

Eks runner branch #23943

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
75b083f
run on eks runner
c-p-b Mar 10, 2023
acfaaa1
retrigger build
c-p-b Mar 10, 2023
c29389a
remote cache
c-p-b Mar 10, 2023
63a0d10
new dagger cli config
c-p-b Mar 15, 2023
48bfe2c
Merge branch 'master' into eks-runner-branch
Mar 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 21 additions & 63 deletions .github/workflows/connector_integration_test_single_dagger.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,38 +10,10 @@ on:
paths:
- "airbyte-integrations/connectors/**"
jobs:
start-runner:
name: "Connectors Integration Test: Start Build EC2 Runner"
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
pipeline-start-timestamp: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
steps:
- name: Get start timestamp
id: get-start-timestamp
run: echo "::set-output name=start-timestamp::$(date +%s)"
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ env.PAT }}
ec2-instance-type: "c5.2xlarge" # https://aws.amazon.com/ec2/instance-types/
connectors_ci:
name: Connectors CI
timeout-minutes: 240 # 4 hours
needs: start-runner
runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
runs-on: medium-runner
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
Expand All @@ -61,7 +33,16 @@ jobs:
run: pip install ./tools/ci_connector_ops\[pipelines]\
- name: Run connectors-ci test-connectors [WORKFLOW DISPATCH]
if: github.event_name == 'workflow_dispatch'
run: connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors ${{ github.event.inputs.test-connectors-options }}
run: |
export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
DAGGER_CLI_COMMIT="67c7e7635cf4ea0e446e2fed522a3e314c960f6a"
DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}"
export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger"
if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then
mkdir -p "$DAGGER_TMP_BINDIR"
curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
fi
connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors ${{ github.event.inputs.test-connectors-options }}
env:
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
Expand All @@ -75,7 +56,16 @@ jobs:
CI_PIPELINE_START_TIMESTAMP: ${{ needs.start-runner.outputs.pipeline-start-timestamp }}
- name: Run connectors-ci test-connectors [PULL REQUESTS]
if: github.event_name == 'pull_request'
run: connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --modified
run: |
export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
DAGGER_CLI_COMMIT="67c7e7635cf4ea0e446e2fed522a3e314c960f6a"
DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}"
export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger"
if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then
mkdir -p "$DAGGER_TMP_BINDIR"
curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
fi
connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --modified
env:
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
Expand All @@ -87,35 +77,3 @@ jobs:
CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }}
CI_CONTEXT: "pull_request"
CI_PIPELINE_START_TIMESTAMP: ${{ needs.start-runner.outputs.pipeline-start-timestamp }}

stop-runner:
name: "Connectors Integration Test: Stop Build EC2 Runner"
timeout-minutes: 10
needs:
- start-runner # required to get output from the start-runner job
- connectors_ci # required to wait when the main job is done
runs-on: ubuntu-latest
# Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
# Thus, we check for skipped here.
if: ${{ always() && needs.start-runner.result != 'skipped'}}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
- name: Stop EC2 runner
uses: supertopher/ec2-github-runner@base64v1.0.10
with:
mode: stop
github-token: ${{ env.PAT }}
label: ${{ needs.start-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}