From 1ea4ffc34fec9534d65ac2e46562103b798c5ec6 Mon Sep 17 00:00:00 2001 From: cgardens Date: Fri, 2 Jul 2021 16:50:17 -0700 Subject: [PATCH 01/21] CORE_ONLY => SUB_BUILD --- .github/workflows/gradle.yml | 183 ++++++++++++++++-- build.gradle | 7 +- .../developing-locally.md | 8 +- settings.gradle | 92 +++++---- tools/bin/acceptance_test.sh | 2 +- 5 files changed, 230 insertions(+), 62 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index a317f6284f92..0fc26f1c0190 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -6,7 +6,7 @@ on: push: jobs: - ## Gradle Build + ## Gradle Build (Connectors Base) # In case of self-hosted EC2 errors, remove this block. start-build-runner: name: Start Build EC2 Runner @@ -87,43 +87,190 @@ jobs: run: python3 -m pip install virtualenv==16.7.9 --user - name: Generate Template scaffold - run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan + run: SUB_BUILD=CONNECTOR_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan - name: Format - run: ./gradlew format --scan --info --stacktrace + run: SUB_BUILD=CONNECTOR_BASE ./gradlew format --scan --info --stacktrace + + - name: Build + run: SUB_BUILD=CONNECTOR_BASE ./gradlew build --scan + + - name: Ensure no file change + run: git status --porcelain && test -z "$(git status --porcelain)" + + - name: Check documentation + if: success() && github.ref == 'refs/heads/master' + run: ./tools/site/link_checker.sh check_docs + + - name: Slack Notification - Failure + if: failure() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildozer + SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg + SLACK_COLOR: DC143C + SLACK_TITLE: "Build failure" + SLACK_FOOTER: "" + + - name: Slack Notification - Success + if: success() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildbot + SLACK_TITLE: "Build Success" + SLACK_FOOTER: "" + # In case of self-hosted EC2 errors, remove this block. + stop-build-runner: + name: Stop Build EC2 Runner + needs: + - start-build-runner # required to get output from the start-runner job + - build # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2.1.0 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + + ## Gradle Build (Platform) + # In case of self-hosted EC2 errors, remove this block. + start-build-platform-runner: + name: Start Build (Platform) EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 Runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2.2.0 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + build-platform: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + needs: start-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner + name: Build (Platform) Airbyte + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + # todo (cgardens) - scope by platform. + - name: Check images exist + run: ./tools/bin/check_images_exist.sh + + - name: Pip Caching + uses: actions/cache@v2 + with: + path: | + ~/.cache/pip + key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}- + + - name: Npm Caching + uses: actions/cache@v2 + with: + path: | + ~/.npm + key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}- + + # this intentionally does not use restore-keys so we don't mess with gradle caching + - name: Gradle and Python Caching + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + **/.venv + key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user + + - name: Format + run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" - name: Build - run: CORE_ONLY=true ./gradlew build --scan + run: SUB_BUILD=PLATFORM ./gradlew build --scan - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" + # todo (cgardens) - scope by platform. - name: Check documentation if: success() && github.ref == 'refs/heads/master' run: ./tools/site/link_checker.sh check_docs -# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. -# - name: Get Docker Space -# run: docker run --rm busybox df -h + # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. + # - name: Get Docker Space + # run: docker run --rm busybox df -h - - name: Build Core Docker Images + - name: Image Cleanup + run: ./tools/bin/clean_images.sh + + - name: Build Platform Docker Images if: success() && github.ref == 'refs/heads/master' - run: ./gradlew composeBuild --scan + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan env: GIT_REVISION: ${{ github.sha }} - - name: Image Cleanup - run: ./tools/bin/clean_images.sh - + # make sure these always run before pushing platform docker images - name: Run End-to-End Acceptance Tests + if: success() && github.ref == 'refs/heads/master' run: ./tools/bin/acceptance_test.sh - name: Automatic Migration Acceptance Test run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan + - name: Push Platform Docker Images + if: success() && github.ref == 'refs/heads/master' + run: | + docker login -u airbytebot -p ${DOCKER_PASSWORD} + VERSION=dev docker-compose -f docker-compose.build.yaml push + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + - name: Slack Notification - Failure if: failure() && github.ref == 'refs/heads/master' uses: rtCamp/action-slack-notify@master @@ -144,8 +291,8 @@ jobs: SLACK_TITLE: "Build Success" SLACK_FOOTER: "" # In case of self-hosted EC2 errors, remove this block. - stop-build-runner: - name: Stop Build EC2 Runner + stop-build-platform-runner: + name: Stop Build (Platform) EC2 Runner needs: - start-build-runner # required to get output from the start-runner job - build # required to wait when the main job is done @@ -218,8 +365,8 @@ jobs: - name: Install Cypress Test Dependencies run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - - name: Build Core Docker Images and Run Tests - run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan + - name: Build Platform Docker Images + run: SUB_BUILD=CONNECTOR_BASE ./gradlew composebuild --scan - name: Run End-to-End Frontend Tests run: ./tools/bin/e2e_test.sh @@ -313,8 +460,8 @@ jobs: HOME: /home/runner CHANGE_MINIKUBE_NONE_USER: true - - name: Build Core Docker Images and Run Tests - run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan - name: Run Logging Tests run: ./tools/bin/cloud_storage_logging_test.sh diff --git a/build.gradle b/build.gradle index b7783d84ffc9..1ed691650a0a 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ def createSpotlessTarget = { pattern -> 'secrets' ] - if (System.getenv().containsKey("CORE_ONLY")) { + if (System.getenv().containsKey("SUB_BUILD")) { excludes.add("airbyte-integrations/connectors") } @@ -237,7 +237,10 @@ task composeBuild { } } } -build.dependsOn(composeBuild) + +if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { + build.dependsOn(composeBuild) +} task('generate') { dependsOn subprojects.collect { it.getTasksByName('generateSeed', true) } diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 147399f024d2..99843815c5c4 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -41,7 +41,7 @@ This will build all the code and run all the unit tests. To compile and build just the core systems: ```bash -CORE_ONLY=1 ./gradlew build +SUB_BUILD=PLATFORM ./gradlew build ``` {% hint style="info" %} @@ -64,7 +64,7 @@ export CPPFLAGS="-I/usr/local/opt/openssl/include" ## Run in `dev` mode with `docker-compose` ```bash -CORE_ONLY=1 ./gradlew build +SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up ``` @@ -77,7 +77,7 @@ In `dev` mode, all data will be persisted in `/tmp/dev_root`. To run acceptance \(end-to-end\) tests, you must have the Airbyte running locally. ```bash -CORE_ONLY=1 ./gradlew build +SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up ./gradlew :airbyte-tests:acceptanceTests ``` @@ -142,7 +142,7 @@ Sometimes you'll want to reset the data in your local environment. One common ca * Rebuild the project ```bash - CORE_ONLY=1 ./gradlew build + SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up -V ``` diff --git a/settings.gradle b/settings.gradle index 8f0d880bd295..c695e5a4b2c1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,48 +13,68 @@ gradleEnterprise { rootProject.name = 'airbyte' -include ':airbyte-analytics' -include ':airbyte-api' -include ':airbyte-cli' -include ':airbyte-cdk:python' +// SUB_BUILD is an enum of , PLATFORM, CONNECTOR_BASE. Blank is equivalent to all. +if(!System.getenv().containsKey("SUB_BUILD")) { + println("Building all of Airbyte.") +} else { + def subBuild = System.getenv().get("SUB_BUILD") + println("Building Airbyte Sub Build: " + subBuild) + if(subBuild != "PLATFORM" && subBuild != "CONNECTOR_BASE") { + throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTOR_BASE", subBuild)) + } +} + +// shared include ':airbyte-commons' include ':airbyte-commons-docker' -include ':airbyte-config:models' -include ':airbyte-config:init' -include ':airbyte-config:persistence' -include ':airbyte-db' -include ':airbyte-e2e-testing' -include ':airbyte-integrations:bases:airbyte-protocol' -include ':airbyte-integrations:bases:base' -include ':airbyte-integrations:bases:base-java' -include ':airbyte-integrations:bases:base-normalization' -include ':airbyte-integrations:bases:base-python' -include ':airbyte-integrations:bases:base-python-test' -include ':airbyte-integrations:bases:base-singer' -include ':airbyte-integrations:bases:base-standard-source-test-file' -include ':airbyte-integrations:bases:source-acceptance-test' -include ':airbyte-integrations:bases:standard-destination-test' -include ':airbyte-integrations:bases:standard-source-test' -include ':airbyte-integrations:connector-templates:generator' -include ':airbyte-integrations:bases:debezium' include ':airbyte-json-validation' -include ':airbyte-migration' -include ':airbyte-notification' include ':airbyte-protocol:models' include ':airbyte-queue' -include ':airbyte-scheduler:app' -include ':airbyte-scheduler:client' -include ':airbyte-scheduler:models' -include ':airbyte-scheduler:persistence' -include ':airbyte-server' -include ':airbyte-webapp' -include ':airbyte-workers' -include ':airbyte-tests' +include ':airbyte-workers' // reused by acceptance tests in connector base. +include ':airbyte-config:models' // reused by acceptance tests in connector base. +include ':airbyte-db' // reused by acceptance tests in connector base. include ':airbyte-test-utils' -include ':tools:code-generator' -if(!System.getenv().containsKey("CORE_ONLY")) { - println "Building all of Airbyte." +// platform +if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { + include ':airbyte-analytics' + include ':airbyte-api' + include ':airbyte-cli' + include ':airbyte-config:init' + include ':airbyte-config:persistence' + include ':airbyte-e2e-testing' + include ':airbyte-migration' + include ':airbyte-notification' + include ':airbyte-scheduler:app' + include ':airbyte-scheduler:client' + include ':airbyte-scheduler:models' + include ':airbyte-scheduler:persistence' + include ':airbyte-server' + include ':airbyte-webapp' + include ':airbyte-tests' +} + +// connectors base +if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CONNECTOR_BASE") { + include ':airbyte-cdk:python' + include ':airbyte-integrations:bases:airbyte-protocol' + include ':airbyte-integrations:bases:base' + include ':airbyte-integrations:bases:base-java' + include ':airbyte-integrations:bases:base-normalization' + include ':airbyte-integrations:bases:base-python' + include ':airbyte-integrations:bases:base-python-test' + include ':airbyte-integrations:bases:base-singer' + include ':airbyte-integrations:bases:base-standard-source-test-file' + include ':airbyte-integrations:bases:source-acceptance-test' + include ':airbyte-integrations:bases:standard-destination-test' + include ':airbyte-integrations:bases:standard-source-test' + include ':airbyte-integrations:connector-templates:generator' + include ':airbyte-integrations:bases:debezium' + include ':tools:code-generator' +} + +// connectors +if(!System.getenv().containsKey("SUB_BUILD")) { // include all connector projects def integrationsPath = rootDir.toPath().resolve('airbyte-integrations/connectors') println integrationsPath @@ -65,6 +85,4 @@ if(!System.getenv().containsKey("CORE_ONLY")) { include ":airbyte-integrations:connectors:${dir.getFileName()}" } } -} else { - println "Building Airbyte Core." } diff --git a/tools/bin/acceptance_test.sh b/tools/bin/acceptance_test.sh index a7a34f63f1bb..4c5ceab594ed 100755 --- a/tools/bin/acceptance_test.sh +++ b/tools/bin/acceptance_test.sh @@ -22,4 +22,4 @@ echo "Waiting for services to begin" sleep 10 # TODO need a better way to wait echo "Running e2e tests via gradle" -./gradlew --no-daemon :airbyte-tests:acceptanceTests --rerun-tasks --scan +SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-tests:acceptanceTests --rerun-tasks --scan From a6f5b5463c2a8884d838d974c6af55308bbb45b1 Mon Sep 17 00:00:00 2001 From: cgardens Date: Fri, 2 Jul 2021 17:02:33 -0700 Subject: [PATCH 02/21] move init to shared --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index c695e5a4b2c1..13aa7b390849 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,6 +28,7 @@ if(!System.getenv().containsKey("SUB_BUILD")) { include ':airbyte-commons' include ':airbyte-commons-docker' include ':airbyte-json-validation' +include ':airbyte-config:init' include ':airbyte-protocol:models' include ':airbyte-queue' include ':airbyte-workers' // reused by acceptance tests in connector base. @@ -40,7 +41,6 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-analytics' include ':airbyte-api' include ':airbyte-cli' - include ':airbyte-config:init' include ':airbyte-config:persistence' include ':airbyte-e2e-testing' include ':airbyte-migration' From 8db334ca2a4569ab48d04148ebf0bf29eb20ec87 Mon Sep 17 00:00:00 2001 From: cgardens Date: Fri, 2 Jul 2021 17:03:19 -0700 Subject: [PATCH 03/21] use platform --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 0fc26f1c0190..32462d6818a2 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -366,7 +366,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - name: Build Platform Docker Images - run: SUB_BUILD=CONNECTOR_BASE ./gradlew composebuild --scan + run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan - name: Run End-to-End Frontend Tests run: ./tools/bin/e2e_test.sh From db6a7d683c880b431a4dd742e58967580f015792 Mon Sep 17 00:00:00 2001 From: cgardens Date: Fri, 2 Jul 2021 17:03:37 -0700 Subject: [PATCH 04/21] right the first time --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 13aa7b390849..c695e5a4b2c1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,7 +28,6 @@ if(!System.getenv().containsKey("SUB_BUILD")) { include ':airbyte-commons' include ':airbyte-commons-docker' include ':airbyte-json-validation' -include ':airbyte-config:init' include ':airbyte-protocol:models' include ':airbyte-queue' include ':airbyte-workers' // reused by acceptance tests in connector base. @@ -41,6 +40,7 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-analytics' include ':airbyte-api' include ':airbyte-cli' + include ':airbyte-config:init' include ':airbyte-config:persistence' include ':airbyte-e2e-testing' include ':airbyte-migration' From a37707c29a29e8cbc2bb7af28424c6352de69a72 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:22:22 -0700 Subject: [PATCH 05/21] naming --- .github/workflows/gradle.yml | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 32462d6818a2..ee6b3c583eb2 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -8,8 +8,8 @@ on: jobs: ## Gradle Build (Connectors Base) # In case of self-hosted EC2 errors, remove this block. - start-build-runner: - name: Start Build EC2 Runner + start-connectors-build-runner: + name: "Connectors Base: Start Build EC2 Runner" runs-on: ubuntu-latest outputs: label: ${{ steps.start-ec2-runner.outputs.label }} @@ -31,11 +31,11 @@ jobs: ec2-instance-type: c5.2xlarge subnet-id: subnet-0469a9e68a379c1d3 security-group-id: sg-0793f3c9413f21970 - build: + build-connector: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: start-build-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner - name: Build Airbyte + needs: start-connectors-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-connectors-build-runner.outputs.label }} # run the job on the newly created runner + name: "Connectors Base: Build" steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -122,10 +122,10 @@ jobs: SLACK_TITLE: "Build Success" SLACK_FOOTER: "" # In case of self-hosted EC2 errors, remove this block. - stop-build-runner: - name: Stop Build EC2 Runner + stop-connectors-build-runner: + name: "Connectors Base: Stop Build EC2 Runner" needs: - - start-build-runner # required to get output from the start-runner job + - start-connectors-build-runner # required to get output from the start-runner job - build # required to wait when the main job is done runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -141,13 +141,13 @@ jobs: with: mode: stop github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + label: ${{ needs.start-connectors-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-connectors-build-runner.outputs.ec2-instance-id }} ## Gradle Build (Platform) # In case of self-hosted EC2 errors, remove this block. - start-build-platform-runner: - name: Start Build (Platform) EC2 Runner + start-platform-build-runner: + name: "Platform: Start Build EC2 Runner" runs-on: ubuntu-latest outputs: label: ${{ steps.start-ec2-runner.outputs.label }} @@ -169,11 +169,11 @@ jobs: ec2-instance-type: c5.2xlarge subnet-id: subnet-0469a9e68a379c1d3 security-group-id: sg-0793f3c9413f21970 - build-platform: + platform-build: # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. needs: start-build-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner - name: Build (Platform) Airbyte + runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner + name: "Platform: Build" steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -291,10 +291,10 @@ jobs: SLACK_TITLE: "Build Success" SLACK_FOOTER: "" # In case of self-hosted EC2 errors, remove this block. - stop-build-platform-runner: - name: Stop Build (Platform) EC2 Runner + stop-platform-build-runner: + name: "Platform: Stop Build EC2 Runner" needs: - - start-build-runner # required to get output from the start-runner job + - start-platform-build-runner # required to get output from the start-runner job - build # required to wait when the main job is done runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -310,8 +310,8 @@ jobs: with: mode: stop github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + label: ${{ needs.start-platform-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} ## Frontend Test ## Gradle Build From 062f70a390e708ad2bea7e808f04083d7fd4b118 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:49:40 -0700 Subject: [PATCH 06/21] fix needs --- .github/workflows/gradle.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ee6b3c583eb2..8a45e5e54e9a 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -8,7 +8,7 @@ on: jobs: ## Gradle Build (Connectors Base) # In case of self-hosted EC2 errors, remove this block. - start-connectors-build-runner: + start-connectors-base-build-runner: name: "Connectors Base: Start Build EC2 Runner" runs-on: ubuntu-latest outputs: @@ -31,10 +31,10 @@ jobs: ec2-instance-type: c5.2xlarge subnet-id: subnet-0469a9e68a379c1d3 security-group-id: sg-0793f3c9413f21970 - build-connector: + build-connectors-base: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: start-connectors-build-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-connectors-build-runner.outputs.label }} # run the job on the newly created runner + needs: start-connectors-base-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner name: "Connectors Base: Build" steps: - name: Checkout Airbyte @@ -122,11 +122,11 @@ jobs: SLACK_TITLE: "Build Success" SLACK_FOOTER: "" # In case of self-hosted EC2 errors, remove this block. - stop-connectors-build-runner: + stop-connectors-base-build-runner: name: "Connectors Base: Stop Build EC2 Runner" needs: - - start-connectors-build-runner # required to get output from the start-runner job - - build # required to wait when the main job is done + - start-connectors-base-build-runner # required to get output from the start-runner job + - build-connectors-base # required to wait when the main job is done runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -141,8 +141,8 @@ jobs: with: mode: stop github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-connectors-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-connectors-build-runner.outputs.ec2-instance-id }} + label: ${{ needs.start-connectors-base-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }} ## Gradle Build (Platform) # In case of self-hosted EC2 errors, remove this block. @@ -295,7 +295,7 @@ jobs: name: "Platform: Stop Build EC2 Runner" needs: - start-platform-build-runner # required to get output from the start-runner job - - build # required to wait when the main job is done + - platform-build # required to wait when the main job is done runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: From 7565cd233d87940185e34fc03da5bb2045b409dc Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:50:51 -0700 Subject: [PATCH 07/21] binary search --- .github/workflows/gradle.yml | 355 ----------------------------------- 1 file changed, 355 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8a45e5e54e9a..1331d20ba16e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -143,358 +143,3 @@ jobs: github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} label: ${{ needs.start-connectors-base-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }} - - ## Gradle Build (Platform) - # In case of self-hosted EC2 errors, remove this block. - start-platform-build-runner: - name: "Platform: Start Build EC2 Runner" - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Start EC2 Runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2.2.0 - with: - mode: start - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - ec2-image-id: ami-04bd6e81239f4f3fb - ec2-instance-type: c5.2xlarge - subnet-id: subnet-0469a9e68a379c1d3 - security-group-id: sg-0793f3c9413f21970 - platform-build: - # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. - needs: start-build-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner - name: "Platform: Build" - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - # todo (cgardens) - scope by platform. - - name: Check images exist - run: ./tools/bin/check_images_exist.sh - - - name: Pip Caching - uses: actions/cache@v2 - with: - path: | - ~/.cache/pip - key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}- - - - name: Npm Caching - uses: actions/cache@v2 - with: - path: | - ~/.npm - key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}- - - # this intentionally does not use restore-keys so we don't mess with gradle caching - - name: Gradle and Python Caching - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - **/.venv - key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} - - - uses: actions/setup-java@v1 - with: - java-version: '14' - - - uses: actions/setup-node@v1 - with: - node-version: '14.7' - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - - name: Format - run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace - - - name: Ensure no file change - run: git status --porcelain && test -z "$(git status --porcelain)" - - - name: Build - run: SUB_BUILD=PLATFORM ./gradlew build --scan - - - name: Ensure no file change - run: git status --porcelain && test -z "$(git status --porcelain)" - - # todo (cgardens) - scope by platform. - - name: Check documentation - if: success() && github.ref == 'refs/heads/master' - run: ./tools/site/link_checker.sh check_docs - - # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. - # - name: Get Docker Space - # run: docker run --rm busybox df -h - - - name: Image Cleanup - run: ./tools/bin/clean_images.sh - - - name: Build Platform Docker Images - if: success() && github.ref == 'refs/heads/master' - run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan - env: - GIT_REVISION: ${{ github.sha }} - - # make sure these always run before pushing platform docker images - - name: Run End-to-End Acceptance Tests - if: success() && github.ref == 'refs/heads/master' - run: ./tools/bin/acceptance_test.sh - - - name: Automatic Migration Acceptance Test - run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan - - - name: Push Platform Docker Images - if: success() && github.ref == 'refs/heads/master' - run: | - docker login -u airbytebot -p ${DOCKER_PASSWORD} - VERSION=dev docker-compose -f docker-compose.build.yaml push - env: - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - - name: Slack Notification - Failure - if: failure() && github.ref == 'refs/heads/master' - uses: rtCamp/action-slack-notify@master - env: - SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} - SLACK_USERNAME: Buildozer - SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg - SLACK_COLOR: DC143C - SLACK_TITLE: "Build failure" - SLACK_FOOTER: "" - - - name: Slack Notification - Success - if: success() && github.ref == 'refs/heads/master' - uses: rtCamp/action-slack-notify@master - env: - SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} - SLACK_USERNAME: Buildbot - SLACK_TITLE: "Build Success" - SLACK_FOOTER: "" - # In case of self-hosted EC2 errors, remove this block. - stop-platform-build-runner: - name: "Platform: Stop Build EC2 Runner" - needs: - - start-platform-build-runner # required to get output from the start-runner job - - platform-build # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-platform-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} - - ## Frontend Test - ## Gradle Build - # In case of self-hosted EC2 errors, remove this block. - start-frontend-test-runner: - name: Start Frontend Test EC2 Runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Start EC2 Runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - ec2-image-id: ami-04bd6e81239f4f3fb - ec2-instance-type: c5.2xlarge - subnet-id: subnet-0469a9e68a379c1d3 - security-group-id: sg-0793f3c9413f21970 - frontend-test: - needs: start-frontend-test-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-frontend-test-runner.outputs.label }} # run the job on the newly created runner - name: Run Frontend Test - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - - uses: actions/setup-java@v1 - with: - java-version: '14' - - - uses: actions/setup-node@v1 - with: - node-version: '14.7' - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - - name: Install Cypress Test Dependencies - run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - - - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan - - - name: Run End-to-End Frontend Tests - run: ./tools/bin/e2e_test.sh - # In case of self-hosted EC2 errors, remove this block. - stop-frontend-test-runner: - name: Stop Frontend Test EC2 Runner - needs: - - start-frontend-test-runner # required to get output from the start-runner job - - frontend-test # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-frontend-test-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-frontend-test-runner.outputs.ec2-instance-id }} - - ## Kube Acceptance Tests - # In case of self-hosted EC2 errors, remove this block. - # Docker acceptance tests run as part of the build job. - start-kube-acceptance-test-runner: - name: Start Kube Acceptance Test EC2 Runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - ec2-image-id: ami-04bd6e81239f4f3fb - ec2-instance-type: c5.2xlarge - subnet-id: subnet-0469a9e68a379c1d3 - security-group-id: sg-0793f3c9413f21970 - kube-acceptance-test: - # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner - name: Acceptance Tests (Kube) - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - - uses: actions/setup-java@v1 - with: - java-version: '14' - - - uses: actions/setup-node@v1 - with: - node-version: '14.7' - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - - name: Fix EC-2 Runner - run: | - mkdir -p /home/runner - - - name: Install socat (required for port forwarding) - run: | - sudo apt-get update - sudo apt-get install socat - - - name: KIND Kubernetes Cluster Setup - uses: helm/kind-action@v1.1.0 - # In case of self-hosted EC2 errors, remove this env block. - env: - USER: root - HOME: /home/runner - CHANGE_MINIKUBE_NONE_USER: true - - - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan - - - name: Run Logging Tests - run: ./tools/bin/cloud_storage_logging_test.sh - env: - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - - - name: Run Kubernetes End-to-End Acceptance Tests - env: - USER: root - HOME: /home/runner - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - run: | - IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh - # In case of self-hosted EC2 errors, remove this block. - stop-kube-acceptance-test-runner: - name: Stop Kube Acceptance Test EC2 Runner - needs: - - start-kube-acceptance-test-runner # required to get output from the start-runner job - - kube-acceptance-test # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} From 0b4d80237f3d988f8a84a32c48f1f694467fcd68 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:52:41 -0700 Subject: [PATCH 08/21] Revert "binary search" This reverts commit 7565cd233d87940185e34fc03da5bb2045b409dc. --- .github/workflows/gradle.yml | 355 +++++++++++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1331d20ba16e..8a45e5e54e9a 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -143,3 +143,358 @@ jobs: github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} label: ${{ needs.start-connectors-base-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }} + + ## Gradle Build (Platform) + # In case of self-hosted EC2 errors, remove this block. + start-platform-build-runner: + name: "Platform: Start Build EC2 Runner" + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 Runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2.2.0 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + platform-build: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + needs: start-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner + name: "Platform: Build" + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + # todo (cgardens) - scope by platform. + - name: Check images exist + run: ./tools/bin/check_images_exist.sh + + - name: Pip Caching + uses: actions/cache@v2 + with: + path: | + ~/.cache/pip + key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}- + + - name: Npm Caching + uses: actions/cache@v2 + with: + path: | + ~/.npm + key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}- + + # this intentionally does not use restore-keys so we don't mess with gradle caching + - name: Gradle and Python Caching + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + **/.venv + key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user + + - name: Format + run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace + + - name: Ensure no file change + run: git status --porcelain && test -z "$(git status --porcelain)" + + - name: Build + run: SUB_BUILD=PLATFORM ./gradlew build --scan + + - name: Ensure no file change + run: git status --porcelain && test -z "$(git status --porcelain)" + + # todo (cgardens) - scope by platform. + - name: Check documentation + if: success() && github.ref == 'refs/heads/master' + run: ./tools/site/link_checker.sh check_docs + + # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. + # - name: Get Docker Space + # run: docker run --rm busybox df -h + + - name: Image Cleanup + run: ./tools/bin/clean_images.sh + + - name: Build Platform Docker Images + if: success() && github.ref == 'refs/heads/master' + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + env: + GIT_REVISION: ${{ github.sha }} + + # make sure these always run before pushing platform docker images + - name: Run End-to-End Acceptance Tests + if: success() && github.ref == 'refs/heads/master' + run: ./tools/bin/acceptance_test.sh + + - name: Automatic Migration Acceptance Test + run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan + + - name: Push Platform Docker Images + if: success() && github.ref == 'refs/heads/master' + run: | + docker login -u airbytebot -p ${DOCKER_PASSWORD} + VERSION=dev docker-compose -f docker-compose.build.yaml push + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Slack Notification - Failure + if: failure() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildozer + SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg + SLACK_COLOR: DC143C + SLACK_TITLE: "Build failure" + SLACK_FOOTER: "" + + - name: Slack Notification - Success + if: success() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildbot + SLACK_TITLE: "Build Success" + SLACK_FOOTER: "" + # In case of self-hosted EC2 errors, remove this block. + stop-platform-build-runner: + name: "Platform: Stop Build EC2 Runner" + needs: + - start-platform-build-runner # required to get output from the start-runner job + - platform-build # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-platform-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} + + ## Frontend Test + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-frontend-test-runner: + name: Start Frontend Test EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 Runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + frontend-test: + needs: start-frontend-test-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-frontend-test-runner.outputs.label }} # run the job on the newly created runner + name: Run Frontend Test + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user + + - name: Install Cypress Test Dependencies + run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan + + - name: Run End-to-End Frontend Tests + run: ./tools/bin/e2e_test.sh + # In case of self-hosted EC2 errors, remove this block. + stop-frontend-test-runner: + name: Stop Frontend Test EC2 Runner + needs: + - start-frontend-test-runner # required to get output from the start-runner job + - frontend-test # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-frontend-test-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-frontend-test-runner.outputs.ec2-instance-id }} + + ## Kube Acceptance Tests + # In case of self-hosted EC2 errors, remove this block. + # Docker acceptance tests run as part of the build job. + start-kube-acceptance-test-runner: + name: Start Kube Acceptance Test EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + kube-acceptance-test: + # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. + needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner + name: Acceptance Tests (Kube) + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user + + - name: Fix EC-2 Runner + run: | + mkdir -p /home/runner + + - name: Install socat (required for port forwarding) + run: | + sudo apt-get update + sudo apt-get install socat + + - name: KIND Kubernetes Cluster Setup + uses: helm/kind-action@v1.1.0 + # In case of self-hosted EC2 errors, remove this env block. + env: + USER: root + HOME: /home/runner + CHANGE_MINIKUBE_NONE_USER: true + + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + + - name: Run Logging Tests + run: ./tools/bin/cloud_storage_logging_test.sh + env: + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} + + - name: Run Kubernetes End-to-End Acceptance Tests + env: + USER: root + HOME: /home/runner + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + run: | + IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh + # In case of self-hosted EC2 errors, remove this block. + stop-kube-acceptance-test-runner: + name: Stop Kube Acceptance Test EC2 Runner + needs: + - start-kube-acceptance-test-runner # required to get output from the start-runner job + - kube-acceptance-test # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} From 286ef0cf7c78261ca2ea5c7539fd209f81b97b3d Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:53:05 -0700 Subject: [PATCH 09/21] binary search 2 --- .github/workflows/gradle.yml | 186 ----------------------------------- 1 file changed, 186 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8a45e5e54e9a..71d0aad5a444 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -312,189 +312,3 @@ jobs: github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} label: ${{ needs.start-platform-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} - - ## Frontend Test - ## Gradle Build - # In case of self-hosted EC2 errors, remove this block. - start-frontend-test-runner: - name: Start Frontend Test EC2 Runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Start EC2 Runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - ec2-image-id: ami-04bd6e81239f4f3fb - ec2-instance-type: c5.2xlarge - subnet-id: subnet-0469a9e68a379c1d3 - security-group-id: sg-0793f3c9413f21970 - frontend-test: - needs: start-frontend-test-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-frontend-test-runner.outputs.label }} # run the job on the newly created runner - name: Run Frontend Test - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - - uses: actions/setup-java@v1 - with: - java-version: '14' - - - uses: actions/setup-node@v1 - with: - node-version: '14.7' - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - - name: Install Cypress Test Dependencies - run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - - - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan - - - name: Run End-to-End Frontend Tests - run: ./tools/bin/e2e_test.sh - # In case of self-hosted EC2 errors, remove this block. - stop-frontend-test-runner: - name: Stop Frontend Test EC2 Runner - needs: - - start-frontend-test-runner # required to get output from the start-runner job - - frontend-test # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-frontend-test-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-frontend-test-runner.outputs.ec2-instance-id }} - - ## Kube Acceptance Tests - # In case of self-hosted EC2 errors, remove this block. - # Docker acceptance tests run as part of the build job. - start-kube-acceptance-test-runner: - name: Start Kube Acceptance Test EC2 Runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - ec2-image-id: ami-04bd6e81239f4f3fb - ec2-instance-type: c5.2xlarge - subnet-id: subnet-0469a9e68a379c1d3 - security-group-id: sg-0793f3c9413f21970 - kube-acceptance-test: - # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner - name: Acceptance Tests (Kube) - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - - uses: actions/setup-java@v1 - with: - java-version: '14' - - - uses: actions/setup-node@v1 - with: - node-version: '14.7' - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install Pyenv - run: python3 -m pip install virtualenv==16.7.9 --user - - - name: Fix EC-2 Runner - run: | - mkdir -p /home/runner - - - name: Install socat (required for port forwarding) - run: | - sudo apt-get update - sudo apt-get install socat - - - name: KIND Kubernetes Cluster Setup - uses: helm/kind-action@v1.1.0 - # In case of self-hosted EC2 errors, remove this env block. - env: - USER: root - HOME: /home/runner - CHANGE_MINIKUBE_NONE_USER: true - - - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan - - - name: Run Logging Tests - run: ./tools/bin/cloud_storage_logging_test.sh - env: - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - - - name: Run Kubernetes End-to-End Acceptance Tests - env: - USER: root - HOME: /home/runner - AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} - run: | - IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh - # In case of self-hosted EC2 errors, remove this block. - stop-kube-acceptance-test-runner: - name: Stop Kube Acceptance Test EC2 Runner - needs: - - start-kube-acceptance-test-runner # required to get output from the start-runner job - - kube-acceptance-test # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} From 7ed4d1fc29c78727d0f512caea9643db0d8ba7a7 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:54:06 -0700 Subject: [PATCH 10/21] fix needs --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 71d0aad5a444..e04e25d22506 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -171,7 +171,7 @@ jobs: security-group-id: sg-0793f3c9413f21970 platform-build: # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. - needs: start-build-runner # required to start the main job when the runner is ready + needs: start-platform-build-runner # required to start the main job when the runner is ready runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner name: "Platform: Build" steps: From 9abed6ebaee3ad3c5f71907fe155e0ccaf48d3df Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 14:54:29 -0700 Subject: [PATCH 11/21] Revert "binary search 2" This reverts commit 286ef0cf7c78261ca2ea5c7539fd209f81b97b3d. --- .github/workflows/gradle.yml | 186 +++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e04e25d22506..ac8d43e7bea9 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -312,3 +312,189 @@ jobs: github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} label: ${{ needs.start-platform-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} + + ## Frontend Test + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-frontend-test-runner: + name: Start Frontend Test EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 Runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + frontend-test: + needs: start-frontend-test-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-frontend-test-runner.outputs.label }} # run the job on the newly created runner + name: Run Frontend Test + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user + + - name: Install Cypress Test Dependencies + run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan + + - name: Run End-to-End Frontend Tests + run: ./tools/bin/e2e_test.sh + # In case of self-hosted EC2 errors, remove this block. + stop-frontend-test-runner: + name: Stop Frontend Test EC2 Runner + needs: + - start-frontend-test-runner # required to get output from the start-runner job + - frontend-test # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-frontend-test-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-frontend-test-runner.outputs.ec2-instance-id }} + + ## Kube Acceptance Tests + # In case of self-hosted EC2 errors, remove this block. + # Docker acceptance tests run as part of the build job. + start-kube-acceptance-test-runner: + name: Start Kube Acceptance Test EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + kube-acceptance-test: + # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. + needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner + name: Acceptance Tests (Kube) + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user + + - name: Fix EC-2 Runner + run: | + mkdir -p /home/runner + + - name: Install socat (required for port forwarding) + run: | + sudo apt-get update + sudo apt-get install socat + + - name: KIND Kubernetes Cluster Setup + uses: helm/kind-action@v1.1.0 + # In case of self-hosted EC2 errors, remove this env block. + env: + USER: root + HOME: /home/runner + CHANGE_MINIKUBE_NONE_USER: true + + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + + - name: Run Logging Tests + run: ./tools/bin/cloud_storage_logging_test.sh + env: + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} + + - name: Run Kubernetes End-to-End Acceptance Tests + env: + USER: root + HOME: /home/runner + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + run: | + IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh + # In case of self-hosted EC2 errors, remove this block. + stop-kube-acceptance-test-runner: + name: Stop Kube Acceptance Test EC2 Runner + needs: + - start-kube-acceptance-test-runner # required to get output from the start-runner job + - kube-acceptance-test # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} From 066a8a578870fe23159fde1a1ea1673e47e03b05 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 16:28:28 -0700 Subject: [PATCH 12/21] docs --- .github/workflows/gradle.yml | 8 +- .../developing-locally.md | 2 +- gradle-readme.md | 89 +++++++++++++++++++ settings.gradle | 8 +- tools/bin/acceptance_test.sh | 2 +- tools/bin/acceptance_test_kube.sh | 4 +- 6 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 gradle-readme.md diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ac8d43e7bea9..e6c0f83bd939 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -87,13 +87,13 @@ jobs: run: python3 -m pip install virtualenv==16.7.9 --user - name: Generate Template scaffold - run: SUB_BUILD=CONNECTOR_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan + run: SUB_BUILD=CONNECTORS_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan - name: Format - run: SUB_BUILD=CONNECTOR_BASE ./gradlew format --scan --info --stacktrace + run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace - name: Build - run: SUB_BUILD=CONNECTOR_BASE ./gradlew build --scan + run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" @@ -261,7 +261,7 @@ jobs: run: ./tools/bin/acceptance_test.sh - name: Automatic Migration Acceptance Test - run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan + run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan - name: Push Platform Docker Images if: success() && github.ref == 'refs/heads/master' diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 99843815c5c4..82f50f6cc247 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -79,7 +79,7 @@ To run acceptance \(end-to-end\) tests, you must have the Airbyte running locall ```bash SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up -./gradlew :airbyte-tests:acceptanceTests +SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests ``` ## Run formatting automation/tests diff --git a/gradle-readme.md b/gradle-readme.md new file mode 100644 index 000000000000..178de3e71bc5 --- /dev/null +++ b/gradle-readme.md @@ -0,0 +1,89 @@ +# Gradle Builds + +## Overview +We have 3 ways of slicing our builds: +1. **Build Everything**: Including every single connectors. +2. **Build Platform**: Build only modules related to the core platform. +3. **Build Connectors Base**: Build only modules related to code infrastructure for connectors. + +**Build Everything** is really not particularly functional as building every single connector at once is really prone to transient errors. As there are more connectors the chance that there is a transient issue while downloading any single dependency starts to get really high. + +In our CI we run **Build Platform** and **Build Connectors Base**. Then separately, on a regular cadence, we build each connector and run its integration tests. + +We split Build Platform and Build Connectors Base from each other for a few reasons: +1. The tech stacks are very different. The Platform is almost entirely Java. Because of differing needs around separating environments, the Platform build can be optimized separately from the Connectors one. +2. We want to the iteration cycles of people working on connectors or the platform faster _and_ independent. e.g. Before this change someone working on a Platform feature needs to run formatting on the entire codebase (including connectors). This led to a lot of cosmetic build failures that obfuscated actually problems. Ideally a failure on the connectors side should not block progress on the platform side. +3. The lifecycles are different. One can safely release the Platform even if parts of Connectors Base is failing (and vice versa). + +## Cheatsheet +Here is a cheatsheet for common gradle commands. + +### Basic Build Syntax +Here is the syntax for running gradle commands on the different parts of the code base that we called out above. + +#### Build Everything +```shell +./gradlew +``` + +#### Build Platform +```shell +SUB_BUILD=PLATFORM ./gradlew +``` + +#### Build Connectors Base +```shell +SUB_BUILD=CONNECTORS_BASE ./gradlew +``` + +### Build +In order to "build" the project. This task includes producing all artifacts and running unit tests (anything called in the `:test` task). It does _not_ include integration tests (anything called in the `:integrationTest` task). + +For example all the following are valid. +```shell +./gradlew build +SUB_BUILD=PLATFORM ./gradlew build +SUB_BUILD=CONNECTORS_BASE ./gradlew build +``` + +### Formatting + +The build system has a custom task called `format`. It is not called as part of `build`. If the command is called on a subset of the project, it will (mostly) target just the included modules. The exception is that `spotless` (a gradle formatter) will always format any file types that it is configured to manage regardless of which sub build is run. `spotless` is relatively fast, so this should not be too much of an annoyance. It can lead to formatting changes in unexpected parts of the code base. + +For example all the following are valid. +```shell +./gradlew format +SUB_BUILD=PLATFORM ./gradlew format +SUB_BUILD=CONNECTORS_BASE ./gradlew format +``` + +### Platform-Specific Commands + +#### Build Artifacts +This command just builds the docker images that are used as artifacts in the platform. It bypasses running tests. + +```shell +SUB_BUILD=PLATFORM ./gradlew composeBuild +``` + +#### Running Tests +The Platform has 2 different levels of tests: Unit Tests and Acceptance Tests. + +##### Unit Tests +Unit Tests can be run using the `:test` task on any submodule. These test class-level behavior. They should avoid using external resources (e.g. calling staging services or pulling resources from the internet). We do allow these tests to spin up local resources (usually in docker containers). For example, we use test containers frequently to spin up test postgres databases. + +##### Acceptance Tests +We split Acceptance Tests into 2 different test suites: +* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test_kube.sh) that is use by the CI. +* Migration Acceptance Tests: These tests make sure the end-to-end process of migrating from one version of Airbyte to the next works. These tests are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan`. These tests do not expect there to be a separate instance of Airbyte running. + +##### Future Work +Our story around "integration testing" or "E2E testing" is a little ambiguous. Our Platform Acceptance Test Suite is getting somewhat unwieldy. It was meant to just be some coarse sanity checks, but over time we have found more need to test interactions between systems more granular. Whether we start supporting a separate class of tests (e.g. integration tests) or figure out how allow for more granular tests in the existing Acceptance Test framework is TBD. + +### Connectors-Specific Commands +#### Running Tests +_coming soon_ +##### Unit Tests +##### Integration Tests +##### Acceptance Tests + diff --git a/settings.gradle b/settings.gradle index c695e5a4b2c1..897839c1b4b0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,14 +13,14 @@ gradleEnterprise { rootProject.name = 'airbyte' -// SUB_BUILD is an enum of , PLATFORM, CONNECTOR_BASE. Blank is equivalent to all. +// SUB_BUILD is an enum of , PLATFORM, CONNECTORS_BASE. Blank is equivalent to all. if(!System.getenv().containsKey("SUB_BUILD")) { println("Building all of Airbyte.") } else { def subBuild = System.getenv().get("SUB_BUILD") println("Building Airbyte Sub Build: " + subBuild) - if(subBuild != "PLATFORM" && subBuild != "CONNECTOR_BASE") { - throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTOR_BASE", subBuild)) + if(subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE") { + throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE", subBuild)) } } @@ -55,7 +55,7 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") } // connectors base -if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CONNECTOR_BASE") { +if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CONNECTORS_BASE") { include ':airbyte-cdk:python' include ':airbyte-integrations:bases:airbyte-protocol' include ':airbyte-integrations:bases:base' diff --git a/tools/bin/acceptance_test.sh b/tools/bin/acceptance_test.sh index 4c5ceab594ed..7da0dfd6dc2f 100755 --- a/tools/bin/acceptance_test.sh +++ b/tools/bin/acceptance_test.sh @@ -22,4 +22,4 @@ echo "Waiting for services to begin" sleep 10 # TODO need a better way to wait echo "Running e2e tests via gradle" -SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-tests:acceptanceTests --rerun-tasks --scan +SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests --rerun-tasks --scan diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh index c28f9c1467b4..a228ff2ba710 100755 --- a/tools/bin/acceptance_test_kube.sh +++ b/tools/bin/acceptance_test_kube.sh @@ -36,7 +36,7 @@ trap "echo 'kube logs:' && print_all_logs" EXIT kubectl port-forward svc/airbyte-server-svc 8001:8001 & echo "Running worker integration tests..." -./gradlew --no-daemon :airbyte-workers:integrationTest --scan +./gradlew :airbyte-workers:integrationTest --scan echo "Running e2e tests via gradle..." -KUBE=true ./gradlew --no-daemon :airbyte-tests:acceptanceTests --scan +KUBE=true SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests --scan From 3014603791b2784c3fd918c340943bb6b396119f Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 16:31:10 -0700 Subject: [PATCH 13/21] add note --- gradle-readme.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gradle-readme.md b/gradle-readme.md index 178de3e71bc5..68adb6a29b77 100644 --- a/gradle-readme.md +++ b/gradle-readme.md @@ -15,6 +15,8 @@ We split Build Platform and Build Connectors Base from each other for a few reas 2. We want to the iteration cycles of people working on connectors or the platform faster _and_ independent. e.g. Before this change someone working on a Platform feature needs to run formatting on the entire codebase (including connectors). This led to a lot of cosmetic build failures that obfuscated actually problems. Ideally a failure on the connectors side should not block progress on the platform side. 3. The lifecycles are different. One can safely release the Platform even if parts of Connectors Base is failing (and vice versa). +Future Work: The next step here is to figure out how to more formally split connectors and platform. Right now we exploit behavior in `settings.gradle` to separate them. This is not a best practice. Ultimately, we want these two builds to be totally separate. We do not know what that will look like yet. + ## Cheatsheet Here is a cheatsheet for common gradle commands. @@ -67,16 +69,23 @@ SUB_BUILD=PLATFORM ./gradlew composeBuild ``` #### Running Tests -The Platform has 2 different levels of tests: Unit Tests and Acceptance Tests. +The Platform has 3 different levels of tests: Unit Tests, Acceptance Tests, Frontend Acceptance Tests. ##### Unit Tests Unit Tests can be run using the `:test` task on any submodule. These test class-level behavior. They should avoid using external resources (e.g. calling staging services or pulling resources from the internet). We do allow these tests to spin up local resources (usually in docker containers). For example, we use test containers frequently to spin up test postgres databases. ##### Acceptance Tests We split Acceptance Tests into 2 different test suites: -* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test_kube.sh) that is use by the CI. +* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test_kube.sh) that is used by the CI. * Migration Acceptance Tests: These tests make sure the end-to-end process of migrating from one version of Airbyte to the next works. These tests are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan`. These tests do not expect there to be a separate instance of Airbyte running. +These tests currently all live in `airbyte-tests` + +##### Frontend Acceptance Tests +These are acceptance tests for the frontend. They are run with `SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-e2e-testing:e2etest`. Like the Platform Acceptance Tests, they expect Airbyte to be running locally. See the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/e2e_test.sh) that is used by the CI. + +These tests currently all live in `airbyte-e2e-testing`. + ##### Future Work Our story around "integration testing" or "E2E testing" is a little ambiguous. Our Platform Acceptance Test Suite is getting somewhat unwieldy. It was meant to just be some coarse sanity checks, but over time we have found more need to test interactions between systems more granular. Whether we start supporting a separate class of tests (e.g. integration tests) or figure out how allow for more granular tests in the existing Acceptance Test framework is TBD. From 964987239d69dc81e30ee5387f689a0d3de761e2 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 16:37:15 -0700 Subject: [PATCH 14/21] make sure tools use sub_builds --- tools/bin/acceptance_test_kube.sh | 2 +- tools/bin/e2e_test.sh | 2 +- tools/bin/release_version.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh index a228ff2ba710..1a56d84edb7f 100755 --- a/tools/bin/acceptance_test_kube.sh +++ b/tools/bin/acceptance_test_kube.sh @@ -36,7 +36,7 @@ trap "echo 'kube logs:' && print_all_logs" EXIT kubectl port-forward svc/airbyte-server-svc 8001:8001 & echo "Running worker integration tests..." -./gradlew :airbyte-workers:integrationTest --scan +SUB_BUILD=PLATFORM ./gradlew :airbyte-workers:integrationTest --scan echo "Running e2e tests via gradle..." KUBE=true SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests --scan diff --git a/tools/bin/e2e_test.sh b/tools/bin/e2e_test.sh index 17412de0a70e..7aeca8cca04c 100755 --- a/tools/bin/e2e_test.sh +++ b/tools/bin/e2e_test.sh @@ -23,5 +23,5 @@ echo "Waiting for services to begin" sleep 30 # TODO need a better way to wait echo "Running e2e tests via gradle" -./gradlew --no-daemon :airbyte-e2e-testing:e2etest +SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-e2e-testing:e2etest diff --git a/tools/bin/release_version.sh b/tools/bin/release_version.sh index 09cc306e6f91..5f93576375e5 100755 --- a/tools/bin/release_version.sh +++ b/tools/bin/release_version.sh @@ -29,7 +29,7 @@ GIT_REVISION=$(git rev-parse HEAD) echo "Bumped version from ${PREV_VERSION} to ${NEW_VERSION}" echo "Building and publishing version $NEW_VERSION for git revision $GIT_REVISION..." -./gradlew clean composeBuild +SUB_BUILD=PLATFORM ./gradlew clean composeBuild VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml build VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml push echo "Completed building and publishing..." From f28a17f215f339fb5cc43f9e9997c77c42c5b66d Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 16:46:01 -0700 Subject: [PATCH 15/21] clean up --- .github/workflows/publish-cdk-command.yml | 2 +- airbyte-config/models/README.md | 2 +- airbyte-migration/README.md | 2 +- .../MigrationAcceptanceTest.java | 2 +- .../developing-locally.md | 13 +-- .../gradle-cheatsheet.md | 95 +++++++++++++++++- gradle-readme.md | 98 ------------------- tools/bin/cloud_storage_logging_test.sh | 2 +- 8 files changed, 101 insertions(+), 115 deletions(-) delete mode 100644 gradle-readme.md diff --git a/.github/workflows/publish-cdk-command.yml b/.github/workflows/publish-cdk-command.yml index 7e9b33f3318b..94152e4358b8 100644 --- a/.github/workflows/publish-cdk-command.yml +++ b/.github/workflows/publish-cdk-command.yml @@ -27,7 +27,7 @@ jobs: - name: Checkout Airbyte uses: actions/checkout@v2 - name: Build CDK Package - run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build + run: SUB_BUILD=CONNECTORS_BASE ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build - name: Add Failure Comment if: github.event.inputs.comment-id && !success() uses: peter-evans/create-or-update-comment@v1 diff --git a/airbyte-config/models/README.md b/airbyte-config/models/README.md index a12c3247e14f..eb5853a318a3 100644 --- a/airbyte-config/models/README.md +++ b/airbyte-config/models/README.md @@ -9,7 +9,7 @@ This module uses `jsonschema2pojo` to generate Java config objects from [json sc ``` - Run the following command under the project root: ```sh - ./gradlew airbyte-config:models:generateJsonSchema2Pojo + SUB_BUILD=PLATFORM ./gradlew airbyte-config:models:generateJsonSchema2Pojo ``` The generated file is under: ``` diff --git a/airbyte-migration/README.md b/airbyte-migration/README.md index 2c7d20f65982..05d15bdb8e0a 100644 --- a/airbyte-migration/README.md +++ b/airbyte-migration/README.md @@ -24,7 +24,7 @@ Run the following command in project root: BUILD_VERSION=$(cat .env | grep VERSION | awk -F"=" '{print $2}') # Build the migration bundle file -./gradlew airbyte-migration:build +SUB_BUILD=PLATFORM ./gradlew airbyte-migration:build # Extract the bundle file tar xf ./airbyte-migration/build/distributions/airbyte-migration-${BUILD_VERSION}.tar --strip-components=1 diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java index 3d606392bc97..161ceb6c8f38 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java @@ -65,7 +65,7 @@ import org.testcontainers.containers.output.OutputFrame; /** - * In order to run this test from intellij, build the docker images via ./gradlew composeBuild and + * In order to run this test from intellij, build the docker images via SUB_BUILD=PLATFORM ./gradlew composeBuild and * replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env file */ public class MigrationAcceptanceTest { diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 82f50f6cc247..fe93a8be978e 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -28,26 +28,21 @@ To start contributing: ## Build with `gradle` -To compile the code and run unit tests: +To compile and build just the platform (not all the connectors): ```bash -./gradlew clean build +SUB_BUILD=PLATFORM ./gradlew build ``` This will build all the code and run all the unit tests. -`./gradlew build` creates all the necessary artifacts \(Webapp, Jars and Docker images\) so that you can run Airbyte locally. Since this builds everything, it can take some time. - -To compile and build just the core systems: +`SUB_BUILD=PLATFORM ./gradlew build` creates all the necessary artifacts \(Webapp, Jars and Docker images\) so that you can run Airbyte locally. Since this builds everything, it can take some time. -```bash -SUB_BUILD=PLATFORM ./gradlew build -``` {% hint style="info" %} Gradle will use all CPU cores by default. If Gradle uses too much/too little CPU, tuning the number of CPU cores it uses to better suit a dev's need can help. -Adjust this by either, 1. Setting an env var: `export GRADLE_OPTS="-Dorg.gradle.workers.max=3"`. 2. Setting a cli option: `./gradlew build --max-workers 3` 3. Setting the `org.gradle.workers.max` property in the `gradle.properties` file. +Adjust this by either, 1. Setting an env var: `export GRADLE_OPTS="-Dorg.gradle.workers.max=3"`. 2. Setting a cli option: `SUB_BUILD=PLATFORM ./gradlew build --max-workers 3` 3. Setting the `org.gradle.workers.max` property in the `gradle.properties` file. A good rule of thumb is to set this to \(\# of cores - 1\). {% endhint %} diff --git a/docs/contributing-to-airbyte/gradle-cheatsheet.md b/docs/contributing-to-airbyte/gradle-cheatsheet.md index 6c65ca745cb2..01a741bcf4cf 100644 --- a/docs/contributing-to-airbyte/gradle-cheatsheet.md +++ b/docs/contributing-to-airbyte/gradle-cheatsheet.md @@ -1,15 +1,104 @@ # Gradle Cheatsheet -## Connector Development +## Overview +We have 3 ways of slicing our builds: +1. **Build Everything**: Including every single connectors. +2. **Build Platform**: Build only modules related to the core platform. +3. **Build Connectors Base**: Build only modules related to code infrastructure for connectors. -### Commands used in CI +**Build Everything** is really not particularly functional as building every single connector at once is really prone to transient errors. As there are more connectors the chance that there is a transient issue while downloading any single dependency starts to get really high. + +In our CI we run **Build Platform** and **Build Connectors Base**. Then separately, on a regular cadence, we build each connector and run its integration tests. + +We split Build Platform and Build Connectors Base from each other for a few reasons: +1. The tech stacks are very different. The Platform is almost entirely Java. Because of differing needs around separating environments, the Platform build can be optimized separately from the Connectors one. +2. We want to the iteration cycles of people working on connectors or the platform faster _and_ independent. e.g. Before this change someone working on a Platform feature needs to run formatting on the entire codebase (including connectors). This led to a lot of cosmetic build failures that obfuscated actually problems. Ideally a failure on the connectors side should not block progress on the platform side. +3. The lifecycles are different. One can safely release the Platform even if parts of Connectors Base is failing (and vice versa). + +Future Work: The next step here is to figure out how to more formally split connectors and platform. Right now we exploit behavior in `settings.gradle` to separate them. This is not a best practice. Ultimately, we want these two builds to be totally separate. We do not know what that will look like yet. + +## Cheatsheet +Here is a cheatsheet for common gradle commands. + +### Basic Build Syntax +Here is the syntax for running gradle commands on the different parts of the code base that we called out above. + +#### Build Everything +```shell +./gradlew +``` + +#### Build Platform +```shell +SUB_BUILD=PLATFORM ./gradlew +``` + +#### Build Connectors Base +```shell +SUB_BUILD=CONNECTORS_BASE ./gradlew +``` + +### Build +In order to "build" the project. This task includes producing all artifacts and running unit tests (anything called in the `:test` task). It does _not_ include integration tests (anything called in the `:integrationTest` task). + +For example all the following are valid. +```shell +./gradlew build +SUB_BUILD=PLATFORM ./gradlew build +SUB_BUILD=CONNECTORS_BASE ./gradlew build +``` + +### Formatting + +The build system has a custom task called `format`. It is not called as part of `build`. If the command is called on a subset of the project, it will (mostly) target just the included modules. The exception is that `spotless` (a gradle formatter) will always format any file types that it is configured to manage regardless of which sub build is run. `spotless` is relatively fast, so this should not be too much of an annoyance. It can lead to formatting changes in unexpected parts of the code base. + +For example all the following are valid. +```shell +./gradlew format +SUB_BUILD=PLATFORM ./gradlew format +SUB_BUILD=CONNECTORS_BASE ./gradlew format +``` + +### Platform-Specific Commands + +#### Build Artifacts +This command just builds the docker images that are used as artifacts in the platform. It bypasses running tests. + +```shell +SUB_BUILD=PLATFORM ./gradlew composeBuild +``` + +#### Running Tests +The Platform has 3 different levels of tests: Unit Tests, Acceptance Tests, Frontend Acceptance Tests. + +##### Unit Tests +Unit Tests can be run using the `:test` task on any submodule. These test class-level behavior. They should avoid using external resources (e.g. calling staging services or pulling resources from the internet). We do allow these tests to spin up local resources (usually in docker containers). For example, we use test containers frequently to spin up test postgres databases. + +##### Acceptance Tests +We split Acceptance Tests into 2 different test suites: +* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test_kube.sh) that is used by the CI. +* Migration Acceptance Tests: These tests make sure the end-to-end process of migrating from one version of Airbyte to the next works. These tests are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan`. These tests do not expect there to be a separate instance of Airbyte running. + +These tests currently all live in `airbyte-tests` + +##### Frontend Acceptance Tests +These are acceptance tests for the frontend. They are run with `SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-e2e-testing:e2etest`. Like the Platform Acceptance Tests, they expect Airbyte to be running locally. See the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/e2e_test.sh) that is used by the CI. + +These tests currently all live in `airbyte-e2e-testing`. + +##### Future Work +Our story around "integration testing" or "E2E testing" is a little ambiguous. Our Platform Acceptance Test Suite is getting somewhat unwieldy. It was meant to just be some coarse sanity checks, but over time we have found more need to test interactions between systems more granular. Whether we start supporting a separate class of tests (e.g. integration tests) or figure out how allow for more granular tests in the existing Acceptance Test framework is TBD. + +### Connectors-Specific Commands (Connector Development) + +#### Commands used in CI All connectors, regardless of implementation language, implement the following interface to allow uniformity in the build system when run from CI: **Build connector, run unit tests, and build Docker image**: `./gradlew :airbyte-integrations:connectors::build` **Run integration tests**: `./gradlew :airbyte-integrations:connectors::integrationTest` -### Python +#### Python The ideal end state for a Python connector developer is that they shouldn't have to know Gradle exists. We're almost there, but today there is only one Gradle command that's needed when developing in Python, used for formatting code. diff --git a/gradle-readme.md b/gradle-readme.md deleted file mode 100644 index 68adb6a29b77..000000000000 --- a/gradle-readme.md +++ /dev/null @@ -1,98 +0,0 @@ -# Gradle Builds - -## Overview -We have 3 ways of slicing our builds: -1. **Build Everything**: Including every single connectors. -2. **Build Platform**: Build only modules related to the core platform. -3. **Build Connectors Base**: Build only modules related to code infrastructure for connectors. - -**Build Everything** is really not particularly functional as building every single connector at once is really prone to transient errors. As there are more connectors the chance that there is a transient issue while downloading any single dependency starts to get really high. - -In our CI we run **Build Platform** and **Build Connectors Base**. Then separately, on a regular cadence, we build each connector and run its integration tests. - -We split Build Platform and Build Connectors Base from each other for a few reasons: -1. The tech stacks are very different. The Platform is almost entirely Java. Because of differing needs around separating environments, the Platform build can be optimized separately from the Connectors one. -2. We want to the iteration cycles of people working on connectors or the platform faster _and_ independent. e.g. Before this change someone working on a Platform feature needs to run formatting on the entire codebase (including connectors). This led to a lot of cosmetic build failures that obfuscated actually problems. Ideally a failure on the connectors side should not block progress on the platform side. -3. The lifecycles are different. One can safely release the Platform even if parts of Connectors Base is failing (and vice versa). - -Future Work: The next step here is to figure out how to more formally split connectors and platform. Right now we exploit behavior in `settings.gradle` to separate them. This is not a best practice. Ultimately, we want these two builds to be totally separate. We do not know what that will look like yet. - -## Cheatsheet -Here is a cheatsheet for common gradle commands. - -### Basic Build Syntax -Here is the syntax for running gradle commands on the different parts of the code base that we called out above. - -#### Build Everything -```shell -./gradlew -``` - -#### Build Platform -```shell -SUB_BUILD=PLATFORM ./gradlew -``` - -#### Build Connectors Base -```shell -SUB_BUILD=CONNECTORS_BASE ./gradlew -``` - -### Build -In order to "build" the project. This task includes producing all artifacts and running unit tests (anything called in the `:test` task). It does _not_ include integration tests (anything called in the `:integrationTest` task). - -For example all the following are valid. -```shell -./gradlew build -SUB_BUILD=PLATFORM ./gradlew build -SUB_BUILD=CONNECTORS_BASE ./gradlew build -``` - -### Formatting - -The build system has a custom task called `format`. It is not called as part of `build`. If the command is called on a subset of the project, it will (mostly) target just the included modules. The exception is that `spotless` (a gradle formatter) will always format any file types that it is configured to manage regardless of which sub build is run. `spotless` is relatively fast, so this should not be too much of an annoyance. It can lead to formatting changes in unexpected parts of the code base. - -For example all the following are valid. -```shell -./gradlew format -SUB_BUILD=PLATFORM ./gradlew format -SUB_BUILD=CONNECTORS_BASE ./gradlew format -``` - -### Platform-Specific Commands - -#### Build Artifacts -This command just builds the docker images that are used as artifacts in the platform. It bypasses running tests. - -```shell -SUB_BUILD=PLATFORM ./gradlew composeBuild -``` - -#### Running Tests -The Platform has 3 different levels of tests: Unit Tests, Acceptance Tests, Frontend Acceptance Tests. - -##### Unit Tests -Unit Tests can be run using the `:test` task on any submodule. These test class-level behavior. They should avoid using external resources (e.g. calling staging services or pulling resources from the internet). We do allow these tests to spin up local resources (usually in docker containers). For example, we use test containers frequently to spin up test postgres databases. - -##### Acceptance Tests -We split Acceptance Tests into 2 different test suites: -* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test_kube.sh) that is used by the CI. -* Migration Acceptance Tests: These tests make sure the end-to-end process of migrating from one version of Airbyte to the next works. These tests are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan`. These tests do not expect there to be a separate instance of Airbyte running. - -These tests currently all live in `airbyte-tests` - -##### Frontend Acceptance Tests -These are acceptance tests for the frontend. They are run with `SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-e2e-testing:e2etest`. Like the Platform Acceptance Tests, they expect Airbyte to be running locally. See the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/e2e_test.sh) that is used by the CI. - -These tests currently all live in `airbyte-e2e-testing`. - -##### Future Work -Our story around "integration testing" or "E2E testing" is a little ambiguous. Our Platform Acceptance Test Suite is getting somewhat unwieldy. It was meant to just be some coarse sanity checks, but over time we have found more need to test interactions between systems more granular. Whether we start supporting a separate class of tests (e.g. integration tests) or figure out how allow for more granular tests in the existing Acceptance Test framework is TBD. - -### Connectors-Specific Commands -#### Running Tests -_coming soon_ -##### Unit Tests -##### Integration Tests -##### Acceptance Tests - diff --git a/tools/bin/cloud_storage_logging_test.sh b/tools/bin/cloud_storage_logging_test.sh index bfec3dd999e0..9808239c1bf5 100755 --- a/tools/bin/cloud_storage_logging_test.sh +++ b/tools/bin/cloud_storage_logging_test.sh @@ -16,4 +16,4 @@ export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" export GCP_STORAGE_BUCKET=airbyte-kube-integration-logging-test echo "Running logging tests.." -./gradlew --no-daemon :airbyte-config:models:integrationTest --scan +SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-config:models:integrationTest --scan From 8e5a1e2e95f2dc5620a2cb40e414dc307c7cbd65 Mon Sep 17 00:00:00 2001 From: cgardens Date: Wed, 14 Jul 2021 16:49:26 -0700 Subject: [PATCH 16/21] format this --- .../MigrationAcceptanceTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java index 161ceb6c8f38..9f4cac779282 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java @@ -65,8 +65,9 @@ import org.testcontainers.containers.output.OutputFrame; /** - * In order to run this test from intellij, build the docker images via SUB_BUILD=PLATFORM ./gradlew composeBuild and - * replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env file + * In order to run this test from intellij, build the docker images via SUB_BUILD=PLATFORM ./gradlew + * composeBuild and replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env + * file */ public class MigrationAcceptanceTest { From d81e6b9ed94dd619c6f679856d4c95f8808139f9 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 09:38:57 -0700 Subject: [PATCH 17/21] remove scaffold --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e6c0f83bd939..d20f4066f05c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -86,8 +86,8 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user - - name: Generate Template scaffold - run: SUB_BUILD=CONNECTORS_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan +# - name: Generate Template scaffold +# run: SUB_BUILD=CONNECTORS_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan - name: Format run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace From 4d14d58e06a55bc4d98e70a0c8a5ff7be1484a5b Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 09:39:32 -0700 Subject: [PATCH 18/21] scaffold without sub build --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index d20f4066f05c..5b9e3a031222 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -86,8 +86,8 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user -# - name: Generate Template scaffold -# run: SUB_BUILD=CONNECTORS_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan + - name: Generate Template scaffold + run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan - name: Format run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace From 25de59a3b5bc870425493c4e393896d74089c2e8 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 10:05:51 -0700 Subject: [PATCH 19/21] split check image script by platform and connectors --- .github/workflows/gradle.yml | 5 ++- tools/bin/check_images_exist.sh | 64 ++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 5b9e3a031222..adda02265f31 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@v2 - name: Check images exist - run: ./tools/bin/check_images_exist.sh + run: ./tools/bin/check_images_exist.sh connectors - name: Pip Caching uses: actions/cache@v2 @@ -178,9 +178,8 @@ jobs: - name: Checkout Airbyte uses: actions/checkout@v2 - # todo (cgardens) - scope by platform. - name: Check images exist - run: ./tools/bin/check_images_exist.sh + run: ./tools/bin/check_images_exist.sh platform - name: Pip Caching uses: actions/cache@v2 diff --git a/tools/bin/check_images_exist.sh b/tools/bin/check_images_exist.sh index d3e1bc1de5a6..1d78f4e3bfd7 100755 --- a/tools/bin/check_images_exist.sh +++ b/tools/bin/check_images_exist.sh @@ -2,30 +2,52 @@ set -e +. tools/lib/lib.sh + function docker_tag_exists() { URL=https://hub.docker.com/v2/repositories/"$1"/tags/"$2" printf "\tURL: %s\n" "$URL" curl --silent -f -lSL "$URL" > /dev/null } -echo "Checking core images exist..." -docker-compose pull || exit 1 - -echo "Checking integration images exist..." -CONFIG_FILES=$(find airbyte-config/init | grep json | grep -v STANDARD_WORKSPACE | grep -v build) -[ -z "$CONFIG_FILES" ] && echo "ERROR: Could not find any config files." && exit 1 - -while IFS= read -r file; do - REPO=$(jq -r .dockerRepository < "$file") - TAG=$(jq -r .dockerImageTag < "$file") - echo "Checking $file..." - printf "\tREPO: %s\n" "$REPO" - printf "\tTAG: %s\n" "$TAG" - if docker_tag_exists "$REPO" "$TAG"; then - printf "\tSTATUS: found\n" - else - printf "\tERROR: not found!\n" && exit 1 - fi -done <<< "$CONFIG_FILES" - -echo "Success! All images exist!" +checkPlatformImages() { + echo "Checking platform images exist..." + docker-compose pull || exit 1 + echo "Success! All platform images exist!" +} + +checkConnectorImages() { + echo "Checking connector images exist..." + + CONFIG_FILES=$(find airbyte-config/init | grep json | grep -v STANDARD_WORKSPACE | grep -v build) + [ -z "$CONFIG_FILES" ] && echo "ERROR: Could not find any config files." && exit 1 + + while IFS= read -r file; do + REPO=$(jq -r .dockerRepository < "$file") + TAG=$(jq -r .dockerImageTag < "$file") + echo "Checking $file..." + printf "\tREPO: %s\n" "$REPO" + printf "\tTAG: %s\n" "$TAG" + if docker_tag_exists "$REPO" "$TAG"; then + printf "\tSTATUS: found\n" + else + printf "\tERROR: not found!\n" && exit 1 + fi + done <<< "$CONFIG_FILES" + + echo "Success! All connector images exist!" +} + +main() { + assert_root + + SUBSET=${1:-all} # default to all. + [[ ! "$SUBSET" =~ ^(all|platform|connectors)$ ]] && echo "Usage ./tools/bin/check_image_exists.sh [all|platform|connectors]" && exit 1 + + echo "checking images for: $SUBSET" + + [[ "$SUBSET" =~ ^(all|platform)$ ]] && checkPlatformImages + [[ "$SUBSET" =~ ^(all|connectors)$ ]] && checkConnectorImages +} + +main "$@" From 4a24d9e0719068f48397880e1e96fc39f2d10d86 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 10:07:42 -0700 Subject: [PATCH 20/21] remove scaffold test --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index adda02265f31..35abaf8d5101 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -86,8 +86,8 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user - - name: Generate Template scaffold - run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan +# - name: Generate Template scaffold +# run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan - name: Format run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace From 74ffb78813717ca50220efc625b8220534f6641f Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 10:14:43 -0700 Subject: [PATCH 21/21] fix status code --- tools/bin/check_images_exist.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/bin/check_images_exist.sh b/tools/bin/check_images_exist.sh index 1d78f4e3bfd7..36c627b832cc 100755 --- a/tools/bin/check_images_exist.sh +++ b/tools/bin/check_images_exist.sh @@ -48,6 +48,8 @@ main() { [[ "$SUBSET" =~ ^(all|platform)$ ]] && checkPlatformImages [[ "$SUBSET" =~ ^(all|connectors)$ ]] && checkConnectorImages + + echo "Image check complete." } main "$@"