diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index d953ba9d5034..a58d6cb7ce81 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -6,10 +6,10 @@ on: push: jobs: - ## Gradle Build + ## Gradle Build (Connectors Base) # In case of self-hosted EC2 errors, remove this block. - start-build-runner: - name: Start Build EC2 Runner + start-connectors-base-build-runner: + name: "Connectors Base: Start Build EC2 Runner" runs-on: ubuntu-latest outputs: label: ${{ steps.start-ec2-runner.outputs.label }} @@ -31,17 +31,17 @@ jobs: ec2-instance-type: c5.2xlarge subnet-id: subnet-0469a9e68a379c1d3 security-group-id: sg-0793f3c9413f21970 - build: + build-connectors-base: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. - needs: start-build-runner # required to start the main job when the runner is ready - runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner - name: Build Airbyte + needs: start-connectors-base-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner + name: "Connectors Base: Build" steps: - name: Checkout Airbyte uses: actions/checkout@v2 - name: Check images exist - run: ./tools/bin/check_images_exist.sh + run: ./tools/bin/check_images_exist.sh connectors - name: Pip Caching uses: actions/cache@v2 @@ -86,43 +86,189 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user - - name: Generate Template scaffold - run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan +# - name: Generate Template scaffold +# run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan + + - name: Format + run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace + + - name: Build + run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan + + - name: Ensure no file change + run: git status --porcelain && test -z "$(git status --porcelain)" + + - name: Check documentation + if: success() && github.ref == 'refs/heads/master' + run: ./tools/site/link_checker.sh check_docs + + - name: Slack Notification - Failure + if: failure() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildozer + SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg + SLACK_COLOR: DC143C + SLACK_TITLE: "Build failure" + SLACK_FOOTER: "" + + - name: Slack Notification - Success + if: success() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildbot + SLACK_TITLE: "Build Success" + SLACK_FOOTER: "" + # In case of self-hosted EC2 errors, remove this block. + stop-connectors-base-build-runner: + name: "Connectors Base: Stop Build EC2 Runner" + needs: + - start-connectors-base-build-runner # required to get output from the start-runner job + - build-connectors-base # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2.1.0 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-connectors-base-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }} + + ## Gradle Build (Platform) + # In case of self-hosted EC2 errors, remove this block. + start-platform-build-runner: + name: "Platform: Start Build EC2 Runner" + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Start EC2 Runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2.2.0 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-04bd6e81239f4f3fb + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0469a9e68a379c1d3 + security-group-id: sg-0793f3c9413f21970 + platform-build: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + needs: start-platform-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner + name: "Platform: Build" + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + - name: Check images exist + run: ./tools/bin/check_images_exist.sh platform + + - name: Pip Caching + uses: actions/cache@v2 + with: + path: | + ~/.cache/pip + key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}- + + - name: Npm Caching + uses: actions/cache@v2 + with: + path: | + ~/.npm + key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}- + + # this intentionally does not use restore-keys so we don't mess with gradle caching + - name: Gradle and Python Caching + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + **/.venv + key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} + + - uses: actions/setup-java@v1 + with: + java-version: '14' + + - uses: actions/setup-node@v1 + with: + node-version: '14.7' + + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install Pyenv + run: python3 -m pip install virtualenv==16.7.9 --user - name: Format - run: ./gradlew format --scan --info --stacktrace + run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" - name: Build - run: CORE_ONLY=true ./gradlew build --scan + run: SUB_BUILD=PLATFORM ./gradlew build --scan - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" + # todo (cgardens) - scope by platform. - name: Check documentation if: success() && github.ref == 'refs/heads/master' run: ./tools/site/link_checker.sh check_docs -# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. -# - name: Get Docker Space -# run: docker run --rm busybox df -h + # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. + # - name: Get Docker Space + # run: docker run --rm busybox df -h + + - name: Image Cleanup + run: ./tools/bin/clean_images.sh - - name: Build Core Docker Images + - name: Build Platform Docker Images if: success() && github.ref == 'refs/heads/master' - run: ./gradlew composeBuild --scan + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan env: GIT_REVISION: ${{ github.sha }} - - name: Image Cleanup - run: ./tools/bin/clean_images.sh - + # make sure these always run before pushing platform docker images - name: Run End-to-End Acceptance Tests + if: success() && github.ref == 'refs/heads/master' run: ./tools/bin/acceptance_test.sh - name: Automatic Migration Acceptance Test - run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i + run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i + + - name: Push Platform Docker Images + if: success() && github.ref == 'refs/heads/master' + run: | + docker login -u airbytebot -p ${DOCKER_PASSWORD} + VERSION=dev docker-compose -f docker-compose.build.yaml push + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - name: Slack Notification - Failure if: failure() && github.ref == 'refs/heads/master' @@ -144,11 +290,11 @@ jobs: SLACK_TITLE: "Build Success" SLACK_FOOTER: "" # In case of self-hosted EC2 errors, remove this block. - stop-build-runner: - name: Stop Build EC2 Runner + stop-platform-build-runner: + name: "Platform: Stop Build EC2 Runner" needs: - - start-build-runner # required to get output from the start-runner job - - build # required to wait when the main job is done + - start-platform-build-runner # required to get output from the start-runner job + - platform-build # required to wait when the main job is done runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -163,8 +309,8 @@ jobs: with: mode: stop github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} - label: ${{ needs.start-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + label: ${{ needs.start-platform-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} ## Frontend Test ## Gradle Build @@ -218,8 +364,8 @@ jobs: - name: Install Cypress Test Dependencies run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - - name: Build Core Docker Images and Run Tests - run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan - name: Run End-to-End Frontend Tests run: ./tools/bin/e2e_test.sh @@ -313,8 +459,8 @@ jobs: HOME: /home/runner CHANGE_MINIKUBE_NONE_USER: true - - name: Build Core Docker Images and Run Tests - run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan + - name: Build Platform Docker Images + run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan - name: Run Logging Tests run: ./tools/bin/cloud_storage_logging_test.sh diff --git a/.github/workflows/publish-cdk-command.yml b/.github/workflows/publish-cdk-command.yml index 7e9b33f3318b..94152e4358b8 100644 --- a/.github/workflows/publish-cdk-command.yml +++ b/.github/workflows/publish-cdk-command.yml @@ -27,7 +27,7 @@ jobs: - name: Checkout Airbyte uses: actions/checkout@v2 - name: Build CDK Package - run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build + run: SUB_BUILD=CONNECTORS_BASE ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build - name: Add Failure Comment if: github.event.inputs.comment-id && !success() uses: peter-evans/create-or-update-comment@v1 diff --git a/airbyte-config/models/README.md b/airbyte-config/models/README.md index a12c3247e14f..eb5853a318a3 100644 --- a/airbyte-config/models/README.md +++ b/airbyte-config/models/README.md @@ -9,7 +9,7 @@ This module uses `jsonschema2pojo` to generate Java config objects from [json sc ``` - Run the following command under the project root: ```sh - ./gradlew airbyte-config:models:generateJsonSchema2Pojo + SUB_BUILD=PLATFORM ./gradlew airbyte-config:models:generateJsonSchema2Pojo ``` The generated file is under: ``` diff --git a/airbyte-migration/README.md b/airbyte-migration/README.md index 2c7d20f65982..05d15bdb8e0a 100644 --- a/airbyte-migration/README.md +++ b/airbyte-migration/README.md @@ -24,7 +24,7 @@ Run the following command in project root: BUILD_VERSION=$(cat .env | grep VERSION | awk -F"=" '{print $2}') # Build the migration bundle file -./gradlew airbyte-migration:build +SUB_BUILD=PLATFORM ./gradlew airbyte-migration:build # Extract the bundle file tar xf ./airbyte-migration/build/distributions/airbyte-migration-${BUILD_VERSION}.tar --strip-components=1 diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java index e863e7bc9c73..27a2b2f7c1c0 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java @@ -65,8 +65,9 @@ import org.testcontainers.containers.output.OutputFrame; /** - * In order to run this test from intellij, build the docker images via ./gradlew composeBuild and - * replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env file + * In order to run this test from intellij, build the docker images via SUB_BUILD=PLATFORM ./gradlew + * composeBuild and replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env + * file */ public class MigrationAcceptanceTest { diff --git a/build.gradle b/build.gradle index b7783d84ffc9..1ed691650a0a 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ def createSpotlessTarget = { pattern -> 'secrets' ] - if (System.getenv().containsKey("CORE_ONLY")) { + if (System.getenv().containsKey("SUB_BUILD")) { excludes.add("airbyte-integrations/connectors") } @@ -237,7 +237,10 @@ task composeBuild { } } } -build.dependsOn(composeBuild) + +if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { + build.dependsOn(composeBuild) +} task('generate') { dependsOn subprojects.collect { it.getTasksByName('generateSeed', true) } diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 147399f024d2..fe93a8be978e 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -28,26 +28,21 @@ To start contributing: ## Build with `gradle` -To compile the code and run unit tests: +To compile and build just the platform (not all the connectors): ```bash -./gradlew clean build +SUB_BUILD=PLATFORM ./gradlew build ``` This will build all the code and run all the unit tests. -`./gradlew build` creates all the necessary artifacts \(Webapp, Jars and Docker images\) so that you can run Airbyte locally. Since this builds everything, it can take some time. +`SUB_BUILD=PLATFORM ./gradlew build` creates all the necessary artifacts \(Webapp, Jars and Docker images\) so that you can run Airbyte locally. Since this builds everything, it can take some time. -To compile and build just the core systems: - -```bash -CORE_ONLY=1 ./gradlew build -``` {% hint style="info" %} Gradle will use all CPU cores by default. If Gradle uses too much/too little CPU, tuning the number of CPU cores it uses to better suit a dev's need can help. -Adjust this by either, 1. Setting an env var: `export GRADLE_OPTS="-Dorg.gradle.workers.max=3"`. 2. Setting a cli option: `./gradlew build --max-workers 3` 3. Setting the `org.gradle.workers.max` property in the `gradle.properties` file. +Adjust this by either, 1. Setting an env var: `export GRADLE_OPTS="-Dorg.gradle.workers.max=3"`. 2. Setting a cli option: `SUB_BUILD=PLATFORM ./gradlew build --max-workers 3` 3. Setting the `org.gradle.workers.max` property in the `gradle.properties` file. A good rule of thumb is to set this to \(\# of cores - 1\). {% endhint %} @@ -64,7 +59,7 @@ export CPPFLAGS="-I/usr/local/opt/openssl/include" ## Run in `dev` mode with `docker-compose` ```bash -CORE_ONLY=1 ./gradlew build +SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up ``` @@ -77,9 +72,9 @@ In `dev` mode, all data will be persisted in `/tmp/dev_root`. To run acceptance \(end-to-end\) tests, you must have the Airbyte running locally. ```bash -CORE_ONLY=1 ./gradlew build +SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up -./gradlew :airbyte-tests:acceptanceTests +SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests ``` ## Run formatting automation/tests @@ -142,7 +137,7 @@ Sometimes you'll want to reset the data in your local environment. One common ca * Rebuild the project ```bash - CORE_ONLY=1 ./gradlew build + SUB_BUILD=PLATFORM ./gradlew build VERSION=dev docker-compose up -V ``` diff --git a/docs/contributing-to-airbyte/gradle-cheatsheet.md b/docs/contributing-to-airbyte/gradle-cheatsheet.md index 6c65ca745cb2..01a741bcf4cf 100644 --- a/docs/contributing-to-airbyte/gradle-cheatsheet.md +++ b/docs/contributing-to-airbyte/gradle-cheatsheet.md @@ -1,15 +1,104 @@ # Gradle Cheatsheet -## Connector Development +## Overview +We have 3 ways of slicing our builds: +1. **Build Everything**: Including every single connectors. +2. **Build Platform**: Build only modules related to the core platform. +3. **Build Connectors Base**: Build only modules related to code infrastructure for connectors. -### Commands used in CI +**Build Everything** is really not particularly functional as building every single connector at once is really prone to transient errors. As there are more connectors the chance that there is a transient issue while downloading any single dependency starts to get really high. + +In our CI we run **Build Platform** and **Build Connectors Base**. Then separately, on a regular cadence, we build each connector and run its integration tests. + +We split Build Platform and Build Connectors Base from each other for a few reasons: +1. The tech stacks are very different. The Platform is almost entirely Java. Because of differing needs around separating environments, the Platform build can be optimized separately from the Connectors one. +2. We want to the iteration cycles of people working on connectors or the platform faster _and_ independent. e.g. Before this change someone working on a Platform feature needs to run formatting on the entire codebase (including connectors). This led to a lot of cosmetic build failures that obfuscated actually problems. Ideally a failure on the connectors side should not block progress on the platform side. +3. The lifecycles are different. One can safely release the Platform even if parts of Connectors Base is failing (and vice versa). + +Future Work: The next step here is to figure out how to more formally split connectors and platform. Right now we exploit behavior in `settings.gradle` to separate them. This is not a best practice. Ultimately, we want these two builds to be totally separate. We do not know what that will look like yet. + +## Cheatsheet +Here is a cheatsheet for common gradle commands. + +### Basic Build Syntax +Here is the syntax for running gradle commands on the different parts of the code base that we called out above. + +#### Build Everything +```shell +./gradlew +``` + +#### Build Platform +```shell +SUB_BUILD=PLATFORM ./gradlew +``` + +#### Build Connectors Base +```shell +SUB_BUILD=CONNECTORS_BASE ./gradlew +``` + +### Build +In order to "build" the project. This task includes producing all artifacts and running unit tests (anything called in the `:test` task). It does _not_ include integration tests (anything called in the `:integrationTest` task). + +For example all the following are valid. +```shell +./gradlew build +SUB_BUILD=PLATFORM ./gradlew build +SUB_BUILD=CONNECTORS_BASE ./gradlew build +``` + +### Formatting + +The build system has a custom task called `format`. It is not called as part of `build`. If the command is called on a subset of the project, it will (mostly) target just the included modules. The exception is that `spotless` (a gradle formatter) will always format any file types that it is configured to manage regardless of which sub build is run. `spotless` is relatively fast, so this should not be too much of an annoyance. It can lead to formatting changes in unexpected parts of the code base. + +For example all the following are valid. +```shell +./gradlew format +SUB_BUILD=PLATFORM ./gradlew format +SUB_BUILD=CONNECTORS_BASE ./gradlew format +``` + +### Platform-Specific Commands + +#### Build Artifacts +This command just builds the docker images that are used as artifacts in the platform. It bypasses running tests. + +```shell +SUB_BUILD=PLATFORM ./gradlew composeBuild +``` + +#### Running Tests +The Platform has 3 different levels of tests: Unit Tests, Acceptance Tests, Frontend Acceptance Tests. + +##### Unit Tests +Unit Tests can be run using the `:test` task on any submodule. These test class-level behavior. They should avoid using external resources (e.g. calling staging services or pulling resources from the internet). We do allow these tests to spin up local resources (usually in docker containers). For example, we use test containers frequently to spin up test postgres databases. + +##### Acceptance Tests +We split Acceptance Tests into 2 different test suites: +* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/acceptance_test_kube.sh) that is used by the CI. +* Migration Acceptance Tests: These tests make sure the end-to-end process of migrating from one version of Airbyte to the next works. These tests are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan`. These tests do not expect there to be a separate instance of Airbyte running. + +These tests currently all live in `airbyte-tests` + +##### Frontend Acceptance Tests +These are acceptance tests for the frontend. They are run with `SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-e2e-testing:e2etest`. Like the Platform Acceptance Tests, they expect Airbyte to be running locally. See the [script](https://github.com/airbytehq/airbyte/blob/master/tools/bin/e2e_test.sh) that is used by the CI. + +These tests currently all live in `airbyte-e2e-testing`. + +##### Future Work +Our story around "integration testing" or "E2E testing" is a little ambiguous. Our Platform Acceptance Test Suite is getting somewhat unwieldy. It was meant to just be some coarse sanity checks, but over time we have found more need to test interactions between systems more granular. Whether we start supporting a separate class of tests (e.g. integration tests) or figure out how allow for more granular tests in the existing Acceptance Test framework is TBD. + +### Connectors-Specific Commands (Connector Development) + +#### Commands used in CI All connectors, regardless of implementation language, implement the following interface to allow uniformity in the build system when run from CI: **Build connector, run unit tests, and build Docker image**: `./gradlew :airbyte-integrations:connectors::build` **Run integration tests**: `./gradlew :airbyte-integrations:connectors::integrationTest` -### Python +#### Python The ideal end state for a Python connector developer is that they shouldn't have to know Gradle exists. We're almost there, but today there is only one Gradle command that's needed when developing in Python, used for formatting code. diff --git a/settings.gradle b/settings.gradle index 8f0d880bd295..897839c1b4b0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,48 +13,68 @@ gradleEnterprise { rootProject.name = 'airbyte' -include ':airbyte-analytics' -include ':airbyte-api' -include ':airbyte-cli' -include ':airbyte-cdk:python' +// SUB_BUILD is an enum of , PLATFORM, CONNECTORS_BASE. Blank is equivalent to all. +if(!System.getenv().containsKey("SUB_BUILD")) { + println("Building all of Airbyte.") +} else { + def subBuild = System.getenv().get("SUB_BUILD") + println("Building Airbyte Sub Build: " + subBuild) + if(subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE") { + throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE", subBuild)) + } +} + +// shared include ':airbyte-commons' include ':airbyte-commons-docker' -include ':airbyte-config:models' -include ':airbyte-config:init' -include ':airbyte-config:persistence' -include ':airbyte-db' -include ':airbyte-e2e-testing' -include ':airbyte-integrations:bases:airbyte-protocol' -include ':airbyte-integrations:bases:base' -include ':airbyte-integrations:bases:base-java' -include ':airbyte-integrations:bases:base-normalization' -include ':airbyte-integrations:bases:base-python' -include ':airbyte-integrations:bases:base-python-test' -include ':airbyte-integrations:bases:base-singer' -include ':airbyte-integrations:bases:base-standard-source-test-file' -include ':airbyte-integrations:bases:source-acceptance-test' -include ':airbyte-integrations:bases:standard-destination-test' -include ':airbyte-integrations:bases:standard-source-test' -include ':airbyte-integrations:connector-templates:generator' -include ':airbyte-integrations:bases:debezium' include ':airbyte-json-validation' -include ':airbyte-migration' -include ':airbyte-notification' include ':airbyte-protocol:models' include ':airbyte-queue' -include ':airbyte-scheduler:app' -include ':airbyte-scheduler:client' -include ':airbyte-scheduler:models' -include ':airbyte-scheduler:persistence' -include ':airbyte-server' -include ':airbyte-webapp' -include ':airbyte-workers' -include ':airbyte-tests' +include ':airbyte-workers' // reused by acceptance tests in connector base. +include ':airbyte-config:models' // reused by acceptance tests in connector base. +include ':airbyte-db' // reused by acceptance tests in connector base. include ':airbyte-test-utils' -include ':tools:code-generator' -if(!System.getenv().containsKey("CORE_ONLY")) { - println "Building all of Airbyte." +// platform +if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { + include ':airbyte-analytics' + include ':airbyte-api' + include ':airbyte-cli' + include ':airbyte-config:init' + include ':airbyte-config:persistence' + include ':airbyte-e2e-testing' + include ':airbyte-migration' + include ':airbyte-notification' + include ':airbyte-scheduler:app' + include ':airbyte-scheduler:client' + include ':airbyte-scheduler:models' + include ':airbyte-scheduler:persistence' + include ':airbyte-server' + include ':airbyte-webapp' + include ':airbyte-tests' +} + +// connectors base +if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CONNECTORS_BASE") { + include ':airbyte-cdk:python' + include ':airbyte-integrations:bases:airbyte-protocol' + include ':airbyte-integrations:bases:base' + include ':airbyte-integrations:bases:base-java' + include ':airbyte-integrations:bases:base-normalization' + include ':airbyte-integrations:bases:base-python' + include ':airbyte-integrations:bases:base-python-test' + include ':airbyte-integrations:bases:base-singer' + include ':airbyte-integrations:bases:base-standard-source-test-file' + include ':airbyte-integrations:bases:source-acceptance-test' + include ':airbyte-integrations:bases:standard-destination-test' + include ':airbyte-integrations:bases:standard-source-test' + include ':airbyte-integrations:connector-templates:generator' + include ':airbyte-integrations:bases:debezium' + include ':tools:code-generator' +} + +// connectors +if(!System.getenv().containsKey("SUB_BUILD")) { // include all connector projects def integrationsPath = rootDir.toPath().resolve('airbyte-integrations/connectors') println integrationsPath @@ -65,6 +85,4 @@ if(!System.getenv().containsKey("CORE_ONLY")) { include ":airbyte-integrations:connectors:${dir.getFileName()}" } } -} else { - println "Building Airbyte Core." } diff --git a/tools/bin/acceptance_test.sh b/tools/bin/acceptance_test.sh index a7a34f63f1bb..7da0dfd6dc2f 100755 --- a/tools/bin/acceptance_test.sh +++ b/tools/bin/acceptance_test.sh @@ -22,4 +22,4 @@ echo "Waiting for services to begin" sleep 10 # TODO need a better way to wait echo "Running e2e tests via gradle" -./gradlew --no-daemon :airbyte-tests:acceptanceTests --rerun-tasks --scan +SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests --rerun-tasks --scan diff --git a/tools/bin/acceptance_test_kube.sh b/tools/bin/acceptance_test_kube.sh index c51bed507c6d..9fc5a45c28a4 100755 --- a/tools/bin/acceptance_test_kube.sh +++ b/tools/bin/acceptance_test_kube.sh @@ -37,7 +37,7 @@ trap "echo 'kube logs:' && print_all_logs" EXIT kubectl port-forward svc/airbyte-server-svc 8001:8001 & echo "Running worker integration tests..." -./gradlew --no-daemon :airbyte-workers:integrationTest --scan +SUB_BUILD=PLATFORM ./gradlew :airbyte-workers:integrationTest --scan echo "Running e2e tests via gradle..." -KUBE=true ./gradlew --no-daemon :airbyte-tests:acceptanceTests --scan +KUBE=true SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests --scan diff --git a/tools/bin/check_images_exist.sh b/tools/bin/check_images_exist.sh index d3e1bc1de5a6..36c627b832cc 100755 --- a/tools/bin/check_images_exist.sh +++ b/tools/bin/check_images_exist.sh @@ -2,30 +2,54 @@ set -e +. tools/lib/lib.sh + function docker_tag_exists() { URL=https://hub.docker.com/v2/repositories/"$1"/tags/"$2" printf "\tURL: %s\n" "$URL" curl --silent -f -lSL "$URL" > /dev/null } -echo "Checking core images exist..." -docker-compose pull || exit 1 - -echo "Checking integration images exist..." -CONFIG_FILES=$(find airbyte-config/init | grep json | grep -v STANDARD_WORKSPACE | grep -v build) -[ -z "$CONFIG_FILES" ] && echo "ERROR: Could not find any config files." && exit 1 - -while IFS= read -r file; do - REPO=$(jq -r .dockerRepository < "$file") - TAG=$(jq -r .dockerImageTag < "$file") - echo "Checking $file..." - printf "\tREPO: %s\n" "$REPO" - printf "\tTAG: %s\n" "$TAG" - if docker_tag_exists "$REPO" "$TAG"; then - printf "\tSTATUS: found\n" - else - printf "\tERROR: not found!\n" && exit 1 - fi -done <<< "$CONFIG_FILES" - -echo "Success! All images exist!" +checkPlatformImages() { + echo "Checking platform images exist..." + docker-compose pull || exit 1 + echo "Success! All platform images exist!" +} + +checkConnectorImages() { + echo "Checking connector images exist..." + + CONFIG_FILES=$(find airbyte-config/init | grep json | grep -v STANDARD_WORKSPACE | grep -v build) + [ -z "$CONFIG_FILES" ] && echo "ERROR: Could not find any config files." && exit 1 + + while IFS= read -r file; do + REPO=$(jq -r .dockerRepository < "$file") + TAG=$(jq -r .dockerImageTag < "$file") + echo "Checking $file..." + printf "\tREPO: %s\n" "$REPO" + printf "\tTAG: %s\n" "$TAG" + if docker_tag_exists "$REPO" "$TAG"; then + printf "\tSTATUS: found\n" + else + printf "\tERROR: not found!\n" && exit 1 + fi + done <<< "$CONFIG_FILES" + + echo "Success! All connector images exist!" +} + +main() { + assert_root + + SUBSET=${1:-all} # default to all. + [[ ! "$SUBSET" =~ ^(all|platform|connectors)$ ]] && echo "Usage ./tools/bin/check_image_exists.sh [all|platform|connectors]" && exit 1 + + echo "checking images for: $SUBSET" + + [[ "$SUBSET" =~ ^(all|platform)$ ]] && checkPlatformImages + [[ "$SUBSET" =~ ^(all|connectors)$ ]] && checkConnectorImages + + echo "Image check complete." +} + +main "$@" diff --git a/tools/bin/cloud_storage_logging_test.sh b/tools/bin/cloud_storage_logging_test.sh index bfec3dd999e0..9808239c1bf5 100755 --- a/tools/bin/cloud_storage_logging_test.sh +++ b/tools/bin/cloud_storage_logging_test.sh @@ -16,4 +16,4 @@ export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" export GCP_STORAGE_BUCKET=airbyte-kube-integration-logging-test echo "Running logging tests.." -./gradlew --no-daemon :airbyte-config:models:integrationTest --scan +SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-config:models:integrationTest --scan diff --git a/tools/bin/e2e_test.sh b/tools/bin/e2e_test.sh index 17412de0a70e..7aeca8cca04c 100755 --- a/tools/bin/e2e_test.sh +++ b/tools/bin/e2e_test.sh @@ -23,5 +23,5 @@ echo "Waiting for services to begin" sleep 30 # TODO need a better way to wait echo "Running e2e tests via gradle" -./gradlew --no-daemon :airbyte-e2e-testing:e2etest +SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-e2e-testing:e2etest diff --git a/tools/bin/release_version.sh b/tools/bin/release_version.sh index 09cc306e6f91..5f93576375e5 100755 --- a/tools/bin/release_version.sh +++ b/tools/bin/release_version.sh @@ -29,7 +29,7 @@ GIT_REVISION=$(git rev-parse HEAD) echo "Bumped version from ${PREV_VERSION} to ${NEW_VERSION}" echo "Building and publishing version $NEW_VERSION for git revision $GIT_REVISION..." -./gradlew clean composeBuild +SUB_BUILD=PLATFORM ./gradlew clean composeBuild VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml build VERSION=$NEW_VERSION GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose.build.yaml push echo "Completed building and publishing..."