RFC: Split Platform and Connector Builds

.github/workflows/gradle.yml

@@ -6,10 +6,10 @@ on:
   push:
 
 jobs:
-  ## Gradle Build
+  ## Gradle Build (Connectors Base)
   # In case of self-hosted EC2 errors, remove this block.
-  start-build-runner:
-    name: Start Build EC2 Runner
+  start-connectors-base-build-runner:
+    name: "Connectors Base: Start Build EC2 Runner"
     runs-on: ubuntu-latest
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
@@ -31,11 +31,11 @@ jobs:
           ec2-instance-type: c5.2xlarge
           subnet-id: subnet-0469a9e68a379c1d3
           security-group-id: sg-0793f3c9413f21970
-  build:
+  build-connectors-base:
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: start-build-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner
-    name: Build Airbyte
+    needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner
+    name: "Connectors Base: Build"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -87,42 +87,189 @@ jobs:
         run: python3 -m pip install virtualenv==16.7.9 --user
 
       - name: Generate Template scaffold
-        run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
 
       - name: Format
-        run: ./gradlew format --scan --info --stacktrace
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+
+      - name: Build
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
+
+      - name: Ensure no file change
+        run: git status --porcelain && test -z "$(git status --porcelain)"
+
+      - name: Check documentation
+        if: success() && github.ref == 'refs/heads/master'
+        run: ./tools/site/link_checker.sh check_docs
+
+      - name: Slack Notification - Failure
+        if: failure() && github.ref == 'refs/heads/master'
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildozer
+          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
+          SLACK_COLOR: DC143C
+          SLACK_TITLE: "Build failure"
+          SLACK_FOOTER: ""
+
+      - name: Slack Notification - Success
+        if: success() && github.ref == 'refs/heads/master'
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildbot
+          SLACK_TITLE: "Build Success"
+          SLACK_FOOTER: ""
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-connectors-base-build-runner:
+    name: "Connectors Base: Stop Build EC2 Runner"
+    needs:
+      - start-connectors-base-build-runner # required to get output from the start-runner job
+      - build-connectors-base # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2.1.0
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}
+
+  ## Gradle Build (Platform)
+  # In case of self-hosted EC2 errors, remove this block.
+  start-platform-build-runner:
+    name: "Platform: Start Build EC2 Runner"
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Start EC2 Runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2.2.0
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          ec2-image-id: ami-04bd6e81239f4f3fb
+          ec2-instance-type: c5.2xlarge
+          subnet-id: subnet-0469a9e68a379c1d3
+          security-group-id: sg-0793f3c9413f21970
+  platform-build:
+    # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
+    needs: start-platform-build-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner
+    name: "Platform: Build"
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      # todo (cgardens) - scope by platform.
+      - name: Check images exist
+        run: ./tools/bin/check_images_exist.sh
+
+      - name: Pip Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+          key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-
+
+      - name: Npm Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.npm
+          key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-
+
+      # this intentionally does not use restore-keys so we don't mess with gradle caching
+      - name: Gradle and Python Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            **/.venv
+          key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '14'
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.7'
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install Pyenv
+        run: python3 -m pip install virtualenv==16.7.9 --user
+
+      - name: Format
+        run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
 
       - name: Ensure no file change
         run: git status --porcelain && test -z "$(git status --porcelain)"
 
       - name: Build
-        run: CORE_ONLY=true ./gradlew build --scan
+        run: SUB_BUILD=PLATFORM ./gradlew build --scan
 
       - name: Ensure no file change
         run: git status --porcelain && test -z "$(git status --porcelain)"
 
+      # todo (cgardens) - scope by platform.
       - name: Check documentation
         if: success() && github.ref == 'refs/heads/master'
         run: ./tools/site/link_checker.sh check_docs
 
-# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
-#      - name: Get Docker Space
-#        run: docker run --rm busybox df -h
+      # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
+      #      - name: Get Docker Space
+      #        run: docker run --rm busybox df -h
+
+      - name: Image Cleanup
+        run: ./tools/bin/clean_images.sh
 
-      - name: Build Core Docker Images
+      - name: Build Platform Docker Images
         if: success() && github.ref == 'refs/heads/master'
-        run: ./gradlew composeBuild --scan
+        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
         env:
           GIT_REVISION: ${{ github.sha }}
 
-      - name: Image Cleanup
-        run: ./tools/bin/clean_images.sh
-
+      # make sure these always run before pushing platform docker images
       - name: Run End-to-End Acceptance Tests
+        if: success() && github.ref == 'refs/heads/master'
         run: ./tools/bin/acceptance_test.sh
 
       - name: Automatic Migration Acceptance Test
-        run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr  -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan
+        run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr  -d "VERSION=") SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan
+
+      - name: Push Platform Docker Images
+        if: success() && github.ref == 'refs/heads/master'
+        run: |
+          docker login -u airbytebot -p ${DOCKER_PASSWORD}
+          VERSION=dev docker-compose -f docker-compose.build.yaml push
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Slack Notification - Failure
         if: failure() && github.ref == 'refs/heads/master'
@@ -144,11 +291,11 @@ jobs:
           SLACK_TITLE: "Build Success"
           SLACK_FOOTER: ""
   # In case of self-hosted EC2 errors, remove this block.
-  stop-build-runner:
-    name: Stop Build EC2 Runner
+  stop-platform-build-runner:
+    name: "Platform: Stop Build EC2 Runner"
     needs:
-      - start-build-runner # required to get output from the start-runner job
-      - build # required to wait when the main job is done
+      - start-platform-build-runner # required to get output from the start-runner job
+      - platform-build # required to wait when the main job is done
     runs-on: ubuntu-latest
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps:
@@ -163,8 +310,8 @@ jobs:
         with:
           mode: stop
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
-          label: ${{ needs.start-build-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }}
+          label: ${{ needs.start-platform-build-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}
 
   ## Frontend Test
   ## Gradle Build
@@ -218,8 +365,8 @@ jobs:
       - name: Install Cypress Test Dependencies
         run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
 
-      - name: Build Core Docker Images and Run Tests
-        run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan
+      - name: Build Platform Docker Images
+        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan
 
       - name: Run End-to-End Frontend Tests
         run: ./tools/bin/e2e_test.sh
@@ -313,8 +460,8 @@ jobs:
           HOME: /home/runner
           CHANGE_MINIKUBE_NONE_USER: true
 
-      - name: Build Core Docker Images and Run Tests
-        run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan
+      - name: Build Platform Docker Images
+        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
 
       - name: Run Logging Tests
         run: ./tools/bin/cloud_storage_logging_test.sh

.github/workflows/publish-cdk-command.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
       - name: Build CDK Package
-        run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
       - name: Add Failure Comment
         if: github.event.inputs.comment-id && !success()
         uses: peter-evans/create-or-update-comment@v1

airbyte-config/models/README.md

@@ -9,7 +9,7 @@ This module uses `jsonschema2pojo` to generate Java config objects from [json sc
   ```
 - Run the following command under the project root:
   ```sh
-  ./gradlew airbyte-config:models:generateJsonSchema2Pojo
+  SUB_BUILD=PLATFORM ./gradlew airbyte-config:models:generateJsonSchema2Pojo
   ```
   The generated file is under:
   ```

airbyte-migration/README.md

@@ -24,7 +24,7 @@ Run the following command in project root:
 BUILD_VERSION=$(cat .env | grep VERSION | awk -F"=" '{print $2}')
 
 # Build the migration bundle file
-./gradlew airbyte-migration:build
+SUB_BUILD=PLATFORM ./gradlew airbyte-migration:build
 
 # Extract the bundle file
 tar xf ./airbyte-migration/build/distributions/airbyte-migration-${BUILD_VERSION}.tar --strip-components=1

airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java

@@ -65,8 +65,9 @@
 import org.testcontainers.containers.output.OutputFrame;
 
 /**
- * In order to run this test from intellij, build the docker images via ./gradlew composeBuild and
- * replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env file
+ * In order to run this test from intellij, build the docker images via SUB_BUILD=PLATFORM ./gradlew
+ * composeBuild and replace System.getenv("MIGRATION_TEST_VERSION") with the version in your .env
+ * file
  */
 public class MigrationAcceptanceTest {
 

build.gradle

@@ -73,7 +73,7 @@ def createSpotlessTarget = { pattern ->
         'secrets'
     ]
 
-    if (System.getenv().containsKey("CORE_ONLY")) {
+    if (System.getenv().containsKey("SUB_BUILD")) {
         excludes.add("airbyte-integrations/connectors")
     }
 
@@ -237,7 +237,10 @@ task composeBuild {
         }
     }
 }
-build.dependsOn(composeBuild)
+
+if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") {
+    build.dependsOn(composeBuild)
+}
 
 task('generate') {
     dependsOn subprojects.collect { it.getTasksByName('generateSeed', true) }