Isolate our build so that the feedback loop is faster

.github/workflows/gradle.yml

@@ -50,6 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       build: ${{ steps.filter.outputs.build }}
+      cdk: ${{ steps.filter.outputs.cdk }}
       cli: ${{ steps.filter.outputs.cli }}
       connectors: ${{ steps.filter.outputs.connectors }}
       db: ${{ steps.filter.outputs.db }}
@@ -69,11 +70,12 @@ jobs:
               - '*.gradle'
               - 'deps.toml'
               - 'airbyte-config/**'
+            cdk:
+              - 'airbyte-cdk/**'
             cli:
               - 'airbyte-api/**'
               - 'octavia-cli/**'
             connectors:
-              - 'airbyte-cdk/**'
               - 'airbyte-integrations/**'
               - 'airbyte-connector-test-harnesses/acceptance-test-harness/**'
             db:
@@ -156,6 +158,67 @@ jobs:
           attempt_limit: 3
           attempt_delay: 5000 # in ms
 
+  cdk-build:
+    needs: changes
+    runs-on: ubuntu-latest
+    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+    if: needs.changes.outputs.cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
+    name: "Airbyte CDK: Build"
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+
+      - name: Cache Build Artifacts
+        uses: ./.github/actions/cache-build-artifacts
+        with:
+          cache-key: ${{ secrets.CACHE_VERSION }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m \
+            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Format
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CDK ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+
+      - name: Commit Formatting Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Automated Commit - Formatting Changes
+          commit_user_name: Octavia Squidington III
+          commit_user_email: octavia-squidington-iii@users.noreply.github.com
+
+      - name: Build
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CDK ./gradlew build --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+
+
   # Connectors Base
   # In case of self-hosted EC2 errors, remove this block.
   start-connectors-base-build-runner:

.github/workflows/publish-cdk-command-manually.yml

@@ -133,7 +133,7 @@ jobs:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
       - name: Build CDK Package
-        run: SUB_BUILD=CONNECTORS_BASE ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
+        run: SUB_BUILD=CDK ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
       - name: Post failure to Slack channel dev-connectors-extensibility
         if: ${{ failure() }}
         uses: slackapi/slack-github-action@v1.23.0

airbyte-cdk/python/README.md

@@ -69,7 +69,7 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies
 ##### Autogenerated files
 If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run:
 ```commandline
-SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+SUB_BUILD=CDK ./gradlew format
 ```
 This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on
 pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateComponentManifestClassFiles`.

docs/contributing-to-airbyte/gradle-cheatsheet.md

@@ -65,6 +65,12 @@ SUB_BUILD=PLATFORM ./gradlew <gradle command>
 SUB_BUILD=CONNECTORS_BASE ./gradlew <gradle command>
 ```
 
+#### Build CDK
+
+```text
+SUB_BUILD=CDK ./gradlew <gradle command>
+```
+
 ### Build
 
 In order to "build" the project. This task includes producing all artifacts and running unit tests \(anything called in the `:test` task\). It does _not_ include integration tests \(anything called in the `:integrationTest` task\).

settings.gradle

@@ -65,31 +65,43 @@ dependencyResolutionManagement {
     }
 }
 
-// SUB_BUILD is an enum of <blank>, PLATFORM, CONNECTORS_BASE, ALL_CONNECTORS and OCTAVIA_CLI. Blank is equivalent to all.
+// SUB_BUILD is an enum of <blank>, PLATFORM, CONNECTORS_BASE, CDK, ALL_CONNECTORS and OCTAVIA_CLI. Blank is equivalent to all.
 if (!System.getenv().containsKey("SUB_BUILD")) {
     println("Building all of Airbyte.")
 } else {
     def subBuild = System.getenv().get("SUB_BUILD")
     println("Building Airbyte Sub Build: " + subBuild)
-    if (subBuild != "CONNECTORS_BASE" && subBuild != "ALL_CONNECTORS" && subBuild != "OCTAVIA_CLI") {
-        throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or CONNECTORS_BASE, ALL_CONNECTORS or OCTAVIA_CLI", subBuild))
+    if (subBuild != "CONNECTORS_BASE" && subBuild != "ALL_CONNECTORS" && subBuild != "OCTAVIA_CLI" && subBuild != "CDK") {
+        throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or CONNECTORS_BASE, CDK, ALL_CONNECTORS or OCTAVIA_CLI", subBuild))
     }
 }
 
-// shared
-include ':airbyte-commons'
-include ':airbyte-api'
-include ':airbyte-commons-cli'
-include ':airbyte-commons-protocol'
-include ':airbyte-config:specs'
-include ':airbyte-config:init'
-include ':airbyte-config:config-models' // reused by acceptance tests in connector base.
-include ':airbyte-db:db-lib' // reused by acceptance tests in connector base.
-include ':airbyte-json-validation'
-include ':airbyte-test-utils'
-
-// airbyte-workers has a lot of dependencies.
-include ':airbyte-connector-test-harnesses:acceptance-test-harness'
+if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CDK" || System.getenv().get("SUB_BUILD") == "ALL_CONNECTORS") {
+    include ':airbyte-commons' // this wouldn't be necessary if it wasn't from https://github.com/airbytehq/airbyte/blob/645558b74aab0b91fda1b4628b37b7095d92b4cc/build.gradle
+    include ':tools:code-generator:airbyteDocker'
+    include ':airbyte-cdk:python'
+}
+
+if (!System.getenv().containsKey("SUB_BUILD") || (System.getenv().containsKey("SUB_BUILD") && System.getenv().get("SUB_BUILD") != "CDK")) {
+    // shared
+    include ':airbyte-commons'
+    include ':airbyte-api'
+    include ':airbyte-commons-cli'
+    include ':airbyte-commons-protocol'
+    include ':airbyte-config:specs'
+    include ':airbyte-config:init'
+    include ':airbyte-config:config-models' // reused by acceptance tests in connector base.
+    include ':airbyte-db:db-lib' // reused by acceptance tests in connector base.
+    include ':airbyte-json-validation'
+    include ':airbyte-test-utils'
+
+    // airbyte-workers has a lot of dependencies.
+    include ':airbyte-commons-worker'
+    include ':airbyte-config:config-persistence' // transitively used by airbyte-workers.
+    include ':airbyte-db:jooq' // transitively used by airbyte-workers.
+    include ':airbyte-persistence:job-persistence' // transitively used by airbyte-workers.
+    include ':airbyte-connector-test-harnesses:acceptance-test-harness'
+}
 
 // connectors base
 if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CONNECTORS_BASE" || System.getenv().get("SUB_BUILD") == "ALL_CONNECTORS") {