mostly incremental builds (#817)

* support cached builds for base -> base-java -> snowflake

* use plugin for image building

* fix matching on COPY from

* remove docker.gradle

* tmp commit

* update connectors

* finish rest of build files

* fix ide errors

* more build fixes

* clean up

* clean up for new soruces

* fix spotless

* fix flake problems

* add recommended empty file

* python caching

* fixes upon review

* clean up docker and build test files

* clean up python

* clean up

* fix integration test dependencies

* fix standard tests

* fix

* remove symlink

* re-add requirements to fix normalizatioon build

* fix symlink

* fix dumbest build problem of all

* add missing integration test def

* fix missing dep

* remove class exclusion

* move trim so null source versions are allowed

* rename map

* fix hardcoded value

* remove unnecessary dep

* use dashes for salesforce package name

* fix typo

* DRY and fix test image name

* Fix edit

* assert string is not empty

* build integration test image only for integrationTest

* move code generator to tools and rename docker build tasks

* make source test depend on integration test build, not the other way

* remove guard because the docker build should exist before the integrationtest is applied

* remove comment

* DRY up airbyte-source-test

* fix plugin compilation

* add missing dependency

* rename getTaggedImage to getDevTaggedImage

* fix test vs main docker build bug

.gitignore

@@ -2,9 +2,9 @@
 .idea
 *.iml
 build
-!tools/build
 .DS_Store
 data
+.dockerversions
 
 secrets
 

airbyte-integrations/bases/airbyte-protocol/build.gradle

@@ -1,14 +1,19 @@
-project.ext.pyModule = 'airbyte_protocol'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+}
+
+airbytePython {
+    moduleDirectory 'airbyte_protocol'
+}
 
 task generateProtocolClassFilesWithoutLicense(type: Exec) {
     environment 'ROOT_DIR', rootDir.absolutePath
     commandLine 'bin/generate-protocol-files.sh'
-    dependsOn ':airbyte-integrations:bases:airbyte-protocol:code-generator:buildImage'
+    dependsOn ':tools:code-generator:airbyteDocker'
 }
 
+// todo: figure out dependencies
 task generateProtocolClassFiles(dependsOn: [generateProtocolClassFilesWithoutLicense, rootProject.spotlessPython])
 rootProject.spotlessPython.mustRunAfter(generateProtocolClassFilesWithoutLicense)
-
 rootProject.build.dependsOn generateProtocolClassFiles

airbyte-integrations/bases/base-java/build.gradle

@@ -1,14 +1,13 @@
 plugins {
     id 'java-library'
+    id 'airbyte-docker'
 }
 
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-
 dependencies {
     implementation 'commons-cli:commons-cli:1.4'
 
     implementation project(':airbyte-config:models')
     implementation project(':airbyte-protocol:models')
-}
 
-buildImage.dependsOn ':airbyte-integrations:bases:base:buildImage'
+    implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs)
+}

airbyte-integrations/bases/base-normalization/build.gradle

@@ -1,13 +1,12 @@
-project.ext.pyModule = 'normalization'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+plugins {
+    id 'airbyte-docker'
+    id 'airbyte-python'
+}
 
-task unitTest(type: PythonTask){
-    command = "setup.py test"
+airbytePython {
+    moduleDirectory 'normalization'
 }
 
-unitTest.dependsOn(installReqs)
-build.dependsOn(unitTest)
-build.dependsOn ':airbyte-integrations:bases:airbyte-protocol:build'
-buildImage.dependsOn ':airbyte-integrations:bases:base:buildImage'
-buildImage.dependsOn ':airbyte-integrations:bases:airbyte-protocol:buildImage'
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs)
+}

airbyte-integrations/bases/base-normalization/setup.py

@@ -34,11 +34,13 @@
     install_requires=["airbyte-protocol", "dbt>=0.18.1", "pyyaml"],
     package_data={"": ["*.yml"]},
     setup_requires=["pytest-runner"],
-    tests_require=["pytest"],
     entry_points={
         "console_scripts": [
             "transform-config=normalization.transform_config.transform:main",
             "transform-catalog=normalization.transform_catalog.transform:main",
         ],
     },
+    extras_require={
+        "tests": ["airbyte-protocol", "pytest"],
+    },
 )

airbyte-integrations/bases/base-python-test/build.gradle

@@ -1,6 +1,12 @@
-project.ext.pyModule = 'base_python_test'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+plugins {
+    id 'airbyte-docker'
+    id 'airbyte-python'
+}
 
-build.dependsOn(':airbyte-integrations:bases:base-python:build')
-buildImage.dependsOn(':airbyte-integrations:bases:base-python:buildImage')
+airbytePython {
+    moduleDirectory 'base_python_test'
+}
+
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
+}

airbyte-integrations/bases/base-python/build.gradle

@@ -1,7 +1,13 @@
-project.ext.pyModule = 'base_python'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+plugins {
+    id 'airbyte-docker'
+    id 'airbyte-python'
+}
 
-build.dependsOn ':airbyte-integrations:bases:airbyte-protocol:build'
-buildImage.dependsOn ':airbyte-integrations:bases:base:buildImage'
-buildImage.dependsOn ':airbyte-integrations:bases:airbyte-protocol:buildImage'
+airbytePython {
+    moduleDirectory 'base_python'
+}
+
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:airbyte-protocol').airbyteDocker.outputs)
+    implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs)
+}

airbyte-integrations/bases/base-singer/build.gradle

@@ -1,7 +1,12 @@
-project.ext.pyModule = 'base_singer'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+plugins {
+    id 'airbyte-docker'
+    id 'airbyte-python'
+}
 
-build.dependsOn ':airbyte-integrations:bases:airbyte-protocol:build'
-build.dependsOn ':airbyte-integrations:bases:base-python:build'
-buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage'
+airbytePython {
+    moduleDirectory 'base_singer'
+}
+
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
+}

airbyte-integrations/bases/base/build.gradle

@@ -1 +1,3 @@
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+plugins {
+    id 'airbyte-docker'
+}

airbyte-integrations/bases/destination-test-lib/build.gradle

@@ -8,5 +8,4 @@ dependencies {
 
     implementation 'org.junit.jupiter:junit-jupiter-api:5.4.2'
     implementation 'org.junit.jupiter:junit-jupiter-params:5.4.2'
-
 }

airbyte-integrations/bases/standard-source-test/build.gradle

@@ -1,8 +1,11 @@
 plugins {
     id 'application'
+    id 'airbyte-docker'
 }
 
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+application {
+    mainClass = 'io.airbyte.integrations.base.TestPythonSourceMain'
+}
 
 dependencies {
     implementation project(':airbyte-config:models')
@@ -15,9 +18,3 @@ dependencies {
     implementation 'org.junit.platform:junit-platform-launcher:1.7.0'
     implementation 'org.junit.jupiter:junit-jupiter-api:5.4.2'
 }
-
-application {
-    mainClass = 'io.airbyte.integrations.base.TestPythonSourceMain'
-}
-
-buildImage.dependsOn(assemble)

airbyte-integrations/connector-templates/java-destination/build.gradle.hbs

@@ -1,22 +1,19 @@
 plugins {
     id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
 }
 
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+application {
+    mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination'
+}
 
 dependencies {
     implementation project(':airbyte-config:models')
     implementation project(':airbyte-protocol:models')
     implementation project(':airbyte-integrations:bases:base-java')
 
     integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib')
-}
 
-application {
-    mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination'
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }
-
-buildImage.dependsOn(assemble)
-buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage')
-integrationTest.dependsOn(buildImage)

airbyte-integrations/connector-templates/source-python/build.gradle.hbs

@@ -1,33 +1,13 @@
-project.ext.pyModule = 'source_{{snakeCase name}}'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle')
-
-
-standardSourceTestPython {
-    ext {
-        imageName = "${extractImageName(project.file('Dockerfile'))}:dev"
-        pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev"
-    }
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+    id 'airbyte-source-test'
 }
 
-task installTestDeps(type: PythonTask){
-    module = "pip"
-    command = "install .[tests]"
+airbytePython {
+    moduleDirectory 'source_{{snakeCase name}}'
 }
 
-task unitTest(type: PythonTask){
-    module = "pytest"
-    command = "unit_tests"
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
 }
-
-unitTest.dependsOn(installTestDeps)
-build.dependsOn(unitTest)
-build.dependsOn ':airbyte-integrations:bases:base-python-test:build'
-buildImage.dependsOn ':airbyte-integrations:bases:base-python:buildImage'
-integrationTest.dependsOn(buildImage)
-
-buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage'
-standardSourceTestPython.dependsOn(buildTestImage)

airbyte-integrations/connector-templates/source-singer/build.gradle.hbs

@@ -1,34 +1,13 @@
-project.ext.pyModule = 'source_{{snakeCase name}}_singer'
-apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle')
-
-
-standardSourceTestPython {
-    ext {
-        imageName = "${extractImageName(project.file('Dockerfile'))}:dev"
-        pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev"
-    }
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+    id 'airbyte-source-test'
 }
 
-task installTestDeps(type: PythonTask){
-    module = "pip"
-    command = "install .[tests]"
+airbytePython {
+    moduleDirectory 'source_{{snakeCase name}}_singer'
 }
 
-task unitTest(type: PythonTask){
-    module = "pytest"
-    command = "unit_tests"
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs)
 }
-
-unitTest.dependsOn(installTestDeps)
-build.dependsOn(unitTest)
-build.dependsOn ':airbyte-integrations:bases:base-python-test:build'
-
-buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage'
-buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage'
-
-integrationTest.dependsOn(buildImage)
-standardSourceTestPython.dependsOn(buildTestImage)

airbyte-integrations/connectors/destination-bigquery/build.gradle

@@ -1,9 +1,12 @@
 plugins {
     id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
 }
 
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+application {
+    mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination'
+}
 
 dependencies {
     implementation 'com.google.cloud:google-cloud-bigquery:1.122.2'
@@ -16,12 +19,5 @@ dependencies {
 
     integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib')
 
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }
-
-application {
-    mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination'
-}
-
-buildImage.dependsOn(assemble)
-buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage')
-integrationTest.dependsOn(buildImage)

airbyte-integrations/connectors/destination-csv/build.gradle

@@ -1,9 +1,12 @@
 plugins {
     id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
 }
 
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+application {
+    mainClass = 'io.airbyte.integrations.destination.csv.CsvDestination'
+}
 
 dependencies {
     implementation 'org.apache.commons:commons-csv:1.4'
@@ -12,12 +15,6 @@ dependencies {
     implementation project(':airbyte-integrations:bases:base-java')
 
     integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib')
-}
 
-application {
-    mainClass = 'io.airbyte.integrations.destination.csv.CsvDestination'
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }
-
-buildImage.dependsOn(assemble)
-buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage')
-integrationTest.dependsOn(buildImage)

airbyte-integrations/connectors/destination-postgres/build.gradle

@@ -1,9 +1,12 @@
 plugins {
     id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
 }
 
-apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+application {
+    mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination'
+}
 
 dependencies {
     implementation project(':airbyte-db')
@@ -15,12 +18,6 @@ dependencies {
 
     integrationTestImplementation project(':airbyte-integrations:bases:destination-test-lib')
     integrationTestImplementation "org.testcontainers:postgresql:1.15.0-rc2"
-}
 
-application {
-    mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination'
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }
-
-buildImage.dependsOn(assemble)
-buildImage.dependsOn(':airbyte-integrations:bases:base-java:buildImage')
-integrationTest.dependsOn(buildImage)