From 16e13473d0db3b0f84842ace1924576d302e4219 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 11:01:03 -0700 Subject: [PATCH 1/9] restore template generator and explicitly declare deps --- .github/workflows/gradle.yml | 4 ++-- settings.gradle | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 14de53889c2b..9fc14e19c82e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -86,8 +86,8 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user -# - name: Generate Template scaffold -# run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan + - name: Generate Template scaffold + run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan - name: Format run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace diff --git a/settings.gradle b/settings.gradle index 897839c1b4b0..deee4cca6a36 100644 --- a/settings.gradle +++ b/settings.gradle @@ -69,6 +69,9 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-integrations:bases:standard-destination-test' include ':airbyte-integrations:bases:standard-source-test' include ':airbyte-integrations:connector-templates:generator' + include ':airbyte-integrations:connectors:source-scaffold-java-jdbc' + include ':airbyte-integrations:connectors:source-scaffold-source-http' + include ':airbyte-integrations:connectors:source-scaffold-source-python' include ':airbyte-integrations:bases:debezium' include ':tools:code-generator' } From b3d6303a56667d1a98a5b5c722cfd962c9959067 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:01:01 -0700 Subject: [PATCH 2/9] change the inclusions --- settings.gradle | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/settings.gradle b/settings.gradle index deee4cca6a36..49a8e943dc6f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -69,11 +69,15 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-integrations:bases:standard-destination-test' include ':airbyte-integrations:bases:standard-source-test' include ':airbyte-integrations:connector-templates:generator' + include ':airbyte-integrations:bases:debezium' + include ':tools:code-generator' + + // dependencies required for the template generator tests to run and format properly. include ':airbyte-integrations:connectors:source-scaffold-java-jdbc' include ':airbyte-integrations:connectors:source-scaffold-source-http' include ':airbyte-integrations:connectors:source-scaffold-source-python' - include ':airbyte-integrations:bases:debezium' - include ':tools:code-generator' + include ':airbyte-integrations:connectors:source-jdbc' + include ':airbyte-integrations:connectors:source-relational-db' } // connectors From e1f2924f95def814de64faf378ba83173670c7d3 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:07:46 -0700 Subject: [PATCH 3/9] only exclude formatting for connectors in platform build --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 1ed691650a0a..448eb94036d6 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ def createSpotlessTarget = { pattern -> 'secrets' ] - if (System.getenv().containsKey("SUB_BUILD")) { + if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { excludes.add("airbyte-integrations/connectors") } From 8d65e0057c461034f672a00f380fdcdab616b963 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:08:23 -0700 Subject: [PATCH 4/9] remove settings meddling --- settings.gradle | 7 ------- 1 file changed, 7 deletions(-) diff --git a/settings.gradle b/settings.gradle index 49a8e943dc6f..897839c1b4b0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -71,13 +71,6 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-integrations:connector-templates:generator' include ':airbyte-integrations:bases:debezium' include ':tools:code-generator' - - // dependencies required for the template generator tests to run and format properly. - include ':airbyte-integrations:connectors:source-scaffold-java-jdbc' - include ':airbyte-integrations:connectors:source-scaffold-source-http' - include ':airbyte-integrations:connectors:source-scaffold-source-python' - include ':airbyte-integrations:connectors:source-jdbc' - include ':airbyte-integrations:connectors:source-relational-db' } // connectors From 54be52c85bf4e3db071be62a22aaa3394014d7c9 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:37:45 -0700 Subject: [PATCH 5/9] desperate --- .github/workflows/gradle.yml | 7 +++++-- settings.gradle | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 9fc14e19c82e..7461ee141fcf 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -92,8 +92,11 @@ jobs: - name: Format run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace - - name: Build - run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan +# - name: Build +# run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan + + - name: Ensure no file change + run: git diff - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" diff --git a/settings.gradle b/settings.gradle index 897839c1b4b0..d94cbd3d1a15 100644 --- a/settings.gradle +++ b/settings.gradle @@ -71,6 +71,13 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-integrations:connector-templates:generator' include ':airbyte-integrations:bases:debezium' include ':tools:code-generator' + + // dependencies required for the template generator tests to run and format properly. +// include ':airbyte-integrations:connectors:source-scaffold-java-jdbc' +// include ':airbyte-integrations:connectors:source-scaffold-source-http' +// include ':airbyte-integrations:connectors:source-scaffold-source-python' +// include ':airbyte-integrations:connectors:source-jdbc' +// include ':airbyte-integrations:connectors:source-relational-db' } // connectors From ea391c1216cfc5efaa39e728535429988f477626 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:45:47 -0700 Subject: [PATCH 6/9] lame brain central --- .../source-scaffold-source-http/integration_tests/acceptance.py | 2 -- .../connectors/source-scaffold-source-http/main.py | 2 -- .../connectors/source-scaffold-source-http/setup.py | 2 -- .../source_scaffold_source_http/source.py | 2 -- .../source-scaffold-source-http/unit_tests/unit_test.py | 2 -- .../integration_tests/acceptance.py | 2 -- .../connectors/source-scaffold-source-python/main.py | 2 -- .../connectors/source-scaffold-source-python/setup.py | 2 -- .../source_scaffold_source_python/source.py | 2 -- .../source-scaffold-source-python/unit_tests/unit_test.py | 2 -- 10 files changed, 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-scaffold-source-http/integration_tests/acceptance.py index eeb4a2d3e02e..df2783d1750f 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/integration_tests/acceptance.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# import pytest diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/main.py b/airbyte-integrations/connectors/source-scaffold-source-http/main.py index 5d7cec274c6e..6e752104685c 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/main.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/main.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# import sys diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py index 5e6bc88d3b56..3041c1fd0ea1 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# from setuptools import find_packages, setup diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/source.py b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/source.py index 1960c953766b..240caca78c33 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/source.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/source.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# from abc import ABC diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/unit_test.py index b8a8150b507f..f03f99f7c46e 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/unit_test.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# def test_example_method(): diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-scaffold-source-python/integration_tests/acceptance.py index 52accc9d8498..545776693290 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/integration_tests/acceptance.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# import pytest diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/main.py b/airbyte-integrations/connectors/source-scaffold-source-python/main.py index 750c40029bae..66dc58763bd6 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/main.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/main.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# import sys diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index 6e1fa0780181..439ca2348f01 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# from setuptools import find_packages, setup diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/source.py b/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/source.py index 89772c429035..37e4a1783cf1 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/source.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/source_scaffold_source_python/source.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# import json diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-scaffold-source-python/unit_tests/unit_test.py index b8a8150b507f..f03f99f7c46e 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/unit_tests/unit_test.py @@ -1,4 +1,3 @@ -# # MIT License # # Copyright (c) 2020 Airbyte @@ -20,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# def test_example_method(): From 02d5cc23711088cb3224b5c16af8a008e0478031 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:53:03 -0700 Subject: [PATCH 7/9] and all was as it should be --- .github/workflows/gradle.yml | 3 --- settings.gradle | 7 ------- 2 files changed, 10 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 7461ee141fcf..480683071cbd 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -95,9 +95,6 @@ jobs: # - name: Build # run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan - - name: Ensure no file change - run: git diff - - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" diff --git a/settings.gradle b/settings.gradle index d94cbd3d1a15..897839c1b4b0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -71,13 +71,6 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-integrations:connector-templates:generator' include ':airbyte-integrations:bases:debezium' include ':tools:code-generator' - - // dependencies required for the template generator tests to run and format properly. -// include ':airbyte-integrations:connectors:source-scaffold-java-jdbc' -// include ':airbyte-integrations:connectors:source-scaffold-source-http' -// include ':airbyte-integrations:connectors:source-scaffold-source-python' -// include ':airbyte-integrations:connectors:source-jdbc' -// include ':airbyte-integrations:connectors:source-relational-db' } // connectors From 6e824ba4c0ded0c47fadf3e1f8ab85be425e5cc3 Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:54:03 -0700 Subject: [PATCH 8/9] have cake; eat too? --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 448eb94036d6..1ed691650a0a 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ def createSpotlessTarget = { pattern -> 'secrets' ] - if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { + if (System.getenv().containsKey("SUB_BUILD")) { excludes.add("airbyte-integrations/connectors") } From a931bab2463c6fda728bbdf61f17c49277c201cd Mon Sep 17 00:00:00 2001 From: cgardens Date: Thu, 15 Jul 2021 13:59:34 -0700 Subject: [PATCH 9/9] max dufus --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 480683071cbd..9fc14e19c82e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -92,8 +92,8 @@ jobs: - name: Format run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace -# - name: Build -# run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan + - name: Build + run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)"