Merge branch 'master' into marcos/test-pr-10419

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.31-alpha
+current_version = 0.35.32-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.31-alpha
+VERSION=0.35.32-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/actions/ci-java-tests/action.yml

@@ -9,11 +9,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install Java
-      uses: actions/setup-java@v1
-      with:
-        java-version: '17'
-
     - name: "Build"
       shell: bash
       run: |

.github/actions/ci-py-tests/action.yml

@@ -27,57 +27,26 @@ runs:
     - name: Build Coverage Reports
       id: build-coverage-reports
       shell: bash
-      working-directory: ${{ inputs.module-folder }}
       run: |
-        virtualenv .venv
-        source .venv/bin/activate
-        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
-        pip install coverage[toml]~=6.2
-        mkdir -p .venv/source-acceptance-test
-        mkdir -p reports
-        SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test
-        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
-        git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done
-        pip install build
-        python -m build .venv/source-acceptance-test
-        pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl
-        [ -f requirements.txt ] && pip install --quiet -r requirements.txt
-        pip install .[tests]
-        coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true
-        coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true
+        GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "unitTest")
+        REPORT_FOLDER="${{ inputs.module-folder }}/coverage/"
+        ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
-        rm -rf .venv
-        echo "::set-output name=coverage-paths::reports/coverage.xml"
+        echo "::set-output name=coverage-paths::coverage/coverage.xml"
 
     - name: Upload coverage to Codecov
-      if: ${{ always() }}
       uses: codecov/codecov-action@v2
+      with:
+        file: ${{ steps.build-coverage-reports.outputs.coverage-paths }}
+        name: "UnitTests of ${{ inputs.module-name }}"
 
     - name: Build Linter Reports
       id: build-linter-reports
       shell: bash
-      working-directory: ${{ inputs.module-folder }}
       run: |
-        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
-        REPORT_FOLDER=reports
-        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
-
-        # run mypy
-        pip install lxml~=4.7 mypy~=0.910 .
-        mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true
-
-        # run black
-        pip install black~=21.12b0
-        XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff
-
-        # run isort
-        pip install isort~=5.10.1
-        cp ${PYPROJECT_CONFIG} ./pyproject.toml
-        isort --diff . | tee reports/isort.diff
-
-        # run flake8
-        pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2
-        pflake8 --exit-zero . | grep ^. | tee reports/flake.txt
+        GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "airbytePythonReport")
+        REPORT_FOLDER="${{ inputs.module-folder }}/reports/"
+        ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
         echo "::set-output name=mypy-logs::reports/mypy.log"
         echo "::set-output name=black-diff::reports/black.diff"

.github/actions/ci-tests-runner/action.yml

@@ -32,13 +32,18 @@ runs:
       with:
         python-version: 3.7
 
+    - name: Install Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: '17'
+
     - name: Tests of CI
       shell: bash
       run: |
         # all CI python packages have the prefix "ci_"
         pip install --quiet tox==3.24.4
-        tox -r -c ./tools/tox_ci.ini
         pip install --quiet -e ./tools/ci_*
+        tox -r -c ./tools/tox_ci.ini
         echo "::echo::off"
 
     - name: Auth with gcloud CLI
@@ -109,6 +114,7 @@ runs:
           [ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}")
           [ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}")
         fi
+        cat ${REPORT_FOLDER}/*
         if [ ${{ inputs.module-lang }} == 'java' ]; then
              [ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java")
              [ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java")
@@ -133,6 +139,7 @@ runs:
         MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
         echo "::set-output name=module_dir::${MODULE_DIR}"
 
+
     - name: SonarQube Scan
 
       uses: sonarsource/sonarqube-scan-action@master
@@ -148,7 +155,7 @@ runs:
             -Dsonar.language=${{ inputs.module-lang }}
             -Dsonar.sourceEncoding=UTF-8
             -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
-            -Dsonar.exclusions=reports/**,*.toml
+            -Dsonar.exclusions=reports/**,*.toml,*_tests/**,setup.py,main.py
             -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
             ${{ steps.sq-options.outputs.options }}
 

.github/workflows/sonar-scan.yml

@@ -72,7 +72,7 @@ jobs:
         sonar-token: ${{ secrets.SONAR_TOKEN }}
         sonar-gcp-access-key: ${{ secrets.GCP_SONAR_SA_KEY }}
         pull-request-id: "${{ github.repository }}/${{ github.event.pull_request.number }}"
-        remove-sonar-project: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
+        remove-sonar-project: ${{  github.event.action == 'closed' }}
 
 
 

airbyte-bootloader/Dockerfile

@@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.31-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.32-alpha.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.31-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.32-alpha/bin/${APPLICATION}"]

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -114,7 +114,7 @@ public void load() throws Exception {
 
       final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase);
       final ConfigRepository configRepository =
-          new ConfigRepository(configPersistence, null, Optional.empty(), Optional.empty());
+          new ConfigRepository(configPersistence, null, Optional.empty(), Optional.empty(), configDatabase);
 
       createWorkspaceIfNoneExists(configRepository);
       LOGGER.info("Default workspace created..");

airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java

@@ -30,6 +30,8 @@
 import io.airbyte.config.persistence.split_secrets.SecretsHelpers;
 import io.airbyte.config.persistence.split_secrets.SecretsHydrator;
 import io.airbyte.config.persistence.split_secrets.SplitSecretConfig;
+import io.airbyte.db.Database;
+import io.airbyte.db.ExceptionWrappingDatabase;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.ConnectorSpecification;
 import io.airbyte.validation.json.JsonSchemaValidator;
@@ -61,15 +63,18 @@ public class ConfigRepository {
   private final SecretsHydrator secretsHydrator;
   private final Optional<SecretPersistence> longLivedSecretPersistence;
   private final Optional<SecretPersistence> ephemeralSecretPersistence;
+  private final ExceptionWrappingDatabase database;
 
   public ConfigRepository(final ConfigPersistence persistence,
                           final SecretsHydrator secretsHydrator,
                           final Optional<SecretPersistence> longLivedSecretPersistence,
-                          final Optional<SecretPersistence> ephemeralSecretPersistence) {
+                          final Optional<SecretPersistence> ephemeralSecretPersistence,
+                          final Database database) {
     this.persistence = persistence;
     this.secretsHydrator = secretsHydrator;
     this.longLivedSecretPersistence = longLivedSecretPersistence;
     this.ephemeralSecretPersistence = ephemeralSecretPersistence;
+    this.database = new ExceptionWrappingDatabase(database);
   }
 
   public StandardWorkspace getStandardWorkspace(final UUID workspaceId, final boolean includeTombstone)

airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryTest.java

@@ -27,6 +27,7 @@
 import io.airbyte.config.State;
 import io.airbyte.config.persistence.split_secrets.MemorySecretPersistence;
 import io.airbyte.config.persistence.split_secrets.NoOpSecretsHydrator;
+import io.airbyte.db.Database;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -47,13 +48,16 @@ class ConfigRepositoryTest {
 
   private ConfigPersistence configPersistence;
   private ConfigRepository configRepository;
+  private Database database;
 
   @BeforeEach
   void setup() {
     configPersistence = mock(ConfigPersistence.class);
+    database = mock(Database.class);
     final var secretPersistence = new MemorySecretPersistence();
     configRepository =
-        spy(new ConfigRepository(configPersistence, new NoOpSecretsHydrator(), Optional.of(secretPersistence), Optional.of(secretPersistence)));
+        spy(new ConfigRepository(configPersistence, new NoOpSecretsHydrator(), Optional.of(secretPersistence), Optional.of(secretPersistence),
+            database));
   }
 
   @AfterEach

airbyte-container-orchestrator/Dockerfile

@@ -26,12 +26,12 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt
 RUN apt-get update && apt-get install -y kubectl
 
 ENV APPLICATION airbyte-container-orchestrator
-ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.31-alpha/bin/${APPLICATION}"
+ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.32-alpha/bin/${APPLICATION}"
 
 WORKDIR /app
 
 # Move orchestrator app
-ADD bin/${APPLICATION}-0.35.31-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.32-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.31-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.32-alpha/bin/${APPLICATION}"]

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumer.java

@@ -36,6 +36,7 @@ public void start() throws Exception {
       AirbyteSentry.executeWithTracing("StartConsumer", this::startTracked,
           Map.of("consumerImpl", FailureTrackingAirbyteMessageConsumer.class.getSimpleName()));
     } catch (final Exception e) {
+      LOGGER.error("Exception while starting consumer", e);
       hasFailed = true;
       throw e;
     }
@@ -48,6 +49,7 @@ public void accept(final AirbyteMessage msg) throws Exception {
     try {
       acceptTracked(msg);
     } catch (final Exception e) {
+      LOGGER.error("Exception while accepting message", e);
       hasFailed = true;
       throw e;
     }

airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java

@@ -22,7 +22,7 @@ class DebeziumConverterUtilsTest {
   @Test
   public void convertDefaultValueTest() {
 
-    RelationalColumn relationalColumn = mock(RelationalColumn.class);
+    final RelationalColumn relationalColumn = mock(RelationalColumn.class);
 
     when(relationalColumn.isOptional()).thenReturn(true);
     Object actualColumnDefaultValue = DebeziumConverterUtils.convertDefaultValue(relationalColumn);
@@ -35,67 +35,67 @@ public void convertDefaultValueTest() {
 
     when(relationalColumn.isOptional()).thenReturn(false);
     when(relationalColumn.hasDefaultValue()).thenReturn(true);
-    String expectedColumnDefaultValue = "default value";
+    final String expectedColumnDefaultValue = "default value";
     when(relationalColumn.defaultValue()).thenReturn(expectedColumnDefaultValue);
     actualColumnDefaultValue = DebeziumConverterUtils.convertDefaultValue(relationalColumn);
     Assertions.assertEquals(actualColumnDefaultValue, expectedColumnDefaultValue);
   }
 
   @Test
   public void convertLocalDate() {
-    LocalDate localDate = LocalDate.of(2021, 1, 1);
+    final LocalDate localDate = LocalDate.of(2021, 1, 1);
 
-    String actual = DebeziumConverterUtils.convertDate(localDate);
+    final String actual = DebeziumConverterUtils.convertDate(localDate);
     Assertions.assertEquals("2021-01-01T00:00:00Z", actual);
   }
 
   @Test
   public void convertTLocalTime() {
-    LocalTime localTime = LocalTime.of(8, 1, 1);
-    String actual = DebeziumConverterUtils.convertDate(localTime);
+    final LocalTime localTime = LocalTime.of(8, 1, 1);
+    final String actual = DebeziumConverterUtils.convertDate(localTime);
     Assertions.assertEquals("08:01:01", actual);
   }
 
   @Test
   public void convertLocalDateTime() {
-    LocalDateTime localDateTime = LocalDateTime.of(2021, 1, 1, 8, 1, 1);
+    final LocalDateTime localDateTime = LocalDateTime.of(2021, 1, 1, 8, 1, 1);
 
-    String actual = DebeziumConverterUtils.convertDate(localDateTime);
+    final String actual = DebeziumConverterUtils.convertDate(localDateTime);
     Assertions.assertEquals("2021-01-01T08:01:01Z", actual);
   }
 
   @Test
   @Disabled
   public void convertDuration() {
-    Duration duration = Duration.ofHours(100_000);
+    final Duration duration = Duration.ofHours(100_000);
 
-    String actual = DebeziumConverterUtils.convertDate(duration);
+    final String actual = DebeziumConverterUtils.convertDate(duration);
     Assertions.assertEquals("1981-05-29T20:00:00Z", actual);
   }
 
   @Test
   public void convertTimestamp() {
-    LocalDateTime localDateTime = LocalDateTime.of(2021, 1, 1, 8, 1, 1);
-    Timestamp timestamp = Timestamp.valueOf(localDateTime);
+    final LocalDateTime localDateTime = LocalDateTime.of(2021, 1, 1, 8, 1, 1);
+    final Timestamp timestamp = Timestamp.valueOf(localDateTime);
 
-    String actual = DebeziumConverterUtils.convertDate(timestamp);
-    Assertions.assertEquals("2021-01-01T08:01:01Z", actual);
+    final String actual = DebeziumConverterUtils.convertDate(timestamp);
+    Assertions.assertEquals("2021-01-01T08:01:01.000000Z", actual);
   }
 
   @Test
   @Disabled
   public void convertNumber() {
-    Number number = 100_000;
+    final Number number = 100_000;
 
-    String actual = DebeziumConverterUtils.convertDate(number);
+    final String actual = DebeziumConverterUtils.convertDate(number);
     Assertions.assertEquals("1970-01-01T03:01:40Z", actual);
   }
 
   @Test
   public void convertStringDateFormat() {
-    String stringValue = "2021-01-01T00:00:00Z";
+    final String stringValue = "2021-01-01T00:00:00Z";
 
-    String actual = DebeziumConverterUtils.convertDate(stringValue);
+    final String actual = DebeziumConverterUtils.convertDate(stringValue);
     Assertions.assertEquals("2021-01-01T00:00:00Z", actual);
   }
 

airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java

@@ -93,13 +93,13 @@ void testInsertRawRecordsInSingleQuery() throws SQLException {
             .put(JavaBaseConstants.COLUMN_NAME_AB_ID, RECORD1_UUID)
             .put(JavaBaseConstants.COLUMN_NAME_DATA, records.get(0).getData())
             .put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, DataTypeUtils
-                .toISO8601String(records.get(0).getEmittedAt()))
+                .toISO8601StringWithMicroseconds(Instant.ofEpochMilli(records.get(0).getEmittedAt())))
             .build()),
         Jsons.jsonNode(ImmutableMap.builder()
             .put(JavaBaseConstants.COLUMN_NAME_AB_ID, RECORD2_UUID)
             .put(JavaBaseConstants.COLUMN_NAME_DATA, records.get(1).getData())
             .put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, DataTypeUtils
-                .toISO8601String(records.get(1).getEmittedAt()))
+                .toISO8601StringWithMicroseconds(Instant.ofEpochMilli(records.get(1).getEmittedAt())))
             .build()));
 
     actualRecords.forEach(