Merge remote-tracking branch 'origin/master' into keu/source-stripe/c…

…all_rate
airbytehq · Nov 14, 2023 · 6f0608e · 6f0608e
2 parents c503ed9 + c642d81
commit 6f0608e
Show file tree

Hide file tree

Showing 389 changed files with 8,565 additions and 6,413 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.33
+current_version = 0.50.34
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -38,6 +38,9 @@
 /airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources
 
 # Java-based destination connectors
+airbyte-cdk/java/airbyte-cdk/db-destinations/ @airbytehq/destinations
+airbyte-cdk/java/airbyte-cdk/s3-destinations/ @airbytehq/destinations
+airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
 /airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations
 /airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations
 /airbyte-integrations/bases/bases-destination-jdbc/ @airbytehq/destinations

diff --git a/.github/workflows/airbyte-ci-release-experiment.yml b/.github/workflows/airbyte-ci-release-experiment.yml
@@ -0,0 +1,32 @@
+name: Connector Ops CI - Experimental Airbyte CI Release
+
+# Note this is a workflow simply to test out if github can build using macos
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest", "macos-latest"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.10
+
+      - run: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+
+      - run: cd airbyte-ci/pipelines/airbyte_ci
+      - run: poetry install --with dev
+      - run: poetry run pyinstaller --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name airbyte-ci-${{ matrix.os }} --onefile pipelines/cli/airbyte_ci.py
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist/*
diff --git a/.github/workflows/tmp-source-postgres-test.yml b/.github/workflows/tmp-source-postgres-test.yml
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -156,6 +156,9 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.4.9   | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors                                                                                                         |
+| 0.4.8   | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database                                                                                                                  |
+| 0.4.7   | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps                                                                                                      |
 | 0.4.5   | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold                                                                                      |
 | 0.4.4   | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler                                                                                                            |
 | 0.4.2   | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation                                                                                                                                  |

diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json
@@ -11,18 +11,39 @@
     },
     "Date": {
       "type": "string",
-      "pattern": "^\\d{4}-\\d{2}-\\d{2}( BC)?$",
-      "description": "RFC 3339\u00a75.6's full-date format, extended with BC era support"
+      "oneOf": [
+        {
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}( BC)?$"
+        },
+        {
+          "enum": ["Infinity", "-Infinity"]
+        }
+      ],
+      "description": "RFC 3339\u00a75.6's full-date format, extended with BC era support and (-)Infinity"
     },
     "TimestampWithTimezone": {
       "type": "string",
-      "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})( BC)?$",
-      "description": "An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339\u00a75.6's date-time format, requiring a \"T\" separator, and extended with BC era support. Note that we do _not_ accept Unix epochs here.\n"
+      "oneOf": [
+        {
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})( BC)?$"
+        },
+        {
+          "enum": ["Infinity", "-Infinity"]
+        }
+      ],
+      "description": "An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339\u00a75.6's date-time format, requiring a \"T\" separator, and extended with BC era support and (-)Infinity. Note that we do _not_ accept Unix epochs here.\n"
     },
     "TimestampWithoutTimezone": {
       "type": "string",
-      "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?( BC)?$",
-      "description": "Also known as a localdatetime, or just datetime. Under RFC 3339\u00a75.6, this would be represented as `full-date \"T\" partial-time`, extended with BC era support.\n"
+      "oneOf": [
+        {
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?( BC)?$"
+        },
+        {
+          "enum": ["Infinity", "-Infinity"]
+        }
+      ],
+      "description": "Also known as a localdatetime, or just datetime. Under RFC 3339\u00a75.6, this would be represented as `full-date \"T\" partial-time`, extended with BC era support and (-)Infinity.\n"
     },
     "TimeWithTimezone": {
       "type": "string",

diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle
@@ -7,6 +7,9 @@ java {
     compileJava {
         options.compilerArgs += "-Xlint:-varargs,-try,-deprecation"
     }
+    compileTestJava {
+        options.compilerArgs += "-Xlint:-try"
+    }
 }
 
 dependencies {

diff --git a/...dk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/...dk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java
@@ -22,6 +22,7 @@ public class EnvVariableFeatureFlags implements FeatureFlags {
   public static final String CONCURRENT_SOURCE_STREAM_READ = "CONCURRENT_SOURCE_STREAM_READ";
   public static final String STRICT_COMPARISON_NORMALIZATION_WORKSPACES = "STRICT_COMPARISON_NORMALIZATION_WORKSPACES";
   public static final String STRICT_COMPARISON_NORMALIZATION_TAG = "STRICT_COMPARISON_NORMALIZATION_TAG";
+  public static final String DEPLOYMENT_MODE = "DEPLOYMENT_MODE";
 
   @Override
   public boolean useStreamCapableState() {
@@ -63,6 +64,11 @@ public String strictComparisonNormalizationTag() {
     return getEnvOrDefault(STRICT_COMPARISON_NORMALIZATION_TAG, "strict_comparison2", (arg) -> arg);
   }
 
+  @Override
+  public String deploymentMode() {
+    return getEnvOrDefault(DEPLOYMENT_MODE, "", (arg) -> arg);
+  }
+
   // TODO: refactor in order to use the same method than the ones in EnvConfigs.java
   public <T> T getEnvOrDefault(final String key, final T defaultValue, final Function<String, T> parser) {
     final String value = System.getenv(key);

diff --git a/...a/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/...a/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java
@@ -49,4 +49,11 @@ public interface FeatureFlags {
    */
   String strictComparisonNormalizationTag();
 
+  /**
+   * Get the deployment mode used to deploy a connector.
+   *
+   * @return empty string for the default deployment mode, "CLOUD" for cloud deployment mode.
+   */
+  String deploymentMode();
+
 }
diff --git a/...te-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java b/...te-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.features;
+
+public class FeatureFlagsWrapper implements FeatureFlags {
+
+  /**
+   * Overrides the {@link FeatureFlags#useStreamCapableState} method in the feature flags.
+   */
+  static public FeatureFlags overridingUseStreamCapableState(
+                                                             final FeatureFlags wrapped,
+                                                             final boolean useStreamCapableState) {
+    return new FeatureFlagsWrapper(wrapped) {
+
+      @Override
+      public boolean useStreamCapableState() {
+        return useStreamCapableState;
+      }
+
+    };
+  }
+
+  /**
+   * Overrides the {@link FeatureFlags#deploymentMode} method in the feature flags.
+   */
+  static public FeatureFlags overridingDeploymentMode(
+                                                      final FeatureFlags wrapped,
+                                                      final String deploymentMode) {
+    return new FeatureFlagsWrapper(wrapped) {
+
+      @Override
+      public String deploymentMode() {
+        return deploymentMode;
+      }
+
+    };
+  }
+
+  private final FeatureFlags wrapped;
+
+  public FeatureFlagsWrapper(FeatureFlags wrapped) {
+    this.wrapped = wrapped;
+  }
+
+  @Override
+  public boolean useStreamCapableState() {
+    return wrapped.useStreamCapableState();
+  }
+
+  @Override
+  public boolean autoDetectSchema() {
+    return wrapped.autoDetectSchema();
+  }
+
+  @Override
+  public boolean logConnectorMessages() {
+    return wrapped.logConnectorMessages();
+  }
+
+  @Override
+  public boolean concurrentSourceStreamRead() {
+    return wrapped.concurrentSourceStreamRead();
+  }
+
+  @Override
+  public boolean applyFieldSelection() {
+    return wrapped.applyFieldSelection();
+  }
+
+  @Override
+  public String fieldSelectionWorkspaces() {
+    return wrapped.fieldSelectionWorkspaces();
+  }
+
+  @Override
+  public String strictComparisonNormalizationWorkspaces() {
+    return wrapped.strictComparisonNormalizationWorkspaces();
+  }
+
+  @Override
+  public String strictComparisonNormalizationTag() {
+    return wrapped.strictComparisonNormalizationTag();
+  }
+
+  @Override
+  public String deploymentMode() {
+    return wrapped.deploymentMode();
+  }
+
+}
diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle
@@ -3,6 +3,9 @@ java {
     compileJava {
         options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked"
     }
+    compileTestJava {
+        options.compilerArgs += "-Xlint:-try,-divzero,-cast"
+    }
 }
 
 configurations.all {

diff --git a/...byte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/...byte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java
@@ -139,12 +139,19 @@ protected void terminate() {
   @VisibleForTesting
   static void addCommonStringsToDeinterpolate() {
     // Add some common strings to deinterpolate, regardless of what the connector is doing
+    STRINGS_TO_DEINTERPOLATE.add("airbyte");
+    STRINGS_TO_DEINTERPOLATE.add("config");
+    STRINGS_TO_DEINTERPOLATE.add("configuration");
     STRINGS_TO_DEINTERPOLATE.add("description");
+    STRINGS_TO_DEINTERPOLATE.add("email");
     STRINGS_TO_DEINTERPOLATE.add("id");
     STRINGS_TO_DEINTERPOLATE.add("location");
+    STRINGS_TO_DEINTERPOLATE.add("message");
     STRINGS_TO_DEINTERPOLATE.add("name");
+    STRINGS_TO_DEINTERPOLATE.add("state");
     STRINGS_TO_DEINTERPOLATE.add("status");
     STRINGS_TO_DEINTERPOLATE.add("type");
+    STRINGS_TO_DEINTERPOLATE.add("userEmail");
   }
 
 }
diff --git a/...re/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java b/...re/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java
@@ -10,6 +10,7 @@
 import io.airbyte.cdk.integrations.base.IntegrationCliParser;
 import io.airbyte.cdk.integrations.base.IntegrationConfig;
 import io.airbyte.cdk.integrations.base.IntegrationRunner;
+import io.airbyte.commons.features.EnvVariableFeatureFlags;
 import io.airbyte.commons.json.Jsons;
 import java.util.function.Supplier;
 import org.slf4j.Logger;
@@ -23,7 +24,7 @@ public class AdaptiveDestinationRunner {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(AdaptiveDestinationRunner.class);
 
-  private static final String DEPLOYMENT_MODE_KEY = "DEPLOYMENT_MODE";
+  private static final String DEPLOYMENT_MODE_KEY = EnvVariableFeatureFlags.DEPLOYMENT_MODE;
   private static final String CLOUD_MODE = "CLOUD";
 
   public static OssDestinationBuilder baseOnEnv() {

diff --git a/...dk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveSourceRunner.java b/...dk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveSourceRunner.java
@@ -6,6 +6,7 @@
 
 import io.airbyte.cdk.integrations.base.IntegrationRunner;
 import io.airbyte.cdk.integrations.base.Source;
+import io.airbyte.commons.features.EnvVariableFeatureFlags;
 import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -17,7 +18,7 @@ public class AdaptiveSourceRunner {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(AdaptiveSourceRunner.class);
 
-  public static final String DEPLOYMENT_MODE_KEY = "DEPLOYMENT_MODE";
+  public static final String DEPLOYMENT_MODE_KEY = EnvVariableFeatureFlags.DEPLOYMENT_MODE;
   public static final String CLOUD_MODE = "CLOUD";
 
   public static OssSourceBuilder baseOnEnv() {

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.4.5
+version=0.4.9