From cf6a42fe82477f9cc44bb9b88de140004d2a8949 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 24 Jan 2024 15:20:45 -0800 Subject: [PATCH] Destinations snowflake, bigquery, redshift: improve error reporting (#34458) --- airbyte-cdk/java/airbyte-cdk/README.md | 1 + .../base/AirbyteExceptionHandler.java | 45 ++++++++++--------- .../src/main/resources/version.properties | 2 +- .../base/AirbyteExceptionHandlerTest.java | 5 ++- .../destination-bigquery/build.gradle | 2 +- .../destination-bigquery/metadata.yaml | 2 +- .../destination-redshift/build.gradle | 2 +- .../destination-redshift/metadata.yaml | 2 +- .../destination-snowflake/build.gradle | 2 +- .../destination-snowflake/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/redshift.md | 13 +++--- docs/integrations/destinations/snowflake.md | 1 + 13 files changed, 45 insertions(+), 35 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index b70ebe6a4479..9eb1651530ee 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -166,6 +166,7 @@ MavenLocal debugging steps: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.14.2 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Handle case-sensitivity in sentry error grouping | | 0.14.1 | 2024-01-24 | [\#34468](https://github.com/airbytehq/airbyte/pull/34468) | Add wait for process to be done before ending sync in destination BaseTDTest | | 0.14.0 | 2024-01-23 | [\#34461](https://github.com/airbytehq/airbyte/pull/34461) | Revert non backward compatible signature changes from 0.13.1 | | 0.13.3 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Denote if destinations fully support Destinations V2 | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java index 64502fb55232..56eecef479ef 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java @@ -71,22 +71,26 @@ public void uncaughtException(final Thread thread, final Throwable throwable) { final Optional deinterpolatableException = ExceptionUtils.getThrowableList(throwable).stream() .filter(t -> THROWABLES_TO_DEINTERPOLATE.stream().anyMatch(deinterpolatableClass -> deinterpolatableClass.isAssignableFrom(t.getClass()))) .findFirst(); + final boolean messageWasMangled; if (deinterpolatableException.isPresent()) { + final String originalMessage = deinterpolatableException.get().getMessage(); mangledMessage = STRINGS_TO_DEINTERPOLATE.stream() // Sort the strings longest to shortest, in case any target string is a substring of another // e.g. "airbyte_internal" should be swapped out before "airbyte" .sorted(Comparator.comparing(String::length).reversed()) - .reduce(deinterpolatableException.get().getMessage(), AirbyteExceptionHandler::deinterpolate); + .reduce(originalMessage, AirbyteExceptionHandler::deinterpolate); + messageWasMangled = !mangledMessage.equals(originalMessage); } else { mangledMessage = throwable.getMessage(); + messageWasMangled = false; } - // If we did not modify the message (either not a deinterpolatable class, or we tried to - // deinterpolate - // but made no changes) then emit our default trace message - if (mangledMessage.equals(throwable.getMessage())) { + if (!messageWasMangled) { + // If we did not modify the message (either not a deinterpolatable class, or we tried to + // deinterpolate but made no changes) then emit our default trace message AirbyteTraceMessageUtility.emitSystemErrorTrace(throwable, logMessage); } else { + // If we did modify the message, then emit a custom trace message AirbyteTraceMessageUtility.emitCustomErrorTrace(throwable.getMessage(), mangledMessage); } @@ -95,7 +99,8 @@ public void uncaughtException(final Thread thread, final Throwable throwable) { @NotNull private static String deinterpolate(final String message, final String targetString) { - final String quotedTarget = '(' + Pattern.quote(targetString) + ')'; + // (?i) makes the pattern case-insensitive + final String quotedTarget = '(' + "(?i)" + Pattern.quote(targetString) + ')'; final String targetRegex = REGEX_PREFIX + quotedTarget + REGEX_SUFFIX; final Pattern pattern = Pattern.compile(targetRegex); final Matcher matcher = pattern.matcher(message); @@ -116,7 +121,7 @@ public static void addThrowableForDeinterpolation(final Class assertEquals(AirbyteTraceMessage.Type.ERROR, traceMessage.getTrace().getType()), - () -> assertEquals("Error happened in arst_foo_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), + () -> assertEquals("Error happened in arst_FOO_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), () -> assertEquals("Error happened in arst_?_?_zxcv (?: ?)", traceMessage.getTrace().getError().getInternalMessage()), () -> assertEquals(AirbyteErrorTraceMessage.FailureType.SYSTEM_ERROR, traceMessage.getTrace().getError().getFailureType()), () -> Assertions.assertNull(traceMessage.getTrace().getError().getStackTrace(), diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 12dbd0073bf8..5d7e94f94f5a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.14.1' + cdkVersionRequired = '0.14.2' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 9b46978b84f7..817f042abb51 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.4.0 + dockerImageTag: 2.4.1 dockerRepository: airbyte/destination-bigquery documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery githubIssueLabel: destination-bigquery diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index a17edaf60d31..c83ec1c8f74f 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.14.0' + cdkVersionRequired = '0.14.2' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 293ea7275ce9..e876c05c13ac 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 2.1.0 + dockerImageTag: 2.1.1 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 84f86b3c3f8a..90dca4c16fc9 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.14.0' + cdkVersionRequired = '0.14.2' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 10a74a7f6dc1..5058e623f284 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.5.1 + dockerImageTag: 3.5.2 dockerRepository: airbyte/destination-snowflake documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake githubIssueLabel: destination-snowflake diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index ee0c6d14e9f4..ad90428b83f6 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -210,6 +210,7 @@ tutorials: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.4.1 | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | | 2.4.0 | 2024-01-24 | [34468](https://github.com/airbytehq/airbyte/pull/34468) | Upgrade CDK to 0.14.0 | | 2.3.31 | 2024-01-22 | [\#34023](https://github.com/airbytehq/airbyte/pull/34023) | Combine DDL operations into a single execution | | 2.3.30 | 2024-01-12 | [\#34226](https://github.com/airbytehq/airbyte/pull/34226) | Upgrade CDK to 0.12.0; Cleanup dependencies | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index cd5c633a3919..b0744ebf7dd3 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -29,7 +29,7 @@ For INSERT strategy: 2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is the recommended loading approach described by Redshift [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_best-practices-single-copy-command.html). - Requires an S3 bucket and credentials. Data is copied into S3 as multiple files with a manifest file. + Requires an S3 bucket and credentials. Data is copied into S3 as multiple files with a manifest file. Airbyte automatically picks an approach depending on the given configuration - if S3 configuration is present, Airbyte will use the COPY strategy and vice versa. @@ -77,8 +77,8 @@ Optional parameters: completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`. NOTE: S3 staging does not use the SSH Tunnel option for copying data, if configured. SSH Tunnel supports the SQL -connection only. S3 is secured through public HTTPS access only. Subsequent typing and deduping queries on final table -are executed over using provided SSH Tunnel configuration. +connection only. S3 is secured through public HTTPS access only. Subsequent typing and deduping queries on final table +are executed over using provided SSH Tunnel configuration. ## Step 1: Set up Redshift @@ -99,10 +99,10 @@ are executed over using provided SSH Tunnel configuration. ### Permissions in Redshift Airbyte writes data into two schemas, whichever schema you want your data to land in, e.g. `my_schema` and a "Raw Data" schema that Airbyte uses to improve ELT reliability. By default, this raw data schema -is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. +is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. Airbyte also needs to query Redshift's -[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for -metadata about the tables airbyte manages. +[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for +metadata about the tables airbyte manages. To ensure the `airbyte_user` has the correction permissions to: - create schemas in your database @@ -237,6 +237,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.1.1 | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | | 2.1.0 | 2024-01-24 | [34467](https://github.com/airbytehq/airbyte/pull/34467) | Upgrade CDK to 0.14.0 | | 2.0.0 | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Destinations V2 | | 0.8.0 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 079ab13786cf..6ab0e8cb81cf 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -246,6 +246,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.5.2 | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting | | 3.5.1 | 2024-01-24 | [\#34501](https://github.com/airbytehq/airbyte/pull/34501) | Internal code changes for Destinations V2 | | 3.5.0 | 2024-01-24 | [\#34462](https://github.com/airbytehq/airbyte/pull/34462) | Upgrade CDK to 0.14.0 | | 3.4.22 | 2024-01-12 | [\#34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies |