From 39acaa0252f15213da45202166ac2f5bd8070863 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 14 Nov 2023 14:27:25 -0800 Subject: [PATCH 1/9] Async Destinations - Make debug messages less scary --- .../destination_async/FlushWorkers.java | 4 +-- .../buffers/BufferManager.java | 28 +++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java index 74231cbe77a9..9438de90cbfe 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java @@ -51,7 +51,7 @@ public class FlushWorkers implements AutoCloseable { private static final long SUPERVISOR_INITIAL_DELAY_SECS = 0L; private static final long SUPERVISOR_PERIOD_SECS = 1L; private static final long DEBUG_INITIAL_DELAY_SECS = 0L; - private static final long DEBUG_PERIOD_SECS = 10L; + private static final long DEBUG_PERIOD_SECS = 60L; private final ScheduledExecutorService supervisorThread; private final ExecutorService workerPool; @@ -135,7 +135,7 @@ private void retrieveWork() { } private void printWorkerInfo() { - final var workerInfo = new StringBuilder().append("WORKER INFO").append(System.lineSeparator()); + final var workerInfo = new StringBuilder().append("[WORKER INFO] "); final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) workerPool; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java index 9d8f2104a044..36217a4978e8 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java @@ -10,6 +10,7 @@ import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.ArrayList; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; @@ -33,6 +34,7 @@ public class BufferManager { private final GlobalAsyncStateManager stateManager; private final ScheduledExecutorService debugLoop; + private static final long DEBUG_PERIOD_SECS = 60L; public static final double MEMORY_LIMIT_RATIO = 0.7; @@ -55,7 +57,7 @@ public BufferManager(final long memoryLimit) { bufferEnqueue = new BufferEnqueue(memoryManager, buffers, stateManager); bufferDequeue = new BufferDequeue(memoryManager, buffers, stateManager); debugLoop = Executors.newSingleThreadScheduledExecutor(); - debugLoop.scheduleAtFixedRate(this::printQueueInfo, 0, 10, TimeUnit.SECONDS); + debugLoop.scheduleAtFixedRate(this::printQueueInfo, 0, DEBUG_PERIOD_SECS, TimeUnit.SECONDS); } public GlobalAsyncStateManager getStateManager() { @@ -91,31 +93,27 @@ public void close() throws Exception { } private void printQueueInfo() { - final var queueInfo = new StringBuilder().append("START OF QUEUE INFO").append(System.lineSeparator()) - .append("This represents an estimation of the size of the elements contain in the in memory buffer.") - .append(System.lineSeparator()); + final var queueInfo = new StringBuilder().append("[QUEUE INFO] "); + final ArrayList messages = new ArrayList<>(); - queueInfo - .append(String.format(" Global Mem Manager -- max: %s, allocated: %s (%s MB), %% used: %s", + messages + .add(String.format("Global: max: %s, allocated: %s (%s MB), %% used: %s, ", AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getMaxMemoryBytes()), AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getCurrentMemoryBytes()), (double) memoryManager.getCurrentMemoryBytes() / 1024 / 1024, - (double) memoryManager.getCurrentMemoryBytes() / memoryManager.getMaxMemoryBytes())) - .append(System.lineSeparator()); + (double) memoryManager.getCurrentMemoryBytes() / memoryManager.getMaxMemoryBytes())); for (final var entry : buffers.entrySet()) { final var queue = entry.getValue(); - queueInfo.append( - String.format(" Queue name: %s, num records: %d, num bytes: %s, allocated bytes: %s", + messages.add( + String.format("Queue `%s`, num records: %d, num bytes: %s, allocated bytes: %s", entry.getKey().getName(), queue.size(), AirbyteFileUtils.byteCountToDisplaySize(queue.getCurrentMemoryUsage()), - AirbyteFileUtils.byteCountToDisplaySize(queue.getMaxMemoryUsage()))) - .append(System.lineSeparator()); + AirbyteFileUtils.byteCountToDisplaySize(queue.getMaxMemoryUsage()))); } - queueInfo.append(stateManager.getMemoryUsageMessage()) - .append(System.lineSeparator()); + messages.add(stateManager.getMemoryUsageMessage()); - queueInfo.append("END OF QUEUE INFO"); + queueInfo.append(String.join(" | ", messages)); log.info(queueInfo.toString()); } From 77c56a8ac0afac67afd3a9f8e0fee5d24fa2652a Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 14 Nov 2023 14:54:32 -0800 Subject: [PATCH 2/9] cdk bumpbs --- .../airbyte-cdk/core/src/main/resources/version.properties | 2 +- .../connectors/destination-bigquery/build.gradle | 4 ++-- .../connectors/destination-snowflake/build.gradle | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 3cf6558a4b17..6c39f216d22b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.4.10 +version=0.4.11 diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 33dab9b167e4..e8b3f81d5a8c 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.4.9' + cdkVersionRequired = '0.4.11' features = ['db-destinations', 's3-destinations'] - useLocalCdk = false + useLocalCdk = true } airbyteJavaConnector.addCdkDependencies() diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index ee416b90798a..e9d89cff4612 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -4,9 +4,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.4.3' + cdkVersionRequired = '0.4.11' features = ['db-destinations', 's3-destinations'] - useLocalCdk = false + useLocalCdk = true } airbyteJavaConnector.addCdkDependencies() From b1124fff1feb196d61a143125cb0247f59ea8ea3 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 14 Nov 2023 14:54:37 -0800 Subject: [PATCH 3/9] copy tweaks --- .../cdk/integrations/destination_async/FlushWorkers.java | 2 +- .../integrations/destination_async/buffers/BufferManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java index 9438de90cbfe..6aa1640bfe29 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java @@ -135,7 +135,7 @@ private void retrieveWork() { } private void printWorkerInfo() { - final var workerInfo = new StringBuilder().append("[WORKER INFO] "); + final var workerInfo = new StringBuilder().append("[ASYNC WORKER INFO] "); final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) workerPool; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java index 36217a4978e8..a7b26f796e63 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java @@ -93,7 +93,7 @@ public void close() throws Exception { } private void printQueueInfo() { - final var queueInfo = new StringBuilder().append("[QUEUE INFO] "); + final var queueInfo = new StringBuilder().append("[ASYNC QUEUE INFO] "); final ArrayList messages = new ArrayList<>(); messages From 61a5cb326b072b4340012e36bf0d4d14b51a33e9 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 14 Nov 2023 14:56:42 -0800 Subject: [PATCH 4/9] remove comma --- .../integrations/destination_async/buffers/BufferManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java index a7b26f796e63..1d824a2b14c0 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java @@ -97,7 +97,7 @@ private void printQueueInfo() { final ArrayList messages = new ArrayList<>(); messages - .add(String.format("Global: max: %s, allocated: %s (%s MB), %% used: %s, ", + .add(String.format("Global: max: %s, allocated: %s (%s MB), %% used: %s", AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getMaxMemoryBytes()), AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getCurrentMemoryBytes()), (double) memoryManager.getCurrentMemoryBytes() / 1024 / 1024, From e979a2f1b1c9ab1ab2ffa3d3fe5e1f32addbb0f9 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 14 Nov 2023 14:59:37 -0800 Subject: [PATCH 5/9] bump versions --- .../destination-bigquery/metadata.yaml | 2 +- .../destination-snowflake/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/snowflake.md | 33 ++++++++++--------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 2891145670f8..7cdb95a0bd77 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.3.15 + dockerImageTag: 2.3.16 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 83487bebfd1c..f37244686991 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.4.10 + dockerImageTag: 3.4.11 dockerRepository: airbyte/destination-snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 4ece99595e00..97bdeff675dc 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -140,6 +140,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 3.4.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up manager logs. | | 2.3.15 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping enhancments | | 2.3.14 | 2023-11-06 | [\#32234](https://github.com/airbytehq/airbyte/pull/32234) | Remove unused config option. | | 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 3216a67bc06b..6ae60871a069 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -156,22 +156,22 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent ### Login and Password -| Field | Description | -|-------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | -| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | -| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | -| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE` | -| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. | -| Username | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER` | -| Password | The password associated with the username. | -| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | -| Disable Final Tables (Optional) | Disables writing final Typed tables See [output schema](#output-schema). WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions | +| Field | Description | +| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | +| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | +| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | +| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE` | +| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. | +| Username | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER` | +| Password | The password associated with the username. | +| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | +| Disable Final Tables (Optional) | Disables writing final Typed tables See [output schema](#output-schema). WARNING! The data format in \_airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions | ### OAuth 2.0 | Field | Description | -|:------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| :---------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | | [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | @@ -208,7 +208,7 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent To use AWS S3 as the cloud storage, enter the information for the S3 bucket you created in Step 2: | Field | Description | -|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | S3 Bucket Name | The name of the staging S3 bucket (Example: `airbyte.staging`). Airbyte will write files to this bucket and read them via statements on Snowflake. | | S3 Bucket Region | The S3 staging bucket region used. | | S3 Key Id \* | The Access Key ID granting access to the S3 staging bucket. Airbyte requires Read and Write permissions for the bucket. | @@ -221,7 +221,7 @@ To use AWS S3 as the cloud storage, enter the information for the S3 bucket you To use a Google Cloud Storage bucket, enter the information for the bucket you created in Step 2: | Field | Description | -|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | GCP Project ID | The name of the GCP project ID for your credentials. (Example: `my-project`) | | GCP Bucket Name | The name of the staging bucket. Airbyte will write files to this bucket and read them via statements on Snowflake. (Example: `airbyte-staging`) | | Google Application Credentials | The contents of the JSON key file that has read/write permissions to the staging GCS bucket. You will separately need to grant bucket access to your Snowflake GCP service account. See the [Google Cloud docs](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) for more information on how to generate a JSON key for your service account. | @@ -233,7 +233,7 @@ Airbyte outputs each stream into its own raw table in `airbyte_internal` schema ### Raw Table schema | Airbyte field | Description | Column type | -|------------------------|--------------------------------------------------------------------|--------------------------| +| ---------------------- | ------------------------------------------------------------------ | ------------------------ | | \_airbyte_raw_id | A UUID assigned to each processed event | VARCHAR | | \_airbyte_extracted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE | | \_airbyte_loaded_at | Timestamp to indicate when the record was loaded into Typed tables | TIMESTAMP WITH TIME ZONE | @@ -276,7 +276,8 @@ Otherwise, make sure to grant the role the required permissions in the desired n ## Changelog | Version | Date | Pull Request | Subject | -|:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| :-------------- | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 3.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up manager logs. | | 3.4.10 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings. | | 3.4.9 | 2023-11-06 | [\#32026](https://github.com/airbytehq/airbyte/pull/32026) | Add separate TRY_CAST transaction to reduce compute usage | | 3.4.8 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Further improve error reporting | From 876542550a1a2a5a34df2173559e1f2e15dda245 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 14 Nov 2023 15:02:22 -0800 Subject: [PATCH 6/9] docs --- airbyte-cdk/java/airbyte-cdk/README.md | 89 +++++++++++---------- docs/integrations/destinations/bigquery.md | 2 +- docs/integrations/destinations/snowflake.md | 2 +- 3 files changed, 47 insertions(+), 46 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 870b06e675e3..26b1f6dc8378 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -2,21 +2,21 @@ This page will walk through the process of developing with the Java CDK. -* [Developing with the Java CDK](#developing-with-the-java-cdk) - * [Intro to the Java CDK](#intro-to-the-java-cdk) - * [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) - * [How is the CDK published?](#how-is-the-cdk-published) - * [Using the Java CDK](#using-the-java-cdk) - * [Building the CDK](#building-the-cdk) - * [Bumping the CDK version](#bumping-the-cdk-version) - * [Publishing the CDK](#publishing-the-cdk) - * [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) - * [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) - * [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) - * [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) - * [Common Debugging Tips](#common-debugging-tips) - * [Changelog](#changelog) - * [Java CDK](#java-cdk) +- [Developing with the Java CDK](#developing-with-the-java-cdk) + - [Intro to the Java CDK](#intro-to-the-java-cdk) + - [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) + - [How is the CDK published?](#how-is-the-cdk-published) + - [Using the Java CDK](#using-the-java-cdk) + - [Building the CDK](#building-the-cdk) + - [Bumping the CDK version](#bumping-the-cdk-version) + - [Publishing the CDK](#publishing-the-cdk) + - [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) + - [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) + - [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) + - [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) + - [Common Debugging Tips](#common-debugging-tips) + - [Changelog](#changelog) + - [Java CDK](#java-cdk) ## Intro to the Java CDK @@ -127,7 +127,7 @@ Once you are done developing and testing your CDK changes: 2. After publishing the CDK, update the `useLocalCdk` setting by running `./gradlew :airbyte-integrations:connectors::disableLocalCdkRefs`. to automatically revert `useLocalCdk` to `false`. 3. You can optionally run `./gradlew :airbyte-integrations:connectors::assertNotUsingLocalCdk` to ensure that the project is not using a local CDK reference. -_Note: You can also use `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._ +_Note: You can also use `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._ ### Developing a connector against a pinned CDK version @@ -154,31 +154,32 @@ MavenLocal debugging steps: ### Java CDK -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector | -| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | -| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | -| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | -| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | -| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | -| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | -| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | -| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | -| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | -| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | -| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | -| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | -| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | -| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | -| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | -| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | -| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | -| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | -| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | -| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | -| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | -| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | -| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | -| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | -| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | +| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector | +| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | +| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | +| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | +| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | +| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | +| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | +| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | +| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | +| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | +| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | +| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | +| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | +| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | +| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | +| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | +| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | +| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | +| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | +| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | +| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | +| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | +| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | +| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | +| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 97bdeff675dc..53d528c5d376 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -140,7 +140,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 3.4.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up manager logs. | +| 3.4.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | | 2.3.15 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping enhancments | | 2.3.14 | 2023-11-06 | [\#32234](https://github.com/airbytehq/airbyte/pull/32234) | Remove unused config option. | | 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 6ae60871a069..6680040989e0 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -277,7 +277,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | | :-------------- | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 3.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up manager logs. | +| 3.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | | 3.4.10 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings. | | 3.4.9 | 2023-11-06 | [\#32026](https://github.com/airbytehq/airbyte/pull/32026) | Add separate TRY_CAST transaction to reduce compute usage | | 3.4.8 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Further improve error reporting | From c301c814ac339440eb81d7f2ad3850417e324f05 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 15 Nov 2023 09:39:42 -0800 Subject: [PATCH 7/9] big changelog entry --- docs/integrations/destinations/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 53d528c5d376..4be09c13e851 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -140,7 +140,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 3.4.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | +| 2.3.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | | 2.3.15 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping enhancments | | 2.3.14 | 2023-11-06 | [\#32234](https://github.com/airbytehq/airbyte/pull/32234) | Remove unused config option. | | 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | From 70a0c0a17fbae8b175ba5bc27c1e66b3dda3cd82 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 15 Nov 2023 09:41:15 -0800 Subject: [PATCH 8/9] fix spaces --- .../cdk/integrations/destination_async/FlushWorkers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java index 6aa1640bfe29..b02ebdf131b1 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java @@ -142,7 +142,7 @@ private void printWorkerInfo() { final int queueSize = threadPoolExecutor.getQueue().size(); final int activeCount = threadPoolExecutor.getActiveCount(); - workerInfo.append(String.format(" Pool queue size: %d, Active threads: %d", queueSize, activeCount)); + workerInfo.append(String.format("Pool queue size: %d, Active threads: %d", queueSize, activeCount)); log.info(workerInfo.toString()); } From 6d9da64ca1ddc6d303fe4c9187680ccab87c9591 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 15 Nov 2023 10:46:47 -0800 Subject: [PATCH 9/9] useLocalCdk = false --- .../connectors/destination-bigquery/build.gradle | 2 +- .../connectors/destination-snowflake/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index e8b3f81d5a8c..78277e3e53f9 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.4.11' features = ['db-destinations', 's3-destinations'] - useLocalCdk = true + useLocalCdk = false } airbyteJavaConnector.addCdkDependencies() diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index e9d89cff4612..0ac5603a6f8a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.4.11' features = ['db-destinations', 's3-destinations'] - useLocalCdk = true + useLocalCdk = false } airbyteJavaConnector.addCdkDependencies()