Community PR: Support CMEK or the GCS destination connector (airbyteh…

…q#21682)

Community PR airbytehq#20351: Support CMEK or the GCS destination connector (airbytehq#21682)

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -139,7 +139,7 @@
 - name: Google Cloud Storage (GCS)
   destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
   dockerRepository: airbyte/destination-gcs
-  dockerImageTag: 0.2.13
+  dockerImageTag: 0.2.14
   documentationUrl: https://docs.airbyte.com/integrations/destinations/gcs
   icon: googlecloudstorage.svg
   resourceRequirements:

airbyte-config/init/src/main/resources/seed/destination_specs.yaml

@@ -2325,7 +2325,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-gcs:0.2.13"
+- dockerImage: "airbyte/destination-gcs:0.2.14"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/gcs"
     connectionSpecification:

airbyte-integrations/connectors/destination-gcs/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.13
+LABEL io.airbyte.version=0.2.14
 LABEL io.airbyte.name=airbyte/destination-gcs

airbyte-integrations/connectors/destination-gcs/sample_files/configured_catalog.json

@@ -0,0 +1,27 @@
+{
+  "streams": [
+    {
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "append",
+      "stream": {
+        "name": "ab-airbyte-testing",
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false,
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "string"
+            },
+            "body": {
+              "type": "string"
+            },
+            "attributes": {
+              "type": ["null", "object"]
+            }
+          }
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/destination-gcs/sample_files/messages.jsonl

@@ -0,0 +1,2 @@
+{"type": "RECORD", "record": {"stream": "ab-airbyte-testing", "data": {"_ab_pk": "my_value", "column2": 221, "column3": "2021-01-01T20:10:22", "column4": 1.214, "column5": [1,2,3]}, "emitted_at": 1626172757000}}
+{"type": "RECORD", "record": {"stream": "ab-airbyte-testing", "data": {"_ab_pk": "my_value2", "column2": 222, "column3": "2021-01-02T22:10:22", "column5": [1,2,null]}, "emitted_at": 1626172757000}}

airbyte-integrations/connectors/destination-gcs/sample_secrets/config.json

@@ -6,5 +6,8 @@
     "credential_type": "HMAC_KEY",
     "hmac_key_access_id": "<access-id>",
     "hmac_key_secret": "<secret>"
+  },
+  "format": {
+    "format_type": "CSV"
   }
 }

airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java

@@ -8,6 +8,7 @@
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy;
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.integrations.BaseConnector;
 import io.airbyte.integrations.base.AirbyteMessageConsumer;
@@ -40,6 +41,8 @@ public GcsDestination() {
   }
 
   public static void main(final String[] args) throws Exception {
+    System.setProperty(SkipMd5CheckStrategy.DISABLE_GET_OBJECT_MD5_VALIDATION_PROPERTY, "true");
+    System.setProperty(SkipMd5CheckStrategy.DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY, "true");
     new IntegrationRunner(new GcsDestination()).run(args);
   }
 

airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java

@@ -15,6 +15,7 @@
 import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig;
 import io.airbyte.integrations.destination.s3.S3DestinationConfig;
 import io.airbyte.integrations.destination.s3.S3DestinationConstants;
+import io.airbyte.integrations.destination.s3.S3StorageOperations;
 import io.airbyte.integrations.destination.s3.S3FormatConfig;
 import io.airbyte.integrations.destination.s3.S3FormatConfigs;
 
@@ -33,14 +34,18 @@ public GcsDestinationConfig(final String bucketName,
                               final String bucketRegion,
                               final GcsCredentialConfig credentialConfig,
                               final S3FormatConfig formatConfig) {
+
     super(GCS_ENDPOINT,
         bucketName,
         bucketPath,
         bucketRegion,
         S3DestinationConstants.DEFAULT_PATH_FORMAT,
         credentialConfig.getS3CredentialConfig().orElseThrow(),
         formatConfig,
-        null);
+        null,
+        null,
+        false,
+        S3StorageOperations.DEFAULT_UPLOAD_THREADS);
 
     this.credentialConfig = credentialConfig;
   }

docs/integrations/destinations/gcs.md

@@ -237,6 +237,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A
 
 | Version | Date | Pull Request                                                | Subject |
 |:--------| :--- |:------------------------------------------------------------| :--- |
+| 0.2.14  | 2023-11-23 | [\#21682](https://github.com/airbytehq/airbyte/pull/21682) | Add support for buckets with Customer-Managed Encryption Key                                                               |
 | 0.2.13  | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087)   | Wrap Authentication Errors as Config Exceptions                                                                          |
 | 0.2.12  | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901)  | Fix logging to GCS |
 | 0.2.11  | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243)  | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |