From 83413b976615a449b238d8de0ab36daf40713fcb Mon Sep 17 00:00:00 2001 From: Itai Admi Date: Tue, 15 Jun 2021 14:41:54 +0300 Subject: [PATCH] S3 destination check to verify prefix access and not full-bucket access --- .../destination/jdbc/copy/s3/S3StreamCopier.java | 9 ++++++++- .../integrations/destination/s3/S3Destination.java | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java index ff95427ebbb3f..8e707fdff4cd0 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java @@ -201,13 +201,20 @@ private void closeAndWaitForUpload() throws IOException { } public static void attemptS3WriteAndDelete(S3Config s3Config) { - final String outputTableName = "_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", ""); + attemptS3WriteAndDelete(s3Config, ""); + } + + public static void attemptS3WriteAndDelete(S3Config s3Config, String bucketPath) { + var prefix = bucketPath.isEmpty() ? "": + bucketPath + (bucketPath.endsWith("/") ? "" : "/"); + final String outputTableName = prefix + "_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", ""); attemptWriteAndDeleteS3Object(s3Config, outputTableName); } private static void attemptWriteAndDeleteS3Object(S3Config s3Config, String outputTableName) { var s3 = getAmazonS3(s3Config); var s3Bucket = s3Config.getBucketName(); + s3.putObject(s3Bucket, outputTableName, "check-content"); s3.deleteObject(s3Bucket, outputTableName); } diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java index e7ad69d347b30..1e66de2bad17e 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java @@ -52,7 +52,7 @@ public static void main(String[] args) throws Exception { @Override public AirbyteConnectionStatus check(JsonNode config) { try { - S3StreamCopier.attemptS3WriteAndDelete(S3Config.getS3Config(config)); + S3StreamCopier.attemptS3WriteAndDelete(S3Config.getS3Config(config), config.get("s3_bucket_path").asText()); return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); } catch (Exception e) { LOGGER.error("Exception attempting to access the S3 bucket: ", e);