#184: Added parameter to ssl enabled (#186)

Fixes #184
exasol · Jan 13, 2022 · 8402dc1 · 8402dc1
1 parent b2d0a65
commit 8402dc1
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 42 deletions.
diff --git a/src/main/scala/com/exasol/cloudetl/bucket/S3Bucket.scala b/src/main/scala/com/exasol/cloudetl/bucket/S3Bucket.scala
@@ -11,6 +11,7 @@ final case class S3Bucket(path: String, params: StorageProperties) extends Bucke
   private[this] val S3_ACCESS_KEY: String = "S3_ACCESS_KEY"
   private[this] val S3_SECRET_KEY: String = "S3_SECRET_KEY"
   private[this] val S3_SESSION_TOKEN: String = "S3_SESSION_TOKEN"
+  private[this] val S3_SSL_ENABLED: String = "S3_SSL_ENABLED"
   private[this] val S3_PATH_STYLE_ACCESS: String = "S3_PATH_STYLE_ACCESS"
   private[this] val S3_CHANGE_DETECTION_MODE: String = "S3_CHANGE_DETECTION_MODE"
 
@@ -50,6 +51,9 @@ final case class S3Bucket(path: String, params: StorageProperties) extends Bucke
     if (properties.containsKey(S3_PATH_STYLE_ACCESS)) {
       conf.set("fs.s3a.path.style.access", properties.getString(S3_PATH_STYLE_ACCESS))
     }
+    if (properties.containsKey(S3_SSL_ENABLED)) {
+      conf.set("fs.s3a.connection.ssl.enabled", properties.getString(S3_SSL_ENABLED))
+    }
 
     val mergedProperties = if (properties.hasNamedConnection()) {
       properties.merge(S3_ACCESS_KEY)

diff --git a/src/test/scala/com/exasol/cloudetl/bucket/S3BucketTest.scala b/src/test/scala/com/exasol/cloudetl/bucket/S3BucketTest.scala
@@ -9,6 +9,7 @@ class S3BucketTest extends AbstractBucketTest {
   private[this] val defaultProperties = Map(
     PATH -> "s3a://my-bucket/",
     FORMAT -> "AVRO",
+    "CONNECTION_NAME" -> "connection_info",
     "S3_ENDPOINT" -> "eu-central-1"
   )
 
@@ -28,93 +29,81 @@ class S3BucketTest extends AbstractBucketTest {
     "fs.s3a.session.token" -> sessionToken
   )
 
-  private[this] def assertConfigurationProperties(
-    bucket: Bucket,
-    extraMappings: Map[String, String]
-  ): Unit = {
+  private[this] def assertConfigurationProperties(bucket: Bucket, extraMappings: Map[String, String]): Unit = {
     assert(bucket.isInstanceOf[S3Bucket])
     val conf = bucket.getConfiguration()
     val defaultMappings = Map(
-      "fs.s3a.impl" -> classOf[S3AFileSystem].getName,
+      "fs.s3a.impl" -> classOf[S3AFileSystem].getName(),
       "fs.s3a.endpoint" -> "eu-central-1"
     )
     (defaultMappings ++ extraMappings).foreach { case (given, expected) =>
       assert(conf.get(given) === expected)
     }
   }
 
+  private[this] def bucketWithDefaultConnectionString(properties: Map[String, String]): Bucket = {
+    val identifier = "S3_ACCESS_KEY=access;S3_SECRET_KEY=secret;S3_SESSION_TOKEN=token"
+    val exaMetadata = mockConnectionInfo("", identifier)
+    getBucket(properties, exaMetadata)
+  }
+
   test("apply throws when no connection name is provided") {
-    properties = defaultProperties
+    val properties = defaultProperties - "CONNECTION_NAME"
     assertNoConnectionName(getBucket(properties).validate())
   }
 
   test("apply throws with access, secret or session token parameters") {
-    properties = accessProperties
-    assertForbiddenProperty(getBucket(properties).validate())
+    assertForbiddenProperty(getBucket(accessProperties).validate())
   }
 
   test("apply returns S3Bucket with secret from connection") {
-    properties = defaultProperties ++ Map(
-      "CONNECTION_NAME" -> "connection_info"
-    )
     val exaMetadata = mockConnectionInfo("access", "S3_SECRET_KEY=secret")
-    val bucket = getBucket(properties, exaMetadata)
+    val bucket = getBucket(defaultProperties, exaMetadata)
     assertConfigurationProperties(bucket, configMappings - "fs.s3a.session.token")
   }
 
   test("apply returns S3Bucket with secret and session token from connection") {
-    properties = defaultProperties ++ Map(
-      "CONNECTION_NAME" -> "connection_info"
-    )
     val exaMetadata = mockConnectionInfo("access", "S3_SECRET_KEY=secret;S3_SESSION_TOKEN=token")
-    val bucket = getBucket(properties, exaMetadata)
+    val bucket = getBucket(defaultProperties, exaMetadata)
     assertConfigurationProperties(bucket, configMappings)
   }
 
   // Access key is encoded in password value of connection object.
   test("apply returns S3Bucket with access and secret from connection") {
-    properties = defaultProperties ++ Map(
-      "CONNECTION_NAME" -> "connection_info"
-    )
     val exaMetadata = mockConnectionInfo("", "S3_ACCESS_KEY=access;S3_SECRET_KEY=secret")
-    val bucket = getBucket(properties, exaMetadata)
+    val bucket = getBucket(defaultProperties, exaMetadata)
     assertConfigurationProperties(bucket, configMappings - "fs.s3a.session.token")
   }
 
-  private[this] def bucketWithDefaultConnectionString(properties: Map[String, String]): Bucket = {
-    val identifier = "S3_ACCESS_KEY=access;S3_SECRET_KEY=secret;S3_SESSION_TOKEN=token"
-    val exaMetadata = mockConnectionInfo("", identifier)
-    getBucket(properties, exaMetadata)
-  }
-
   test("apply returns S3Bucket with access, secret and session token from connection") {
-    properties = defaultProperties ++ Map(
-      "CONNECTION_NAME" -> "connection_info"
-    )
-    assertConfigurationProperties(bucketWithDefaultConnectionString(properties), configMappings)
+    assertConfigurationProperties(bucketWithDefaultConnectionString(defaultProperties), configMappings)
   }
 
   test("apply returns S3Bucket with change detection mode") {
-    properties = defaultProperties ++ Map(
-      "S3_CHANGE_DETECTION_MODE" -> "none",
-      "CONNECTION_NAME" -> "connection_info"
-    )
+    val properties = defaultProperties ++ Map("S3_CHANGE_DETECTION_MODE" -> "none")
     val extraConfigs = configMappings ++ Map("fs.s3a.change.detection.mode" -> "none")
     assertConfigurationProperties(bucketWithDefaultConnectionString(properties), extraConfigs)
   }
 
   test("apply returns S3Bucket with path style access") {
-    properties = defaultProperties ++ Map(
-      "S3_PATH_STYLE_ACCESS" -> "true",
-      "CONNECTION_NAME" -> "connection_info"
-    )
+    val properties = defaultProperties ++ Map("S3_PATH_STYLE_ACCESS" -> "true")
     val extraConfigs = configMappings ++ Map("fs.s3a.path.style.access" -> "true")
     assertConfigurationProperties(bucketWithDefaultConnectionString(properties), extraConfigs)
   }
 
+  test("apply returns S3Bucket with ssl enabled by default") {
+    val extraConfigs = configMappings ++ Map("fs.s3a.connection.ssl.enabled" -> "true")
+    assertConfigurationProperties(bucketWithDefaultConnectionString(defaultProperties), extraConfigs)
+  }
+
+  test("apply returns S3Bucket with ssl enabled by user") {
+    val properties = defaultProperties ++ Map("S3_SSL_ENABLED" -> "falsy")
+    val extraConfigs = configMappings ++ Map("fs.s3a.connection.ssl.enabled" -> "falsy")
+    assertConfigurationProperties(bucketWithDefaultConnectionString(properties), extraConfigs)
+  }
+
   test("proxy settings should be added when present") {
-    properties = defaultProperties ++ Map(
-      "CONNECTION_NAME" -> "connection_info",
+    val properties = defaultProperties ++ Map(
       "PROXY_HOST" -> "my_proxy.net",
       "PROXY_PORT" -> "2345"
     )
@@ -126,8 +115,7 @@ class S3BucketTest extends AbstractBucketTest {
   }
 
   test("proxy settings should not be added without host") {
-    properties = defaultProperties ++ Map(
-      "CONNECTION_NAME" -> "connection_info",
+    val properties = defaultProperties ++ Map(
       "PROXY_USERNAME" -> "myUser",
       "PROXY_PASSWORD" -> "mySecretPassword",
       "PROXY_PORT" -> "2345"