Merge pull request #295 from NDLANO/awss3region

Add properties for AWS S3 regions, rather than auto-detecting
NDLANO · Sep 8, 2023 · 5b4487c · 5b4487c
2 parents e28098f + f7f594f
commit 5b4487c
Show file tree

Hide file tree

Showing 8 changed files with 36 additions and 23 deletions.
diff --git a/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala b/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala
@@ -8,8 +8,9 @@
 
 package no.ndla.audioapi
 
+import com.amazonaws.regions.Regions
 import com.typesafe.scalalogging.StrictLogging
-import no.ndla.common.Environment.prop
+import no.ndla.common.Environment.{prop, propToAwsRegion}
 import no.ndla.common.configuration.{BaseProps, HasBaseProps}
 import no.ndla.network.{AuthUser, Domains}
 import no.ndla.common.secrets.PropertyKeys
@@ -42,7 +43,8 @@ class AudioApiProperties extends BaseProps with StrictLogging {
 
   val MaxAudioFileSizeBytes: Int = 1024 * 1024 * 100 // 100 MiB
 
-  val StorageName: String = propOrElse("AUDIO_FILE_S3_BUCKET", s"$Environment.audio.ndla")
+  val StorageName: String    = propOrElse("AUDIO_FILE_S3_BUCKET", s"$Environment.audio.ndla")
+  val StorageRegion: Regions = propToAwsRegion("AUDIO_FILE_S3_BUCKET_REGION")
 
   val SearchServer: String                 = propOrElse("SEARCH_SERVER", "http://search-audio-api.ndla-local")
   val RunWithSignedSearchRequests: Boolean = propOrElse("RUN_WITH_SIGNED_SEARCH_REQUESTS", "true").toBoolean

diff --git a/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala b/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala
@@ -10,7 +10,6 @@ package no.ndla.audioapi
 
 import cats.data.Kleisli
 import cats.effect.IO
-import com.amazonaws.regions.Regions
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import com.zaxxer.hikari.HikariDataSource
 import no.ndla.audioapi.controller._
@@ -71,13 +70,10 @@ class ComponentRegistry(properties: AudioApiProperties)
   override val dataSource: HikariDataSource = DataSource.getHikariDataSource
   DataSource.connectToDatabase()
 
-  val currentRegion: Option[Regions] = Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName))
-
-  val amazonClient: AmazonS3 =
-    AmazonS3ClientBuilder
-      .standard()
-      .withRegion(currentRegion.getOrElse(Regions.EU_CENTRAL_1))
-      .build()
+  val amazonClient: AmazonS3 = AmazonS3ClientBuilder
+    .standard()
+    .withRegion(props.StorageRegion)
+    .build()
 
   lazy val audioRepository  = new AudioRepository
   lazy val seriesRepository = new SeriesRepository

diff --git a/common/src/main/scala/no/ndla/common/Environment.scala b/common/src/main/scala/no/ndla/common/Environment.scala
@@ -8,6 +8,8 @@
 
 package no.ndla.common
 
+import com.amazonaws.regions.Regions
+
 import scala.jdk.CollectionConverters.MapHasAsScala
 import scala.util.Properties.propOrElse
 import scala.util.Properties.propOrNone
@@ -20,6 +22,19 @@ object Environment {
   /** UNSAFE: Will throw [[EnvironmentNotFoundException]] if property is not found */
   def prop(key: String): String = propOrElse(key, throw EnvironmentNotFoundException(key))
 
+  /** Will try to derive aws-region from ec2 instance metadata if `auto` is specified in the property If another string
+    * is passed, it will be attempted to be parsed as a aws region. Otherwise the default region will be used.
+    */
+  def propToAwsRegion(key: String, defaultRegion: Regions = Regions.EU_WEST_1): Regions = {
+    val specifiedRegion = propOrNone(key)
+    specifiedRegion
+      .flatMap {
+        case "auto" => Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName))
+        case str    => Option(Regions.fromName(str))
+      }
+      .getOrElse(defaultRegion)
+  }
+
   def booleanPropOrFalse(key: String): Boolean = {
     propOrNone(key).flatMap(_.toBooleanOption).getOrElse(false)
   }

diff --git a/draft-api/src/main/scala/no/ndla/draftapi/ComponentRegistry.scala b/draft-api/src/main/scala/no/ndla/draftapi/ComponentRegistry.scala
@@ -7,7 +7,6 @@
 
 package no.ndla.draftapi
 
-import com.amazonaws.regions.Regions
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import com.typesafe.scalalogging.StrictLogging
 import com.zaxxer.hikari.HikariDataSource
@@ -110,13 +109,12 @@ class ComponentRegistry(properties: DraftApiProperties)
   lazy val writeService           = new WriteService
   lazy val reindexClient          = new ReindexClient
 
-  lazy val fileStorage               = new FileStorageService
-  val currentRegion: Option[Regions] = Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName))
+  lazy val fileStorage = new FileStorageService
 
   val amazonClient: AmazonS3 =
     AmazonS3ClientBuilder
       .standard()
-      .withRegion(currentRegion.getOrElse(Regions.EU_WEST_1))
+      .withRegion(props.AttachmentStorageRegion)
       .build()
 
   var e4sClient: NdlaE4sClient = Elastic4sClientFactory.getClient(props.SearchServer)

diff --git a/draft-api/src/main/scala/no/ndla/draftapi/DraftApiProperties.scala b/draft-api/src/main/scala/no/ndla/draftapi/DraftApiProperties.scala
@@ -7,8 +7,9 @@
 
 package no.ndla.draftapi
 
+import com.amazonaws.regions.Regions
 import com.typesafe.scalalogging.StrictLogging
-import no.ndla.common.Environment.prop
+import no.ndla.common.Environment.{prop, propToAwsRegion}
 import no.ndla.common.configuration.{BaseProps, HasBaseProps}
 import no.ndla.common.secrets.PropertyKeys
 import no.ndla.network.{AuthUser, Domains}
@@ -87,6 +88,8 @@ class DraftApiProperties extends BaseProps with StrictLogging {
   def AttachmentStorageName: String =
     propOrElse("ARTICLE_ATTACHMENT_S3_BUCKET", s"$Environment.article-attachments.ndla")
 
+  def AttachmentStorageRegion: Regions = propToAwsRegion("ARTICLE_ATTACHMENT_S3_BUCKET")
+
   def H5PAddress: String = propOrElse(
     "NDLA_H5P_ADDRESS",
     Map(

diff --git a/image-api/src/main/scala/no/ndla/imageapi/ComponentRegistry.scala b/image-api/src/main/scala/no/ndla/imageapi/ComponentRegistry.scala
@@ -8,7 +8,6 @@
 
 package no.ndla.imageapi
 
-import com.amazonaws.regions.Regions
 import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 import no.ndla.common.Clock
 import no.ndla.common.configuration.BaseComponentRegistry
@@ -79,13 +78,10 @@ class ComponentRegistry(properties: ImageApiProperties)
 
   implicit val swagger: ImageSwagger = new ImageSwagger
 
-  val currentRegion: Option[Regions] =
-    Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName))
-
   val amazonClient: AmazonS3 =
     AmazonS3ClientBuilder
       .standard()
-      .withRegion(currentRegion.getOrElse(Regions.EU_CENTRAL_1))
+      .withRegion(props.StorageRegion)
       .build()
 
   lazy val imageIndexService      = new ImageIndexService

diff --git a/image-api/src/main/scala/no/ndla/imageapi/ImageApiProperties.scala b/image-api/src/main/scala/no/ndla/imageapi/ImageApiProperties.scala
@@ -8,8 +8,9 @@
 
 package no.ndla.imageapi
 
+import com.amazonaws.regions.Regions
 import com.typesafe.scalalogging.StrictLogging
-import no.ndla.common.Environment.prop
+import no.ndla.common.Environment.{prop, propToAwsRegion}
 import no.ndla.common.configuration.{BaseProps, HasBaseProps}
 import no.ndla.network.{AuthUser, Domains}
 import no.ndla.common.secrets.PropertyKeys
@@ -103,7 +104,8 @@ class ImageApiProperties extends BaseProps with StrictLogging {
   def MetaPort: Int        = prop(PropertyKeys.MetaPortKey).toInt
   def MetaSchema: String   = prop(PropertyKeys.MetaSchemaKey)
 
-  val StorageName: String = propOrElse("IMAGE_FILE_S3_BUCKET", s"$Environment.images.ndla")
+  val StorageName: String    = propOrElse("IMAGE_FILE_S3_BUCKET", s"$Environment.images.ndla")
+  val StorageRegion: Regions = propToAwsRegion("IMAGE_FILE_S3_BUCKET_REGION")
 
   val SearchIndex: String    = propOrElse("SEARCH_INDEX_NAME", "images")
   val SearchDocument         = "image"

diff --git a/project/commonlib.scala b/project/commonlib.scala
@@ -23,7 +23,8 @@ object commonlib extends Module {
       "org.scala-lang"    % "scala-compiler"    % ScalaV,
       "org.eclipse.jetty" % "jetty-webapp"      % JettyV  % "compile",
       "org.eclipse.jetty" % "jetty-plus"        % JettyV  % "container",
-      "javax.servlet"     % "javax.servlet-api" % "4.0.1" % "container;provided;test"
+      "javax.servlet"     % "javax.servlet-api" % "4.0.1" % "container;provided;test",
+      "com.amazonaws"     % "aws-java-sdk-s3"   % AwsSdkV
     ),
     melody,
     scalatra,