From f7f594f12d7389a974c6a1ac20c59a2c1c0102d5 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Wed, 6 Sep 2023 20:42:58 +0200 Subject: [PATCH] Add properties for AWS S3 regions This patch adds a environment variable for the s3 buckets region. Instead of attempting to find the region with ec2 metadata by default, we simply default to `eu-west-1`. This makes more sense as there is really no technical reason why the s3 bucket has to exist in the same region as the application. The old behavior is still available by specifying `auto` as the environment variable, but i believe the default region with no environment variable specified will be sufficient. --- .../no/ndla/audioapi/AudioApiProperties.scala | 6 ++++-- .../no/ndla/audioapi/ComponentRegistry.scala | 12 ++++-------- .../main/scala/no/ndla/common/Environment.scala | 15 +++++++++++++++ .../no/ndla/draftapi/ComponentRegistry.scala | 6 ++---- .../no/ndla/draftapi/DraftApiProperties.scala | 5 ++++- .../no/ndla/imageapi/ComponentRegistry.scala | 6 +----- .../no/ndla/imageapi/ImageApiProperties.scala | 6 ++++-- project/commonlib.scala | 3 ++- 8 files changed, 36 insertions(+), 23 deletions(-) diff --git a/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala b/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala index 96201a2e8c..65e222220f 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala @@ -8,8 +8,9 @@ package no.ndla.audioapi +import com.amazonaws.regions.Regions import com.typesafe.scalalogging.StrictLogging -import no.ndla.common.Environment.prop +import no.ndla.common.Environment.{prop, propToAwsRegion} import no.ndla.common.configuration.{BaseProps, HasBaseProps} import no.ndla.network.{AuthUser, Domains} import no.ndla.common.secrets.PropertyKeys @@ -42,7 +43,8 @@ class AudioApiProperties extends BaseProps with StrictLogging { val MaxAudioFileSizeBytes: Int = 1024 * 1024 * 100 // 100 MiB - val StorageName: String = propOrElse("AUDIO_FILE_S3_BUCKET", s"$Environment.audio.ndla") + val StorageName: String = propOrElse("AUDIO_FILE_S3_BUCKET", s"$Environment.audio.ndla") + val StorageRegion: Regions = propToAwsRegion("AUDIO_FILE_S3_BUCKET_REGION") val SearchServer: String = propOrElse("SEARCH_SERVER", "http://search-audio-api.ndla-local") val RunWithSignedSearchRequests: Boolean = propOrElse("RUN_WITH_SIGNED_SEARCH_REQUESTS", "true").toBoolean diff --git a/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala b/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala index 3b3c28364b..a42462bb08 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala @@ -10,7 +10,6 @@ package no.ndla.audioapi import cats.data.Kleisli import cats.effect.IO -import com.amazonaws.regions.Regions import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} import com.zaxxer.hikari.HikariDataSource import no.ndla.audioapi.controller._ @@ -71,13 +70,10 @@ class ComponentRegistry(properties: AudioApiProperties) override val dataSource: HikariDataSource = DataSource.getHikariDataSource DataSource.connectToDatabase() - val currentRegion: Option[Regions] = Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName)) - - val amazonClient: AmazonS3 = - AmazonS3ClientBuilder - .standard() - .withRegion(currentRegion.getOrElse(Regions.EU_CENTRAL_1)) - .build() + val amazonClient: AmazonS3 = AmazonS3ClientBuilder + .standard() + .withRegion(props.StorageRegion) + .build() lazy val audioRepository = new AudioRepository lazy val seriesRepository = new SeriesRepository diff --git a/common/src/main/scala/no/ndla/common/Environment.scala b/common/src/main/scala/no/ndla/common/Environment.scala index c7d4a24775..866f06ba21 100644 --- a/common/src/main/scala/no/ndla/common/Environment.scala +++ b/common/src/main/scala/no/ndla/common/Environment.scala @@ -8,6 +8,8 @@ package no.ndla.common +import com.amazonaws.regions.Regions + import scala.jdk.CollectionConverters.MapHasAsScala import scala.util.Properties.propOrElse import scala.util.Properties.propOrNone @@ -20,6 +22,19 @@ object Environment { /** UNSAFE: Will throw [[EnvironmentNotFoundException]] if property is not found */ def prop(key: String): String = propOrElse(key, throw EnvironmentNotFoundException(key)) + /** Will try to derive aws-region from ec2 instance metadata if `auto` is specified in the property If another string + * is passed, it will be attempted to be parsed as a aws region. Otherwise the default region will be used. + */ + def propToAwsRegion(key: String, defaultRegion: Regions = Regions.EU_WEST_1): Regions = { + val specifiedRegion = propOrNone(key) + specifiedRegion + .flatMap { + case "auto" => Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName)) + case str => Option(Regions.fromName(str)) + } + .getOrElse(defaultRegion) + } + def booleanPropOrFalse(key: String): Boolean = { propOrNone(key).flatMap(_.toBooleanOption).getOrElse(false) } diff --git a/draft-api/src/main/scala/no/ndla/draftapi/ComponentRegistry.scala b/draft-api/src/main/scala/no/ndla/draftapi/ComponentRegistry.scala index 25c39f010e..3f310c7b86 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/ComponentRegistry.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/ComponentRegistry.scala @@ -7,7 +7,6 @@ package no.ndla.draftapi -import com.amazonaws.regions.Regions import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} import com.typesafe.scalalogging.StrictLogging import com.zaxxer.hikari.HikariDataSource @@ -110,13 +109,12 @@ class ComponentRegistry(properties: DraftApiProperties) lazy val writeService = new WriteService lazy val reindexClient = new ReindexClient - lazy val fileStorage = new FileStorageService - val currentRegion: Option[Regions] = Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName)) + lazy val fileStorage = new FileStorageService val amazonClient: AmazonS3 = AmazonS3ClientBuilder .standard() - .withRegion(currentRegion.getOrElse(Regions.EU_WEST_1)) + .withRegion(props.AttachmentStorageRegion) .build() var e4sClient: NdlaE4sClient = Elastic4sClientFactory.getClient(props.SearchServer) diff --git a/draft-api/src/main/scala/no/ndla/draftapi/DraftApiProperties.scala b/draft-api/src/main/scala/no/ndla/draftapi/DraftApiProperties.scala index aeb8c756c7..05acaa3c07 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/DraftApiProperties.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/DraftApiProperties.scala @@ -7,8 +7,9 @@ package no.ndla.draftapi +import com.amazonaws.regions.Regions import com.typesafe.scalalogging.StrictLogging -import no.ndla.common.Environment.prop +import no.ndla.common.Environment.{prop, propToAwsRegion} import no.ndla.common.configuration.{BaseProps, HasBaseProps} import no.ndla.common.secrets.PropertyKeys import no.ndla.network.{AuthUser, Domains} @@ -87,6 +88,8 @@ class DraftApiProperties extends BaseProps with StrictLogging { def AttachmentStorageName: String = propOrElse("ARTICLE_ATTACHMENT_S3_BUCKET", s"$Environment.article-attachments.ndla") + def AttachmentStorageRegion: Regions = propToAwsRegion("ARTICLE_ATTACHMENT_S3_BUCKET") + def H5PAddress: String = propOrElse( "NDLA_H5P_ADDRESS", Map( diff --git a/image-api/src/main/scala/no/ndla/imageapi/ComponentRegistry.scala b/image-api/src/main/scala/no/ndla/imageapi/ComponentRegistry.scala index d944421e06..66c0e23359 100644 --- a/image-api/src/main/scala/no/ndla/imageapi/ComponentRegistry.scala +++ b/image-api/src/main/scala/no/ndla/imageapi/ComponentRegistry.scala @@ -8,7 +8,6 @@ package no.ndla.imageapi -import com.amazonaws.regions.Regions import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} import no.ndla.common.Clock import no.ndla.common.configuration.BaseComponentRegistry @@ -79,13 +78,10 @@ class ComponentRegistry(properties: ImageApiProperties) implicit val swagger: ImageSwagger = new ImageSwagger - val currentRegion: Option[Regions] = - Option(Regions.getCurrentRegion).map(region => Regions.fromName(region.getName)) - val amazonClient: AmazonS3 = AmazonS3ClientBuilder .standard() - .withRegion(currentRegion.getOrElse(Regions.EU_CENTRAL_1)) + .withRegion(props.StorageRegion) .build() lazy val imageIndexService = new ImageIndexService diff --git a/image-api/src/main/scala/no/ndla/imageapi/ImageApiProperties.scala b/image-api/src/main/scala/no/ndla/imageapi/ImageApiProperties.scala index 7aff1ea0db..2130be284f 100644 --- a/image-api/src/main/scala/no/ndla/imageapi/ImageApiProperties.scala +++ b/image-api/src/main/scala/no/ndla/imageapi/ImageApiProperties.scala @@ -8,8 +8,9 @@ package no.ndla.imageapi +import com.amazonaws.regions.Regions import com.typesafe.scalalogging.StrictLogging -import no.ndla.common.Environment.prop +import no.ndla.common.Environment.{prop, propToAwsRegion} import no.ndla.common.configuration.{BaseProps, HasBaseProps} import no.ndla.network.{AuthUser, Domains} import no.ndla.common.secrets.PropertyKeys @@ -103,7 +104,8 @@ class ImageApiProperties extends BaseProps with StrictLogging { def MetaPort: Int = prop(PropertyKeys.MetaPortKey).toInt def MetaSchema: String = prop(PropertyKeys.MetaSchemaKey) - val StorageName: String = propOrElse("IMAGE_FILE_S3_BUCKET", s"$Environment.images.ndla") + val StorageName: String = propOrElse("IMAGE_FILE_S3_BUCKET", s"$Environment.images.ndla") + val StorageRegion: Regions = propToAwsRegion("IMAGE_FILE_S3_BUCKET_REGION") val SearchIndex: String = propOrElse("SEARCH_INDEX_NAME", "images") val SearchDocument = "image" diff --git a/project/commonlib.scala b/project/commonlib.scala index 08d63b411d..f1d8c59c1e 100644 --- a/project/commonlib.scala +++ b/project/commonlib.scala @@ -23,7 +23,8 @@ object commonlib extends Module { "org.scala-lang" % "scala-compiler" % ScalaV, "org.eclipse.jetty" % "jetty-webapp" % JettyV % "compile", "org.eclipse.jetty" % "jetty-plus" % JettyV % "container", - "javax.servlet" % "javax.servlet-api" % "4.0.1" % "container;provided;test" + "javax.servlet" % "javax.servlet-api" % "4.0.1" % "container;provided;test", + "com.amazonaws" % "aws-java-sdk-s3" % AwsSdkV ), melody, scalatra,