s3: Support S3 compatible services like Google Storage

reposerver/src/main/resources/application.conf

@@ -17,6 +17,7 @@ storage {
     bucketId = ${?TUF_REPOSERVER_AWS_BUCKET_ID}
     region = "eu-central-1"
     region = ${?TUF_REPOSERVER_AWS_REGION}
+    endpointUrl = ${?TUF_REPOSERVER_S3_URL}
   }
 }
 

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala

@@ -39,7 +39,8 @@ trait Settings {
     val secretKey = _config.getString("storage.s3.secretKey")
     val bucketId = _config.getString("storage.s3.bucketId")
     val region = Regions.fromName(_config.getString("storage.s3.region"))
-    new S3Credentials(accessKey, secretKey, bucketId, region)
+    val endpointUrl = _config.getString("storage.s3.endpointUrl")
+    new S3Credentials(accessKey, secretKey, bucketId, region, endpointUrl)
   }
 
   lazy val useS3 = _config.getString("storage.type").equals("s3")

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala

@@ -12,6 +12,7 @@ import akka.util.ByteString
 import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, TargetFilename}
 import com.advancedtelematic.tuf.reposerver.target_store.TargetStoreEngine.{TargetRedirect, TargetRetrieveResult, TargetStoreResult}
 import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider}
+import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import com.amazonaws.services.s3.model.{CannedAccessControlList, PutObjectRequest}
@@ -29,10 +30,18 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
-  private lazy val s3client = AmazonS3ClientBuilder.standard()
-    .withCredentials(credentials)
-    .withRegion(credentials.region)
-    .build()
+  protected lazy val s3client = {
+    if(credentials.endpointUrl.length() > 0) {
+      log.info(s"Using custom S3 url: ${credentials.endpointUrl}")
+      AmazonS3ClientBuilder.standard()
+        .withCredentials(credentials)
+        .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(credentials.endpointUrl, credentials.region.getName()))
+    } else {
+      AmazonS3ClientBuilder.standard()
+        .withCredentials(credentials)
+        .withRegion(credentials.region)
+    }
+  }.build()
 
   override def store(repoId: RepoId, filename: TargetFilename, fileData: Source[ByteString, Any]): Future[TargetStoreResult] = {
     val tempFile = File.createTempFile("s3file", ".tmp")
@@ -91,7 +100,7 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor
   }
 }
 
-class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions)
+class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions, val endpointUrl: String)
   extends AWSCredentials with AWSCredentialsProvider {
   override def getAWSAccessKeyId: String = accessKey
 

reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala

@@ -4,14 +4,15 @@ import akka.http.scaladsl.model.Multipart.FormData.BodyPart
 import akka.http.scaladsl.model.{HttpEntity, Multipart, StatusCodes}
 import akka.util.ByteString
 import com.advancedtelematic.libtuf.data.TufDataType.RepoId
-import com.advancedtelematic.tuf.reposerver.target_store.{S3TargetStoreEngine, TargetStore}
+import com.advancedtelematic.tuf.reposerver.target_store.{S3Credentials, S3TargetStoreEngine, TargetStore}
 import com.advancedtelematic.tuf.reposerver.util._
 import org.scalatest.{BeforeAndAfterAll, Inspectors}
 import org.scalatest.concurrent.PatienceConfiguration
 import org.scalatest.prop.Whenever
 import RepoId._
 import cats.syntax.show._
 import com.advancedtelematic.tuf.reposerver.Settings
+import com.amazonaws.regions.Regions
 
 class S3StorageResourceIntegrationSpec extends TufReposerverSpec
     with ResourceSpec with BeforeAndAfterAll with Inspectors with Whenever with PatienceConfiguration {
@@ -50,4 +51,33 @@ class S3StorageResourceIntegrationSpec extends TufReposerverSpec
       header("Location").get.value() should include("amazonaws.com")
     }
   }
+
+  test("uploading a target changes targets json with custom endpoint url") {
+    lazy val credentials = new S3Credentials("", "", "", Regions.fromName("us-central-1"), "https://storage.googleapis.com")
+    lazy val s3Storage = new S3TargetStoreEngine(credentials)
+    lazy val targetStore = new TargetStore(fakeKeyserverClient, s3Storage, fakeHttpClient, messageBusPublisher)
+
+    pending // Needs valid s3 credentials to run
+
+    val repoId = RepoId.generate()
+    fakeKeyserverClient.createRoot(repoId).futureValue
+
+    val entity = HttpEntity(ByteString("""
+                                         |Like all the men of the Library, in my younger days I traveled;
+                                         |I have journeyed in quest of a book, perhaps the catalog of catalogs.
+                                       """.stripMargin))
+
+    val fileBodyPart = BodyPart("file", entity, Map("filename" -> "babel.txt"))
+
+    val form = Multipart.FormData(fileBodyPart)
+
+    Put(s"/repo/${repoId.show}/targets/some/target/funky/thing?name=pkgname&version=pkgversion&desc=wat", form) ~> routes ~> check {
+      status shouldBe StatusCodes.OK
+    }
+
+    Get(s"/repo/${repoId.show}/targets/some/target/funky/thing") ~> routes ~> check {
+      status shouldBe StatusCodes.Found
+      header("Location").get.value() should include("amazonaws.com")
+    }
+  }
 }