From aaa7cd716779499da6ad7c81c331274c77f1a895 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 18 Jun 2018 19:47:24 +0530 Subject: [PATCH 01/14] Initial implementation --- common/scala/build.gradle | 2 + .../scala/src/main/resources/reference.conf | 2 + .../src/main/resources/s3-reference.conf | 74 +++++++ .../main/scala/whisk/core/WhiskConfig.scala | 2 + .../core/database/s3/S3AttachmentStore.scala | 186 ++++++++++++++++++ tests/build.gradle | 2 + .../s3/S3AttachmentStoreMinioTests.scala | 137 +++++++++++++ 7 files changed, 405 insertions(+) create mode 100644 common/scala/src/main/resources/s3-reference.conf create mode 100644 common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala create mode 100644 tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala diff --git a/common/scala/build.gradle b/common/scala/build.gradle index 9669b580a1e..326423bc169 100644 --- a/common/scala/build.gradle +++ b/common/scala/build.gradle @@ -75,6 +75,8 @@ dependencies { compile 'io.reactivex:rxscala_2.11:0.26.5' compile 'io.reactivex:rxjava-reactive-streams:1.2.1' compile 'com.microsoft.azure:azure-cosmosdb:2.0.0' + + compile 'com.lightbend.akka:akka-stream-alpakka-s3_2.11:0.19' scoverage gradle.scoverage.deps } diff --git a/common/scala/src/main/resources/reference.conf b/common/scala/src/main/resources/reference.conf index 2f2d898ae32..d600c349d67 100644 --- a/common/scala/src/main/resources/reference.conf +++ b/common/scala/src/main/resources/reference.conf @@ -1,6 +1,8 @@ # Licensed to the Apache Software Foundation (ASF) under one or more contributor # license agreements; and to You under the Apache License, Version 2.0. +include "s3-reference.conf" + whisk.spi { ArtifactStoreProvider = whisk.core.database.CouchDbStoreProvider ActivationStoreProvider = whisk.core.entity.ArtifactActivationStoreProvider diff --git a/common/scala/src/main/resources/s3-reference.conf b/common/scala/src/main/resources/s3-reference.conf new file mode 100644 index 00000000000..16b035fcf06 --- /dev/null +++ b/common/scala/src/main/resources/s3-reference.conf @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements; and to You under the Apache License, Version 2.0. + +whisk { + s3 { + # See https://developer.lightbend.com/docs/alpakka/current/s3.html#usage + alpakka { + # whether the buffer request chunks (up to 5MB each) to "memory" or "disk" + buffer = "memory" + + # location for temporary files, if buffer is set to "disk". If empty, uses the standard java temp path. + disk-buffer-path = "" + + proxy { + # hostname of the proxy. If undefined ("") proxy is not enabled. + host = "" + port = 8000 + + # if "secure" is set to "true" then HTTPS will be used for all requests to S3, otherwise HTTP will be used + secure = true + } + + # default values for AWS configuration. If credentials and/or region are not specified when creating S3Client, + # these values will be used. + aws { + # If this section is absent, the fallback behavior is to use the + # com.amazonaws.auth.DefaultAWSCredentialsProviderChain instance to resolve credentials + credentials { + # supported providers: + # anon - anonymous requests ("no auth") + # static - static credentials, + # required params: + # access-key-id + # secret-access-key + # optional: + # token + # default: as described in com.amazonaws.auth.DefaultAWSCredentialsProviderChain docs, + # attempts to get the credentials from either: + # - environment variables + # - system properties + # - credentials file + # - EC2 credentials service + # - IAM / metadata + provider = default + } + + # If this section is absent, the fallback behavior is to use the + # com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain instance to resolve region + region { + # supported providers: + # static - static credentials, + # required params: + # default-region + # default: as described in com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain docs, + # attempts to get the region from either: + # - environment variables + # - system properties + # - progile file + # - EC2 metadata + provider = default + } + } + + # Enable path style access to s3, i.e. "https://s3-eu-west-1.amazonaws.com/my.bucket/myobject" + # Default is virtual-hosted style. + # When using virtual hosted–style buckets with SSL, the S3 wild card certificate only matches buckets that do not contain periods. + # Buckets containing periods will lead to certificate errors. In those cases it's useful to enable path-style access. + path-style-access = true + + # Custom endpoint url, used for alternate s3 implementations + # endpoint-url = null + } + } +} \ No newline at end of file diff --git a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala index fd6eeec7e5b..25e19a45eed 100644 --- a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala @@ -245,4 +245,6 @@ object ConfigKeys { val containerProxy = "whisk.container-proxy" val containerProxyTimeouts = s"$containerProxy.timeouts" + val s3 = "whisk.s3" + } diff --git a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala new file mode 100644 index 00000000000..849aa36c6ac --- /dev/null +++ b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.s3 + +import akka.actor.ActorSystem +import akka.http.scaladsl.model.ContentType +import akka.stream.ActorMaterializer +import akka.stream.alpakka.s3.scaladsl.S3Client +import akka.stream.alpakka.s3.{S3Exception, S3Settings} +import akka.stream.scaladsl.{Sink, Source} +import akka.util.ByteString +import com.typesafe.config.Config +import pureconfig.loadConfigOrThrow +import whisk.common.LoggingMarkers.{DATABASE_ATTS_DELETE, DATABASE_ATT_DELETE, DATABASE_ATT_GET, DATABASE_ATT_SAVE} +import whisk.common.{Logging, TransactionId} +import whisk.core.ConfigKeys +import whisk.core.database.StoreUtils._ +import whisk.core.database._ +import whisk.core.entity.DocId + +import scala.concurrent.{ExecutionContext, Future} +import scala.reflect.ClassTag + +object S3AttachmentStoreProvider extends AttachmentStoreProvider { + val alpakkaConfigKey = s"${ConfigKeys.s3}.alpakka" + case class S3Config(bucket: String) { + def prefixFor[D](implicit tag: ClassTag[D]): String = { + tag.runtimeClass.getSimpleName.toLowerCase + } + } + + override def makeStore[D <: DocumentSerializer: ClassTag]()(implicit actorSystem: ActorSystem, + logging: Logging, + materializer: ActorMaterializer): AttachmentStore = { + val client = new S3Client(S3Settings(alpakkaConfigKey)) + val config = loadConfigOrThrow[S3Config](ConfigKeys.s3) + new S3AttachmentStore(client, config.bucket, config.prefixFor[D]) + } + + def makeStore[D <: DocumentSerializer: ClassTag](config: Config)(implicit actorSystem: ActorSystem, + logging: Logging, + materializer: ActorMaterializer): AttachmentStore = { + val client = new S3Client(S3Settings(config, alpakkaConfigKey)) + val s3config = loadConfigOrThrow[S3Config](config, ConfigKeys.s3) + new S3AttachmentStore(client, s3config.bucket, s3config.prefixFor[D]) + } + +} +class S3AttachmentStore(client: S3Client, bucket: String, prefix: String)(implicit system: ActorSystem, + logging: Logging, + materializer: ActorMaterializer) + extends AttachmentStore { + private val metaContentType = "content-type" + private val amzMetaContentType = s"x-amz-meta-$metaContentType" + + override val scheme = "s3" + + override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher + + override protected[core] def attach( + docId: DocId, + name: String, + contentType: ContentType, + docStream: Source[ByteString, _])(implicit transid: TransactionId): Future[AttachResult] = { + require(name != null, "name undefined") + val start = + transid.started(this, DATABASE_ATT_SAVE, s"[ATT_PUT] uploading attachment '$name' of document 'id: $docId'") + + //A possible optimization for small attachments < 5MB can be to use putObject instead of multipartUpload + //and thus use 1 remote call instead of 3 + val f = docStream + .runWith(combinedSink(client.multipartUpload(bucket, objectKey(docId, name), contentType))) + .map(r => AttachResult(r.digest, r.length)) + + f.onSuccess({ + case _ => + transid + .finished(this, start, s"[ATT_PUT] '$prefix' completed uploading attachment '$name' of document 'id: $docId'") + }) + + reportFailure( + f, + start, + failure => s"[ATT_PUT] '$prefix' internal error, name: '$name', doc: '$docId', failure: '${failure.getMessage}'") + } + + override protected[core] def readAttachment[T](docId: DocId, name: String, sink: Sink[ByteString, Future[T]])( + implicit transid: TransactionId): Future[T] = { + require(name != null, "name undefined") + val start = + transid.started( + this, + DATABASE_ATT_GET, + s"[ATT_GET] '$prefix' finding attachment '$name' of document 'id: $docId'") + val (source, _) = client.download(bucket, objectKey(docId, name)) + + val f = source.runWith(sink) + + val g = f.transform( + { s => + transid + .finished(this, start, s"[ATT_GET] '$prefix' completed: found attachment '$name' of document 'id: $docId'") + s + }, { + case s: S3Exception if s.code == "NoSuchKey" => + transid + .finished( + this, + start, + s"[ATT_GET] '$prefix', retrieving attachment '$name' of document 'id: $docId'; not found.") + NoDocumentException("Not found on 'readAttachment'.") + case e => e + }) + + reportFailure( + g, + start, + failure => + s"[ATT_GET] '$prefix' internal error, name: '$name', doc: 'id: $docId', failure: '${failure.getMessage}'") + } + + override protected[core] def deleteAttachments(docId: DocId)(implicit transid: TransactionId): Future[Boolean] = { + val start = + transid.started(this, DATABASE_ATTS_DELETE, s"[ATT_DELETE] deleting attachments of document 'id: $docId'") + + //S3 provides API to delete multiple objects in single call however alpakka client + //currently does not support that and also in current usage 1 docs has at most 1 attachment + //so current approach would also involve 2 remote calls + val f = client + .listBucket(bucket, Some(objectKeyPrefix(docId))) + .mapAsync(1)(bc => client.deleteObject(bc.bucketName, bc.key)) + .runWith(Sink.seq) + .map(_ => true) + + f.onSuccess { + case _ => + transid.finished(this, start, s"[ATTS_DELETE] completed: deleting attachments of document 'id: $docId'") + } + + reportFailure( + f, + start, + failure => s"[ATTS_DELETE] '$prefix' internal error, doc: '$docId', failure: '${failure.getMessage}'") + } + + override protected[core] def deleteAttachment(docId: DocId, name: String)( + implicit transid: TransactionId): Future[Boolean] = { + val start = + transid.started(this, DATABASE_ATT_DELETE, s"[ATT_DELETE] deleting attachment '$name' of document 'id: $docId'") + + val f = client + .deleteObject(bucket, objectKey(docId, name)) + .map(_ => true) + + f.onSuccess { + case _ => + transid.finished(this, start, s"[ATT_DELETE] completed: deleting attachment '$name' of document 'id: $docId'") + } + + reportFailure( + f, + start, + failure => s"[ATT_DELETE] '$prefix' internal error, doc: '$docId', failure: '${failure.getMessage}'") + } + + override def shutdown(): Unit = {} + + private def objectKey(id: DocId, name: String): String = s"$prefix/${id.id}/$name" + + private def objectKeyPrefix(id: DocId): String = s"$prefix/${id.id}" +} diff --git a/tests/build.gradle b/tests/build.gradle index b90c99d102d..05561e7e8f2 100644 --- a/tests/build.gradle +++ b/tests/build.gradle @@ -148,6 +148,8 @@ dependencies { compile 'io.opentracing:opentracing-mock:0.31.0' compile "org.apache.curator:curator-test:${gradle.curator.version}" + compile "com.amazonaws:aws-java-sdk-s3:1.11.295" + compile project(':common:scala') compile project(':core:controller') compile project(':core:invoker') diff --git a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala new file mode 100644 index 00000000000..05d15c0f9a2 --- /dev/null +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.s3 + +import java.io.File +import java.net.ServerSocket + +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration +import com.amazonaws.services.s3.AmazonS3ClientBuilder +import com.typesafe.config.ConfigFactory +import common.{SimpleExec, WhiskProperties} +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +import org.scalatest.{BeforeAndAfterAll, FlatSpec} +import whisk.common.TransactionId +import whisk.core.database.AttachmentStore +import whisk.core.database.test.AttachmentStoreBehaviors +import whisk.core.entity.WhiskEntity + +import scala.concurrent.Future +import scala.concurrent.duration._ + +@RunWith(classOf[JUnitRunner]) +class S3AttachmentStoreMinioTests extends FlatSpec with AttachmentStoreBehaviors with BeforeAndAfterAll { + override lazy val store: AttachmentStore = { + val config = ConfigFactory.parseString(s""" + |whisk { + | s3 { + | alpakka { + | aws { + | credentials { + | provider = static + | access-key-id = "$accessKey" + | secret-access-key = "$secretAccessKey" + | } + | region { + | provider = static + | default-region = us-west-2 + | } + | } + | endpoint-url = "http://localhost:$port" + | } + | bucket = "$bucket" + | } + |} + """.stripMargin).withFallback(ConfigFactory.load()) + S3AttachmentStoreProvider.makeStore[WhiskEntity](config) + } + + override def storeType: String = "S3" + + override def garbageCollectAttachments: Boolean = false + + private val accessKey = "TESTKEY" + private val secretAccessKey = "TESTSECRET" + private val port = freePort() + private val bucket = "test-ow-travis" + + override def afterAll(): Unit = { + super.afterAll() + val containerId = dockerExec("ps -q --filter ancestor=minio/minio") + containerId.split("\n").map(_.trim).foreach(id => dockerExec(s"stop $id")) + println(s"Stopped minio container") + } + + def createTestBucket(): Unit = { + val endpoint = new EndpointConfiguration(s"http://localhost:$port", "us-west-2") + val client = AmazonS3ClientBuilder.standard + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(endpoint) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretAccessKey))) + .build + + retry(() => Future.successful(client.createBucket(bucket)), 1.minute) + println(s"Created bucket $bucket") + } + + override protected def beforeAll(): Unit = { + super.beforeAll() + implicit val tid: TransactionId = transid() + dockerExec( + s"run -d -e MINIO_ACCESS_KEY=$accessKey -e MINIO_SECRET_KEY=$secretAccessKey -p $port:9000 minio/minio server /data") + println(s"Started minio on $port") + createTestBucket() + } + + private def dockerExec(cmd: String): String = { + implicit val tid: TransactionId = transid() + val command = s"$dockerCmd $cmd" + val cmdSeq = command.split(" ").map(_.trim).filter(_.nonEmpty) + val (out, err, code) = SimpleExec.syncRunCmd(cmdSeq) + assert(code == 0, s"Error occurred for command '$command'. Exit code: $code, Error: $err") + out + } + + //Taken from ActionContainer + private lazy val dockerBin: String = { + List("/usr/bin/docker", "/usr/local/bin/docker") + .find { bin => + new File(bin).isFile + } + .getOrElse(???) // This fails if the docker binary couldn't be located. + } + + private lazy val dockerCmd: String = { + val version = WhiskProperties.getProperty("whisk.version.name") + // Check if we are running on docker-machine env. + val hostStr = if (version.toLowerCase().contains("mac")) { + s" --host tcp://${WhiskProperties.getMainDockerEndpoint} " + } else { + "" + } + s"$dockerBin $hostStr" + } + + private def freePort(): Int = { + val socket = new ServerSocket(0) + try socket.getLocalPort + finally if (socket != null) socket.close() + } +} From 9b231ecab2a5073f87b1ee5c07c105e72cf9e8d4 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 18 Jun 2018 19:48:40 +0530 Subject: [PATCH 02/14] Remove unused variables --- .../main/scala/whisk/core/database/s3/S3AttachmentStore.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala index 849aa36c6ac..d29482015db 100644 --- a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala +++ b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala @@ -65,9 +65,6 @@ class S3AttachmentStore(client: S3Client, bucket: String, prefix: String)(implic logging: Logging, materializer: ActorMaterializer) extends AttachmentStore { - private val metaContentType = "content-type" - private val amzMetaContentType = s"x-amz-meta-$metaContentType" - override val scheme = "s3" override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher From 11d676bf1d7d58e230fe96a1a1bcc1660dba402e Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 18 Jun 2018 20:40:17 +0530 Subject: [PATCH 03/14] Add Minio based tests --- .../core/database/s3/S3AttachmentStore.scala | 14 +- .../MemoryArtifactStoreBehaviorBase.scala | 61 ++++++++ .../memory/MemoryArtifactStoreTests.scala | 30 +--- .../s3/S3AttachmentStoreMinioTests.scala | 110 +------------- .../s3/S3MemoryArtifactStoreTests.scala | 38 +++++ .../whisk/core/database/s3/S3Minio.scala | 134 ++++++++++++++++++ 6 files changed, 249 insertions(+), 138 deletions(-) create mode 100644 tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala create mode 100644 tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala create mode 100644 tests/src/test/scala/whisk/core/database/s3/S3Minio.scala diff --git a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala index d29482015db..570ce2af4ab 100644 --- a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala +++ b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala @@ -65,7 +65,8 @@ class S3AttachmentStore(client: S3Client, bucket: String, prefix: String)(implic logging: Logging, materializer: ActorMaterializer) extends AttachmentStore { - override val scheme = "s3" + //TODO Use 's3s' for now as due to some bug `Uri.from(scheme="s3") is causing issue + override val scheme = "s3s" override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher @@ -114,7 +115,7 @@ class S3AttachmentStore(client: S3Client, bucket: String, prefix: String)(implic .finished(this, start, s"[ATT_GET] '$prefix' completed: found attachment '$name' of document 'id: $docId'") s }, { - case s: S3Exception if s.code == "NoSuchKey" => + case s: Throwable if isMissingKeyException(s) => transid .finished( this, @@ -180,4 +181,13 @@ class S3AttachmentStore(client: S3Client, bucket: String, prefix: String)(implic private def objectKey(id: DocId, name: String): String = s"$prefix/${id.id}/$name" private def objectKeyPrefix(id: DocId): String = s"$prefix/${id.id}" + + private def isMissingKeyException(e: Throwable): Boolean = { + //In some case S3Exception is a sub cause. So need to recurse + e match { + case s: S3Exception if s.code == "NoSuchKey" => true + case t if t != null && isMissingKeyException(t.getCause) => true + case _ => false + } + } } diff --git a/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala b/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala new file mode 100644 index 00000000000..72b2f8e64a8 --- /dev/null +++ b/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreBehaviorBase.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.memory + +import org.scalatest.FlatSpec +import whisk.core.database.{ArtifactStore, AttachmentStore, DocumentSerializer} +import whisk.core.database.test.behavior.ArtifactStoreBehaviorBase +import whisk.core.entity.{ + DocumentReader, + WhiskActivation, + WhiskAuth, + WhiskDocumentReader, + WhiskEntity, + WhiskEntityJsonFormat +} + +import scala.reflect.{classTag, ClassTag} + +trait MemoryArtifactStoreBehaviorBase extends FlatSpec with ArtifactStoreBehaviorBase { + override def storeType = "Memory" + + override lazy val authStore = { + implicit val docReader: DocumentReader = WhiskDocumentReader + MemoryArtifactStoreProvider.makeArtifactStore[WhiskAuth](getAttachmentStore[WhiskAuth]()) + } + + override lazy val entityStore = + MemoryArtifactStoreProvider.makeArtifactStore[WhiskEntity](getAttachmentStore[WhiskEntity]())( + classTag[WhiskEntity], + WhiskEntityJsonFormat, + WhiskDocumentReader, + actorSystem, + logging, + materializer) + + override lazy val activationStore = { + implicit val docReader: DocumentReader = WhiskDocumentReader + MemoryArtifactStoreProvider.makeArtifactStore[WhiskActivation](getAttachmentStore[WhiskActivation]()) + } + + override protected def getAttachmentStore(store: ArtifactStore[_]) = + Some(store.asInstanceOf[MemoryArtifactStore[_]].attachmentStore) + + protected def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore = + MemoryAttachmentStoreProvider.makeStore() +} diff --git a/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreTests.scala b/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreTests.scala index c47daaacead..e9860a02ec2 100644 --- a/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreTests.scala +++ b/tests/src/test/scala/whisk/core/database/memory/MemoryArtifactStoreTests.scala @@ -20,35 +20,7 @@ package whisk.core.database.memory import org.junit.runner.RunWith import org.scalatest.FlatSpec import org.scalatest.junit.JUnitRunner -import whisk.core.database.ArtifactStore import whisk.core.database.test.behavior.ArtifactStoreBehavior -import whisk.core.entity._ - -import scala.reflect.classTag @RunWith(classOf[JUnitRunner]) -class MemoryArtifactStoreTests extends FlatSpec with ArtifactStoreBehavior { - override def storeType = "Memory" - - override val authStore = { - implicit val docReader: DocumentReader = WhiskDocumentReader - MemoryArtifactStoreProvider.makeStore[WhiskAuth]() - } - - override val entityStore = - MemoryArtifactStoreProvider.makeStore[WhiskEntity]()( - classTag[WhiskEntity], - WhiskEntityJsonFormat, - WhiskDocumentReader, - actorSystem, - logging, - materializer) - - override val activationStore = { - implicit val docReader: DocumentReader = WhiskDocumentReader - MemoryArtifactStoreProvider.makeStore[WhiskActivation]() - } - - override protected def getAttachmentStore(store: ArtifactStore[_]) = - Some(store.asInstanceOf[MemoryArtifactStore[_]].attachmentStore) -} +class MemoryArtifactStoreTests extends FlatSpec with MemoryArtifactStoreBehaviorBase with ArtifactStoreBehavior diff --git a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala index 05d15c0f9a2..d61c3e43e66 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala @@ -17,121 +17,17 @@ package whisk.core.database.s3 -import java.io.File -import java.net.ServerSocket - -import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration -import com.amazonaws.services.s3.AmazonS3ClientBuilder -import com.typesafe.config.ConfigFactory -import common.{SimpleExec, WhiskProperties} import org.junit.runner.RunWith +import org.scalatest.FlatSpec import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfterAll, FlatSpec} -import whisk.common.TransactionId -import whisk.core.database.AttachmentStore import whisk.core.database.test.AttachmentStoreBehaviors import whisk.core.entity.WhiskEntity -import scala.concurrent.Future -import scala.concurrent.duration._ - @RunWith(classOf[JUnitRunner]) -class S3AttachmentStoreMinioTests extends FlatSpec with AttachmentStoreBehaviors with BeforeAndAfterAll { - override lazy val store: AttachmentStore = { - val config = ConfigFactory.parseString(s""" - |whisk { - | s3 { - | alpakka { - | aws { - | credentials { - | provider = static - | access-key-id = "$accessKey" - | secret-access-key = "$secretAccessKey" - | } - | region { - | provider = static - | default-region = us-west-2 - | } - | } - | endpoint-url = "http://localhost:$port" - | } - | bucket = "$bucket" - | } - |} - """.stripMargin).withFallback(ConfigFactory.load()) - S3AttachmentStoreProvider.makeStore[WhiskEntity](config) - } +class S3AttachmentStoreMinioTests extends FlatSpec with AttachmentStoreBehaviors with S3Minio { + override lazy val store = makeS3Store[WhiskEntity] override def storeType: String = "S3" override def garbageCollectAttachments: Boolean = false - - private val accessKey = "TESTKEY" - private val secretAccessKey = "TESTSECRET" - private val port = freePort() - private val bucket = "test-ow-travis" - - override def afterAll(): Unit = { - super.afterAll() - val containerId = dockerExec("ps -q --filter ancestor=minio/minio") - containerId.split("\n").map(_.trim).foreach(id => dockerExec(s"stop $id")) - println(s"Stopped minio container") - } - - def createTestBucket(): Unit = { - val endpoint = new EndpointConfiguration(s"http://localhost:$port", "us-west-2") - val client = AmazonS3ClientBuilder.standard - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration(endpoint) - .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretAccessKey))) - .build - - retry(() => Future.successful(client.createBucket(bucket)), 1.minute) - println(s"Created bucket $bucket") - } - - override protected def beforeAll(): Unit = { - super.beforeAll() - implicit val tid: TransactionId = transid() - dockerExec( - s"run -d -e MINIO_ACCESS_KEY=$accessKey -e MINIO_SECRET_KEY=$secretAccessKey -p $port:9000 minio/minio server /data") - println(s"Started minio on $port") - createTestBucket() - } - - private def dockerExec(cmd: String): String = { - implicit val tid: TransactionId = transid() - val command = s"$dockerCmd $cmd" - val cmdSeq = command.split(" ").map(_.trim).filter(_.nonEmpty) - val (out, err, code) = SimpleExec.syncRunCmd(cmdSeq) - assert(code == 0, s"Error occurred for command '$command'. Exit code: $code, Error: $err") - out - } - - //Taken from ActionContainer - private lazy val dockerBin: String = { - List("/usr/bin/docker", "/usr/local/bin/docker") - .find { bin => - new File(bin).isFile - } - .getOrElse(???) // This fails if the docker binary couldn't be located. - } - - private lazy val dockerCmd: String = { - val version = WhiskProperties.getProperty("whisk.version.name") - // Check if we are running on docker-machine env. - val hostStr = if (version.toLowerCase().contains("mac")) { - s" --host tcp://${WhiskProperties.getMainDockerEndpoint} " - } else { - "" - } - s"$dockerBin $hostStr" - } - - private def freePort(): Int = { - val socket = new ServerSocket(0) - try socket.getLocalPort - finally if (socket != null) socket.close() - } } diff --git a/tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala new file mode 100644 index 00000000000..9a375d1ae64 --- /dev/null +++ b/tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.s3 + +import org.junit.runner.RunWith +import org.scalatest.FlatSpec +import org.scalatest.junit.JUnitRunner +import whisk.core.database.memory.MemoryArtifactStoreBehaviorBase +import whisk.core.database.test.behavior.ArtifactStoreAttachmentBehaviors +import whisk.core.database.{AttachmentStore, DocumentSerializer} + +import scala.reflect.ClassTag + +@RunWith(classOf[JUnitRunner]) +class S3MemoryArtifactStoreTests + extends FlatSpec + with MemoryArtifactStoreBehaviorBase + with ArtifactStoreAttachmentBehaviors + with S3Minio { + + override def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore = + makeS3Store[D]() +} diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala new file mode 100644 index 00000000000..0ad76fbf19b --- /dev/null +++ b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.s3 + +import java.io.File +import java.net.ServerSocket + +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration +import com.amazonaws.services.s3.AmazonS3ClientBuilder +import com.typesafe.config.ConfigFactory +import common.{SimpleExec, StreamLogging, WhiskProperties} +import org.scalatest.{BeforeAndAfterAll, FlatSpec} +import whisk.common.{Logging, TransactionId} +import whisk.core.database.test.DbUtils +import whisk.core.database.{AttachmentStore, DocumentSerializer} + +import scala.concurrent.Future +import scala.concurrent.duration._ +import scala.reflect.ClassTag + +trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLogging { + def makeS3Store[D <: DocumentSerializer: ClassTag]()(implicit actorSystem: ActorSystem, + logging: Logging, + materializer: ActorMaterializer): AttachmentStore = { + val config = ConfigFactory.parseString(s""" + |whisk { + | s3 { + | alpakka { + | aws { + | credentials { + | provider = static + | access-key-id = "$accessKey" + | secret-access-key = "$secretAccessKey" + | } + | region { + | provider = static + | default-region = us-west-2 + | } + | } + | endpoint-url = "http://localhost:$port" + | } + | bucket = "$bucket" + | } + |} + """.stripMargin).withFallback(ConfigFactory.load()) + S3AttachmentStoreProvider.makeStore[D](config) + } + + private val accessKey = "TESTKEY" + private val secretAccessKey = "TESTSECRET" + private val port = freePort() + private val bucket = "test-ow-travis" + + override def afterAll(): Unit = { + super.afterAll() + val containerId = dockerExec("ps -q --filter ancestor=minio/minio") + containerId.split("\n").map(_.trim).foreach(id => dockerExec(s"stop $id")) + println(s"Stopped minio container") + } + + def createTestBucket(): Unit = { + val endpoint = new EndpointConfiguration(s"http://localhost:$port", "us-west-2") + val client = AmazonS3ClientBuilder.standard + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(endpoint) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretAccessKey))) + .build + + retry(() => Future.successful(client.createBucket(bucket)), 1.minute) + println(s"Created bucket $bucket") + } + + override protected def beforeAll(): Unit = { + super.beforeAll() + implicit val tid: TransactionId = transid() + dockerExec( + s"run -d -e MINIO_ACCESS_KEY=$accessKey -e MINIO_SECRET_KEY=$secretAccessKey -p $port:9000 minio/minio server /data") + println(s"Started minio on $port") + createTestBucket() + } + + private def dockerExec(cmd: String): String = { + implicit val tid: TransactionId = TransactionId.testing + val command = s"$dockerCmd $cmd" + val cmdSeq = command.split(" ").map(_.trim).filter(_.nonEmpty) + val (out, err, code) = SimpleExec.syncRunCmd(cmdSeq) + assert(code == 0, s"Error occurred for command '$command'. Exit code: $code, Error: $err") + out + } + + //Taken from ActionContainer + private lazy val dockerBin: String = { + List("/usr/bin/docker", "/usr/local/bin/docker") + .find { bin => + new File(bin).isFile + } + .getOrElse(???) // This fails if the docker binary couldn't be located. + } + + private lazy val dockerCmd: String = { + val version = WhiskProperties.getProperty("whisk.version.name") + // Check if we are running on docker-machine env. + val hostStr = if (version.toLowerCase().contains("mac")) { + s" --host tcp://${WhiskProperties.getMainDockerEndpoint} " + } else { + "" + } + s"$dockerBin $hostStr" + } + + private def freePort(): Int = { + val socket = new ServerSocket(0) + try socket.getLocalPort + finally if (socket != null) socket.close() + } +} From e4727ab26492176ba88ee1044736092477125a0a Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Mon, 18 Jun 2018 20:47:57 +0530 Subject: [PATCH 04/14] add new line --- common/scala/src/main/resources/s3-reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/scala/src/main/resources/s3-reference.conf b/common/scala/src/main/resources/s3-reference.conf index 16b035fcf06..62519023b2b 100644 --- a/common/scala/src/main/resources/s3-reference.conf +++ b/common/scala/src/main/resources/s3-reference.conf @@ -71,4 +71,4 @@ whisk { # endpoint-url = null } } -} \ No newline at end of file +} From 64443b458b94abdabfc6fe72e6ec6bac36f5000a Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 09:59:22 +0530 Subject: [PATCH 05/14] Move config to whisk.db.s3 --- .../src/main/resources/s3-reference.conf | 120 +++++++++--------- .../main/scala/whisk/core/WhiskConfig.scala | 2 +- .../whisk/core/database/s3/S3Minio.scala | 4 +- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/common/scala/src/main/resources/s3-reference.conf b/common/scala/src/main/resources/s3-reference.conf index 62519023b2b..ec34bec9a21 100644 --- a/common/scala/src/main/resources/s3-reference.conf +++ b/common/scala/src/main/resources/s3-reference.conf @@ -2,73 +2,75 @@ # license agreements; and to You under the Apache License, Version 2.0. whisk { - s3 { - # See https://developer.lightbend.com/docs/alpakka/current/s3.html#usage - alpakka { - # whether the buffer request chunks (up to 5MB each) to "memory" or "disk" - buffer = "memory" + db { + s3 { + # See https://developer.lightbend.com/docs/alpakka/current/s3.html#usage + alpakka { + # whether the buffer request chunks (up to 5MB each) to "memory" or "disk" + buffer = "memory" - # location for temporary files, if buffer is set to "disk". If empty, uses the standard java temp path. - disk-buffer-path = "" + # location for temporary files, if buffer is set to "disk". If empty, uses the standard java temp path. + disk-buffer-path = "" - proxy { - # hostname of the proxy. If undefined ("") proxy is not enabled. - host = "" - port = 8000 + proxy { + # hostname of the proxy. If undefined ("") proxy is not enabled. + host = "" + port = 8000 - # if "secure" is set to "true" then HTTPS will be used for all requests to S3, otherwise HTTP will be used - secure = true - } - - # default values for AWS configuration. If credentials and/or region are not specified when creating S3Client, - # these values will be used. - aws { - # If this section is absent, the fallback behavior is to use the - # com.amazonaws.auth.DefaultAWSCredentialsProviderChain instance to resolve credentials - credentials { - # supported providers: - # anon - anonymous requests ("no auth") - # static - static credentials, - # required params: - # access-key-id - # secret-access-key - # optional: - # token - # default: as described in com.amazonaws.auth.DefaultAWSCredentialsProviderChain docs, - # attempts to get the credentials from either: - # - environment variables - # - system properties - # - credentials file - # - EC2 credentials service - # - IAM / metadata - provider = default + # if "secure" is set to "true" then HTTPS will be used for all requests to S3, otherwise HTTP will be used + secure = true } - # If this section is absent, the fallback behavior is to use the - # com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain instance to resolve region - region { - # supported providers: - # static - static credentials, - # required params: - # default-region - # default: as described in com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain docs, - # attempts to get the region from either: - # - environment variables - # - system properties - # - progile file - # - EC2 metadata - provider = default + # default values for AWS configuration. If credentials and/or region are not specified when creating S3Client, + # these values will be used. + aws { + # If this section is absent, the fallback behavior is to use the + # com.amazonaws.auth.DefaultAWSCredentialsProviderChain instance to resolve credentials + credentials { + # supported providers: + # anon - anonymous requests ("no auth") + # static - static credentials, + # required params: + # access-key-id + # secret-access-key + # optional: + # token + # default: as described in com.amazonaws.auth.DefaultAWSCredentialsProviderChain docs, + # attempts to get the credentials from either: + # - environment variables + # - system properties + # - credentials file + # - EC2 credentials service + # - IAM / metadata + provider = default + } + + # If this section is absent, the fallback behavior is to use the + # com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain instance to resolve region + region { + # supported providers: + # static - static credentials, + # required params: + # default-region + # default: as described in com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain docs, + # attempts to get the region from either: + # - environment variables + # - system properties + # - progile file + # - EC2 metadata + provider = default + } } - } - # Enable path style access to s3, i.e. "https://s3-eu-west-1.amazonaws.com/my.bucket/myobject" - # Default is virtual-hosted style. - # When using virtual hosted–style buckets with SSL, the S3 wild card certificate only matches buckets that do not contain periods. - # Buckets containing periods will lead to certificate errors. In those cases it's useful to enable path-style access. - path-style-access = true + # Enable path style access to s3, i.e. "https://s3-eu-west-1.amazonaws.com/my.bucket/myobject" + # Default is virtual-hosted style. + # When using virtual hosted–style buckets with SSL, the S3 wild card certificate only matches buckets that do not contain periods. + # Buckets containing periods will lead to certificate errors. In those cases it's useful to enable path-style access. + path-style-access = true - # Custom endpoint url, used for alternate s3 implementations - # endpoint-url = null + # Custom endpoint url, used for alternate s3 implementations + # endpoint-url = null + } } } } diff --git a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala index 25e19a45eed..61032508ddc 100644 --- a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala @@ -245,6 +245,6 @@ object ConfigKeys { val containerProxy = "whisk.container-proxy" val containerProxyTimeouts = s"$containerProxy.timeouts" - val s3 = "whisk.s3" + val s3 = s"$db.s3" } diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala index 0ad76fbf19b..625ec48ae12 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala @@ -42,7 +42,8 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLo materializer: ActorMaterializer): AttachmentStore = { val config = ConfigFactory.parseString(s""" |whisk { - | s3 { + | db{ + | s3 { | alpakka { | aws { | credentials { @@ -58,6 +59,7 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLo | endpoint-url = "http://localhost:$port" | } | bucket = "$bucket" + | } | } |} """.stripMargin).withFallback(ConfigFactory.load()) From e881ac4e864f1bb5a0f48807b7acddbedf46c9f0 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 11:01:04 +0530 Subject: [PATCH 06/14] Exclude unused dependencies aws sdk pulls in quite a few dependencies which are not used with akka s3 based usage as it uses its own http client instead of commons http client. So those can be safely excluded --- common/scala/build.gradle | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/scala/build.gradle b/common/scala/build.gradle index 326423bc169..a6f0c6fba8e 100644 --- a/common/scala/build.gradle +++ b/common/scala/build.gradle @@ -76,7 +76,12 @@ dependencies { compile 'io.reactivex:rxjava-reactive-streams:1.2.1' compile 'com.microsoft.azure:azure-cosmosdb:2.0.0' - compile 'com.lightbend.akka:akka-stream-alpakka-s3_2.11:0.19' + compile ('com.lightbend.akka:akka-stream-alpakka-s3_2.11:0.19') { + exclude group: 'commons-logging' + exclude group: 'org.apache.httpcomponents' //Not used as alpakka uses akka-http + exclude group: 'com.fasterxml.jackson.core' + exclude group: 'com.fasterxml.jackson.dataformat' + } scoverage gradle.scoverage.deps } From e6b8e4828c994979719743d83da1f516916ec965 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 11:19:39 +0530 Subject: [PATCH 07/14] retry create bucket --- .../whisk/core/database/s3/S3Minio.scala | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala index 625ec48ae12..56802ae6346 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala @@ -32,7 +32,6 @@ import whisk.common.{Logging, TransactionId} import whisk.core.database.test.DbUtils import whisk.core.database.{AttachmentStore, DocumentSerializer} -import scala.concurrent.Future import scala.concurrent.duration._ import scala.reflect.ClassTag @@ -71,6 +70,15 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLo private val port = freePort() private val bucket = "test-ow-travis" + override protected def beforeAll(): Unit = { + implicit val tid: TransactionId = transid() + dockerExec( + s"run -d -e MINIO_ACCESS_KEY=$accessKey -e MINIO_SECRET_KEY=$secretAccessKey -p $port:9000 minio/minio server /data") + println(s"Started minio on $port") + createTestBucket() + super.beforeAll() + } + override def afterAll(): Unit = { super.afterAll() val containerId = dockerExec("ps -q --filter ancestor=minio/minio") @@ -86,19 +94,10 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLo .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretAccessKey))) .build - retry(() => Future.successful(client.createBucket(bucket)), 1.minute) + whisk.utils.retry(client.createBucket(bucket), 6, Some(1.minute)) println(s"Created bucket $bucket") } - override protected def beforeAll(): Unit = { - super.beforeAll() - implicit val tid: TransactionId = transid() - dockerExec( - s"run -d -e MINIO_ACCESS_KEY=$accessKey -e MINIO_SECRET_KEY=$secretAccessKey -p $port:9000 minio/minio server /data") - println(s"Started minio on $port") - createTestBucket() - } - private def dockerExec(cmd: String): String = { implicit val tid: TransactionId = TransactionId.testing val command = s"$dockerCmd $cmd" From 712bad9b4115c014ee732a74a6b30abf851178f5 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 11:22:36 +0530 Subject: [PATCH 08/14] Remove unused DBUtils trait --- tests/src/test/scala/whisk/core/database/s3/S3Minio.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala index 56802ae6346..9d49329e2bf 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala @@ -29,13 +29,12 @@ import com.typesafe.config.ConfigFactory import common.{SimpleExec, StreamLogging, WhiskProperties} import org.scalatest.{BeforeAndAfterAll, FlatSpec} import whisk.common.{Logging, TransactionId} -import whisk.core.database.test.DbUtils import whisk.core.database.{AttachmentStore, DocumentSerializer} import scala.concurrent.duration._ import scala.reflect.ClassTag -trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLogging { +trait S3Minio extends FlatSpec with BeforeAndAfterAll with StreamLogging { def makeS3Store[D <: DocumentSerializer: ClassTag]()(implicit actorSystem: ActorSystem, logging: Logging, materializer: ActorMaterializer): AttachmentStore = { @@ -71,12 +70,11 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with DbUtils with StreamLo private val bucket = "test-ow-travis" override protected def beforeAll(): Unit = { - implicit val tid: TransactionId = transid() + super.beforeAll() dockerExec( s"run -d -e MINIO_ACCESS_KEY=$accessKey -e MINIO_SECRET_KEY=$secretAccessKey -p $port:9000 minio/minio server /data") println(s"Started minio on $port") createTestBucket() - super.beforeAll() } override def afterAll(): Unit = { From a7b185ef8a7985b94cf9c0bae59216c2a04cd462 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 12:32:03 +0530 Subject: [PATCH 09/14] Use version 2 of list api --- common/scala/src/main/resources/s3-reference.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/scala/src/main/resources/s3-reference.conf b/common/scala/src/main/resources/s3-reference.conf index ec34bec9a21..15d29fe6fef 100644 --- a/common/scala/src/main/resources/s3-reference.conf +++ b/common/scala/src/main/resources/s3-reference.conf @@ -70,6 +70,10 @@ whisk { # Custom endpoint url, used for alternate s3 implementations # endpoint-url = null + + # Which version of the list bucket api to use. Set to 1 to use the old style version 1 API. + # By default the newer version 2 api is used. + list-bucket-api-version = 2 } } } From 30cfe721c796ac6b23011c4bc32b5e34ddb417fa Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 14:52:23 +0530 Subject: [PATCH 10/14] Consolidate S3 tests in a base trait --- ...la => S3AttachmentStoreBehaviorBase.scala} | 26 ++++++++++++++----- .../s3/S3AttachmentStoreMinioTests.scala | 6 +---- .../scala/whisk/core/database/s3/S3Aws.scala | 22 ++++++++++++++++ 3 files changed, 43 insertions(+), 11 deletions(-) rename tests/src/test/scala/whisk/core/database/s3/{S3MemoryArtifactStoreTests.scala => S3AttachmentStoreBehaviorBase.scala} (61%) create mode 100644 tests/src/test/scala/whisk/core/database/s3/S3Aws.scala diff --git a/tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala similarity index 61% rename from tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala rename to tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala index 9a375d1ae64..e8ebcb58709 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3MemoryArtifactStoreTests.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala @@ -17,22 +17,36 @@ package whisk.core.database.s3 -import org.junit.runner.RunWith +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import org.scalatest.FlatSpec -import org.scalatest.junit.JUnitRunner +import whisk.common.Logging +import whisk.core.database.{AttachmentStore, DocumentSerializer} import whisk.core.database.memory.MemoryArtifactStoreBehaviorBase +import whisk.core.database.test.AttachmentStoreBehaviors import whisk.core.database.test.behavior.ArtifactStoreAttachmentBehaviors -import whisk.core.database.{AttachmentStore, DocumentSerializer} +import whisk.core.entity.WhiskEntity import scala.reflect.ClassTag +import scala.util.Random -@RunWith(classOf[JUnitRunner]) -class S3MemoryArtifactStoreTests +trait S3AttachmentStoreBehaviorBase extends FlatSpec with MemoryArtifactStoreBehaviorBase with ArtifactStoreAttachmentBehaviors - with S3Minio { + with AttachmentStoreBehaviors { + override lazy val store = makeS3Store[WhiskEntity] + + override implicit val materializer: ActorMaterializer = ActorMaterializer() + + override val prefix = s"attachmentTCK_${Random.alphanumeric.take(4).mkString}" + + override def storeType: String = "S3" override def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore = makeS3Store[D]() + + def makeS3Store[D <: DocumentSerializer: ClassTag]()(implicit actorSystem: ActorSystem, + logging: Logging, + materializer: ActorMaterializer): AttachmentStore } diff --git a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala index d61c3e43e66..3438aad42d6 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala @@ -18,16 +18,12 @@ package whisk.core.database.s3 import org.junit.runner.RunWith -import org.scalatest.FlatSpec import org.scalatest.junit.JUnitRunner -import whisk.core.database.test.AttachmentStoreBehaviors import whisk.core.entity.WhiskEntity @RunWith(classOf[JUnitRunner]) -class S3AttachmentStoreMinioTests extends FlatSpec with AttachmentStoreBehaviors with S3Minio { +class S3AttachmentStoreMinioTests extends S3AttachmentStoreBehaviorBase with S3Minio { override lazy val store = makeS3Store[WhiskEntity] - override def storeType: String = "S3" - override def garbageCollectAttachments: Boolean = false } diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala b/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala new file mode 100644 index 00000000000..85792c99a2a --- /dev/null +++ b/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.s3 + +import org.scalatest.FlatSpec + +trait S3Aws extends FlatSpec {} From 402044ce732d2edb1a5a3d4be490065cda3a3026 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 19 Jun 2018 15:11:34 +0530 Subject: [PATCH 11/14] Add test against S3 --- .../s3/S3AttachmentStoreAwsTests.scala | 29 ++++++++++ .../s3/S3AttachmentStoreBehaviorBase.scala | 2 - .../s3/S3AttachmentStoreMinioTests.scala | 2 + .../scala/whisk/core/database/s3/S3Aws.scala | 55 ++++++++++++++++++- 4 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreAwsTests.scala diff --git a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreAwsTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreAwsTests.scala new file mode 100644 index 00000000000..2e99f58fe6b --- /dev/null +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreAwsTests.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database.s3 + +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +import whisk.core.entity.WhiskEntity + +@RunWith(classOf[JUnitRunner]) +class S3AttachmentStoreAwsTests extends S3AttachmentStoreBehaviorBase with S3Aws { + override lazy val store = makeS3Store[WhiskEntity] + + override def storeType: String = "S3" +} diff --git a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala index e8ebcb58709..48def3960f4 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreBehaviorBase.scala @@ -41,8 +41,6 @@ trait S3AttachmentStoreBehaviorBase override val prefix = s"attachmentTCK_${Random.alphanumeric.take(4).mkString}" - override def storeType: String = "S3" - override def getAttachmentStore[D <: DocumentSerializer: ClassTag](): AttachmentStore = makeS3Store[D]() diff --git a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala index 3438aad42d6..07cf4b497f3 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3AttachmentStoreMinioTests.scala @@ -25,5 +25,7 @@ import whisk.core.entity.WhiskEntity class S3AttachmentStoreMinioTests extends S3AttachmentStoreBehaviorBase with S3Minio { override lazy val store = makeS3Store[WhiskEntity] + override def storeType: String = "S3Minio" + override def garbageCollectAttachments: Boolean = false } diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala b/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala index 85792c99a2a..be1efa6848d 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala @@ -17,6 +17,59 @@ package whisk.core.database.s3 +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer +import com.typesafe.config.ConfigFactory import org.scalatest.FlatSpec +import whisk.common.Logging +import whisk.core.database.{AttachmentStore, DocumentSerializer} -trait S3Aws extends FlatSpec {} +import scala.reflect.ClassTag + +trait S3Aws extends FlatSpec { + def makeS3Store[D <: DocumentSerializer: ClassTag]()(implicit actorSystem: ActorSystem, + logging: Logging, + materializer: ActorMaterializer): AttachmentStore = { + val config = ConfigFactory.parseString(s""" + |whisk { + | db { + | s3 { + | alpakka { + | aws { + | credentials { + | provider = static + | access-key-id = "$accessKeyId" + | secret-access-key = "$secretAccessKey" + | } + | region { + | provider = static + | default-region = "$region" + | } + | } + | } + | bucket = "$bucket" + | } + | } + |} + """.stripMargin).withFallback(ConfigFactory.load()) + S3AttachmentStoreProvider.makeStore[D](config) + } + + override protected def withFixture(test: NoArgTest) = { + assume( + secretAccessKey != null, + s"'AWS_SECRET_ACCESS_KEY' env not configured. Configure following " + + s"env variables for test to run. 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_REGION'") + + require(accessKeyId != null, "'AWS_ACCESS_KEY_ID' env variable not set") + require(region != null, "'AWS_REGION' env variable not set") + + super.withFixture(test) + } + + val bucket = "test-ow-travis" + + val accessKeyId = System.getenv("AWS_ACCESS_KEY_ID") + val secretAccessKey = System.getenv("AWS_SECRET_ACCESS_KEY") + val region = System.getenv("AWS_REGION") +} From 1f53bb9ace6c9cf6a47cc920a256828ddab05d02 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Thu, 21 Jun 2018 15:09:12 +0530 Subject: [PATCH 12/14] Adapt Uri construction to workaround akka http bug --- .../main/scala/whisk/core/database/AttachmentSupport.scala | 5 ++++- .../main/scala/whisk/core/database/CouchDbRestStore.scala | 4 ++-- .../scala/whisk/core/database/s3/S3AttachmentStore.scala | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/common/scala/src/main/scala/whisk/core/database/AttachmentSupport.scala b/common/scala/src/main/scala/whisk/core/database/AttachmentSupport.scala index 72f7754d389..600e8b676aa 100644 --- a/common/scala/src/main/scala/whisk/core/database/AttachmentSupport.scala +++ b/common/scala/src/main/scala/whisk/core/database/AttachmentSupport.scala @@ -102,10 +102,13 @@ trait AttachmentSupport[DocumentAbstraction <: DocumentSerializer] extends Defau protected[database] def uriOf(bytesOrSource: Either[ByteString, Source[ByteString, _]], path: => String): Uri = { bytesOrSource match { case Left(bytes) => Uri.from(scheme = MemScheme, path = encode(bytes)) - case Right(_) => Uri.from(scheme = attachmentScheme, path = path) + case Right(_) => uriFrom(scheme = attachmentScheme, path = path) } } + //Not using Uri.from due to https://github.com/akka/akka-http/issues/2080 + protected[database] def uriFrom(scheme: String, path: String): Uri = Uri(s"$scheme:$path") + /** * Constructs a source from inlined attachment contents */ diff --git a/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala b/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala index 1e43eb7ad26..ab31052165b 100644 --- a/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala +++ b/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala @@ -374,7 +374,7 @@ class CouchDbRestStore[DocumentAbstraction <: DocumentSerializer](dbProtocol: St docStream: Source[ByteString, _])(implicit transid: TransactionId): Future[(DocInfo, Attached)] = { if (maxInlineSize.toBytes == 0) { - val uri = Uri.from(scheme = attachmentScheme, path = UUID().asString) + val uri = uriFrom(scheme = attachmentScheme, path = UUID().asString) for { attached <- Future.successful(Attached(uri.toString, contentType)) i1 <- put(update(doc, attached)) @@ -549,7 +549,7 @@ class CouchDbRestStore[DocumentAbstraction <: DocumentSerializer](dbProtocol: St */ private def getAttachmentName(name: String): String = { Try(java.util.UUID.fromString(name)) - .map(_ => Uri.from(scheme = attachmentScheme, path = name).toString) + .map(_ => uriFrom(scheme = attachmentScheme, path = name).toString) .getOrElse(name) } diff --git a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala index 570ce2af4ab..fee6c5f8420 100644 --- a/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala +++ b/common/scala/src/main/scala/whisk/core/database/s3/S3AttachmentStore.scala @@ -65,8 +65,7 @@ class S3AttachmentStore(client: S3Client, bucket: String, prefix: String)(implic logging: Logging, materializer: ActorMaterializer) extends AttachmentStore { - //TODO Use 's3s' for now as due to some bug `Uri.from(scheme="s3") is causing issue - override val scheme = "s3s" + override val scheme = "s3" override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher From bdfe88d1dc098a94a8a916c8a56e11e027467530 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Thu, 21 Jun 2018 15:11:15 +0530 Subject: [PATCH 13/14] Change config namespace to `whisk.s3` from `whisk.db.s3` --- .../src/main/resources/s3-reference.conf | 126 +++++++++--------- .../main/scala/whisk/core/WhiskConfig.scala | 2 +- .../scala/whisk/core/database/s3/S3Aws.scala | 2 - .../whisk/core/database/s3/S3Minio.scala | 2 - 4 files changed, 63 insertions(+), 69 deletions(-) diff --git a/common/scala/src/main/resources/s3-reference.conf b/common/scala/src/main/resources/s3-reference.conf index 15d29fe6fef..92bc23ec8c0 100644 --- a/common/scala/src/main/resources/s3-reference.conf +++ b/common/scala/src/main/resources/s3-reference.conf @@ -2,79 +2,77 @@ # license agreements; and to You under the Apache License, Version 2.0. whisk { - db { - s3 { - # See https://developer.lightbend.com/docs/alpakka/current/s3.html#usage - alpakka { - # whether the buffer request chunks (up to 5MB each) to "memory" or "disk" - buffer = "memory" + s3 { + # See https://developer.lightbend.com/docs/alpakka/current/s3.html#usage + alpakka { + # whether the buffer request chunks (up to 5MB each) to "memory" or "disk" + buffer = "memory" - # location for temporary files, if buffer is set to "disk". If empty, uses the standard java temp path. - disk-buffer-path = "" + # location for temporary files, if buffer is set to "disk". If empty, uses the standard java temp path. + disk-buffer-path = "" - proxy { - # hostname of the proxy. If undefined ("") proxy is not enabled. - host = "" - port = 8000 + proxy { + # hostname of the proxy. If undefined ("") proxy is not enabled. + host = "" + port = 8000 - # if "secure" is set to "true" then HTTPS will be used for all requests to S3, otherwise HTTP will be used - secure = true - } + # if "secure" is set to "true" then HTTPS will be used for all requests to S3, otherwise HTTP will be used + secure = true + } - # default values for AWS configuration. If credentials and/or region are not specified when creating S3Client, - # these values will be used. - aws { - # If this section is absent, the fallback behavior is to use the - # com.amazonaws.auth.DefaultAWSCredentialsProviderChain instance to resolve credentials - credentials { - # supported providers: - # anon - anonymous requests ("no auth") - # static - static credentials, - # required params: - # access-key-id - # secret-access-key - # optional: - # token - # default: as described in com.amazonaws.auth.DefaultAWSCredentialsProviderChain docs, - # attempts to get the credentials from either: - # - environment variables - # - system properties - # - credentials file - # - EC2 credentials service - # - IAM / metadata - provider = default - } + # default values for AWS configuration. If credentials and/or region are not specified when creating S3Client, + # these values will be used. + aws { + # If this section is absent, the fallback behavior is to use the + # com.amazonaws.auth.DefaultAWSCredentialsProviderChain instance to resolve credentials + credentials { + # supported providers: + # anon - anonymous requests ("no auth") + # static - static credentials, + # required params: + # access-key-id + # secret-access-key + # optional: + # token + # default: as described in com.amazonaws.auth.DefaultAWSCredentialsProviderChain docs, + # attempts to get the credentials from either: + # - environment variables + # - system properties + # - credentials file + # - EC2 credentials service + # - IAM / metadata + provider = default + } - # If this section is absent, the fallback behavior is to use the - # com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain instance to resolve region - region { - # supported providers: - # static - static credentials, - # required params: - # default-region - # default: as described in com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain docs, - # attempts to get the region from either: - # - environment variables - # - system properties - # - progile file - # - EC2 metadata - provider = default - } + # If this section is absent, the fallback behavior is to use the + # com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain instance to resolve region + region { + # supported providers: + # static - static credentials, + # required params: + # default-region + # default: as described in com.amazonaws.regions.AwsRegionProvider.DefaultAwsRegionProviderChain docs, + # attempts to get the region from either: + # - environment variables + # - system properties + # - progile file + # - EC2 metadata + provider = default } + } - # Enable path style access to s3, i.e. "https://s3-eu-west-1.amazonaws.com/my.bucket/myobject" - # Default is virtual-hosted style. - # When using virtual hosted–style buckets with SSL, the S3 wild card certificate only matches buckets that do not contain periods. - # Buckets containing periods will lead to certificate errors. In those cases it's useful to enable path-style access. - path-style-access = true + # Enable path style access to s3, i.e. "https://s3-eu-west-1.amazonaws.com/my.bucket/myobject" + # Default is virtual-hosted style. + # When using virtual hosted–style buckets with SSL, the S3 wild card certificate only matches buckets that do not contain periods. + # Buckets containing periods will lead to certificate errors. In those cases it's useful to enable path-style access. + path-style-access = true - # Custom endpoint url, used for alternate s3 implementations - # endpoint-url = null + # Custom endpoint url, used for alternate s3 implementations + # endpoint-url = null - # Which version of the list bucket api to use. Set to 1 to use the old style version 1 API. - # By default the newer version 2 api is used. - list-bucket-api-version = 2 - } + # Which version of the list bucket api to use. Set to 1 to use the old style version 1 API. + # By default the newer version 2 api is used. + list-bucket-api-version = 2 } } } diff --git a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala index 61032508ddc..25e19a45eed 100644 --- a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala @@ -245,6 +245,6 @@ object ConfigKeys { val containerProxy = "whisk.container-proxy" val containerProxyTimeouts = s"$containerProxy.timeouts" - val s3 = s"$db.s3" + val s3 = "whisk.s3" } diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala b/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala index be1efa6848d..59c1cbe3e6c 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Aws.scala @@ -32,7 +32,6 @@ trait S3Aws extends FlatSpec { materializer: ActorMaterializer): AttachmentStore = { val config = ConfigFactory.parseString(s""" |whisk { - | db { | s3 { | alpakka { | aws { @@ -49,7 +48,6 @@ trait S3Aws extends FlatSpec { | } | bucket = "$bucket" | } - | } |} """.stripMargin).withFallback(ConfigFactory.load()) S3AttachmentStoreProvider.makeStore[D](config) diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala index 9d49329e2bf..b5391ef4c34 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala @@ -40,7 +40,6 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with StreamLogging { materializer: ActorMaterializer): AttachmentStore = { val config = ConfigFactory.parseString(s""" |whisk { - | db{ | s3 { | alpakka { | aws { @@ -58,7 +57,6 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with StreamLogging { | } | bucket = "$bucket" | } - | } |} """.stripMargin).withFallback(ConfigFactory.load()) S3AttachmentStoreProvider.makeStore[D](config) From 59664527c5cc335b648f60e1603fe931d4622fc6 Mon Sep 17 00:00:00 2001 From: Chetan Mehrotra Date: Tue, 26 Jun 2018 10:11:27 +0530 Subject: [PATCH 14/14] Use DockerCommand from ActionContainer --- .../actionContainers/ActionContainer.scala | 2 +- .../whisk/core/database/s3/S3Minio.scala | 26 +++---------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/tests/src/test/scala/actionContainers/ActionContainer.scala b/tests/src/test/scala/actionContainers/ActionContainer.scala index a16888917ef..b96423140c0 100644 --- a/tests/src/test/scala/actionContainers/ActionContainer.scala +++ b/tests/src/test/scala/actionContainers/ActionContainer.scala @@ -91,7 +91,7 @@ object ActionContainer { }.get // This fails if the docker binary couldn't be located. } - private lazy val dockerCmd: String = { + lazy val dockerCmd: String = { /* * The docker host is set to a provided property 'docker.host' if it's * available; otherwise we check with WhiskProperties to see whether we are diff --git a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala index b5391ef4c34..0c59e68a8c5 100644 --- a/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala +++ b/tests/src/test/scala/whisk/core/database/s3/S3Minio.scala @@ -17,16 +17,16 @@ package whisk.core.database.s3 -import java.io.File import java.net.ServerSocket +import actionContainers.ActionContainer import akka.actor.ActorSystem import akka.stream.ActorMaterializer import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.typesafe.config.ConfigFactory -import common.{SimpleExec, StreamLogging, WhiskProperties} +import common.{SimpleExec, StreamLogging} import org.scalatest.{BeforeAndAfterAll, FlatSpec} import whisk.common.{Logging, TransactionId} import whisk.core.database.{AttachmentStore, DocumentSerializer} @@ -96,33 +96,13 @@ trait S3Minio extends FlatSpec with BeforeAndAfterAll with StreamLogging { private def dockerExec(cmd: String): String = { implicit val tid: TransactionId = TransactionId.testing - val command = s"$dockerCmd $cmd" + val command = s"${ActionContainer.dockerCmd} $cmd" val cmdSeq = command.split(" ").map(_.trim).filter(_.nonEmpty) val (out, err, code) = SimpleExec.syncRunCmd(cmdSeq) assert(code == 0, s"Error occurred for command '$command'. Exit code: $code, Error: $err") out } - //Taken from ActionContainer - private lazy val dockerBin: String = { - List("/usr/bin/docker", "/usr/local/bin/docker") - .find { bin => - new File(bin).isFile - } - .getOrElse(???) // This fails if the docker binary couldn't be located. - } - - private lazy val dockerCmd: String = { - val version = WhiskProperties.getProperty("whisk.version.name") - // Check if we are running on docker-machine env. - val hostStr = if (version.toLowerCase().contains("mac")) { - s" --host tcp://${WhiskProperties.getMainDockerEndpoint} " - } else { - "" - } - s"$dockerBin $hostStr" - } - private def freePort(): Int = { val socket = new ServerSocket(0) try socket.getLocalPort