Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport #1923] AWS S3: make raw request URI contains path and query #1963

Merged
merged 1 commit into from
Oct 8, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -217,13 +217,13 @@ import scala.concurrent.{ExecutionContext, Future}
}

private def rawRequestUri(uri: Uri): String = {
val rawUri = uri.toString
val rawUri = uri.toHttpRequestTargetOriginForm.toString
val rawPath = uri.path.toString()

if (rawPath.contains("+")) {
val fixedPath = rawPath.replaceAll("\\+", "%2B")

rawUri.replace(rawPath, fixedPath)
require(rawUri startsWith rawPath)
fixedPath + rawUri.drop(rawPath.length)
} else {
rawUri
}
Expand Down
4 changes: 3 additions & 1 deletion s3/src/test/resources/logback-test.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@
</encoder>
</appender>

<logger name="org.eclipse.jetty.io" level="info"/>
<logger name="org.eclipse.jetty.util" level="info"/>
<logger name="org.eclipse.jetty.servlet" level="warn"/>

<root level="warn">
<appender-ref ref="FILE" />
</root>
</configuration>
</configuration>
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ class DiskBufferSpec(_system: ActorSystem)

implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system).withDebugLogging(true))

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

"DiskBuffer" should
"emit a chunk on its output containing the concatenation of all input values" in {
val result = Source(Vector(ByteString(1, 2, 3, 4, 5), ByteString(6, 7, 8, 9, 10, 11, 12), ByteString(13, 14)))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import akka.stream.ActorMaterializer
import akka.stream.alpakka.s3.headers.{CannedAcl, ServerSideEncryption, StorageClass}
import akka.stream.alpakka.s3.{ApiVersion, BufferType, MemoryBufferType, MetaHeaders, S3Headers, S3Settings}
import akka.stream.scaladsl.Source
import akka.testkit.{SocketUtil, TestProbe}
import akka.testkit.{SocketUtil, TestKit, TestProbe}
import com.amazonaws.auth.{AWSCredentialsProvider, AWSStaticCredentialsProvider, AnonymousAWSCredentials}
import com.amazonaws.regions.AwsRegionProvider
import org.scalatest.concurrent.ScalaFutures
Expand Down Expand Up @@ -184,7 +184,7 @@ class HttpRequestsSpec extends FlatSpec with Matchers with ScalaFutures {
val req = HttpRequests.getDownloadRequest(location)
req.uri.authority.host.toString shouldEqual "bucket.s3.amazonaws.com"
req.uri.path.toString shouldEqual "/test%20folder/1%20+%202%20=%203"
req.headers should contain(`Raw-Request-URI`("https://bucket.s3.amazonaws.com/test%20folder/1%20%2B%202%20=%203"))
req.headers should contain(`Raw-Request-URI`("/test%20folder/1%20%2B%202%20=%203"))
}

it should "support download requests with keys containing spaces with path-style access in other regions" in {
Expand Down Expand Up @@ -375,7 +375,7 @@ class HttpRequestsSpec extends FlatSpec with Matchers with ScalaFutures {
probe.expectMsgType[HttpRequest]

materializer.shutdown()
system.terminate()
TestKit.shutdownActorSystem(system)
}

it should "add two (source, range) headers to multipart upload (copy) request when byte range populated" in {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,24 @@ import akka.stream.{ActorMaterializer, ActorMaterializerSettings}
import akka.stream.alpakka.s3.ListBucketResultContents
import akka.testkit.TestKit
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{FlatSpecLike, Matchers}
import org.scalatest.{BeforeAndAfterAll, FlatSpecLike, Matchers}

import scala.collection.immutable.Seq

class MarshallingSpec(_system: ActorSystem) extends TestKit(_system) with FlatSpecLike with Matchers with ScalaFutures {
class MarshallingSpec(_system: ActorSystem)
extends TestKit(_system)
with FlatSpecLike
with Matchers
with ScalaFutures
with BeforeAndAfterAll {

def this() = this(ActorSystem("MarshallingSpec"))

implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system).withDebugLogging(true))
implicit val ec = materializer.executionContext

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

val xmlString = """<?xml version="1.0" encoding="UTF-8"?>
|<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
| <Name>bucket</Name>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ class MemoryBufferSpec(_system: ActorSystem)

implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system).withDebugLogging(true))

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

"MemoryBuffer" should "emit a chunk on its output containg the concatenation of all input values" in {
val result = Source(Vector(ByteString(1, 2, 3, 4, 5), ByteString(6, 7, 8, 9, 10, 11, 12), ByteString(13, 14)))
.via(new MemoryBuffer(200))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@ import akka.util.ByteString
import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.amazonaws.regions.AwsRegionProvider
import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures}
import org.scalatest.{FlatSpecLike, Matchers, PrivateMethodTester}
import org.scalatest.{BeforeAndAfterAll, FlatSpecLike, Matchers, PrivateMethodTester}

import scala.concurrent.Future

class S3StreamSpec(_system: ActorSystem)
extends TestKit(_system)
with FlatSpecLike
with Matchers
with BeforeAndAfterAll
with PrivateMethodTester
with ScalaFutures
with IntegrationPatience {
Expand All @@ -33,6 +34,8 @@ class S3StreamSpec(_system: ActorSystem)
def this() = this(ActorSystem("S3StreamSpec"))
implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system).withDebugLogging(true))

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

"Non-ranged downloads" should "have two (host and synthetic raw-request-uri) headers" in {

val requestHeaders = PrivateMethod[HttpRequest]('requestHeaders)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ class SplitAfterSizeSpec(_system: ActorSystem)

implicit val materializer = ActorMaterializer(ActorMaterializerSettings(system).withDebugLogging(true))

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

final val MaxChunkSize = 1024
"SplitAfterSize" should "yield a single empty substream on no input" in assertAllStagesStopped {
Source
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,19 @@ import com.amazonaws.auth.{
BasicAWSCredentials,
BasicSessionCredentials
}
import org.scalatest.{FlatSpecLike, Matchers}
import org.scalatest.{BeforeAndAfterAll, FlatSpecLike, Matchers}
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.OptionValues._
import org.scalatest.time.{Millis, Seconds, Span}

import scala.compat.java8.OptionConverters._

class SignerSpec(_system: ActorSystem) extends TestKit(_system) with FlatSpecLike with Matchers with ScalaFutures {
class SignerSpec(_system: ActorSystem)
extends TestKit(_system)
with FlatSpecLike
with Matchers
with BeforeAndAfterAll
with ScalaFutures {
def this() = this(ActorSystem("SignerSpec"))

implicit val defaultPatience =
Expand All @@ -38,6 +43,8 @@ class SignerSpec(_system: ActorSystem) extends TestKit(_system) with FlatSpecLik
new BasicAWSCredentials("AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY")
)

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

def signingKey(dateTime: ZonedDateTime) =
SigningKey(dateTime, credentials, CredentialScope(dateTime.toLocalDate, "us-east-1", "iam"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ class StreamUtilsSpec(_system: ActorSystem)
implicit val defaultPatience =
PatienceConfig(timeout = Span(5, Seconds), interval = Span(30, Millis))

override protected def afterAll(): Unit = {
fs.close()
TestKit.shutdownActorSystem(system)
}

val fs = Jimfs.newFileSystem("FileSourceSpec", Configuration.unix())

val TestText = {
Expand Down Expand Up @@ -78,7 +83,4 @@ class StreamUtilsSpec(_system: ActorSystem)
result should contain theSameElementsInOrderAs dis.getMessageDigest.digest()
}
}

override def afterAll(): Unit = fs.close()

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package akka.stream.alpakka.s3.scaladsl

import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.testkit.TestKit
import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures}
import org.scalatest._

Expand All @@ -19,4 +20,7 @@ trait S3ClientIntegrationSpec

implicit val system: ActorSystem
implicit val materializer = ActorMaterializer()

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(system)

}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import akka.stream.ActorMaterializer
import akka.stream.alpakka.s3.BucketAccess.{AccessGranted, NotExists}
import akka.stream.alpakka.s3._
import akka.stream.scaladsl.{Keep, Sink, Source}
import akka.testkit.TestKit
import akka.util.ByteString
import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.typesafe.config.ConfigFactory
Expand Down Expand Up @@ -47,6 +48,8 @@ trait S3IntegrationSpec extends FlatSpecLike with BeforeAndAfterAll with Matcher
val objectValue = "Some String"
val metaHeaders: Map[String, String] = Map("location" -> "Africa", "datatype" -> "image")

override protected def afterAll(): Unit = TestKit.shutdownActorSystem(actorSystem)

def config() = ConfigFactory.parseString("""
|alpakka.s3.aws.region {
| provider = static
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import com.typesafe.config.ConfigFactory

abstract class S3WireMockBase(_system: ActorSystem, _wireMockServer: WireMockServer) extends TestKit(_system) {

def this(mock: WireMockServer) =
private def this(mock: WireMockServer) =
this(ActorSystem(getCallerName(getClass), config(mock.port()).withFallback(ConfigFactory.load())), mock)
def this() = this(initServer())

Expand Down