diff --git a/.changes/d3ce7511-6fb2-4435-8f46-db724551b384.json b/.changes/d3ce7511-6fb2-4435-8f46-db724551b384.json new file mode 100644 index 0000000000..aa89d9167e --- /dev/null +++ b/.changes/d3ce7511-6fb2-4435-8f46-db724551b384.json @@ -0,0 +1,5 @@ +{ + "id": "d3ce7511-6fb2-4435-8f46-db724551b384", + "type": "misc", + "description": "Add telemetry provider configuration to `DefaultAwsSigner`" +} \ No newline at end of file diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 3187339faf..365a70b22b 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -73,7 +73,7 @@ jobs: - name: Save Test Reports if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-reports-${{ matrix.os }} path: '**/build/reports' @@ -107,7 +107,7 @@ jobs: - name: Save Test Reports if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-reports-${{ matrix.os }} path: '**/build/reports' diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c4a65548..2e925d0943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [1.4.2] - 01/28/2025 + +### Fixes +* Ignore hop-by-hop headers when signing requests + ## [1.4.1] - 01/16/2025 ## [1.4.0] - 01/15/2025 diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 201ffc877f..4f246e1b99 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,8 +39,9 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - // FIXME: This trait fails smithy validator - // @required + + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `document` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -65,8 +66,8 @@ structure TestStruct { @required nestedListValue: NestedList - // FIXME: This trait fails smithy validator - // @required + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. nested: Nested @required @@ -97,8 +98,7 @@ union MyUnion { } structure Nested { - // FIXME: This trait fails smithy validator - // @required + @required a: String } diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt index 6e5834f7d1..bd41afcade 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt @@ -40,13 +40,7 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator() // val targetedTest = TestMemberDelta(setOf("RestJsonComplexErrorWithNoMessage"), TestContainmentMode.RUN_TESTS) val ignoredTests = TestMemberDelta( - setOf( - "AwsJson10ClientErrorCorrectsWithDefaultValuesWhenServerFailsToSerializeRequiredValues", - "RestJsonNullAndEmptyHeaders", - "NullAndEmptyHeaders", - "RpcV2CborClientPopulatesDefaultsValuesWhenMissingInResponse", - "RpcV2CborClientPopulatesDefaultValuesInInput", - ), + setOf(), ) val requestTestBuilder = HttpProtocolUnitTestRequestGenerator.Builder() diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt index e6dbd0dca1..f35570dfeb 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt @@ -193,7 +193,7 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli } else { // only use @default if type is `T` shape.getTrait()?.let { - defaultValue(it.getDefaultValue(targetShape)) + setDefaultValue(it, targetShape) } } } @@ -219,9 +219,10 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli } } - private fun DefaultTrait.getDefaultValue(targetShape: Shape): String? { - val node = toNode() - return when { + private fun Symbol.Builder.setDefaultValue(defaultTrait: DefaultTrait, targetShape: Shape) { + val node = defaultTrait.toNode() + + val defaultValue = when { node.toString() == "null" -> null // Check if target is an enum before treating the default like a regular number/string @@ -235,13 +236,20 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli "${enumSymbol.fullName}.fromValue($arg)" } - targetShape.isBlobShape && targetShape.isStreaming -> - node - .toString() - .takeUnless { it.isEmpty() } - ?.let { "ByteStream.fromString(${it.dq()})" } + targetShape.isBlobShape -> { + addReferences(RuntimeTypes.Core.Text.Encoding.decodeBase64) - targetShape.isBlobShape -> "${node.toString().dq()}.encodeToByteArray()" + if (targetShape.isStreaming) { + node.toString() + .takeUnless { it.isEmpty() } + ?.let { + addReferences(RuntimeTypes.Core.Content.ByteStream) + "ByteStream.fromString(${it.dq()}.decodeBase64())" + } + } else { + "${node.toString().dq()}.decodeBase64().encodeToByteArray()" + } + } targetShape.isDocumentShape -> getDefaultValueForDocument(node) targetShape.isTimestampShape -> getDefaultValueForTimestamp(node.asNumberNode().get()) @@ -252,6 +260,8 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli node.isStringNode -> node.toString().dq() else -> node.toString() } + + defaultValue(defaultValue) } private fun getDefaultValueForTimestamp(node: NumberNode): String { diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt index c34eb5ab9d..5d90f376d2 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt @@ -248,6 +248,7 @@ class StructureGenerator( } else { memberSymbol } + write("public var #L: #E", memberName, builderMemberSymbol) } write("") diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt index 932b2a796a..5dfc519962 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt @@ -157,9 +157,8 @@ class HttpStringValuesMapSerializer( val paramName = binding.locationName // addAll collection parameter 2 val param2 = if (mapFnContents.isEmpty()) "input.$memberName" else "input.$memberName.map { $mapFnContents }" - val nullCheck = if (memberSymbol.isNullable) "?" else "" writer.write( - "if (input.#L$nullCheck.isNotEmpty() == true) #L(#S, #L)", + "if (input.#L != null) #L(#S, #L)", memberName, binding.location.addAllFnName, paramName, @@ -174,8 +173,7 @@ class HttpStringValuesMapSerializer( val paramName = binding.locationName val memberSymbol = symbolProvider.toSymbol(binding.member) - // NOTE: query parameters are allowed to be empty, whereas headers should omit empty string - // values from serde + // NOTE: query parameters are allowed to be empty if ((location == HttpBinding.Location.QUERY || location == HttpBinding.Location.HEADER) && binding.member.hasTrait()) { // Call the idempotency token function if no supplied value. writer.addImport(RuntimeTypes.SmithyClient.IdempotencyTokenProviderExt) @@ -185,18 +183,7 @@ class HttpStringValuesMapSerializer( paramName, ) } else { - val nullCheck = - if (location == HttpBinding.Location.QUERY || - memberTarget.hasTrait< - @Suppress("DEPRECATION") - software.amazon.smithy.model.traits.EnumTrait, - >() - ) { - if (memberSymbol.isNullable) "input.$memberName != null" else "" - } else { - val nullCheck = if (memberSymbol.isNullable) "?" else "" - "input.$memberName$nullCheck.isNotEmpty() == true" - } + val nullCheck = if (memberSymbol.isNullable) "input.$memberName != null" else "" val cond = defaultCheck(binding.member) ?: nullCheck diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt index 0cc4d2cb14..bb90fbc0fb 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt @@ -182,7 +182,7 @@ class SymbolProviderTest { "double,2.71828,2.71828", "byte,10,10.toByte()", "string,\"hello\",\"hello\"", - "blob,\"abcdefg\",\"abcdefg\".encodeToByteArray()", + "blob,\"abcdefg\",\"abcdefg\".decodeBase64().encodeToByteArray()", "boolean,true,true", "bigInteger,5,5", "bigDecimal,9.0123456789,9.0123456789", diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt index 3c15579dab..6f8040e7dc 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt @@ -57,8 +57,8 @@ internal class SmokeTestOperationSerializer: HttpSerializer.NonStreaming ()V + public final fun build ()Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner; + public final fun getTelemetryProvider ()Laws/smithy/kotlin/runtime/telemetry/TelemetryProvider; + public final fun setTelemetryProvider (Laws/smithy/kotlin/runtime/telemetry/TelemetryProvider;)V +} + public final class aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSignerKt { + public static final fun DefaultAwsSigner (Lkotlin/jvm/functions/Function1;)Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner; public static final fun getDefaultAwsSigner ()Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner; } diff --git a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt index 69d742de79..c7c47fe5a2 100644 --- a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt +++ b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt @@ -58,16 +58,27 @@ internal interface Canonicalizer { ): CanonicalRequest } -// Taken from https://github.com/awslabs/aws-c-auth/blob/dd505b55fd46222834f35c6e54165d8cbebbfaaa/source/aws_signing.c#L118-L156 private val skipHeaders = setOf( - "connection", "expect", // https://github.com/awslabs/aws-sdk-kotlin/issues/862 + + // Taken from https://github.com/awslabs/aws-c-auth/blob/274a1d21330731cc51bb742794adc70ada5f4380/source/aws_signing.c#L121-L164 "sec-websocket-key", "sec-websocket-protocol", "sec-websocket-version", - "upgrade", "user-agent", "x-amzn-trace-id", + + // Taken from https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1. These are "hop-by-hop" headers which may + // be modified/removed by intervening proxies or caches. These are unsafe to sign because if they change they render + // the signature invalid. + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", ) internal class DefaultCanonicalizer(private val sha256Supplier: HashSupplier = ::Sha256) : Canonicalizer { diff --git a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt index df613d31b0..791d8fce0a 100644 --- a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt +++ b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt @@ -4,8 +4,10 @@ */ package aws.smithy.kotlin.runtime.auth.awssigning +import aws.smithy.kotlin.runtime.ExperimentalApi import aws.smithy.kotlin.runtime.http.Headers import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.TimestampFormat import kotlin.coroutines.coroutineContext @@ -13,13 +15,30 @@ import kotlin.coroutines.coroutineContext /** The default implementation of [AwsSigner] */ public val DefaultAwsSigner: AwsSigner = DefaultAwsSignerImpl() +/** Creates a customized instance of [AwsSigner] */ +@Suppress("ktlint:standard:function-naming") +public fun DefaultAwsSigner(block: DefaultAwsSignerBuilder.() -> Unit): AwsSigner = + DefaultAwsSignerBuilder().apply(block).build() + +/** A builder class for creating instances of [AwsSigner] using the default implementation */ +public class DefaultAwsSignerBuilder { + public var telemetryProvider: TelemetryProvider? = null + + public fun build(): AwsSigner = DefaultAwsSignerImpl( + telemetryProvider = telemetryProvider, + ) +} + +@OptIn(ExperimentalApi::class) internal class DefaultAwsSignerImpl( private val canonicalizer: Canonicalizer = Canonicalizer.Default, private val signatureCalculator: SignatureCalculator = SignatureCalculator.Default, private val requestMutator: RequestMutator = RequestMutator.Default, + private val telemetryProvider: TelemetryProvider? = null, ) : AwsSigner { override suspend fun sign(request: HttpRequest, config: AwsSigningConfig): AwsSigningResult { - val logger = coroutineContext.logger() + val logger = telemetryProvider?.loggerProvider?.getOrCreateLogger("DefaultAwsSigner") + ?: coroutineContext.logger() // TODO: implement SigV4a if (config.algorithm != AwsSigningAlgorithm.SIGV4) { @@ -52,7 +71,8 @@ internal class DefaultAwsSignerImpl( prevSignature: ByteArray, config: AwsSigningConfig, ): AwsSigningResult { - val logger = coroutineContext.logger() + val logger = telemetryProvider?.loggerProvider?.getOrCreateLogger("DefaultAwsSigner") + ?: coroutineContext.logger() val stringToSign = signatureCalculator.chunkStringToSign(chunkBody, prevSignature, config) logger.trace { "Chunk string to sign:\n$stringToSign" } @@ -70,7 +90,8 @@ internal class DefaultAwsSignerImpl( prevSignature: ByteArray, config: AwsSigningConfig, ): AwsSigningResult { - val logger = coroutineContext.logger() + val logger = telemetryProvider?.loggerProvider?.getOrCreateLogger("DefaultAwsSigner") + ?: coroutineContext.logger() // FIXME - can we share canonicalization code more than we are..., also this reduce is inefficient. // canonicalize the headers diff --git a/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt b/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt index e5047ee56b..734371cb10 100644 --- a/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt +++ b/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt @@ -7,8 +7,12 @@ package aws.smithy.kotlin.runtime.auth.awssigning import aws.smithy.kotlin.runtime.IgnoreNative import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.tests.DEFAULT_TEST_CREDENTIALS -import aws.smithy.kotlin.runtime.http.* -import aws.smithy.kotlin.runtime.http.request.* +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.HttpMethod +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.headers +import aws.smithy.kotlin.runtime.http.request.url import aws.smithy.kotlin.runtime.net.Host import aws.smithy.kotlin.runtime.net.url.Url import aws.smithy.kotlin.runtime.time.Instant @@ -142,6 +146,7 @@ class DefaultCanonicalizerTest { // These should not be signed set("Expect", "100-continue") set("X-Amzn-Trace-Id", "qux") + set("Transfer-Encoding", "chunked") } body = HttpBody.Empty } diff --git a/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt b/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt index 86d5744a62..51be65b92a 100644 --- a/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt +++ b/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt @@ -13,7 +13,9 @@ import aws.smithy.kotlin.runtime.auth.awssigning.DefaultAwsSigner import aws.smithy.kotlin.runtime.auth.awssigning.internal.AWS_CHUNKED_THRESHOLD import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get -import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.HttpMethod +import aws.smithy.kotlin.runtime.http.SdkHttpClient import aws.smithy.kotlin.runtime.http.operation.* import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder @@ -154,8 +156,8 @@ public abstract class AwsHttpSignerTestBase( val op = buildOperation(streaming = true, replayable = false, requestBody = "a".repeat(AWS_CHUNKED_THRESHOLD + 1)) val expectedDate = "20201016T195600Z" val expectedSig = "AWS4-HMAC-SHA256 Credential=AKID/20201016/us-east-1/demo/aws4_request, " + - "SignedHeaders=content-encoding;host;transfer-encoding;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + - "Signature=ac341b9b248a0b23d2fcd9f7e805f4eb0b8a1b789bb23a8ec6adc6c48dd084ad" + "SignedHeaders=content-encoding;host;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + + "Signature=ef06c95647c4d2daa6c89ac90274f1c780777cba8eaab772df6d8009def3eb8f" val signed = getSignedRequest(op) assertEquals(expectedDate, signed.headers["X-Amz-Date"]) @@ -168,8 +170,8 @@ public abstract class AwsHttpSignerTestBase( val op = buildOperation(streaming = true, replayable = true, requestBody = "a".repeat(AWS_CHUNKED_THRESHOLD + 1)) val expectedDate = "20201016T195600Z" val expectedSig = "AWS4-HMAC-SHA256 Credential=AKID/20201016/us-east-1/demo/aws4_request, " + - "SignedHeaders=content-encoding;host;transfer-encoding;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + - "Signature=ac341b9b248a0b23d2fcd9f7e805f4eb0b8a1b789bb23a8ec6adc6c48dd084ad" + "SignedHeaders=content-encoding;host;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + + "Signature=ef06c95647c4d2daa6c89ac90274f1c780777cba8eaab772df6d8009def3eb8f" val signed = getSignedRequest(op) assertEquals(expectedDate, signed.headers["X-Amz-Date"]) @@ -183,8 +185,8 @@ public abstract class AwsHttpSignerTestBase( val expectedDate = "20201016T195600Z" // should have same signature as testSignAwsChunkedStreamNonReplayable(), except for the hash, since the body is different val expectedSig = "AWS4-HMAC-SHA256 Credential=AKID/20201016/us-east-1/demo/aws4_request, " + - "SignedHeaders=content-encoding;host;transfer-encoding;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + - "Signature=3f0277123c9ed8a8858f793886a0ac0fcb457bc54401ffc22d470f373397cff0" + "SignedHeaders=content-encoding;host;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + + "Signature=a902702b57057a864bf41cc22ee846a1b7bd047e22784367ec6a459f6791330e" val signed = getSignedRequest(op) assertEquals(expectedDate, signed.headers["X-Amz-Date"]) diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt index 0ea8ae5e66..8047b870c3 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt @@ -30,7 +30,7 @@ public object MetricsInterceptor : Interceptor { } val originalResponse = chain.proceed(request) - val response = if (originalResponse.body == null || originalResponse.body?.contentLength() == 0L) { + val response = if (originalResponse.body.contentLength() == 0L) { originalResponse } else { originalResponse.newBuilder() diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt index 061c22a13d..a20387d0a0 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt @@ -15,6 +15,7 @@ import aws.smithy.kotlin.runtime.net.TlsVersion import aws.smithy.kotlin.runtime.operation.ExecutionContext import aws.smithy.kotlin.runtime.time.Instant import aws.smithy.kotlin.runtime.time.fromEpochMilliseconds +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.job import okhttp3.* import okhttp3.ConnectionPool @@ -47,6 +48,7 @@ public class OkHttpEngine( private val connectionIdleMonitor = config.connectionIdlePollingInterval?.let { ConnectionIdleMonitor(it) } private val client = config.buildClientWithConnectionListener(metrics, connectionIdleMonitor) + @OptIn(ExperimentalCoroutinesApi::class) override suspend fun roundTrip(context: ExecutionContext, request: HttpRequest): HttpCall { val callContext = callContext() diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt index c287228fdf..181ed400c3 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt @@ -104,7 +104,7 @@ public fun Headers.toOkHttpHeaders(): OkHttpHeaders = OkHttpHeaders.Builder().al @InternalApi public fun OkHttpResponse.toSdkResponse(): HttpResponse { val sdkHeaders = OkHttpHeadersAdapter(headers) - val httpBody = if (body == null || body!!.contentLength() == 0L) { + val httpBody = if (body.contentLength() == 0L) { HttpBody.Empty } else { object : HttpBody.SourceContent() { diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt index ff59823201..026bbd942c 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt @@ -99,7 +99,7 @@ private suspend fun Call.executeAsync(): Response = call: Call, response: Response, ) { - continuation.resume(response) { + continuation.resume(response) { cause, _, _ -> response.closeQuietly() } } diff --git a/runtime/protocol/http-client/build.gradle.kts b/runtime/protocol/http-client/build.gradle.kts index d55e5ffe41..b485f29393 100644 --- a/runtime/protocol/http-client/build.gradle.kts +++ b/runtime/protocol/http-client/build.gradle.kts @@ -27,6 +27,7 @@ kotlin { commonTest { dependencies { implementation(libs.kotlinx.coroutines.test) + implementation(project(":runtime:runtime-core")) implementation(project(":runtime:protocol:http-test")) } } diff --git a/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/HttpChecksumRequiredInterceptorTest.kt b/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/HttpChecksumRequiredInterceptorTest.kt index bc54d0f349..8634c0735a 100644 --- a/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/HttpChecksumRequiredInterceptorTest.kt +++ b/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/HttpChecksumRequiredInterceptorTest.kt @@ -68,7 +68,7 @@ class HttpChecksumRequiredInterceptorTest { assertEquals(expected, call.request.headers["x-amz-checksum-crc32"]) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native Implementation + @IgnoreNative @Test fun itSetsHeaderForNonBytesContent() = runTest { val req = HttpRequestBuilder().apply { diff --git a/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTest.kt b/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTest.kt index dc847a8839..725989fe98 100644 --- a/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTest.kt +++ b/runtime/protocol/http-client/common/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTest.kt @@ -9,6 +9,7 @@ import aws.smithy.kotlin.runtime.IgnoreNative import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.compression.CompressionAlgorithm import aws.smithy.kotlin.runtime.compression.Gzip +import aws.smithy.kotlin.runtime.compression.decompressGzipBytes import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext import aws.smithy.kotlin.runtime.http.operation.newTestOperation @@ -23,8 +24,6 @@ import kotlin.test.assertContentEquals import kotlin.test.assertEquals import kotlin.test.assertFailsWith -internal expect fun decompressGzipBytes(bytes: ByteArray): ByteArray - class RequestCompressionInterceptorTest { private val client = SdkHttpClient(TestEngine()) diff --git a/runtime/protocol/http-client/jvm/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTestJvm.kt b/runtime/protocol/http-client/jvm/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTestJvm.kt deleted file mode 100644 index a30c707658..0000000000 --- a/runtime/protocol/http-client/jvm/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTestJvm.kt +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.smithy.kotlin.runtime.http.interceptors - -import java.util.zip.GZIPInputStream - -internal actual fun decompressGzipBytes(bytes: ByteArray): ByteArray = - GZIPInputStream(bytes.inputStream()).use { it.readBytes() } diff --git a/runtime/protocol/http-client/native/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTestNative.kt b/runtime/protocol/http-client/native/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTestNative.kt deleted file mode 100644 index 7d55f18341..0000000000 --- a/runtime/protocol/http-client/native/test/aws/smithy/kotlin/runtime/http/interceptors/RequestCompressionInterceptorTestNative.kt +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.smithy.kotlin.runtime.http.interceptors - -internal actual fun decompressGzipBytes(bytes: ByteArray): ByteArray { - TODO("Not yet implemented. Can we write a pure Kotlin implementation?") -} diff --git a/runtime/runtime-core/api/runtime-core.api b/runtime/runtime-core/api/runtime-core.api index 835afc0e39..b864b68dce 100644 --- a/runtime/runtime-core/api/runtime-core.api +++ b/runtime/runtime-core/api/runtime-core.api @@ -330,6 +330,10 @@ public final class aws/smithy/kotlin/runtime/compression/Gzip : aws/smithy/kotli public fun getId ()Ljava/lang/String; } +public final class aws/smithy/kotlin/runtime/compression/GzipTestUtilsJvmKt { + public static final fun decompressGzipBytes ([B)[B +} + public final class aws/smithy/kotlin/runtime/config/EnvironmentSetting { public static final field Companion Laws/smithy/kotlin/runtime/config/EnvironmentSetting$Companion; public fun (Lkotlin/jvm/functions/Function1;Ljava/lang/String;Ljava/lang/String;Ljava/lang/Object;)V @@ -465,6 +469,8 @@ public final class aws/smithy/kotlin/runtime/content/ByteStreamJVMKt { } public final class aws/smithy/kotlin/runtime/content/ByteStreamKt { + public static final fun asByteStream (Ljava/lang/String;)Laws/smithy/kotlin/runtime/content/ByteStream; + public static final fun asByteStream ([B)Laws/smithy/kotlin/runtime/content/ByteStream; public static final fun cancel (Laws/smithy/kotlin/runtime/content/ByteStream;)V public static final fun decodeToString (Laws/smithy/kotlin/runtime/content/ByteStream;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static final fun toByteArray (Laws/smithy/kotlin/runtime/content/ByteStream;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipTestUtils.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/compression/GzipTestUtils.kt similarity index 52% rename from runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipTestUtils.kt rename to runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/compression/GzipTestUtils.kt index 511fe59f43..31b014acd0 100644 --- a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipTestUtils.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/compression/GzipTestUtils.kt @@ -2,9 +2,12 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.smithy.kotlin.runtime.io +package aws.smithy.kotlin.runtime.compression + +import aws.smithy.kotlin.runtime.InternalApi /** * Decompresses a [ByteArray] compressed using the gzip format */ -internal expect fun decompressGzipBytes(bytes: ByteArray): ByteArray +@InternalApi +public expect fun decompressGzipBytes(bytes: ByteArray): ByteArray diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/content/ByteStream.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/content/ByteStream.kt index 58f0fa1b4c..75fc36c00d 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/content/ByteStream.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/content/ByteStream.kt @@ -7,7 +7,6 @@ package aws.smithy.kotlin.runtime.content import aws.smithy.kotlin.runtime.io.* import aws.smithy.kotlin.runtime.io.internal.SdkDispatchers import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.IO import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch @@ -197,3 +196,13 @@ private fun SdkSource.toFlow(bufferSize: Long): Flow = flow { emit(sink.readByteArray()) } } + +/** + * Convert this [String] to a [ByteStream] + */ +public fun String.asByteStream(): ByteStream = ByteStream.fromString(this) + +/** + * Convert this [ByteArray] to a [ByteStream] + */ +public fun ByteArray.asByteStream(): ByteStream = ByteStream.fromBytes(this) diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt index 7a3a569899..9af6977d29 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt @@ -99,10 +99,17 @@ public fun String.decodeBase64Bytes(): ByteArray = encodeToByteArray().decodeBas * Decode [ByteArray] from base64 format */ public fun ByteArray.decodeBase64(): ByteArray { - val encoded = this + // Calculate the padding needed to make the length a multiple of 4 + val remainder = size % 4 + val encoded: ByteArray = if (remainder == 0) { + this + } else { + this + ByteArray(4 - remainder) { BASE64_PAD.code.toByte() } + } + val decodedLen = base64DecodedLen(encoded) val decoded = ByteArray(decodedLen) - val blockCnt = size / 4 + val blockCnt = encoded.size / 4 var readIdx = 0 var writeIdx = 0 diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt index 9b691a79a2..c8437b2b33 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt @@ -62,8 +62,7 @@ public fun Any?.type(): String = when (this) { is List<*>, is Array<*> -> "array" is Number -> "number" is Any -> "object" - null -> "null" - else -> throw Exception("Undetected type for: $this") + else -> "null" } // Collection `flattenIfPossible` functions diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/compression/GzipTest.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/compression/GzipTest.kt new file mode 100644 index 0000000000..0125eae489 --- /dev/null +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/compression/GzipTest.kt @@ -0,0 +1,41 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.smithy.kotlin.runtime.compression + +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.toByteArray +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertContentEquals + +class GzipTest { + @Test + fun testCompress() = runTest { + val payload = "Hello World".encodeToByteArray() + val byteStream = ByteStream.fromBytes(payload) + + val compressed = Gzip() + .compress(byteStream) + .toByteArray() + + val decompressedBytes = decompressGzipBytes(compressed) + assertContentEquals(payload, decompressedBytes) + } + + @Test + fun testCompressEmptyByteArray() = runTest { + val payload = ByteArray(0) + val byteStream = ByteStream.fromBytes(payload) + + val compressed = Gzip() + .compress(byteStream) + .toByteArray() + + val decompressedBytes = decompressGzipBytes(compressed) + + assertContentEquals(payload, decompressedBytes) + } +} diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipByteReadChannelTest.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipByteReadChannelTest.kt index ad1c9bf5b6..259b4a3aab 100644 --- a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipByteReadChannelTest.kt +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipByteReadChannelTest.kt @@ -4,7 +4,7 @@ */ package aws.smithy.kotlin.runtime.io -import aws.smithy.kotlin.runtime.IgnoreNative +import aws.smithy.kotlin.runtime.compression.decompressGzipBytes import aws.smithy.kotlin.runtime.hashing.crc32 import kotlinx.coroutines.test.runTest import kotlin.test.Test @@ -12,7 +12,6 @@ import kotlin.test.assertContentEquals import kotlin.test.assertEquals class GzipByteReadChannelTest { - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadAll() = runTest { val payload = "Hello World" @@ -32,7 +31,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadToBuffer() = runTest { val payload = "Hello World".repeat(1600) @@ -51,7 +49,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadRemaining() = runTest { val payload = "Hello World".repeat(1600) @@ -71,7 +68,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testRead() = runTest { val payload = "Hello World" @@ -92,7 +88,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadLargeBody() = runTest { val payload = "Hello World".repeat(1600) @@ -113,7 +108,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadLargeLimit() = runTest { val payload = "Hello World" @@ -134,7 +128,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadLargeBodyLargeLimit() = runTest { val payload = "Hello World".repeat(1600) @@ -155,7 +148,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testIsClosedForRead() = runTest { val payload = "Hello World" @@ -178,7 +170,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testIsClosedForReadLargeBody() = runTest { val payload = "Hello World".repeat(1600) @@ -201,7 +192,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testIsClosedForReadLargeLimit() = runTest { val payload = "Hello World" @@ -224,7 +214,6 @@ class GzipByteReadChannelTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testIsClosedForReadLargeBodyLargeLimit() = runTest { val payload = "Hello World".repeat(1600) diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipSdkSourceTest.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipSdkSourceTest.kt index f36b671894..179673c28a 100644 --- a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipSdkSourceTest.kt +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/io/GzipSdkSourceTest.kt @@ -4,7 +4,7 @@ */ package aws.smithy.kotlin.runtime.io -import aws.smithy.kotlin.runtime.IgnoreNative +import aws.smithy.kotlin.runtime.compression.decompressGzipBytes import aws.smithy.kotlin.runtime.hashing.crc32 import kotlinx.coroutines.test.runTest import kotlin.test.Test @@ -12,7 +12,6 @@ import kotlin.test.assertContentEquals import kotlin.test.assertEquals class GzipSdkSourceTest { - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadToByteArray() = runTest { val payload = "Hello World" @@ -30,7 +29,6 @@ class GzipSdkSourceTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testRead() = runTest { val payload = "Hello World" @@ -51,7 +49,6 @@ class GzipSdkSourceTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadLargeBody() = runTest { val payload = "Hello World".repeat(1600) @@ -73,7 +70,6 @@ class GzipSdkSourceTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadLargeLimit() = runTest { val payload = "Hello World" @@ -94,7 +90,6 @@ class GzipSdkSourceTest { assertEquals(bytesHash, decompressedBytes.crc32()) } - @IgnoreNative // FIXME Re-enable after Kotlin/Native implementation @Test fun testReadLargeBodyLargeLimit() = runTest { val payload = "Hello World".repeat(1600) diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt index 4c2ada2010..956fabee2e 100644 --- a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt @@ -76,14 +76,6 @@ class Base64Test { ex.message!!.shouldContain("decode base64: invalid input byte: 45") } - @Test - fun decodeNonMultipleOf4() { - val ex = assertFails { - "Zm9vY=".decodeBase64() - } - ex.message!!.shouldContain("invalid base64 string of length 6; not a multiple of 4") - } - @Test fun decodeInvalidPadding() { val ex = assertFails { @@ -116,4 +108,24 @@ class Base64Test { assertEquals(encoded, decoded.encodeBase64()) assertEquals(decoded, encoded.decodeBase64()) } + + @Test + fun testUnpaddedInputs() { + // from https://github.com/smithy-lang/smithy/pull/2502 + val input = "v2hkZWZhdWx0c79tZGVmYXVsdFN0cmluZ2JoaW5kZWZhdWx0Qm9vbGVhbvVrZGVmYXVsdExpc3Sf/3BkZWZhdWx0VGltZXN0YW1wwQBrZGVmYXVsdEJsb2JDYWJja2RlZmF1bHRCeXRlAWxkZWZhdWx0U2hvcnQBbmRlZmF1bHRJbnRlZ2VyCmtkZWZhdWx0TG9uZxhkbGRlZmF1bHRGbG9hdPo/gAAAbWRlZmF1bHREb3VibGX6P4AAAGpkZWZhdWx0TWFwv/9rZGVmYXVsdEVudW1jRk9PbmRlZmF1bHRJbnRFbnVtAWtlbXB0eVN0cmluZ2BsZmFsc2VCb29sZWFu9GllbXB0eUJsb2JAaHplcm9CeXRlAGl6ZXJvU2hvcnQAa3plcm9JbnRlZ2VyAGh6ZXJvTG9uZwBpemVyb0Zsb2F0+gAAAABqemVyb0RvdWJsZfoAAAAA//8" + input.decodeBase64() + + val inputs = mapOf( + "YQ" to "a", + "Yg" to "b", + "YWI" to "ab", + "YWJj" to "abc", + "SGVsbG8gd29ybGQ" to "Hello world", + ) + + inputs.forEach { (encoded, expected) -> + val actual = encoded.decodeBase64() + assertEquals(expected, actual) + } + } } diff --git a/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/io/GzipTestUtilsJVM.kt b/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/compression/GzipTestUtilsJvm.kt similarity index 57% rename from runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/io/GzipTestUtilsJVM.kt rename to runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/compression/GzipTestUtilsJvm.kt index 1d879aa925..bd2b274962 100644 --- a/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/io/GzipTestUtilsJVM.kt +++ b/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/compression/GzipTestUtilsJvm.kt @@ -2,12 +2,15 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.smithy.kotlin.runtime.io +package aws.smithy.kotlin.runtime.compression +import aws.smithy.kotlin.runtime.InternalApi +import aws.smithy.kotlin.runtime.io.use import java.util.zip.GZIPInputStream /** * Decompresses a [ByteArray] compressed using the gzip format */ -internal actual fun decompressGzipBytes(bytes: ByteArray): ByteArray = +@InternalApi +public actual fun decompressGzipBytes(bytes: ByteArray): ByteArray = GZIPInputStream(bytes.inputStream()).use { it.readBytes() } diff --git a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipCompressor.kt b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipCompressor.kt new file mode 100644 index 0000000000..954cfb3bd5 --- /dev/null +++ b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipCompressor.kt @@ -0,0 +1,128 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.smithy.kotlin.runtime.compression + +import aws.sdk.kotlin.crt.Closeable +import aws.smithy.kotlin.runtime.io.SdkBuffer +import kotlinx.cinterop.* +import platform.zlib.* + +private const val DEFAULT_WINDOW_BITS = 15 // Default window bits +private const val WINDOW_BITS_GZIP_OFFSET = 16 // Gzip offset for window bits +private const val MEM_LEVEL = 8 // Default memory level + +/** + * Streaming-style gzip compressor, implemented using zlib bindings + */ +internal class GzipCompressor : Closeable { + companion object { + internal const val BUFFER_SIZE = 16384 + } + + private val stream = nativeHeap.alloc() + private val outputBuffer = SdkBuffer() + internal var isClosed = false + + internal val availableForRead: Int + get() = outputBuffer.size.toInt() + + init { + // Initialize deflate with gzip encoding + val initResult = deflateInit2_( + stream.ptr, + Z_BEST_COMPRESSION, + Z_DEFLATED, + DEFAULT_WINDOW_BITS + WINDOW_BITS_GZIP_OFFSET, + MEM_LEVEL, + Z_DEFAULT_STRATEGY, + ZLIB_VERSION, + sizeOf().toInt(), + ) + + check(initResult == Z_OK) { "Failed to initialize zlib deflate with error code $initResult: ${zError(initResult)!!.toKString()}" } + } + + /** + * Update the compressor with [input] bytes + */ + fun update(input: ByteArray) = memScoped { + check(!isClosed) { "Compressor is closed" } + + val inputPin = input.pin() + + stream.next_in = inputPin.addressOf(0).reinterpret() + stream.avail_in = input.size.toUInt() + + val compressionBuffer = ByteArray(BUFFER_SIZE) + + while (stream.avail_in > 0u) { + val outputPin = compressionBuffer.pin() + stream.next_out = outputPin.addressOf(0).reinterpret() + stream.avail_out = BUFFER_SIZE.toUInt() + + val deflateResult = deflate(stream.ptr, Z_NO_FLUSH) + check(deflateResult == Z_OK) { "Deflate failed with error code $deflateResult" } + + val bytesWritten = BUFFER_SIZE - stream.avail_out.toInt() + outputBuffer.write(compressionBuffer, 0, bytesWritten) + + outputPin.unpin() + } + + inputPin.unpin() + } + + /** + * Consume [count] gzip-compressed bytes. + */ + fun consume(count: Int): ByteArray { + check(!isClosed) { "Compressor is closed" } + require(count in 0..availableForRead) { + "Count must be between 0 and $availableForRead, got $count" + } + + return outputBuffer.readByteArray(count.toLong()) + } + + /** + * Flush the compressor and return the terminal sequence of bytes that represent the end of the gzip compression. + */ + fun flush(): ByteArray { + check(!isClosed) { "Compressor is closed" } + + memScoped { + val compressionBuffer = ByteArray(BUFFER_SIZE) + var deflateResult: Int? = null + var outputLength = 0L + + do { + val outputPin = compressionBuffer.pin() + stream.next_out = outputPin.addressOf(0).reinterpret() + stream.avail_out = BUFFER_SIZE.toUInt() + + deflateResult = deflate(stream.ptr, Z_FINISH) + check(deflateResult == Z_OK || deflateResult == Z_STREAM_END) { "Deflate failed during finish with error code $deflateResult" } + + val bytesWritten = BUFFER_SIZE - stream.avail_out.toInt() + outputBuffer.write(compressionBuffer, 0, bytesWritten) + + outputLength += bytesWritten.toLong() + outputPin.unpin() + } while (deflateResult != Z_STREAM_END) + + return outputBuffer.readByteArray(outputLength) + } + } + + override fun close() { + if (isClosed) { + return + } + + deflateEnd(stream.ptr) + nativeHeap.free(stream.ptr) + isClosed = true + } +} diff --git a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt index a9fc036399..f82274317a 100644 --- a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt +++ b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipNative.kt @@ -5,6 +5,14 @@ package aws.smithy.kotlin.runtime.compression import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.asByteStream +import aws.smithy.kotlin.runtime.io.GzipByteReadChannel +import aws.smithy.kotlin.runtime.io.GzipSdkSource +import aws.smithy.kotlin.runtime.io.SdkByteReadChannel +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.buffer +import aws.smithy.kotlin.runtime.io.source +import aws.smithy.kotlin.runtime.io.use /** * The gzip compression algorithm. @@ -13,12 +21,31 @@ import aws.smithy.kotlin.runtime.content.ByteStream * See: https://en.wikipedia.org/wiki/Gzip */ public actual class Gzip : CompressionAlgorithm { - actual override val id: String - get() = TODO("Not yet implemented") - actual override val contentEncoding: String - get() = TODO("Not yet implemented") + actual override val id: String = "gzip" + actual override val contentEncoding: String = "gzip" - actual override fun compress(stream: ByteStream): ByteStream { - TODO("Not yet implemented") + actual override fun compress(stream: ByteStream): ByteStream = when (stream) { + is ByteStream.ChannelStream -> object : ByteStream.ChannelStream() { + override fun readFrom(): SdkByteReadChannel = GzipByteReadChannel(stream.readFrom()) + override val contentLength: Long? = null + override val isOneShot: Boolean = stream.isOneShot + } + + is ByteStream.SourceStream -> object : ByteStream.SourceStream() { + override fun readFrom(): SdkSource = GzipSdkSource(stream.readFrom()) + override val contentLength: Long? = null + override val isOneShot: Boolean = stream.isOneShot + } + + is ByteStream.Buffer -> { + val bytes = stream.bytes() + if (bytes.isEmpty()) { + stream + } else { + GzipSdkSource(bytes.source()).use { + it.buffer().readByteArray().asByteStream() + } + } + } } } diff --git a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipTestUtilsNative.kt b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipTestUtilsNative.kt new file mode 100644 index 0000000000..6015984fb3 --- /dev/null +++ b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/compression/GzipTestUtilsNative.kt @@ -0,0 +1,76 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.smithy.kotlin.runtime.compression + +import aws.smithy.kotlin.runtime.InternalApi +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.pin +import kotlinx.cinterop.ptr +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.sizeOf +import kotlinx.cinterop.toKString +import platform.zlib.* + +/** + * Decompresses a byte array compressed using the gzip format + */ +@InternalApi +public actual fun decompressGzipBytes(bytes: ByteArray): ByteArray { + if (bytes.isEmpty()) { + return bytes + } + + val decompressedBuffer = UByteArray(bytes.size * 2).pin() + + memScoped { + val zStream = alloc() + + // Initialize the inflate context for gzip decoding + val result = inflateInit2_( + strm = zStream.ptr, + windowBits = 31, + version = ZLIB_VERSION, + stream_size = sizeOf().toInt(), + ) + if (result != Z_OK) { + throw IllegalStateException("inflateInit2_ failed with error code $result") + } + + try { + val bytesPinned = bytes.pin() + + zStream.next_in = bytesPinned.addressOf(0).getPointer(memScope).reinterpret() + zStream.avail_in = bytes.size.toUInt() + + val output = mutableListOf() + while (zStream.avail_in > 0u) { + zStream.next_out = decompressedBuffer.addressOf(0) + zStream.avail_out = decompressedBuffer.get().size.toUInt() + + val inflateResult = inflate(zStream.ptr, Z_NO_FLUSH) + + when (inflateResult) { + Z_OK, Z_STREAM_END -> { + val chunkSize = decompressedBuffer.get().size.toUInt() - zStream.avail_out + output.addAll(decompressedBuffer.get().copyOf(chunkSize.toInt())) + } + else -> { + throw IllegalStateException("Decompression failed with error code $inflateResult: ${zError(inflateResult)!!.toKString()}") + } + } + + if (inflateResult == Z_STREAM_END) { + break + } + } + + return output.toUByteArray().toByteArray() + } finally { + inflateEnd(zStream.ptr) + } + } +} diff --git a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipByteReadChannelNative.kt b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipByteReadChannelNative.kt index f8ca725e14..4fadeacbb9 100644 --- a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipByteReadChannelNative.kt +++ b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipByteReadChannelNative.kt @@ -5,26 +5,62 @@ package aws.smithy.kotlin.runtime.io import aws.smithy.kotlin.runtime.InternalApi +import aws.smithy.kotlin.runtime.compression.GzipCompressor /** - * Wraps the SdkByteReadChannel so that it compresses into gzip format with each read. + * Wraps an [SdkByteReadChannel], compressing bytes read into GZIP format. + * @param channel the [SdkByteReadChannel] to compress the contents of */ @InternalApi -public actual class GzipByteReadChannel actual constructor(channel: SdkByteReadChannel) : SdkByteReadChannel { +public actual class GzipByteReadChannel actual constructor(public val channel: SdkByteReadChannel) : SdkByteReadChannel { + private val compressor = GzipCompressor() + actual override val availableForRead: Int - get() = TODO("Not yet implemented") + get() = compressor.availableForRead + actual override val isClosedForRead: Boolean - get() = TODO("Not yet implemented") + get() = channel.isClosedForRead && compressor.isClosed + actual override val isClosedForWrite: Boolean - get() = TODO("Not yet implemented") + get() = channel.isClosedForWrite + actual override val closedCause: Throwable? - get() = TODO("Not yet implemented") + get() = channel.closedCause actual override suspend fun read(sink: SdkBuffer, limit: Long): Long { - TODO("Not yet implemented") + require(limit >= 0L) + if (limit == 0L) return 0L + if (compressor.isClosed) return -1L + + // If no compressed bytes are available, attempt to refill the compressor + if (compressor.availableForRead == 0 && !channel.isClosedForRead) { + val temp = SdkBuffer() + val rc = channel.read(temp, GzipCompressor.BUFFER_SIZE.toLong()) + + if (rc > 0) { + val input = temp.readByteArray(rc) + compressor.update(input) + } + } + + // If still no data is available and the channel is closed, we've hit EOF. Close the compressor and write the remaining bytes + if (compressor.availableForRead == 0 && channel.isClosedForRead) { + val terminationBytes = compressor.flush() + sink.write(terminationBytes) + return terminationBytes.size.toLong().also { + compressor.close() + } + } + + // Read compressed bytes from the compressor + val bytesToRead = minOf(limit, compressor.availableForRead.toLong()) + val compressed = compressor.consume(bytesToRead.toInt()) + sink.write(compressed) + return compressed.size.toLong() } actual override fun cancel(cause: Throwable?): Boolean { - TODO("Not yet implemented") + compressor.close() + return channel.cancel(cause) } } diff --git a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipSdkSourceNative.kt b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipSdkSourceNative.kt index f38cfd45d1..39055dd4d9 100644 --- a/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipSdkSourceNative.kt +++ b/runtime/runtime-core/native/src/aws/smithy/kotlin/runtime/io/GzipSdkSourceNative.kt @@ -5,17 +5,50 @@ package aws.smithy.kotlin.runtime.io import aws.smithy.kotlin.runtime.InternalApi +import aws.smithy.kotlin.runtime.compression.GzipCompressor /** - * Wraps the SdkSource so that it compresses into gzip format with each read. + * Wraps an [SdkSource], compressing bytes read into GZIP format. + * @param source the [SdkSource] to compress the contents of */ @InternalApi -public actual class GzipSdkSource actual constructor(source: SdkSource) : SdkSource { +public actual class GzipSdkSource actual constructor(public val source: SdkSource) : SdkSource { + private val compressor = GzipCompressor() + actual override fun read(sink: SdkBuffer, limit: Long): Long { - TODO("Not yet implemented") + require(limit >= 0L) + if (limit == 0L) return 0L + if (compressor.isClosed) return -1L + + // If no compressed bytes are available, attempt to refill the compressor + if (compressor.availableForRead == 0) { + val temp = SdkBuffer() + val rc = source.read(temp, GzipCompressor.BUFFER_SIZE.toLong()) + + if (rc > 0) { + val input = temp.readByteArray(rc) + compressor.update(input) + } + } + + // If still no data is available, we've hit EOF. Close the compressor and write the remaining bytes + if (compressor.availableForRead == 0) { + val terminationBytes = compressor.flush() + sink.write(terminationBytes) + return terminationBytes.size.toLong().also { + compressor.close() + } + } + + // Read compressed bytes from the compressor + val bytesToRead = minOf(limit, compressor.availableForRead.toLong()) + val compressed = compressor.consume(bytesToRead.toInt()) + sink.write(compressed) + return compressed.size.toLong() } actual override fun close() { - TODO("Not yet implemented") + compressor.close() + source.close() } } diff --git a/runtime/runtime-core/native/test/aws/smithy/kotlin/runtime/io/GzipTestUtilsNative.kt b/runtime/runtime-core/native/test/aws/smithy/kotlin/runtime/io/GzipTestUtilsNative.kt deleted file mode 100644 index 198808f770..0000000000 --- a/runtime/runtime-core/native/test/aws/smithy/kotlin/runtime/io/GzipTestUtilsNative.kt +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.smithy.kotlin.runtime.io - -/** - * Decompresses a byte array compressed using the gzip format - */ -internal actual fun decompressGzipBytes(bytes: ByteArray): ByteArray { - TODO("Not yet implemented") -}