From 45750232626bbe4e279142e486be8bbbfccab33f Mon Sep 17 00:00:00 2001 From: Matas Date: Wed, 18 Dec 2024 14:20:08 -0500 Subject: [PATCH 1/7] misc: enhance support for replayable instances of `InputStream` (#1197) --- .../2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json | 8 ++++++ .../kotlin/runtime/content/ByteStreamJVM.kt | 23 +++++++++++++-- .../runtime/content/ByteStreamJVMTest.kt | 28 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json diff --git a/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json b/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json new file mode 100644 index 0000000000..fb16fa5843 --- /dev/null +++ b/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json @@ -0,0 +1,8 @@ +{ + "id": "2de8162f-e5a0-4618-b00d-8da3cfdbc6a2", + "type": "feature", + "description": "Enhance support for replayable instances of `InputStream`", + "issues": [ + "https://github.com/awslabs/aws-sdk-kotlin/issues/1473" + ] +} \ No newline at end of file diff --git a/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt b/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt index 1d6124357a..5647ac15af 100644 --- a/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt +++ b/runtime/runtime-core/jvm/src/aws/smithy/kotlin/runtime/content/ByteStreamJVM.kt @@ -114,11 +114,30 @@ public fun ByteStream.Companion.fromInputStream( * @param contentLength If specified, indicates how many bytes remain in this stream. Defaults to `null`. */ public fun InputStream.asByteStream(contentLength: Long? = null): ByteStream.SourceStream { - val source = source() + if (markSupported() && contentLength != null) { + mark(contentLength.toInt()) + } + return object : ByteStream.SourceStream() { override val contentLength: Long? = contentLength override val isOneShot: Boolean = !markSupported() - override fun readFrom(): SdkSource = source + override fun readFrom(): SdkSource { + if (markSupported() && contentLength != null) { + reset() + mark(contentLength.toInt()) + return object : SdkSource by source() { + /* + * This is a no-op close to prevent body hashing from closing the underlying InputStream, which causes + * `IOException: Stream closed` on subsequent reads. Consider making [ByteStream.ChannelStream]/[ByteStream.SourceStream] + * (or possibly even [ByteStream] itself) implement [Closeable] to better handle closing streams. + * This should allow us to clean up our usage of [ByteStream.cancel()]. + */ + override fun close() { } + } + } + + return source() + } } } diff --git a/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt b/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt index 054387b2e5..e8324fb11f 100644 --- a/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt +++ b/runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/content/ByteStreamJVMTest.kt @@ -5,8 +5,11 @@ package aws.smithy.kotlin.runtime.content +import aws.smithy.kotlin.runtime.io.readToByteArray import aws.smithy.kotlin.runtime.testing.RandomTempFile import kotlinx.coroutines.test.runTest +import java.io.BufferedInputStream +import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.InputStream import java.io.OutputStream @@ -228,6 +231,31 @@ class ByteStreamJVMTest { assertFalse(sos.closed) } + // https://github.com/awslabs/aws-sdk-kotlin/issues/1473 + @Test + fun testReplayableInputStreamAsByteStream() = runTest { + val content = "Hello, Bytes!".encodeToByteArray() + val byteArrayIns = ByteArrayInputStream(content) + val nonReplayableIns = NonReplayableInputStream(byteArrayIns) + + // buffer the non-replayable stream, making it replayable... + val bufferedIns = BufferedInputStream(nonReplayableIns) + + val byteStream = bufferedIns.asByteStream(content.size.toLong()) + + // Test that it can be read at least twice (e.g. once for hashing the body, once for transmitting the body) + assertContentEquals(content, byteStream.readFrom().use { it.readToByteArray() }) + assertContentEquals(content, byteStream.readFrom().use { it.readToByteArray() }) + } + + private class NonReplayableInputStream(val inputStream: InputStream) : InputStream() { + override fun markSupported(): Boolean = false // not replayable + + override fun read(): Int = inputStream.read() + override fun mark(readlimit: Int) = inputStream.mark(readlimit) + override fun reset() = inputStream.reset() + } + private class StatusTrackingOutputStream(val os: OutputStream) : OutputStream() { var closed: Boolean = false From b0a4bacc3626bf2c9e8cfe9607ef8d5ece03ee4e Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Wed, 18 Dec 2024 19:23:06 +0000 Subject: [PATCH 2/7] chore: release 1.3.31 --- .changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json | 8 -------- CHANGELOG.md | 5 +++++ gradle.properties | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) delete mode 100644 .changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json diff --git a/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json b/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json deleted file mode 100644 index fb16fa5843..0000000000 --- a/.changes/2de8162f-e5a0-4618-b00d-8da3cfdbc6a2.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id": "2de8162f-e5a0-4618-b00d-8da3cfdbc6a2", - "type": "feature", - "description": "Enhance support for replayable instances of `InputStream`", - "issues": [ - "https://github.com/awslabs/aws-sdk-kotlin/issues/1473" - ] -} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5633a30d73..5473a56d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [1.3.31] - 12/18/2024 + +### Features +* [#1473](https://github.com/awslabs/aws-sdk-kotlin/issues/1473) Enhance support for replayable instances of `InputStream` + ## [1.3.30] - 12/16/2024 ## [1.3.29] - 12/12/2024 diff --git a/gradle.properties b/gradle.properties index 3628520e5a..cf967edd0b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.31-SNAPSHOT +sdkVersion=1.3.31 # codegen -codegenVersion=0.33.31-SNAPSHOT \ No newline at end of file +codegenVersion=0.33.31 \ No newline at end of file From 80f453814d8b0c4e20ef7e729d47a364ca8b5753 Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Wed, 18 Dec 2024 19:23:08 +0000 Subject: [PATCH 3/7] chore: bump snapshot version to 1.3.32-SNAPSHOT --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index cf967edd0b..f902a36719 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.31 +sdkVersion=1.3.32-SNAPSHOT # codegen -codegenVersion=0.33.31 \ No newline at end of file +codegenVersion=0.33.32-SNAPSHOT \ No newline at end of file From e9d16a98b8030468e35a05f63c79cae2444e6759 Mon Sep 17 00:00:00 2001 From: Matas Date: Wed, 18 Dec 2024 16:28:30 -0500 Subject: [PATCH 4/7] fix: CBOR protocol test assertions / blob serialization (#1198) --- .changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json | 5 +++++ .../protocol/HttpProtocolUnitTestRequestGenerator.kt | 4 ++-- .../codegen/rendering/serde/SerializeStructGenerator.kt | 1 - .../codegen/rendering/serde/SerializeStructGeneratorTest.kt | 2 +- .../src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 .changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json diff --git a/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json b/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json new file mode 100644 index 0000000000..0fad5e0349 --- /dev/null +++ b/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json @@ -0,0 +1,5 @@ +{ + "id": "1021e75a-45f3-4f3a-820c-700d9ec6e782", + "type": "bugfix", + "description": "Fix serialization of CBOR blobs" +} \ No newline at end of file diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt index 74bd7db782..6fcc47c3e5 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestRequestGenerator.kt @@ -117,11 +117,11 @@ open class HttpProtocolUnitTestRequestGenerator protected constructor(builder: B write("return") } write("requireNotNull(expectedBytes) { #S }", "expected application/cbor body cannot be null") - write("requireNotNull(expectedBytes) { #S }", "actual application/cbor body cannot be null") + write("requireNotNull(actualBytes) { #S }", "actual application/cbor body cannot be null") write("") write("val expectedRequest = #L(#T(expectedBytes))", inputDeserializer.name, RuntimeTypes.Serde.SerdeCbor.CborDeserializer) - write("val actualRequest = #L(#T(expectedBytes))", inputDeserializer.name, RuntimeTypes.Serde.SerdeCbor.CborDeserializer) + write("val actualRequest = #L(#T(actualBytes))", inputDeserializer.name, RuntimeTypes.Serde.SerdeCbor.CborDeserializer) write("assertEquals(expectedRequest, actualRequest)") } writer.write("") diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt index e7f2e261ef..d8810044a5 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGenerator.kt @@ -647,7 +647,6 @@ open class SerializeStructGenerator( val target = member.targetOrSelf(ctx.model) val encoded = when { - target.type == ShapeType.BLOB -> writer.format("#L.#T()", identifier, RuntimeTypes.Core.Text.Encoding.encodeBase64String) target.type == ShapeType.TIMESTAMP -> { writer.addImport(RuntimeTypes.Core.TimestampFormat) val tsFormat = member diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt index 40d9c9db7d..347da007cb 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/SerializeStructGeneratorTest.kt @@ -1822,7 +1822,7 @@ class SerializeStructGeneratorTest { val expected = """ serializer.serializeStruct(OBJ_DESCRIPTOR) { - input.fooBlob?.let { field(FOOBLOB_DESCRIPTOR, it.encodeBase64String()) } + input.fooBlob?.let { field(FOOBLOB_DESCRIPTOR, it) } } """.trimIndent() diff --git a/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt b/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt index 35618cb49e..d1356f848c 100644 --- a/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt +++ b/runtime/serde/serde-xml/common/src/aws/smithy/kotlin/runtime/serde/xml/XmlSerializer.kt @@ -134,7 +134,7 @@ public class XmlSerializer(private val xmlWriter: XmlStreamWriter = xmlStreamWri field(descriptor, value.format(format)) override fun field(descriptor: SdkFieldDescriptor, value: ByteArray): Unit = - field(descriptor, value) + field(descriptor, value.encodeBase64String()) override fun field(descriptor: SdkFieldDescriptor, value: Document?): Unit = throw SerializationException( "cannot serialize field ${descriptor.serialName}; Document type is not supported by xml encoding", From 48849a1208fe0bb2c42c06c49b54b9c419104378 Mon Sep 17 00:00:00 2001 From: Matas Date: Mon, 6 Jan 2025 16:34:11 -0500 Subject: [PATCH 5/7] fix: correctly serialize subset of shape's members when configured (#1199) --- .../core/QueryHttpBindingProtocolGenerator.kt | 2 +- .../codegen/aws/protocols/RpcV2CborTest.kt | 17 +++++++++++++++++ .../rendering/serde/CborSerializerGenerator.kt | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/QueryHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/QueryHttpBindingProtocolGenerator.kt index 62f8f7ebe1..db97349f84 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/QueryHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/QueryHttpBindingProtocolGenerator.kt @@ -145,7 +145,7 @@ abstract class AbstractQueryFormUrlSerializerGenerator( return shape.documentSerializer(ctx.settings, symbol, members) { writer -> writer.openBlock("internal fun #identifier.name:L(serializer: #T, input: #T) {", RuntimeTypes.Serde.Serializer, symbol) .call { - renderSerializerBody(ctx, shape, shape.members().toList(), writer) + renderSerializerBody(ctx, shape, members.toList(), writer) } .closeBlock("}") } diff --git a/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/RpcV2CborTest.kt b/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/RpcV2CborTest.kt index 17a7c0f3d1..0114185004 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/RpcV2CborTest.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/RpcV2CborTest.kt @@ -4,6 +4,7 @@ */ package software.amazon.smithy.kotlin.codegen.aws.protocols +import io.kotest.matchers.string.shouldNotContain import software.amazon.smithy.kotlin.codegen.test.* import kotlin.test.Test @@ -145,4 +146,20 @@ class RpcV2CborTest { val serializeBody = serializer.lines(" override suspend fun serialize(context: ExecutionContext, input: PutFooStreamingRequest): HttpRequestBuilder {", "}") serializeBody.shouldContainOnlyOnceWithDiff("""builder.headers.setMissing("Content-Type", "application/vnd.amazon.eventstream")""") } + + @Test + fun testEventStreamInitialRequestDoesNotSerializeStreamMember() { + val ctx = model.newTestContext("CborExample") + + val generator = RpcV2Cbor() + generator.generateProtocolClient(ctx.generationCtx) + + ctx.generationCtx.delegator.finalize() + ctx.generationCtx.delegator.flushWriters() + + val documentSerializer = ctx.manifest.expectFileString("/src/main/kotlin/com/test/serde/PutFooStreamingRequestDocumentSerializer.kt") + + val serializeBody = documentSerializer.lines(" serializer.serializeStruct(OBJ_DESCRIPTOR) {", "}") + serializeBody.shouldNotContain("input.messages") // `messages` is the stream member and should not be serialized in the initial request + } } diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/CborSerializerGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/CborSerializerGenerator.kt index 1fdb7850a7..545a484a90 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/CborSerializerGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/serde/CborSerializerGenerator.kt @@ -108,7 +108,7 @@ class CborSerializerGenerator( val symbol = ctx.symbolProvider.toSymbol(shape) return shape.documentSerializer(ctx.settings, symbol, members) { writer -> writer.withBlock("internal fun #identifier.name:L(serializer: #T, input: #T) {", "}", RuntimeTypes.Serde.Serializer, symbol) { - call { renderSerializerBody(ctx, shape, shape.members().toList(), writer) } + call { renderSerializerBody(ctx, shape, members.toList(), writer) } } } } From ded3a4bb03060f28f3ce085fed3a9627f7297ae5 Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Mon, 6 Jan 2025 21:45:24 +0000 Subject: [PATCH 6/7] chore: release 1.3.32 --- .changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json | 5 ----- CHANGELOG.md | 5 +++++ gradle.properties | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 .changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json diff --git a/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json b/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json deleted file mode 100644 index 0fad5e0349..0000000000 --- a/.changes/1021e75a-45f3-4f3a-820c-700d9ec6e782.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "1021e75a-45f3-4f3a-820c-700d9ec6e782", - "type": "bugfix", - "description": "Fix serialization of CBOR blobs" -} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5473a56d69..a52fe756c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [1.3.32] - 01/06/2025 + +### Fixes +* Fix serialization of CBOR blobs + ## [1.3.31] - 12/18/2024 ### Features diff --git a/gradle.properties b/gradle.properties index f902a36719..eda623b285 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.32-SNAPSHOT +sdkVersion=1.3.32 # codegen -codegenVersion=0.33.32-SNAPSHOT \ No newline at end of file +codegenVersion=0.33.32 \ No newline at end of file From d97e8ba056d5f0c9869226854638dd486e06f1fa Mon Sep 17 00:00:00 2001 From: aws-sdk-kotlin-ci Date: Mon, 6 Jan 2025 21:45:25 +0000 Subject: [PATCH 7/7] chore: bump snapshot version to 1.3.33-SNAPSHOT --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index eda623b285..b49ffff27f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ kotlinx.atomicfu.enableNativeIrTransformation=false org.gradle.jvmargs=-Xmx2G -XX:MaxMetaspaceSize=1G # SDK -sdkVersion=1.3.32 +sdkVersion=1.3.33-SNAPSHOT # codegen -codegenVersion=0.33.32 \ No newline at end of file +codegenVersion=0.33.33-SNAPSHOT \ No newline at end of file