From 95e148054d80a7d9282930fe8aadd81e54537cec Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 17 Jan 2025 13:06:04 -0500 Subject: [PATCH 01/16] Allow serializing empty headers --- .../core/AwsHttpBindingProtocolGenerator.kt | 2 -- .../protocol/HttpStringValuesMapSerializer.kt | 19 +++---------------- .../HttpBindingProtocolGeneratorTest.kt | 8 ++++---- .../HttpStringValuesMapSerializerTest.kt | 16 ++++++++-------- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt index 6e5834f7d1..8cfbd5e03d 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt @@ -42,8 +42,6 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator() val ignoredTests = TestMemberDelta( setOf( "AwsJson10ClientErrorCorrectsWithDefaultValuesWhenServerFailsToSerializeRequiredValues", - "RestJsonNullAndEmptyHeaders", - "NullAndEmptyHeaders", "RpcV2CborClientPopulatesDefaultsValuesWhenMissingInResponse", "RpcV2CborClientPopulatesDefaultValuesInInput", ), diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt index 932b2a796a..5dfc519962 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt @@ -157,9 +157,8 @@ class HttpStringValuesMapSerializer( val paramName = binding.locationName // addAll collection parameter 2 val param2 = if (mapFnContents.isEmpty()) "input.$memberName" else "input.$memberName.map { $mapFnContents }" - val nullCheck = if (memberSymbol.isNullable) "?" else "" writer.write( - "if (input.#L$nullCheck.isNotEmpty() == true) #L(#S, #L)", + "if (input.#L != null) #L(#S, #L)", memberName, binding.location.addAllFnName, paramName, @@ -174,8 +173,7 @@ class HttpStringValuesMapSerializer( val paramName = binding.locationName val memberSymbol = symbolProvider.toSymbol(binding.member) - // NOTE: query parameters are allowed to be empty, whereas headers should omit empty string - // values from serde + // NOTE: query parameters are allowed to be empty if ((location == HttpBinding.Location.QUERY || location == HttpBinding.Location.HEADER) && binding.member.hasTrait()) { // Call the idempotency token function if no supplied value. writer.addImport(RuntimeTypes.SmithyClient.IdempotencyTokenProviderExt) @@ -185,18 +183,7 @@ class HttpStringValuesMapSerializer( paramName, ) } else { - val nullCheck = - if (location == HttpBinding.Location.QUERY || - memberTarget.hasTrait< - @Suppress("DEPRECATION") - software.amazon.smithy.model.traits.EnumTrait, - >() - ) { - if (memberSymbol.isNullable) "input.$memberName != null" else "" - } else { - val nullCheck = if (memberSymbol.isNullable) "?" else "" - "input.$memberName$nullCheck.isNotEmpty() == true" - } + val nullCheck = if (memberSymbol.isNullable) "input.$memberName != null" else "" val cond = defaultCheck(binding.member) ?: nullCheck diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt index 3c15579dab..6f8040e7dc 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt @@ -57,8 +57,8 @@ internal class SmokeTestOperationSerializer: HttpSerializer.NonStreaming Date: Tue, 21 Jan 2025 10:27:00 -0500 Subject: [PATCH 02/16] Add explanations for other failing protocol tests --- .../aws/protocols/core/AwsHttpBindingProtocolGenerator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt index 8cfbd5e03d..1e42b858a0 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt @@ -41,9 +41,9 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator() val ignoredTests = TestMemberDelta( setOf( - "AwsJson10ClientErrorCorrectsWithDefaultValuesWhenServerFailsToSerializeRequiredValues", - "RpcV2CborClientPopulatesDefaultsValuesWhenMissingInResponse", - "RpcV2CborClientPopulatesDefaultValuesInInput", + "AwsJson10ClientErrorCorrectsWithDefaultValuesWhenServerFailsToSerializeRequiredValues", // FIXME Protocol test seems to be wrong. Default value of the member is "YmxvYg==" but they are expecting it to be decoded "blob". + "RpcV2CborClientPopulatesDefaultsValuesWhenMissingInResponse", // FIXME Protocol test seems to be wrong. Default value of member is "YWJj" but they are expecting it to be decoded "abc". + "RpcV2CborClientPopulatesDefaultValuesInInput", // FIXME Bug in protocol test, malformed Base64 body. Fix will be in the next Smithy release: https://github.com/smithy-lang/smithy/pull/2502 ), ) From 0b506f3d221be49cb3a3d94693bbe98be9c18402 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 10:59:13 -0500 Subject: [PATCH 03/16] Add `@required` back --- .../protocol-tests/model/error-correction-tests.smithy | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 201ffc877f..8b0999384d 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,8 +39,7 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - // FIXME: This trait fails smithy validator - // @required + @required document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -65,8 +64,7 @@ structure TestStruct { @required nestedListValue: NestedList - // FIXME: This trait fails smithy validator - // @required + @required nested: Nested @required @@ -97,8 +95,7 @@ union MyUnion { } structure Nested { - // FIXME: This trait fails smithy validator - // @required + @required a: String } From 6db0e69b96a6bd7834efd944419f5f1269f1fda6 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 11:42:21 -0500 Subject: [PATCH 04/16] Fill in correct default values --- codegen/protocol-tests/model/error-correction-tests.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 8b0999384d..6ec3dbf306 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -128,8 +128,8 @@ apply SayHello @httpResponseTests([ listValue: [], mapValue: {}, nestedListValue: [], - document: null, - nested: null, + document: {}, + nested: { a: "" }, timestampValue: 0 }, code: 200, @@ -151,7 +151,7 @@ apply SayHelloXml @httpResponseTests([ listValue: [], mapValue: {}, nestedListValue: [], - nested: null, + nested: { a: "" }, timestampValue: 0 }, code: 200, From b78964bb92f035d04b5ec8502155f0c126dfc3e3 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 12:15:57 -0500 Subject: [PATCH 05/16] Revert "Fill in correct default values" This reverts commit 6db0e69b96a6bd7834efd944419f5f1269f1fda6. --- codegen/protocol-tests/model/error-correction-tests.smithy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 6ec3dbf306..8b0999384d 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -128,8 +128,8 @@ apply SayHello @httpResponseTests([ listValue: [], mapValue: {}, nestedListValue: [], - document: {}, - nested: { a: "" }, + document: null, + nested: null, timestampValue: 0 }, code: 200, @@ -151,7 +151,7 @@ apply SayHelloXml @httpResponseTests([ listValue: [], mapValue: {}, nestedListValue: [], - nested: { a: "" }, + nested: null, timestampValue: 0 }, code: 200, From 48350f205ecc5a2a1570d4ff8c43ca07359560b7 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 12:16:06 -0500 Subject: [PATCH 06/16] Revert "Add `@required` back" This reverts commit 0b506f3d221be49cb3a3d94693bbe98be9c18402. --- .../protocol-tests/model/error-correction-tests.smithy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 8b0999384d..201ffc877f 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,7 +39,8 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - @required + // FIXME: This trait fails smithy validator + // @required document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -64,7 +65,8 @@ structure TestStruct { @required nestedListValue: NestedList - @required + // FIXME: This trait fails smithy validator + // @required nested: Nested @required @@ -95,7 +97,8 @@ union MyUnion { } structure Nested { - @required + // FIXME: This trait fails smithy validator + // @required a: String } From ed41af78b3527faaa330de241ccb0b49da850cdc Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 12:18:19 -0500 Subject: [PATCH 07/16] Add explanation to why `@required` is removed --- .../model/error-correction-tests.smithy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 201ffc877f..86b43b5da5 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,8 +39,9 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - // FIXME: This trait fails smithy validator - // @required + + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -65,8 +66,8 @@ structure TestStruct { @required nestedListValue: NestedList - // FIXME: This trait fails smithy validator - // @required + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. nested: Nested @required @@ -97,8 +98,7 @@ union MyUnion { } structure Nested { - // FIXME: This trait fails smithy validator - // @required + @required a: String } From 1333c24f1306b75fa67d9ae7a1b2773459ea18c7 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 12:41:59 -0500 Subject: [PATCH 08/16] Upgrade to Smithy v1.54.0 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d8724016e..31f5d17afa 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -18,7 +18,7 @@ micrometer-version = "1.14.2" binary-compatibility-validator-version = "0.16.3" # codegen -smithy-version = "1.53.0" +smithy-version = "1.54.0" smithy-gradle-version = "0.9.0" # testing From 3333049753431651540ac7e480d90c34098635ca Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 12:47:20 -0500 Subject: [PATCH 09/16] Base64-decode default blob values --- .../aws/protocols/core/AwsHttpBindingProtocolGenerator.kt | 2 -- .../smithy/kotlin/codegen/core/KotlinSymbolProvider.kt | 2 +- .../smithy/kotlin/codegen/rendering/StructureGenerator.kt | 5 +++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt index 1e42b858a0..de8aa59124 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt @@ -41,8 +41,6 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator() val ignoredTests = TestMemberDelta( setOf( - "AwsJson10ClientErrorCorrectsWithDefaultValuesWhenServerFailsToSerializeRequiredValues", // FIXME Protocol test seems to be wrong. Default value of the member is "YmxvYg==" but they are expecting it to be decoded "blob". - "RpcV2CborClientPopulatesDefaultsValuesWhenMissingInResponse", // FIXME Protocol test seems to be wrong. Default value of member is "YWJj" but they are expecting it to be decoded "abc". "RpcV2CborClientPopulatesDefaultValuesInInput", // FIXME Bug in protocol test, malformed Base64 body. Fix will be in the next Smithy release: https://github.com/smithy-lang/smithy/pull/2502 ), ) diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt index e6dbd0dca1..b7d743309c 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt @@ -241,7 +241,7 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli .takeUnless { it.isEmpty() } ?.let { "ByteStream.fromString(${it.dq()})" } - targetShape.isBlobShape -> "${node.toString().dq()}.encodeToByteArray()" + targetShape.isBlobShape -> "${node.toString().dq()}.decodeBase64().encodeToByteArray()" targetShape.isDocumentShape -> getDefaultValueForDocument(node) targetShape.isTimestampShape -> getDefaultValueForTimestamp(node.asNumberNode().get()) diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt index c34eb5ab9d..96a8284508 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt @@ -248,6 +248,11 @@ class StructureGenerator( } else { memberSymbol } + + if (builderMemberSymbol.shape is BlobShape && builderMemberSymbol.isNotNullable) { + writer.addImport(RuntimeTypes.Core.Text.Encoding.decodeBase64) + } + write("public var #L: #E", memberName, builderMemberSymbol) } write("") From 0bab19735651121cd325a0bcc6920bc3b36c95a3 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 13:38:18 -0500 Subject: [PATCH 10/16] Fix test --- .../amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt index 0cc4d2cb14..bb90fbc0fb 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt @@ -182,7 +182,7 @@ class SymbolProviderTest { "double,2.71828,2.71828", "byte,10,10.toByte()", "string,\"hello\",\"hello\"", - "blob,\"abcdefg\",\"abcdefg\".encodeToByteArray()", + "blob,\"abcdefg\",\"abcdefg\".decodeBase64().encodeToByteArray()", "boolean,true,true", "bigInteger,5,5", "bigDecimal,9.0123456789,9.0123456789", From 9f0ffe6d7662d21cd8869956d79e9a861cf3d9aa Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 16:00:35 -0500 Subject: [PATCH 11/16] Add support for unpadded Base64 inputs --- .../kotlin/runtime/text/encoding/Base64.kt | 12 ++++++-- .../runtime/text/encoding/Base64Test.kt | 28 +++++++++++++------ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt index 7a3a569899..aa75e9fcdc 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt @@ -99,10 +99,18 @@ public fun String.decodeBase64Bytes(): ByteArray = encodeToByteArray().decodeBas * Decode [ByteArray] from base64 format */ public fun ByteArray.decodeBase64(): ByteArray { - val encoded = this + + // Calculate the padding needed to make the length a multiple of 4 + val remainder = size % 4 + val encoded: ByteArray = if (remainder == 0) { + this + } else { + this + ByteArray(4 - remainder) { BASE64_PAD.code.toByte() } + } + val decodedLen = base64DecodedLen(encoded) val decoded = ByteArray(decodedLen) - val blockCnt = size / 4 + val blockCnt = encoded.size / 4 var readIdx = 0 var writeIdx = 0 diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt index 4c2ada2010..956fabee2e 100644 --- a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt @@ -76,14 +76,6 @@ class Base64Test { ex.message!!.shouldContain("decode base64: invalid input byte: 45") } - @Test - fun decodeNonMultipleOf4() { - val ex = assertFails { - "Zm9vY=".decodeBase64() - } - ex.message!!.shouldContain("invalid base64 string of length 6; not a multiple of 4") - } - @Test fun decodeInvalidPadding() { val ex = assertFails { @@ -116,4 +108,24 @@ class Base64Test { assertEquals(encoded, decoded.encodeBase64()) assertEquals(decoded, encoded.decodeBase64()) } + + @Test + fun testUnpaddedInputs() { + // from https://github.com/smithy-lang/smithy/pull/2502 + val input = "v2hkZWZhdWx0c79tZGVmYXVsdFN0cmluZ2JoaW5kZWZhdWx0Qm9vbGVhbvVrZGVmYXVsdExpc3Sf/3BkZWZhdWx0VGltZXN0YW1wwQBrZGVmYXVsdEJsb2JDYWJja2RlZmF1bHRCeXRlAWxkZWZhdWx0U2hvcnQBbmRlZmF1bHRJbnRlZ2VyCmtkZWZhdWx0TG9uZxhkbGRlZmF1bHRGbG9hdPo/gAAAbWRlZmF1bHREb3VibGX6P4AAAGpkZWZhdWx0TWFwv/9rZGVmYXVsdEVudW1jRk9PbmRlZmF1bHRJbnRFbnVtAWtlbXB0eVN0cmluZ2BsZmFsc2VCb29sZWFu9GllbXB0eUJsb2JAaHplcm9CeXRlAGl6ZXJvU2hvcnQAa3plcm9JbnRlZ2VyAGh6ZXJvTG9uZwBpemVyb0Zsb2F0+gAAAABqemVyb0RvdWJsZfoAAAAA//8" + input.decodeBase64() + + val inputs = mapOf( + "YQ" to "a", + "Yg" to "b", + "YWI" to "ab", + "YWJj" to "abc", + "SGVsbG8gd29ybGQ" to "Hello world", + ) + + inputs.forEach { (encoded, expected) -> + val actual = encoded.decodeBase64() + assertEquals(expected, actual) + } + } } From 514455c9436f8d9500f33ee4f126342d10b9525b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 16:00:56 -0500 Subject: [PATCH 12/16] git push --- .../common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt index aa75e9fcdc..9af6977d29 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt @@ -99,7 +99,6 @@ public fun String.decodeBase64Bytes(): ByteArray = encodeToByteArray().decodeBas * Decode [ByteArray] from base64 format */ public fun ByteArray.decodeBase64(): ByteArray { - // Calculate the padding needed to make the length a multiple of 4 val remainder = size % 4 val encoded: ByteArray = if (remainder == 0) { From 81dc2a1e610b1c6f11e2fe5b0a8e18d8873bacef Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 16:11:59 -0500 Subject: [PATCH 13/16] Re-enable `RpcV2CborClientPopulatesDefaultValuesInInput` test --- .../aws/protocols/core/AwsHttpBindingProtocolGenerator.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt index de8aa59124..bd41afcade 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt @@ -40,9 +40,7 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator() // val targetedTest = TestMemberDelta(setOf("RestJsonComplexErrorWithNoMessage"), TestContainmentMode.RUN_TESTS) val ignoredTests = TestMemberDelta( - setOf( - "RpcV2CborClientPopulatesDefaultValuesInInput", // FIXME Bug in protocol test, malformed Base64 body. Fix will be in the next Smithy release: https://github.com/smithy-lang/smithy/pull/2502 - ), + setOf(), ) val requestTestBuilder = HttpProtocolUnitTestRequestGenerator.Builder() From 314902773c2eadb5ce93ec5ec2dac5aad211431d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 21 Jan 2025 16:22:28 -0500 Subject: [PATCH 14/16] Revert `@required` changes --- codegen/protocol-tests/model/error-correction-tests.smithy | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 86b43b5da5..8b0999384d 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,9 +39,7 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - - // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. - // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. + @required document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -66,8 +64,7 @@ structure TestStruct { @required nestedListValue: NestedList - // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. - // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. + @required nested: Nested @required From 04a33e9abf1ddfd91c310056732046aac4d8fa79 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 22 Jan 2025 10:48:30 -0500 Subject: [PATCH 15/16] `Symbol.Builder.addReferences` --- .../codegen/core/KotlinSymbolProvider.kt | 30 ++++++++++++------- .../codegen/rendering/StructureGenerator.kt | 4 --- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt index b7d743309c..f35570dfeb 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt @@ -193,7 +193,7 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli } else { // only use @default if type is `T` shape.getTrait()?.let { - defaultValue(it.getDefaultValue(targetShape)) + setDefaultValue(it, targetShape) } } } @@ -219,9 +219,10 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli } } - private fun DefaultTrait.getDefaultValue(targetShape: Shape): String? { - val node = toNode() - return when { + private fun Symbol.Builder.setDefaultValue(defaultTrait: DefaultTrait, targetShape: Shape) { + val node = defaultTrait.toNode() + + val defaultValue = when { node.toString() == "null" -> null // Check if target is an enum before treating the default like a regular number/string @@ -235,13 +236,20 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli "${enumSymbol.fullName}.fromValue($arg)" } - targetShape.isBlobShape && targetShape.isStreaming -> - node - .toString() - .takeUnless { it.isEmpty() } - ?.let { "ByteStream.fromString(${it.dq()})" } + targetShape.isBlobShape -> { + addReferences(RuntimeTypes.Core.Text.Encoding.decodeBase64) - targetShape.isBlobShape -> "${node.toString().dq()}.decodeBase64().encodeToByteArray()" + if (targetShape.isStreaming) { + node.toString() + .takeUnless { it.isEmpty() } + ?.let { + addReferences(RuntimeTypes.Core.Content.ByteStream) + "ByteStream.fromString(${it.dq()}.decodeBase64())" + } + } else { + "${node.toString().dq()}.decodeBase64().encodeToByteArray()" + } + } targetShape.isDocumentShape -> getDefaultValueForDocument(node) targetShape.isTimestampShape -> getDefaultValueForTimestamp(node.asNumberNode().get()) @@ -252,6 +260,8 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli node.isStringNode -> node.toString().dq() else -> node.toString() } + + defaultValue(defaultValue) } private fun getDefaultValueForTimestamp(node: NumberNode): String { diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt index 96a8284508..5d90f376d2 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt @@ -249,10 +249,6 @@ class StructureGenerator( memberSymbol } - if (builderMemberSymbol.shape is BlobShape && builderMemberSymbol.isNotNullable) { - writer.addImport(RuntimeTypes.Core.Text.Encoding.decodeBase64) - } - write("public var #L: #E", memberName, builderMemberSymbol) } write("") From 189edc7fc0eb5ecfd33e317cd8219b8efc2680e9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 22 Jan 2025 10:48:41 -0500 Subject: [PATCH 16/16] Revert `@required` removal --- codegen/protocol-tests/model/error-correction-tests.smithy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 8b0999384d..4f246e1b99 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,7 +39,9 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - @required + + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `document` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -64,7 +66,8 @@ structure TestStruct { @required nestedListValue: NestedList - @required + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. nested: Nested @required