feat: add ability to run protocol tests as unit tests of a service

.github/workflows/codegen-build-test.yml

@@ -61,5 +61,19 @@ jobs:
         run: ./scripts/ci_steps/codegen_sdk.sh
       - name: Build SDK with Unit Tests
         run: swift build --build-tests
+
+      # Configure credentials for use during special protocol tests run against real clients
+      # Since environment variables are modified by some tests profiles are used instead
+      - name: Create AWS config directory
+        run: mkdir -p ~/.aws
+      - name: Create AWS config file
+        run: |
+          echo "[default]" > ~/.aws/config
+          echo "region=us-west-2" >> ~/.aws/config
+      - name: Create AWS credentials file
+        run: |
+          echo "[default]" > ~/.aws/credentials
+          echo "aws_access_key_id = test-key-id" >> ~/.aws/credentials
+          echo "aws_secret_access_key = test-secret-access-key" >> ~/.aws/credentials
       - name: Run Unit Tests
         run: swift test --skip-build

.github/workflows/continuous-integration.yml

@@ -7,6 +7,9 @@ on:
 env:
   AWS_SWIFT_SDK_USE_LOCAL_DEPS: 1
 
+permissions:
+  id-token: write
+
 jobs:
   apple:
     runs-on: ${{ matrix.runner }}
@@ -109,8 +112,9 @@ jobs:
         run: |
           cd codegen/
           set -o pipefail && \
-          NSUnbufferedIO=YES xcodebuild \
+          xcodebuild \
             -scheme aws-sdk-swift-protocol-tests-Package \
+            -testPlan ProtocolTestPlan \
             -destination '${{ matrix.destination }}' \
             test 2>&1 \
             | xcbeautify
@@ -173,6 +177,11 @@ jobs:
         run: ./scripts/ci_steps/install_native_linux_dependencies.sh
       - name: Tools Versions
         run: ./scripts/ci_steps/log_tool_versions.sh
+      - name: Configure AWS Credentials for Integration Tests
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.INTEGRATION_TEST_ROLE_ARN }}
+          aws-region: us-west-2
       - name: Prepare Protocol & Unit Tests
         run: |
           ./scripts/ci_steps/prepare_protocol_and_unit_tests.sh

Sources/Core/AWSClientRuntime/Middlewares/Sha256TreeHashMiddleware.swift

@@ -7,6 +7,7 @@ import AwsCommonRuntimeKit
 import ClientRuntime
 import SmithyHTTPAPI
 import struct Foundation.Data
+import struct Smithy.AttributeKey
 
 public struct Sha256TreeHashMiddleware<OperationStackInput, OperationStackOutput>: Middleware {
     public let id: String = "Sha256TreeHash"
@@ -24,11 +25,11 @@ public struct Sha256TreeHashMiddleware<OperationStackInput, OperationStackOutput
           Self.MInput == H.Input,
           Self.MOutput == H.Output {
               let request = input.build()
-              try await addHashes(request: request, builder: input)
+              try await addHashes(request: request, builder: input, context: context)
               return try await next.handle(context: context, input: input)
           }
 
-    private func addHashes(request: SdkHttpRequest, builder: SdkHttpRequestBuilder) async throws {
+    private func addHashes(request: SdkHttpRequest, builder: SdkHttpRequestBuilder, context: Context) async throws {
         switch request.body {
         case .data(let data):
             guard let data = data else {
@@ -57,7 +58,8 @@ public struct Sha256TreeHashMiddleware<OperationStackInput, OperationStackOutput
             let (linearHash, treeHash) = try computeHashes(data: streamBytes)
             if let treeHash = treeHash, let linearHash = linearHash {
                 builder.withHeader(name: X_AMZ_SHA256_TREE_HASH_HEADER_NAME, value: treeHash)
-                builder.withHeader(name: X_AMZ_CONTENT_SHA256_HEADER_NAME, value: linearHash)
+                // provide the value but let CRT add the SHA256 header during signing
+                context.attributes.set(key: AttributeKey(name: X_AMZ_CONTENT_SHA256_HEADER_NAME), value: linearHash)
             }
         case .noStream:
             break
@@ -110,10 +112,10 @@ extension Sha256TreeHashMiddleware: HttpInterceptor {
     public typealias InputType = OperationStackInput
     public typealias OutputType = OperationStackOutput
 
-    public func modifyBeforeTransmit(context: some MutableRequest<Self.InputType, Self.RequestType>) async throws {
+    public func modifyBeforeSigning(context: some MutableRequest<Self.InputType, Self.RequestType>) async throws {
         let request = context.getRequest()
         let builder = request.toBuilder()
-        try await addHashes(request: request, builder: builder)
+        try await addHashes(request: request, builder: builder, context: context.getAttributes())
         context.updateRequest(updated: builder.build())
     }
 }

Sources/Core/AWSSDKHTTPAuth/AWSSigV4Signer.swift

@@ -126,11 +126,13 @@ public class AWSSigV4Signer: SmithyHTTPAuthAPI.Signer {
         // Determine signed body value
         let checksumIsPresent = signingProperties.get(key: SigningPropertyKeys.checksum) != nil
         let isChunkedEligibleStream = signingProperties.get(key: SigningPropertyKeys.isChunkedEligibleStream) ?? false
+        let preComputedSha256 = signingProperties.get(key: AttributeKey<String>(name: "SignedBodyValue"))
 
         let signedBodyValue: AWSSignedBodyValue = determineSignedBodyValue(
             checksumIsPresent: checksumIsPresent,
             isChunkedEligbleStream: isChunkedEligibleStream,
-            isUnsignedBody: unsignedBody
+            isUnsignedBody: unsignedBody,
+            preComputedSha256: preComputedSha256
         )
 
         let flags: SigningFlags = SigningFlags(
@@ -240,11 +242,18 @@ public class AWSSigV4Signer: SmithyHTTPAuthAPI.Signer {
     private func determineSignedBodyValue(
         checksumIsPresent: Bool,
         isChunkedEligbleStream: Bool,
-        isUnsignedBody: Bool
+        isUnsignedBody: Bool,
+        preComputedSha256: String?
     ) -> AWSSignedBodyValue {
         if !isChunkedEligbleStream {
             // Normal Payloads, Event Streams, etc.
-            return isUnsignedBody ? .unsignedPayload : .empty
+            if isUnsignedBody {
+                return .unsignedPayload
+            } else if let sha256 = preComputedSha256 {
+                return .precomputed(sha256)
+            } else {
+                return .empty
+            }
         }
 
         // streaming + eligible for chunked transfer

Sources/Services/AWSS3/Sources/AWSS3/Endpoints.swift

@@ -11,6 +11,7 @@ import class ClientRuntime.EndpointsRequestContext
 import let AWSClientRuntime.awsPartitionJSON
 import protocol ClientRuntime.EndpointsRequestContextProviding
 import struct ClientRuntime.DefaultEndpointResolver
+import struct ClientRuntime.StaticEndpointResolver
 import struct SmithyHTTPAPI.Endpoint
 
 public struct EndpointParams {
@@ -150,3 +151,7 @@ extension DefaultEndpointResolver {
 }
 
 extension DefaultEndpointResolver: EndpointResolver {}
+
+typealias StaticEndpointResolver = ClientRuntime.StaticEndpointResolver<EndpointParams>
+
+extension StaticEndpointResolver: EndpointResolver {}

Sources/Services/AWSS3/Tests/AWSS3Tests/DeleteObjectTaggingRequestTest.swift

@@ -0,0 +1,73 @@
+// Code generated by smithy-swift-codegen. DO NOT EDIT!
+
+@testable import AWSS3
+import ClientRuntime
+import Smithy
+import SmithyTestUtil
+import XCTest
+
+
+class DeleteObjectTaggingRequestTest: HttpRequestTestBase {
+    /// S3 clients should escape special characters in Object Keys when the Object Key is used as a URI label binding.
+    func testProtocol_S3EscapeObjectKeyInUriLabel() async throws {
+        let urlPrefix = urlPrefixFromHost(host: "s3.us-west-2.amazonaws.com")
+        let hostOnly = hostOnlyFromHost(host: "s3.us-west-2.amazonaws.com")
+        let expected = buildExpectedHttpRequest(
+            method: .delete,
+            path: "/my%20key.txt",
+            queryParams: [
+                "tagging"
+            ],
+            body: nil,
+            host: "s3.us-west-2.amazonaws.com",
+            resolvedHost: "mybucket.s3.us-west-2.amazonaws.com"
+        )
+
+        let config = try await S3Client.S3ClientConfiguration()
+        config.region = "us-west-2"
+        config.httpClientEngine = ProtocolTestClient()
+        config.idempotencyTokenGenerator = ProtocolTestIdempotencyTokenGenerator()
+        let client = S3Client(config: config)
+
+        let input = DeleteObjectTaggingInput(
+            bucket: "mybucket",
+            key: "my key.txt"
+        )
+        do {
+            _ = try await client.deleteObjectTagging(input: input)
+        } catch TestCheckError.actual(let actual) {
+            try await self.assertEqual(expected, actual)
+        }
+    }
+    /// S3 clients should preserve an Object Key representing a path when the Object Key is used as a URI label binding, but still escape special characters.
+    func testProtocol_S3EscapePathObjectKeyInUriLabel() async throws {
+        let urlPrefix = urlPrefixFromHost(host: "s3.us-west-2.amazonaws.com")
+        let hostOnly = hostOnlyFromHost(host: "s3.us-west-2.amazonaws.com")
+        let expected = buildExpectedHttpRequest(
+            method: .delete,
+            path: "/foo/bar/my%20key.txt",
+            queryParams: [
+                "tagging"
+            ],
+            body: nil,
+            host: "s3.us-west-2.amazonaws.com",
+            resolvedHost: "mybucket.s3.us-west-2.amazonaws.com"
+        )
+
+        let config = try await S3Client.S3ClientConfiguration()
+        config.region = "us-west-2"
+        config.httpClientEngine = ProtocolTestClient()
+        config.idempotencyTokenGenerator = ProtocolTestIdempotencyTokenGenerator()
+        let client = S3Client(config: config)
+
+        let input = DeleteObjectTaggingInput(
+            bucket: "mybucket",
+            key: "foo/bar/my key.txt"
+        )
+        do {
+            _ = try await client.deleteObjectTagging(input: input)
+        } catch TestCheckError.actual(let actual) {
+            try await self.assertEqual(expected, actual)
+        }
+    }
+}

Sources/Services/AWSS3/Tests/AWSS3Tests/GetObjectRequestTest.swift

@@ -0,0 +1,68 @@
+// Code generated by smithy-swift-codegen. DO NOT EDIT!
+
+@testable import AWSS3
+import ClientRuntime
+import Foundation
+import Smithy
+import SmithyTestUtil
+import XCTest
+
+
+class GetObjectRequestTest: HttpRequestTestBase {
+    /// S3 clients should not remove dot segments from request paths.
+    func testProtocol_S3PreservesLeadingDotSegmentInUriLabel() async throws {
+        let urlPrefix = urlPrefixFromHost(host: "s3.us-west-2.amazonaws.com")
+        let hostOnly = hostOnlyFromHost(host: "s3.us-west-2.amazonaws.com")
+        let expected = buildExpectedHttpRequest(
+            method: .get,
+            path: "/../key.txt",
+            body: nil,
+            host: "s3.us-west-2.amazonaws.com",
+            resolvedHost: "mybucket.s3.us-west-2.amazonaws.com"
+        )
+
+        let config = try await S3Client.S3ClientConfiguration()
+        config.region = "us-west-2"
+        config.httpClientEngine = ProtocolTestClient()
+        config.idempotencyTokenGenerator = ProtocolTestIdempotencyTokenGenerator()
+        let client = S3Client(config: config)
+
+        let input = GetObjectInput(
+            bucket: "mybucket",
+            key: "../key.txt"
+        )
+        do {
+            _ = try await client.getObject(input: input)
+        } catch TestCheckError.actual(let actual) {
+            try await self.assertEqual(expected, actual)
+        }
+    }
+    /// S3 clients should not remove dot segments from request paths.
+    func testProtocol_S3PreservesEmbeddedDotSegmentInUriLabel() async throws {
+        let urlPrefix = urlPrefixFromHost(host: "s3.us-west-2.amazonaws.com")
+        let hostOnly = hostOnlyFromHost(host: "s3.us-west-2.amazonaws.com")
+        let expected = buildExpectedHttpRequest(
+            method: .get,
+            path: "/foo/../key.txt",
+            body: nil,
+            host: "s3.us-west-2.amazonaws.com",
+            resolvedHost: "mybucket.s3.us-west-2.amazonaws.com"
+        )
+
+        let config = try await S3Client.S3ClientConfiguration()
+        config.region = "us-west-2"
+        config.httpClientEngine = ProtocolTestClient()
+        config.idempotencyTokenGenerator = ProtocolTestIdempotencyTokenGenerator()
+        let client = S3Client(config: config)
+
+        let input = GetObjectInput(
+            bucket: "mybucket",
+            key: "foo/../key.txt"
+        )
+        do {
+            _ = try await client.getObject(input: input)
+        } catch TestCheckError.actual(let actual) {
+            try await self.assertEqual(expected, actual)
+        }
+    }
+}

Tests/AdditionalServiceTests/AWSS3/models/s3-tests.smithy

@@ -0,0 +1,81 @@
+$version: "1.0"
+
+namespace com.amazonaws.s3
+use smithy.test#httpRequestTests
+
+apply DeleteObjectTagging @httpRequestTests([
+    {
+        "id": "Protocol_S3EscapeObjectKeyInUriLabel",
+        "documentation": "    S3 clients should escape special characters in Object Keys\n    when the Object Key is used as a URI label binding.\n",
+        "protocol": "aws.protocols#restXml",
+        "method": "DELETE",
+        "uri": "/my%20key.txt",
+        "host": "s3.us-west-2.amazonaws.com",
+        "resolvedHost": "mybucket.s3.us-west-2.amazonaws.com",
+        "body": "",
+        "queryParams": [
+            "tagging"
+        ],
+        "params": {
+            "Bucket": "mybucket",
+            "Key": "my key.txt"
+        }
+    },
+    {
+        "id": "Protocol_S3EscapePathObjectKeyInUriLabel",
+        "documentation": "    S3 clients should preserve an Object Key representing a path\n    when the Object Key is used as a URI label binding, but still\n    escape special characters.\n",
+        "protocol": "aws.protocols#restXml",
+        "method": "DELETE",
+        "uri": "/foo/bar/my%20key.txt",
+        "host": "s3.us-west-2.amazonaws.com",
+        "resolvedHost": "mybucket.s3.us-west-2.amazonaws.com",
+        "body": "",
+        "queryParams": [
+            "tagging"
+        ],
+        "params": {
+            "Bucket": "mybucket",
+            "Key": "foo/bar/my key.txt"
+        }
+    }
+])
+@http(
+    method: "DELETE",
+    uri: "/{Bucket}/{Key+}?tagging",
+    code: 204
+)
+operation DeleteObjectTagging {
+    input: DeleteObjectTaggingRequest
+    output: DeleteObjectTaggingOutput
+}
+
+apply GetObject @httpRequestTests([
+    {
+        "id": "Protocol_S3PreservesLeadingDotSegmentInUriLabel",
+        "documentation": "    S3 clients should not remove dot segments from request paths.\n",
+        "protocol": "aws.protocols#restXml",
+        "method": "GET",
+        "uri": "/../key.txt",
+        "host": "s3.us-west-2.amazonaws.com",
+        "resolvedHost": "mybucket.s3.us-west-2.amazonaws.com",
+        "body": "",
+        "params": {
+            "Bucket": "mybucket",
+            "Key": "../key.txt"
+        }
+    },
+    {
+        "id": "Protocol_S3PreservesEmbeddedDotSegmentInUriLabel",
+        "documentation": "    S3 clients should not remove dot segments from request paths.\n",
+        "protocol": "aws.protocols#restXml",
+        "method": "GET",
+        "uri": "/foo/../key.txt",
+        "host": "s3.us-west-2.amazonaws.com",
+        "resolvedHost": "mybucket.s3.us-west-2.amazonaws.com",
+        "body": "",
+        "params": {
+            "Bucket": "mybucket",
+            "Key": "foo/../key.txt"
+        }
+    }
+])

Tests/Core/AWSClientRuntimeTests/Middlewares/Sha256TreeHashMiddlewareTests.swift

@@ -30,7 +30,7 @@ class Sha256TreeHashMiddlewareTests: XCTestCase {
         let output = OperationOutput<MockOutput>(httpResponse: mockHttpResponse, output: mockOutput)
         stack.finalizeStep.intercept(position: .after, middleware: Sha256TreeHashMiddleware<MockStreamInput, MockOutput>())
         _ = try await stack.handleMiddleware(context: context, input: streamInput, next: MockHandler(handleCallback: { context, input in
-            let linear = input.headers.value(for: "X-Amz-Content-Sha256")
+            let linear = context.get(key: AttributeKey<String>(name: "X-Amz-Content-Sha256"))
             XCTAssertEqual(linear, "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60")
             let treeHash = input.headers.value(for: "X-Amz-Sha256-Tree-Hash")
             XCTAssertEqual(treeHash, "a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455")