From 65cd999e20552e987eca784c0cf01d4dcf3c51bb Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 25 Aug 2024 15:26:59 -0700 Subject: [PATCH 01/32] Added request_checksum_calculation to SdkConfig --- aws/rust-runtime/Cargo.lock | 7 +- aws/rust-runtime/aws-config/Cargo.lock | 247 ++++++++++++++++++ aws/rust-runtime/aws-config/Cargo.toml | 1 + .../aws-config/src/default_provider.rs | 3 + .../request_checksum_calculation.rs | 128 +++++++++ aws/rust-runtime/aws-config/src/lib.rs | 19 +- aws/rust-runtime/aws-types/Cargo.toml | 3 +- aws/rust-runtime/aws-types/src/sdk_config.rs | 29 ++ .../rustsdk/HttpRequestChecksumDecorator.kt | 73 +++++- rust-runtime/Cargo.lock | 5 +- rust-runtime/aws-smithy-checksums/Cargo.toml | 2 +- .../aws-smithy-checksums/src/error.rs | 31 +++ rust-runtime/aws-smithy-checksums/src/lib.rs | 44 +++- 13 files changed, 576 insertions(+), 16 deletions(-) create mode 100644 aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index 1774e88d38..4e773efdd1 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -240,7 +240,7 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.12" +version = "0.60.13" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" dependencies = [ "base64-simd", "bytes", @@ -367,10 +367,11 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", + "aws-smithy-checksums", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", diff --git a/aws/rust-runtime/aws-config/Cargo.lock b/aws/rust-runtime/aws-config/Cargo.lock index f849dcd596..990c977d1b 100644 --- a/aws/rust-runtime/aws-config/Cargo.lock +++ b/aws/rust-runtime/aws-config/Cargo.lock @@ -46,6 +46,7 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" version = "1.5.6" +version = "1.5.6" dependencies = [ "aws-credential-types", "aws-runtime", @@ -53,6 +54,7 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", + "aws-smithy-checksums", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", @@ -80,6 +82,7 @@ dependencies = [ [[package]] name = "aws-credential-types" version = "1.2.1" +version = "1.2.1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -96,6 +99,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-runtime", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -104,6 +108,8 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "once_cell", + "http-body 0.4.6", + "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -195,12 +201,32 @@ dependencies = [ [[package]] name = "aws-smithy-async" version = "1.2.1" +version = "1.2.1" dependencies = [ "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + [[package]] name = "aws-smithy-http" version = "0.60.11" @@ -212,6 +238,7 @@ dependencies = [ "futures-core", "http 0.2.12", "http-body 0.4.6", + "http-body 0.4.6", "once_cell", "percent-encoding", "pin-project-lite", @@ -222,6 +249,7 @@ dependencies = [ [[package]] name = "aws-smithy-json" version = "0.60.7" +version = "0.60.7" dependencies = [ "aws-smithy-types", ] @@ -229,16 +257,20 @@ dependencies = [ [[package]] name = "aws-smithy-protocol-test" version = "0.62.0" +version = "0.62.0" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", "base64-simd", "cbor-diag", + "base64-simd", + "cbor-diag", "http 0.2.12", "pretty_assertions", "regex-lite", "roxmltree", "serde_cbor", + "serde_cbor", "serde_json", "thiserror", ] @@ -246,6 +278,7 @@ dependencies = [ [[package]] name = "aws-smithy-query" version = "0.60.7" +version = "0.60.7" dependencies = [ "aws-smithy-types", "urlencoding", @@ -254,6 +287,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" version = "1.7.1" +version = "1.7.1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -267,9 +301,13 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", "hyper", "hyper-rustls", "indexmap", + "indexmap", "once_cell", "pin-project-lite", "pin-utils", @@ -284,6 +322,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" version = "1.7.2" +version = "1.7.2" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -308,6 +347,10 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "http-body-util", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "itoa", "num-integer", "pin-project-lite", @@ -327,9 +370,11 @@ dependencies = [ [[package]] name = "aws-types" version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", + "aws-smithy-checksums", "aws-smithy-runtime-api", "aws-smithy-types", "rustc_version", @@ -391,6 +436,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bytes" version = "1.7.1" @@ -426,6 +480,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "cbor-diag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +dependencies = [ + "bs58", + "chrono", + "data-encoding", + "half 2.4.1", + "nom", + "num-bigint", + "num-rational", + "num-traits", + "separator", + "url", + "uuid", +] + [[package]] name = "cc" version = "1.1.16" @@ -450,6 +523,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "num-traits", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -481,6 +563,30 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -497,6 +603,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + [[package]] name = "deranged" version = "0.3.11" @@ -669,6 +781,22 @@ dependencies = [ "crunchy", ] +[[package]] +name = "half" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -752,6 +880,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.9.4" @@ -777,6 +928,7 @@ dependencies = [ "h2", "http 0.2.12", "http-body 0.4.6", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -824,6 +976,7 @@ dependencies = [ "equivalent", "hashbrown", "serde", + "serde", ] [[package]] @@ -869,12 +1022,29 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -912,6 +1082,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -932,6 +1112,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -958,6 +1148,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1306,6 +1507,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" +[[package]] +name = "separator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" + [[package]] name = "serde" version = "1.0.209" @@ -1325,6 +1532,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half 1.8.3", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.209" @@ -1342,6 +1559,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ + "indexmap", "indexmap", "itoa", "memchr", @@ -1349,6 +1567,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1512,6 +1741,24 @@ dependencies = [ "tinyvec_macros", ] +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + [[package]] name = "tinyvec_macros" version = "0.1.1" diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index bedb7d0bcc..86c290cea9 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -33,6 +33,7 @@ aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" } aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client"] } aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] } aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" } +aws-smithy-checksums = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-checksums" } aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" } bytes = "1.1.0" http = "0.2.4" diff --git a/aws/rust-runtime/aws-config/src/default_provider.rs b/aws/rust-runtime/aws-config/src/default_provider.rs index fceb869fb0..07e3ad4f57 100644 --- a/aws/rust-runtime/aws-config/src/default_provider.rs +++ b/aws/rust-runtime/aws-config/src/default_provider.rs @@ -63,3 +63,6 @@ pub mod disable_request_compression; /// Default "request minimum compression size bytes" provider chain pub mod request_min_compression_size_bytes; + +/// Default "request checksum calculation" provider chain +pub mod request_checksum_calculation; diff --git a/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs b/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs new file mode 100644 index 0000000000..73fa5ad7ec --- /dev/null +++ b/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs @@ -0,0 +1,128 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::provider_config::ProviderConfig; +use aws_runtime::env_config::EnvConfigValue; +use aws_smithy_types::error::display::DisplayErrorContext; +use aws_types::sdk_config::RequestChecksumCalculation; +use std::str::FromStr; + +mod env { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "AWS_REQUEST_CHECKSUM_CALCULATION"; +} + +mod profile_key { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "request_checksum_calculation"; +} + +/// Load the value for `request_checksum_calculation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `request_checksum_calculation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn request_checksum_calculation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::REQUEST_CHECKSUM_CALCULATION) + .profile(profile_key::REQUEST_CHECKSUM_CALCULATION) + .validate(&env, profiles, RequestChecksumCalculation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for request_checksum_calculation setting"), + ) + // WhenSupported is the default setting + .unwrap_or(Some(RequestChecksumCalculation::WhenSupported)); + + // request_checksum_calculation should always have a non-None value + loaded.xor(Some(RequestChecksumCalculation::WhenSupported)) +} + +#[cfg(test)] +mod test { + use crate::default_provider::request_checksum_calculation::request_checksum_calculation_provider; + #[allow(deprecated)] + use crate::profile::profile_file::{ProfileFileKind, ProfileFiles}; + use crate::provider_config::ProviderConfig; + use aws_types::os_shim_internal::{Env, Fs}; + use aws_types::sdk_config::RequestChecksumCalculation; + use tracing_test::traced_test; + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "not-a-valid-value", + )])); + assert_eq!(request_checksum_calculation_provider(&conf).await, None); + assert!(logs_contain( + "invalid value for request_checksum_calculation setting" + )); + assert!(logs_contain("AWS_REQUEST_CHECKSUM_CALCULATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "WHEN_SUPPORTED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } +} diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index 49bfad8d6a..fd761ea546 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -234,13 +234,14 @@ mod loader { use aws_types::docs_for; use aws_types::origin::Origin; use aws_types::os_shim_internal::{Env, Fs}; + use aws_types::sdk_config::RequestChecksumCalculation; use aws_types::sdk_config::SharedHttpClient; use aws_types::SdkConfig; use crate::default_provider::{ app_name, credentials, disable_request_compression, endpoint_url, - ignore_configured_endpoint_urls as ignore_ep, region, request_min_compression_size_bytes, - retry_config, timeout_config, use_dual_stack, use_fips, + ignore_configured_endpoint_urls as ignore_ep, region, request_checksum_calculation, + request_min_compression_size_bytes, retry_config, timeout_config, use_dual_stack, use_fips, }; use crate::meta::region::ProvideRegion; #[allow(deprecated)] @@ -289,6 +290,7 @@ mod loader { env: Option, fs: Option, behavior_version: Option, + request_checksum_calculation: Option, } impl ConfigLoader { @@ -908,6 +910,19 @@ mod loader { Some(user_cache) => Some(user_cache), }; + let request_checksum_calculation = if let Some(request_checksum_calculation) = + self.request_checksum_calculation + { + println!("LNJ checksum explicitly set on config loader: {request_checksum_calculation:#?}"); + Some(request_checksum_calculation) + } else { + println!("LNJ loading checksum from request_checksum_calculation_provider"); + request_checksum_calculation::request_checksum_calculation_provider(&conf).await + }; + + println!("LNJ final value of checksum {request_checksum_calculation:#?}"); + + builder.set_request_checksum_calculation(request_checksum_calculation); builder.set_identity_cache(identity_cache); builder.set_credentials_provider(credentials_provider); builder.set_token_provider(token_provider); diff --git a/aws/rust-runtime/aws-types/Cargo.toml b/aws/rust-runtime/aws-types/Cargo.toml index 694b8676a5..fc4a514b2b 100644 --- a/aws/rust-runtime/aws-types/Cargo.toml +++ b/aws/rust-runtime/aws-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-types" -version = "1.3.3" +version = "1.3.4" authors = ["AWS Rust SDK Team ", "Russell Cohen "] description = "Cross-service types for the AWS SDK." edition = "2021" @@ -17,6 +17,7 @@ aws-smithy-async = { path = "../../../rust-runtime/aws-smithy-async" } aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" } aws-smithy-runtime = { path = "../../../rust-runtime/aws-smithy-runtime", optional = true } aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] } +aws-smithy-checksums = {path = "../../../rust-runtime/aws-smithy-checksums"} tracing = "0.1" # cargo does not support optional test dependencies, so to completely disable rustls # we need to add the webpki-roots feature here. diff --git a/aws/rust-runtime/aws-types/src/sdk_config.rs b/aws/rust-runtime/aws-types/src/sdk_config.rs index c26d8741be..81242e270f 100644 --- a/aws/rust-runtime/aws-types/src/sdk_config.rs +++ b/aws/rust-runtime/aws-types/src/sdk_config.rs @@ -19,6 +19,7 @@ pub use aws_credential_types::provider::SharedCredentialsProvider; use aws_smithy_async::rt::sleep::AsyncSleep; pub use aws_smithy_async::rt::sleep::SharedAsyncSleep; pub use aws_smithy_async::time::{SharedTimeSource, TimeSource}; +pub use aws_smithy_checksums::RequestChecksumCalculation; use aws_smithy_runtime_api::client::behavior_version::BehaviorVersion; use aws_smithy_runtime_api::client::http::HttpClient; pub use aws_smithy_runtime_api::client::http::SharedHttpClient; @@ -92,6 +93,7 @@ pub struct SdkConfig { config_origins: HashMap<&'static str, Origin>, disable_request_compression: Option, request_min_compression_size_bytes: Option, + request_checksum_calculation: Option, } /// Builder for AWS Shared Configuration @@ -120,6 +122,7 @@ pub struct Builder { config_origins: HashMap<&'static str, Origin>, disable_request_compression: Option, request_min_compression_size_bytes: Option, + request_checksum_calculation: Option, } impl Builder { @@ -174,6 +177,30 @@ impl Builder { self } + /// Set the checksum calculation strategy to use when making requests. + /// # Examples + /// ``` + /// use aws_types::SdkConfig; + /// use aws_smithy_checksums::RequestChecksumCalculation; + /// let config = SdkConfig::builder().request_checksum_calculation(RequestChecksumCalculation::WhenSupported).build(); + /// ``` + pub fn request_checksum_calculation( + mut self, + request_checksum_calculation: RequestChecksumCalculation, + ) -> Self { + self.set_request_checksum_calculation(Some(request_checksum_calculation)); + self + } + + /// Set the checksum calculation strategy to use when making requests. + pub fn set_request_checksum_calculation( + &mut self, + request_checksum_calculation: Option, + ) -> &mut Self { + self.request_checksum_calculation = request_checksum_calculation; + self + } + /// Set the retry_config for the builder /// /// _Note:_ Retries require a sleep implementation in order to work. When enabling retry, make @@ -720,6 +747,7 @@ impl Builder { config_origins: self.config_origins, disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, + request_checksum_calculation: self.request_checksum_calculation, } } } @@ -933,6 +961,7 @@ impl SdkConfig { config_origins: self.config_origins, disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, + request_checksum_calculation: self.request_checksum_calculation, } } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 2794fc0295..8628106c9d 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -20,6 +20,7 @@ import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.smithy.generators.BuilderCustomization import software.amazon.smithy.rust.codegen.core.smithy.generators.operationBuildError import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait @@ -51,8 +52,18 @@ class HttpRequestChecksumDecorator : ClientCodegenDecorator { operation: OperationShape, baseCustomizations: List, ): List = baseCustomizations + HttpRequestChecksumCustomization(codegenContext, operation) + + override fun builderCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List { + return baseCustomizations + } } +/** + * Extract the name of the operation's input member that indicates which checksum algorithm to use + */ private fun HttpChecksumTrait.requestAlgorithmMember( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -64,6 +75,9 @@ private fun HttpChecksumTrait.requestAlgorithmMember( return codegenContext.symbolProvider.toMemberName(checksumAlgorithmMemberShape) } +/** + * Extract the name of the operation's input member that indicates which checksum algorithm to use + */ private fun HttpChecksumTrait.checksumAlgorithmToStr( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -76,14 +90,14 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( if (requestAlgorithmMember != null) { if (isRequestChecksumRequired) { // Checksums are required, fall back to MD5 - rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("md5"));""") + rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") } else { // Checksums aren't required, don't set a fallback rust("let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str());") } } else if (isRequestChecksumRequired) { - // Checksums are required but a user can't set one, so we set MD5 for them - rust("""let checksum_algorithm = Some("md5");""") + // Checksums are required but a user can't set one, so we set crc32 for them + rust("""let checksum_algorithm = Some("crc32");""") } rustTemplate( @@ -150,7 +164,58 @@ class HttpRequestChecksumCustomization( } } } - else -> { } + + else -> {} } } } + +// +// class HttpRequestChecksumsBuilderCustomization( +// private val codegenContext: ClientCodegenContext, +// private val operationShape: OperationShape, +// ) : BuilderCustomization() { +// +// private val runtimeConfig = codegenContext.runtimeConfig +// +// override fun section(section: BuilderSection): Writable = +// writable { +// // Get the `HttpChecksumTrait`, returning early if this `OperationShape` doesn't have one +// val checksumTrait = operationShape.getTrait() ?: return@writable +// val fieldName = "request_checksum_calculation" +// +// when (section) { +// is BuilderSection.AdditionalFields -> { +// rustTemplate( +// "$fieldName: Option,", +// "BLAH" to runtimeConfig.smithyRuntimeCrate(), +// ) +// } +// +// is BuilderSection.AdditionalMethods -> { +// rust( +// """ +// pub(crate) fn $fieldName(mut self, $fieldName: impl Into) -> Self { +// self.$fieldName = Some($fieldName.into()); +// self +// } +// +// pub(crate) fn set_$fieldName(&mut self, $fieldName: Option) -> &mut Self { +// self.$fieldName = $fieldName; +// self +// } +// """, +// ) +// } +// +// is BuilderSection.AdditionalDebugFields -> { +// rust("""${section.formatterName}.field("$fieldName", &self.$fieldName);""") +// } +// +// is BuilderSection.AdditionalFieldsInBuild -> { +// rust("$fieldName: self.$fieldName,") +// } +// } +// +// } +// } diff --git a/rust-runtime/Cargo.lock b/rust-runtime/Cargo.lock index 5373a5e3ca..71d7b2804a 100644 --- a/rust-runtime/Cargo.lock +++ b/rust-runtime/Cargo.lock @@ -344,12 +344,13 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" +version = "0.60.13" version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" dependencies = [ - "aws-smithy-http 0.60.10", - "aws-smithy-types 1.2.4", + "aws-smithy-http 0.60.11", + "aws-smithy-types 1.2.6", "bytes", "crc32c", "crc32fast", diff --git a/rust-runtime/aws-smithy-checksums/Cargo.toml b/rust-runtime/aws-smithy-checksums/Cargo.toml index 5c7a6b8a01..0888246992 100644 --- a/rust-runtime/aws-smithy-checksums/Cargo.toml +++ b/rust-runtime/aws-smithy-checksums/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-checksums" -version = "0.60.12" +version = "0.60.13" authors = [ "AWS Rust SDK Team ", "Zelda Hessler ", diff --git a/rust-runtime/aws-smithy-checksums/src/error.rs b/rust-runtime/aws-smithy-checksums/src/error.rs index 6a93cddbb7..c8aca74980 100644 --- a/rust-runtime/aws-smithy-checksums/src/error.rs +++ b/rust-runtime/aws-smithy-checksums/src/error.rs @@ -36,3 +36,34 @@ impl fmt::Display for UnknownChecksumAlgorithmError { } impl Error for UnknownChecksumAlgorithmError {} + +/// A checksum algorithm was unknown +#[derive(Debug)] +pub struct UnknownRequestChecksumCalculationError { + request_checksum_calculation: String, +} + +impl UnknownRequestChecksumCalculationError { + pub(crate) fn new(request_checksum_calculation: impl Into) -> Self { + Self { + request_checksum_calculation: request_checksum_calculation.into(), + } + } + + /// The checksum algorithm that is unknown + pub fn request_checksum_calculation(&self) -> &str { + &self.request_checksum_calculation + } +} + +impl fmt::Display for UnknownRequestChecksumCalculationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown request_checksum_calculation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.request_checksum_calculation + ) + } +} + +impl Error for UnknownRequestChecksumCalculationError {} diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index e969918dd7..f022646bda 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -17,6 +17,8 @@ //! Checksum calculation and verification callbacks. use crate::error::UnknownChecksumAlgorithmError; +use crate::error::UnknownRequestChecksumCalculationError; +use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use bytes::Bytes; use std::str::FromStr; @@ -33,9 +35,11 @@ pub const MD5_NAME: &str = "md5"; /// We only support checksum calculation and validation for these checksum algorithms. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum ChecksumAlgorithm { Crc32, Crc32c, + #[deprecated] Md5, Sha1, Sha256, @@ -49,7 +53,6 @@ impl FromStr for ChecksumAlgorithm { /// - "crc32c" /// - "sha1" /// - "sha256" - /// - "md5" /// /// Passing an invalid name will return an error. fn from_str(checksum_algorithm: &str) -> Result { @@ -62,7 +65,8 @@ impl FromStr for ChecksumAlgorithm { } else if checksum_algorithm.eq_ignore_ascii_case(SHA_256_NAME) { Ok(Self::Sha256) } else if checksum_algorithm.eq_ignore_ascii_case(MD5_NAME) { - Ok(Self::Md5) + // MD5 is now an alias for the default Crc32 since it is deprecated + Ok(Self::Crc32) } else { Err(UnknownChecksumAlgorithmError::new(checksum_algorithm)) } @@ -75,7 +79,8 @@ impl ChecksumAlgorithm { match self { Self::Crc32 => Box::::default(), Self::Crc32c => Box::::default(), - Self::Md5 => Box::::default(), + #[allow(deprecated)] + Self::Md5 => Box::::default(), Self::Sha1 => Box::::default(), Self::Sha256 => Box::::default(), } @@ -86,6 +91,7 @@ impl ChecksumAlgorithm { match self { Self::Crc32 => CRC_32_NAME, Self::Crc32c => CRC_32_C_NAME, + #[allow(deprecated)] Self::Md5 => MD5_NAME, Self::Sha1 => SHA_1_NAME, Self::Sha256 => SHA_256_NAME, @@ -93,6 +99,38 @@ impl ChecksumAlgorithm { } } +// Valid RequestChecksumCalculation names +pub const WHEN_SUPPORTED: &str = "when_supported"; +pub const WHEN_REQUIRED: &str = "when_required"; + +/// Configure checksum behavior +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RequestChecksumCalculation { + WhenSupported, + WhenRequired, +} + +impl Storable for RequestChecksumCalculation { + type Storer = StoreReplace; +} + +impl FromStr for RequestChecksumCalculation { + type Err = UnknownRequestChecksumCalculationError; + + fn from_str(request_checksum_calculation: &str) -> Result { + if request_checksum_calculation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if request_checksum_calculation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownRequestChecksumCalculationError::new( + request_checksum_calculation, + )) + } + } +} + /// Types implementing this trait can calculate checksums. /// /// Checksum algorithms are used to validate the integrity of data. Structs that implement this trait From d0a5c29cb4f6300674c9e937eae516a46cb25624 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 27 Aug 2024 19:37:09 -0700 Subject: [PATCH 02/32] Working version of flexible checksums for requests --- .../request_checksum_calculation.rs | 6 +- .../src/http_request_checksum.rs | 64 ++++-- aws/rust-runtime/aws-types/src/sdk_config.rs | 5 + .../rustsdk/HttpRequestChecksumDecorator.kt | 187 ++++++++++++------ 4 files changed, 189 insertions(+), 73 deletions(-) diff --git a/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs b/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs index 73fa5ad7ec..c310d61469 100644 --- a/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs +++ b/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs @@ -37,10 +37,10 @@ pub async fn request_checksum_calculation_provider( .map_err( |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for request_checksum_calculation setting"), ) - // WhenSupported is the default setting - .unwrap_or(Some(RequestChecksumCalculation::WhenSupported)); + .unwrap_or(None); - // request_checksum_calculation should always have a non-None value + // request_checksum_calculation should always have a non-None value and the + // default is WhenSupported loaded.xor(Some(RequestChecksumCalculation::WhenSupported)) } diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 3e010068c6..6f8c9bed4b 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -10,8 +10,8 @@ use aws_runtime::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions}; use aws_runtime::{auth::SigV4OperationSigningConfig, content_encoding::header_value::AWS_CHUNKED}; use aws_sigv4::http_request::SignableBody; -use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; +use aws_smithy_checksums::{ChecksumAlgorithm, RequestChecksumCalculation}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ BeforeSerializationInterceptorContextRef, BeforeTransmitInterceptorContextMut, Input, @@ -55,6 +55,8 @@ impl std::error::Error for Error {} #[derive(Debug)] struct RequestChecksumInterceptorState { checksum_algorithm: Option, + /// This value is set in the model on the `httpChecksum` trait + request_checksum_required: bool, } impl Storable for RequestChecksumInterceptorState { type Storer = StoreReplace; @@ -117,7 +119,7 @@ impl RequestChecksumInterceptor { impl Intercept for RequestChecksumInterceptor where - AP: Fn(&Input) -> Result, BoxError> + Send + Sync, + AP: Fn(&Input) -> Result<(Option, bool), BoxError> + Send + Sync, { fn name(&self) -> &'static str { "RequestChecksumInterceptor" @@ -130,9 +132,11 @@ where cfg: &mut ConfigBag, ) -> Result<(), BoxError> { let checksum_algorithm = (self.algorithm_provider)(context.input())?; - let mut layer = Layer::new("RequestChecksumInterceptor"); - layer.store_put(RequestChecksumInterceptorState { checksum_algorithm }); + layer.store_put(RequestChecksumInterceptorState { + checksum_algorithm: checksum_algorithm.0, + request_checksum_required: checksum_algorithm.1, + }); cfg.push_layer(layer); Ok(()) @@ -151,10 +155,32 @@ where .load::() .expect("set in `read_before_serialization`"); + // This value is from the trait, but is needed for runtimem logic + let request_checksum_required = state.request_checksum_required; + + // This value is set by the user on the SdkConfig to indicate their preference + let request_checksum_calculation = cfg + .load::() + .expect("set from service config"); + + // Determine if we actually calculate the checksum. If the user setting is WhenSupported (the default) + // we always calculate it. If it is WhenRequired we only calculate it if the checksum is marked required + // on the trait. + let calculate_checksum = match request_checksum_calculation { + RequestChecksumCalculation::WhenSupported => true, + RequestChecksumCalculation::WhenRequired => request_checksum_required, + _ => true, + }; + + // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg); - if let Some(checksum_algorithm) = checksum_algorithm { - let request = context.request_mut(); - add_checksum_for_request_body(request, checksum_algorithm, cfg)?; + + if calculate_checksum { + // If a checksum algorithm is set we calculate the checksum. + if let Some(checksum_algorithm) = checksum_algorithm { + let request = context.request_mut(); + add_checksum_for_request_body(request, checksum_algorithm, cfg)?; + } } Ok(()) @@ -179,13 +205,18 @@ fn add_checksum_for_request_body( match request.body().bytes() { // Body is in-memory: read it and insert the checksum as a header. Some(data) => { - tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); let mut checksum = checksum_algorithm.into_impl(); - checksum.update(data); - request - .headers_mut() - .insert(checksum.header_name(), checksum.header_value()); + // If the header has not already been set we set it. If it was already set by the user + // we do nothing and maintain their set value. + if let None = request.headers().get(checksum.header_name()) { + tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); + checksum.update(data); + + request + .headers_mut() + .insert(checksum.header_name(), checksum.header_value()); + } } // Body is streaming: wrap the body so it will emit a checksum as a trailer. None => { @@ -205,6 +236,13 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( request: &mut HttpRequest, checksum_algorithm: ChecksumAlgorithm, ) -> Result<(), BuildError> { + let checksum = checksum_algorithm.into_impl(); + + // If the user already set the header value then do nothing and return early + if let Some(_) = request.headers().get(checksum.header_name()) { + return Ok(()); + } + let original_body_size = request .body() .size_hint() @@ -236,7 +274,7 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( headers.insert( http::header::HeaderName::from_static("x-amz-trailer"), - checksum_algorithm.into_impl().header_name(), + checksum.header_name(), ); headers.insert( diff --git a/aws/rust-runtime/aws-types/src/sdk_config.rs b/aws/rust-runtime/aws-types/src/sdk_config.rs index 81242e270f..aace99b416 100644 --- a/aws/rust-runtime/aws-types/src/sdk_config.rs +++ b/aws/rust-runtime/aws-types/src/sdk_config.rs @@ -894,6 +894,11 @@ impl SdkConfig { self.disable_request_compression } + /// Configured checksum request option. + pub fn request_checksum_calculation(&self) -> Option { + self.request_checksum_calculation + } + /// Configured minimum request compression size. pub fn request_min_compression_size_bytes(&self) -> Option { self.request_min_compression_size_bytes diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 8628106c9d..943bfaeec8 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -8,9 +8,12 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Visibility import software.amazon.smithy.rust.codegen.core.rustlang.Writable @@ -20,12 +23,16 @@ import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.smithy.generators.BuilderCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.AdHocCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.adhocCustomization import software.amazon.smithy.rust.codegen.core.smithy.generators.operationBuildError import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.hasTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull +import kotlin.jvm.optionals.getOrNull internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = RuntimeType.forInlineDependency( @@ -53,12 +60,28 @@ class HttpRequestChecksumDecorator : ClientCodegenDecorator { baseCustomizations: List, ): List = baseCustomizations + HttpRequestChecksumCustomization(codegenContext, operation) - override fun builderCustomizations( + override fun configCustomizations( codegenContext: ClientCodegenContext, - baseCustomizations: List, - ): List { - return baseCustomizations - } + baseCustomizations: List, + ): List = baseCustomizations + HttpRequestChecksumConfigCustomization(codegenContext) + + /** + * Copy the `request_checksum_calculation` value from the `SdkConfig` to the client config + */ + override fun extraSections(codegenContext: ClientCodegenContext): List = + if (!serviceHasHttpChecksumOperation(codegenContext)) { + listOf() + } else { + listOf( + adhocCustomization { section -> + rust( + """ + ${section.serviceConfigBuilder}.set_request_checksum_calculation(${section.sdkConfig}.request_checksum_calculation()); + """, + ) + }, + ) + } } /** @@ -89,7 +112,7 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( return { if (requestAlgorithmMember != null) { if (isRequestChecksumRequired) { - // Checksums are required, fall back to MD5 + // Checksums are required, fall back to crc32 rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") } else { // Checksums aren't required, don't set a fallback @@ -100,6 +123,7 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( rust("""let checksum_algorithm = Some("crc32");""") } + // Parse the checksum_algorithm to a ChecksumAlgorithm enum rustTemplate( """ let checksum_algorithm = match checksum_algorithm { @@ -121,6 +145,11 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( // This generator was implemented based on this spec: // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums + +/** + * Calculate the checksum algorithm based on the trait's `requestAlgorithmMember`. Then instantiate an + * (inlineable) `http_request_checksum` interceptor with that checksum algorithm. + */ class HttpRequestChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, @@ -133,6 +162,7 @@ class HttpRequestChecksumCustomization( val checksumTrait = operationShape.getTrait() ?: return@writable val requestAlgorithmMember = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + val requestChecksumRequired = checksumTrait.isRequestChecksumRequired when (section) { is OperationSection.AdditionalInterceptors -> { @@ -145,7 +175,7 @@ class HttpRequestChecksumCustomization( let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); let checksum_algorithm = input.$requestAlgorithmMember(); #{checksum_algorithm_to_str} - #{Result}::<_, #{BoxError}>::Ok(checksum_algorithm) + #{Result}::<_, #{BoxError}>::Ok((checksum_algorithm, $requestChecksumRequired)) }) """, *preludeScope, @@ -170,52 +200,95 @@ class HttpRequestChecksumCustomization( } } -// -// class HttpRequestChecksumsBuilderCustomization( -// private val codegenContext: ClientCodegenContext, -// private val operationShape: OperationShape, -// ) : BuilderCustomization() { -// -// private val runtimeConfig = codegenContext.runtimeConfig -// -// override fun section(section: BuilderSection): Writable = -// writable { -// // Get the `HttpChecksumTrait`, returning early if this `OperationShape` doesn't have one -// val checksumTrait = operationShape.getTrait() ?: return@writable -// val fieldName = "request_checksum_calculation" -// -// when (section) { -// is BuilderSection.AdditionalFields -> { -// rustTemplate( -// "$fieldName: Option,", -// "BLAH" to runtimeConfig.smithyRuntimeCrate(), -// ) -// } -// -// is BuilderSection.AdditionalMethods -> { -// rust( -// """ -// pub(crate) fn $fieldName(mut self, $fieldName: impl Into) -> Self { -// self.$fieldName = Some($fieldName.into()); -// self -// } -// -// pub(crate) fn set_$fieldName(&mut self, $fieldName: Option) -> &mut Self { -// self.$fieldName = $fieldName; -// self -// } -// """, -// ) -// } -// -// is BuilderSection.AdditionalDebugFields -> { -// rust("""${section.formatterName}.field("$fieldName", &self.$fieldName);""") -// } -// -// is BuilderSection.AdditionalFieldsInBuild -> { -// rust("$fieldName: self.$fieldName,") -// } -// } -// -// } -// } +/** + * Add a `request_checksum_calculation;` field to Service config. + */ +class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientCodegenContext) : + NamedCustomization() { + private val rc = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "RequestChecksumCalculation" to + configReexport( + RuntimeType.smithyChecksums(rc) + .resolve("RequestChecksumCalculation"), + ), + ) + + override fun section(section: ServiceConfig): Writable { + // If the service contains no operations with the httpChecksum trait we return early + if (!serviceHasHttpChecksumOperation(codegenContext)) { + return emptySection + } + + // Otherwise we write the necessary sections to the service config + return when (section) { + is ServiceConfig.ConfigImpl -> + writable { + rustTemplate( + """ + /// Return a reference to the request_checksum_calculation value contained in this config, if any. + pub fn request_checksum_calculation(&self) -> #{Option}<&#{RequestChecksumCalculation}> { + self.config.load::<#{RequestChecksumCalculation}>() + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderImpl -> + writable { + rustTemplate( + """ + /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) + /// to configure checksum behavior. + pub fn request_checksum_calculation( + mut self, + request_checksum_calculation: #{RequestChecksumCalculation} + ) -> Self { + self.set_request_checksum_calculation(#{Some}(request_checksum_calculation)); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) + /// to configure checksum behavior. + pub fn set_request_checksum_calculation( + &mut self, + request_checksum_calculation: #{Option}<#{RequestChecksumCalculation}> + ) -> &mut Self { + self.config.store_or_unset(request_checksum_calculation); + self + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_request_checksum_calculation(${section.configBag}.load::<#{RequestChecksumCalculation}>().cloned());", + *codegenScope, + ) + } + + else -> emptySection + } + } +} + +/** + * Determine if the current service contains any operations with the HttpChecksum trait + */ +private fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext) = + codegenContext.serviceShape.allOperations + .mapNotNull { codegenContext.model.getShape(it).getOrNull() } + .any { + it.hasTrait() + } From 6cac2a962de4ad33a841c5c9a4b80160ad4ded5a Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 28 Aug 2024 20:39:54 -0700 Subject: [PATCH 03/32] Add S3 MultiPartUpload decorator It sets default values for the ChecksumAlgorithm field (although that doesn't actually seem to be reflected in the code at the moment) --- .../src/http_request_checksum.rs | 16 +++--- .../smithy/rustsdk/AwsCodegenDecorator.kt | 2 + .../rustsdk/HttpRequestChecksumDecorator.kt | 13 ++--- .../s3/S3MultiPartUploadDecorator.kt | 54 +++++++++++++++++++ 4 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 6f8c9bed4b..b1c15fd52e 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -164,8 +164,8 @@ where .expect("set from service config"); // Determine if we actually calculate the checksum. If the user setting is WhenSupported (the default) - // we always calculate it. If it is WhenRequired we only calculate it if the checksum is marked required - // on the trait. + // we always calculate it (because this interceptor isn't added if it isn't supported). If it is + // WhenRequired we only calculate it if the checksum is marked required on the trait. let calculate_checksum = match request_checksum_calculation { RequestChecksumCalculation::WhenSupported => true, RequestChecksumCalculation::WhenRequired => request_checksum_required, @@ -173,14 +173,14 @@ where }; // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) - let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg); + // If we have made it this far without a checksum being set we set the default as Crc32 + let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg) + .unwrap_or(ChecksumAlgorithm::Crc32); + // Calculate the checksum if necessary if calculate_checksum { - // If a checksum algorithm is set we calculate the checksum. - if let Some(checksum_algorithm) = checksum_algorithm { - let request = context.request_mut(); - add_checksum_for_request_body(request, checksum_algorithm, cfg)?; - } + let request = context.request_mut(); + add_checksum_for_request_body(request, checksum_algorithm, cfg)?; } Ok(()) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index f27ca2c45b..5686778c37 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -24,6 +24,7 @@ import software.amazon.smithy.rustsdk.customize.s3.S3Decorator import software.amazon.smithy.rustsdk.customize.s3.S3ExpiresDecorator import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator +import software.amazon.smithy.rustsdk.customize.s3.S3MultiPartUploadDecorator import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator import software.amazon.smithy.rustsdk.customize.sso.SSODecorator import software.amazon.smithy.rustsdk.customize.sts.STSDecorator @@ -83,6 +84,7 @@ val DECORATORS: List = S3ExtendedRequestIdDecorator(), IsTruncatedPaginatorDecorator(), S3ExpiresDecorator(), + S3MultiPartUploadDecorator(), ), S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"), STSDecorator().onlyApplyTo("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 943bfaeec8..4ff20f7856 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -110,17 +110,12 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( val isRequestChecksumRequired = this.isRequestChecksumRequired return { - if (requestAlgorithmMember != null) { - if (isRequestChecksumRequired) { - // Checksums are required, fall back to crc32 - rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") - } else { - // Checksums aren't required, don't set a fallback - rust("let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str());") - } - } else if (isRequestChecksumRequired) { + if (requestAlgorithmMember == null && isRequestChecksumRequired) { // Checksums are required but a user can't set one, so we set crc32 for them rust("""let checksum_algorithm = Some("crc32");""") + } else { + // Use checksum algo set by user or crc32 if one has not been set + rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") } // Parse the checksum_algorithm to a ChecksumAlgorithm enum diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt new file mode 100644 index 0000000000..b717e431b4 --- /dev/null +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt @@ -0,0 +1,54 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk.customize.s3 + +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.shapes.EnumShape +import software.amazon.smithy.model.shapes.MemberShape +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.Shape +import software.amazon.smithy.model.shapes.ShapeId +import software.amazon.smithy.model.traits.DefaultTrait +import software.amazon.smithy.model.transform.ModelTransformer +import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings +import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator +import software.amazon.smithy.rust.codegen.core.util.letIf +import java.util.logging.Logger + +/** + * Add a default value to CreateMultiPartUploadRequest's ChecksumAlgorithm + */ +class S3MultiPartUploadDecorator : ClientCodegenDecorator { + override val name: String = "S3MultiPartUpload" + override val order: Byte = 0 + private val logger: Logger = Logger.getLogger(javaClass.name) + private val defaultAlgorithm = "CRC32" + private val targetEnumName = "ChecksumAlgorithm" + private val requestName = "CreateMultipartUploadRequest" + private val s3Namespace = "com.amazonaws.s3" + + private fun isChecksumAlgorithm(shape: Shape): Boolean = + shape is EnumShape && shape.id == ShapeId.from("$s3Namespace#$targetEnumName") + + private fun isChecksumAlgoRequestMember(shape: Shape): Boolean = + shape is MemberShape && shape.id == ShapeId.from("$s3Namespace#$requestName$$targetEnumName") + + override fun transformModel( + service: ServiceShape, + model: Model, + settings: ClientRustSettings, + ): Model = + ModelTransformer.create().mapShapes(model) { shape -> + shape.letIf(isChecksumAlgorithm(shape)) { + // Update root checksum algo enum with a default + (shape as EnumShape).toBuilder().addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() + }.letIf(isChecksumAlgoRequestMember(shape)) { + // Update the default on the CreateMPURequest shape + (shape as MemberShape).toBuilder().addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() + } + } +} From 9c7ae97c91e69a2549638b7e7581582cc6400fbc Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 28 Aug 2024 22:07:23 -0700 Subject: [PATCH 04/32] Add response_checksum_validation to sdk_config Value is not yet added to ConfigBag --- .../aws-config/src/default_provider.rs | 2 +- .../src/default_provider/checksums.rs | 238 ++++++++++++++++++ .../request_checksum_calculation.rs | 128 ---------- aws/rust-runtime/aws-config/src/lib.rs | 8 +- aws/rust-runtime/aws-types/src/sdk_config.rs | 37 ++- .../aws-smithy-checksums/src/error.rs | 35 ++- rust-runtime/aws-smithy-checksums/src/lib.rs | 36 ++- 7 files changed, 344 insertions(+), 140 deletions(-) create mode 100644 aws/rust-runtime/aws-config/src/default_provider/checksums.rs delete mode 100644 aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs diff --git a/aws/rust-runtime/aws-config/src/default_provider.rs b/aws/rust-runtime/aws-config/src/default_provider.rs index 07e3ad4f57..c872bf9a12 100644 --- a/aws/rust-runtime/aws-config/src/default_provider.rs +++ b/aws/rust-runtime/aws-config/src/default_provider.rs @@ -65,4 +65,4 @@ pub mod disable_request_compression; pub mod request_min_compression_size_bytes; /// Default "request checksum calculation" provider chain -pub mod request_checksum_calculation; +pub mod checksums; diff --git a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs new file mode 100644 index 0000000000..605d8ce015 --- /dev/null +++ b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs @@ -0,0 +1,238 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::provider_config::ProviderConfig; +use aws_runtime::env_config::EnvConfigValue; +use aws_smithy_types::error::display::DisplayErrorContext; +use aws_types::sdk_config::{RequestChecksumCalculation, ResponseChecksumValidation}; +use std::str::FromStr; + +mod env { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "AWS_REQUEST_CHECKSUM_CALCULATION"; + pub(super) const RESPONSE_CHECKSUM_VALIDATION: &str = "AWS_RESPONSE_CHECKSUM_VALIDATION"; +} + +mod profile_key { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "request_checksum_calculation"; + pub(super) const RESPONSE_CHECKSUM_VALIDATION: &str = "response_checksum_validation"; +} + +/// Load the value for `request_checksum_calculation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `request_checksum_calculation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn request_checksum_calculation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::REQUEST_CHECKSUM_CALCULATION) + .profile(profile_key::REQUEST_CHECKSUM_CALCULATION) + .validate(&env, profiles, RequestChecksumCalculation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for request_checksum_calculation setting"), + ) + .unwrap_or(None); + + // request_checksum_calculation should always have a non-None value and the + // default is WhenSupported + loaded.or(Some(RequestChecksumCalculation::WhenSupported)) +} + +/// Load the value for `response_checksum_validation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `response_checksum_validation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn response_checksum_validation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::RESPONSE_CHECKSUM_VALIDATION) + .profile(profile_key::RESPONSE_CHECKSUM_VALIDATION) + .validate(&env, profiles, ResponseChecksumValidation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for response_checksum_validation setting"), + ) + .unwrap_or(None); + + // response_checksum_validation should always have a non-None value and the + // default is WhenSupported + loaded.or(Some(ResponseChecksumValidation::WhenSupported)) +} + +#[cfg(test)] +mod test { + use crate::default_provider::checksums::{ + request_checksum_calculation_provider, response_checksum_validation_provider, + }; + #[allow(deprecated)] + use crate::profile::profile_file::{ProfileFileKind, ProfileFiles}; + use crate::provider_config::ProviderConfig; + use aws_smithy_checksums::ResponseChecksumValidation; + use aws_types::os_shim_internal::{Env, Fs}; + use aws_types::sdk_config::RequestChecksumCalculation; + use tracing_test::traced_test; + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value_request() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "not-a-valid-value", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + assert!(logs_contain( + "invalid value for request_checksum_calculation setting" + )); + assert!(logs_contain("AWS_REQUEST_CHECKSUM_CALCULATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority_request() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "WHEN_REQUIRED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_SUPPORTED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works_request() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value_response() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_RESPONSE_CHECKSUM_VALIDATION", + "not-a-valid-value", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + assert!(logs_contain( + "invalid value for response_checksum_validation setting" + )); + assert!(logs_contain("AWS_RESPONSE_CHECKSUM_VALIDATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority_response() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_RESPONSE_CHECKSUM_VALIDATION", + "WHEN_SUPPORTED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\response_checksum_validation = WHEN_REQUIRED", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works_response() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nresponse_checksum_validation = WHEN_REQUIRED", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenRequired) + ); + } +} diff --git a/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs b/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs deleted file mode 100644 index c310d61469..0000000000 --- a/aws/rust-runtime/aws-config/src/default_provider/request_checksum_calculation.rs +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use crate::provider_config::ProviderConfig; -use aws_runtime::env_config::EnvConfigValue; -use aws_smithy_types::error::display::DisplayErrorContext; -use aws_types::sdk_config::RequestChecksumCalculation; -use std::str::FromStr; - -mod env { - pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "AWS_REQUEST_CHECKSUM_CALCULATION"; -} - -mod profile_key { - pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "request_checksum_calculation"; -} - -/// Load the value for `request_checksum_calculation` -/// -/// This checks the following sources: -/// 1. The environment variable `AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_SUPPORTED/WHEN_REQUIRED` -/// 2. The profile key `request_checksum_calculation=WHEN_SUPPORTED/WHEN_REQUIRED` -/// -/// If invalid values are found, the provider will return `None` and an error will be logged. -pub async fn request_checksum_calculation_provider( - provider_config: &ProviderConfig, -) -> Option { - let env = provider_config.env(); - let profiles = provider_config.profile().await; - - let loaded = EnvConfigValue::new() - .env(env::REQUEST_CHECKSUM_CALCULATION) - .profile(profile_key::REQUEST_CHECKSUM_CALCULATION) - .validate(&env, profiles, RequestChecksumCalculation::from_str) - .map_err( - |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for request_checksum_calculation setting"), - ) - .unwrap_or(None); - - // request_checksum_calculation should always have a non-None value and the - // default is WhenSupported - loaded.xor(Some(RequestChecksumCalculation::WhenSupported)) -} - -#[cfg(test)] -mod test { - use crate::default_provider::request_checksum_calculation::request_checksum_calculation_provider; - #[allow(deprecated)] - use crate::profile::profile_file::{ProfileFileKind, ProfileFiles}; - use crate::provider_config::ProviderConfig; - use aws_types::os_shim_internal::{Env, Fs}; - use aws_types::sdk_config::RequestChecksumCalculation; - use tracing_test::traced_test; - - #[tokio::test] - #[traced_test] - async fn log_error_on_invalid_value() { - let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( - "AWS_REQUEST_CHECKSUM_CALCULATION", - "not-a-valid-value", - )])); - assert_eq!(request_checksum_calculation_provider(&conf).await, None); - assert!(logs_contain( - "invalid value for request_checksum_calculation setting" - )); - assert!(logs_contain("AWS_REQUEST_CHECKSUM_CALCULATION")); - } - - #[tokio::test] - #[traced_test] - async fn environment_priority() { - let conf = ProviderConfig::empty() - .with_env(Env::from_slice(&[( - "AWS_REQUEST_CHECKSUM_CALCULATION", - "WHEN_SUPPORTED", - )])) - .with_profile_config( - Some( - #[allow(deprecated)] - ProfileFiles::builder() - .with_file( - #[allow(deprecated)] - ProfileFileKind::Config, - "conf", - ) - .build(), - ), - None, - ) - .with_fs(Fs::from_slice(&[( - "conf", - "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", - )])); - assert_eq!( - request_checksum_calculation_provider(&conf).await, - Some(RequestChecksumCalculation::WhenSupported) - ); - } - - #[tokio::test] - #[traced_test] - async fn profile_works() { - let conf = ProviderConfig::empty() - .with_profile_config( - Some( - #[allow(deprecated)] - ProfileFiles::builder() - .with_file( - #[allow(deprecated)] - ProfileFileKind::Config, - "conf", - ) - .build(), - ), - None, - ) - .with_fs(Fs::from_slice(&[( - "conf", - "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", - )])); - assert_eq!( - request_checksum_calculation_provider(&conf).await, - Some(RequestChecksumCalculation::WhenRequired) - ); - } -} diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index fd761ea546..8d9a329743 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -239,9 +239,9 @@ mod loader { use aws_types::SdkConfig; use crate::default_provider::{ - app_name, credentials, disable_request_compression, endpoint_url, - ignore_configured_endpoint_urls as ignore_ep, region, request_checksum_calculation, - request_min_compression_size_bytes, retry_config, timeout_config, use_dual_stack, use_fips, + app_name, checksums, credentials, disable_request_compression, endpoint_url, + ignore_configured_endpoint_urls as ignore_ep, region, request_min_compression_size_bytes, + retry_config, timeout_config, use_dual_stack, use_fips, }; use crate::meta::region::ProvideRegion; #[allow(deprecated)] @@ -917,7 +917,7 @@ mod loader { Some(request_checksum_calculation) } else { println!("LNJ loading checksum from request_checksum_calculation_provider"); - request_checksum_calculation::request_checksum_calculation_provider(&conf).await + checksums::request_checksum_calculation_provider(&conf).await }; println!("LNJ final value of checksum {request_checksum_calculation:#?}"); diff --git a/aws/rust-runtime/aws-types/src/sdk_config.rs b/aws/rust-runtime/aws-types/src/sdk_config.rs index aace99b416..923cb47be0 100644 --- a/aws/rust-runtime/aws-types/src/sdk_config.rs +++ b/aws/rust-runtime/aws-types/src/sdk_config.rs @@ -19,7 +19,7 @@ pub use aws_credential_types::provider::SharedCredentialsProvider; use aws_smithy_async::rt::sleep::AsyncSleep; pub use aws_smithy_async::rt::sleep::SharedAsyncSleep; pub use aws_smithy_async::time::{SharedTimeSource, TimeSource}; -pub use aws_smithy_checksums::RequestChecksumCalculation; +pub use aws_smithy_checksums::{RequestChecksumCalculation, ResponseChecksumValidation}; use aws_smithy_runtime_api::client::behavior_version::BehaviorVersion; use aws_smithy_runtime_api::client::http::HttpClient; pub use aws_smithy_runtime_api::client::http::SharedHttpClient; @@ -94,6 +94,7 @@ pub struct SdkConfig { disable_request_compression: Option, request_min_compression_size_bytes: Option, request_checksum_calculation: Option, + response_checksum_validation: Option, } /// Builder for AWS Shared Configuration @@ -123,6 +124,7 @@ pub struct Builder { disable_request_compression: Option, request_min_compression_size_bytes: Option, request_checksum_calculation: Option, + response_checksum_validation: Option, } impl Builder { @@ -201,6 +203,30 @@ impl Builder { self } + /// Set the checksum calculation strategy to use for responses. + /// # Examples + /// ``` + /// use aws_types::SdkConfig; + /// use aws_smithy_checksums::ResponseChecksumValidation; + /// let config = SdkConfig::builder().response_checksum_validation(ResponseChecksumValidation::WhenSupported).build(); + /// ``` + pub fn response_checksum_validation( + mut self, + response_checksum_validation: ResponseChecksumValidation, + ) -> Self { + self.set_response_checksum_validation(Some(response_checksum_validation)); + self + } + + /// Set the checksum calculation strategy to use for responses. + pub fn set_response_checksum_validation( + &mut self, + response_checksum_validation: Option, + ) -> &mut Self { + self.response_checksum_validation = response_checksum_validation; + self + } + /// Set the retry_config for the builder /// /// _Note:_ Retries require a sleep implementation in order to work. When enabling retry, make @@ -748,6 +774,7 @@ impl Builder { disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, request_checksum_calculation: self.request_checksum_calculation, + response_checksum_validation: self.response_checksum_validation, } } } @@ -894,11 +921,16 @@ impl SdkConfig { self.disable_request_compression } - /// Configured checksum request option. + /// Configured checksum request behavior. pub fn request_checksum_calculation(&self) -> Option { self.request_checksum_calculation } + /// Configured checksum response behavior. + pub fn response_checksum_validation(&self) -> Option { + self.response_checksum_validation + } + /// Configured minimum request compression size. pub fn request_min_compression_size_bytes(&self) -> Option { self.request_min_compression_size_bytes @@ -967,6 +999,7 @@ impl SdkConfig { disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, request_checksum_calculation: self.request_checksum_calculation, + response_checksum_validation: self.response_checksum_validation, } } } diff --git a/rust-runtime/aws-smithy-checksums/src/error.rs b/rust-runtime/aws-smithy-checksums/src/error.rs index c8aca74980..f184001829 100644 --- a/rust-runtime/aws-smithy-checksums/src/error.rs +++ b/rust-runtime/aws-smithy-checksums/src/error.rs @@ -37,7 +37,7 @@ impl fmt::Display for UnknownChecksumAlgorithmError { impl Error for UnknownChecksumAlgorithmError {} -/// A checksum algorithm was unknown +/// Unknown setting for `request_checksum_calculation` #[derive(Debug)] pub struct UnknownRequestChecksumCalculationError { request_checksum_calculation: String, @@ -50,7 +50,7 @@ impl UnknownRequestChecksumCalculationError { } } - /// The checksum algorithm that is unknown + /// The unknown value pub fn request_checksum_calculation(&self) -> &str { &self.request_checksum_calculation } @@ -67,3 +67,34 @@ impl fmt::Display for UnknownRequestChecksumCalculationError { } impl Error for UnknownRequestChecksumCalculationError {} + +/// Unknown setting for `response_checksum_validation` +#[derive(Debug)] +pub struct UnknownResponseChecksumValidationError { + response_checksum_validation: String, +} + +impl UnknownResponseChecksumValidationError { + pub(crate) fn new(response_checksum_validation: impl Into) -> Self { + Self { + response_checksum_validation: response_checksum_validation.into(), + } + } + + /// The unknown value + pub fn response_checksum_validation(&self) -> &str { + &self.response_checksum_validation + } +} + +impl fmt::Display for UnknownResponseChecksumValidationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown response_checksum_validation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.response_checksum_validation + ) + } +} + +impl Error for UnknownResponseChecksumValidationError {} diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index f022646bda..4644b90b3b 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -17,8 +17,10 @@ //! Checksum calculation and verification callbacks. use crate::error::UnknownChecksumAlgorithmError; -use crate::error::UnknownRequestChecksumCalculationError; -use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; +use crate::error::{ + UnknownRequestChecksumCalculationError, UnknownResponseChecksumValidationError, +}; +use aws_smithy_types::config_bag::{Storable, StoreReplace}; use bytes::Bytes; use std::str::FromStr; @@ -103,7 +105,7 @@ impl ChecksumAlgorithm { pub const WHEN_SUPPORTED: &str = "when_supported"; pub const WHEN_REQUIRED: &str = "when_required"; -/// Configure checksum behavior +/// Configure checksum behavior for requests #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum RequestChecksumCalculation { @@ -131,6 +133,34 @@ impl FromStr for RequestChecksumCalculation { } } +/// Configure checksum behavior for responses +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ResponseChecksumValidation { + WhenSupported, + WhenRequired, +} + +impl Storable for ResponseChecksumValidation { + type Storer = StoreReplace; +} + +impl FromStr for ResponseChecksumValidation { + type Err = UnknownResponseChecksumValidationError; + + fn from_str(response_checksum_validation: &str) -> Result { + if response_checksum_validation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if response_checksum_validation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownResponseChecksumValidationError::new( + response_checksum_validation, + )) + } + } +} + /// Types implementing this trait can calculate checksums. /// /// Checksum algorithms are used to validate the integrity of data. Structs that implement this trait From fd8dc487df92dcc7ac0550fbb0e4ae0cd26bb7a6 Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 29 Aug 2024 14:11:40 -0700 Subject: [PATCH 05/32] Cleaning up response_checksum_validation logic --- aws/rust-runtime/aws-config/src/lib.rs | 26 +++++++++++--------- rust-runtime/aws-smithy-checksums/src/lib.rs | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index 8d9a329743..7c753501d1 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -222,6 +222,7 @@ mod loader { use aws_credential_types::Credentials; use aws_smithy_async::rt::sleep::{default_async_sleep, AsyncSleep, SharedAsyncSleep}; use aws_smithy_async::time::{SharedTimeSource, TimeSource}; + use aws_smithy_checksums::{RequestChecksumCalculation, ResponseChecksumValidation}; use aws_smithy_runtime::client::identity::IdentityCache; use aws_smithy_runtime_api::client::behavior_version::BehaviorVersion; use aws_smithy_runtime_api::client::http::HttpClient; @@ -234,7 +235,6 @@ mod loader { use aws_types::docs_for; use aws_types::origin::Origin; use aws_types::os_shim_internal::{Env, Fs}; - use aws_types::sdk_config::RequestChecksumCalculation; use aws_types::sdk_config::SharedHttpClient; use aws_types::SdkConfig; @@ -291,6 +291,7 @@ mod loader { fs: Option, behavior_version: Option, request_checksum_calculation: Option, + response_checksum_validation: Option, } impl ConfigLoader { @@ -910,19 +911,22 @@ mod loader { Some(user_cache) => Some(user_cache), }; - let request_checksum_calculation = if let Some(request_checksum_calculation) = - self.request_checksum_calculation - { - println!("LNJ checksum explicitly set on config loader: {request_checksum_calculation:#?}"); - Some(request_checksum_calculation) - } else { - println!("LNJ loading checksum from request_checksum_calculation_provider"); - checksums::request_checksum_calculation_provider(&conf).await - }; + let request_checksum_calculation = + if let Some(request_checksum_calculation) = self.request_checksum_calculation { + Some(request_checksum_calculation) + } else { + checksums::request_checksum_calculation_provider(&conf).await + }; - println!("LNJ final value of checksum {request_checksum_calculation:#?}"); + let response_checksum_validation = + if let Some(response_checksum_validation) = self.response_checksum_validation { + Some(response_checksum_validation) + } else { + checksums::response_checksum_validation_provider(&conf).await + }; builder.set_request_checksum_calculation(request_checksum_calculation); + builder.set_response_checksum_validation(response_checksum_validation); builder.set_identity_cache(identity_cache); builder.set_credentials_provider(credentials_provider); builder.set_token_provider(token_provider); diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index 4644b90b3b..1a598e51bf 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -101,7 +101,7 @@ impl ChecksumAlgorithm { } } -// Valid RequestChecksumCalculation names +// Valid names for RequestChecksumCalculation and ResponseChecksumValidation pub const WHEN_SUPPORTED: &str = "when_supported"; pub const WHEN_REQUIRED: &str = "when_required"; From a40952a2f8b6501772c48f8758cbbe092fa38a1b Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 31 Aug 2024 22:26:08 -0700 Subject: [PATCH 06/32] Finished HttpResponseChecksum functionality Still need to implement tests from SEP --- .../src/http_request_checksum.rs | 3 +- .../src/http_response_checksum.rs | 36 ++++- .../rustsdk/HttpRequestChecksumDecorator.kt | 7 +- .../rustsdk/HttpResponseChecksumDecorator.kt | 138 +++++++++++++++- ...nseChecksumMutationInterceptorGenerator.kt | 147 ++++++++++++++++++ .../smithy/generators/OperationGenerator.kt | 10 +- 6 files changed, 325 insertions(+), 16 deletions(-) create mode 100644 codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index b1c15fd52e..a10d25d981 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -167,9 +167,8 @@ where // we always calculate it (because this interceptor isn't added if it isn't supported). If it is // WhenRequired we only calculate it if the checksum is marked required on the trait. let calculate_checksum = match request_checksum_calculation { - RequestChecksumCalculation::WhenSupported => true, RequestChecksumCalculation::WhenRequired => request_checksum_required, - _ => true, + RequestChecksumCalculation::WhenSupported | _ => true, }; // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 1236c9fe50..5f53f5c8d4 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -7,10 +7,11 @@ //! Interceptor for handling Smithy `@httpChecksum` response checksumming -use aws_smithy_checksums::ChecksumAlgorithm; +use aws_smithy_checksums::{ChecksumAlgorithm, ResponseChecksumValidation}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextRef, Input, + BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, + BeforeSerializationInterceptorContextRef, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; @@ -60,9 +61,9 @@ where "ResponseChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { @@ -72,6 +73,8 @@ where layer.store_put(ResponseChecksumInterceptorState { validation_enabled }); cfg.push_layer(layer); + // let req = context.input_mut().; + Ok(()) } @@ -81,17 +84,39 @@ where _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { + println!("LNJ INSIDE RESPONSE CHECKSUM INTERCEPTOR"); let state = cfg .load::() .expect("set in `read_before_serialization`"); - if state.validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + let response_checksum_validation = cfg + .load::() + .expect("set from service config"); + + // If validation has not been explicitly enabled we check the ResponseChecksumValidation + // from the SdkConfig. If it is WhenSupported (or unknown) we enable validation and if it + // is WhenRequired we leave it disabled. + let validation_enabled = if !state.validation_enabled { + match response_checksum_validation { + ResponseChecksumValidation::WhenRequired => false, + ResponseChecksumValidation::WhenSupported | _ => true, + } + } else { + true + }; + + if validation_enabled { + println!("LNJ INSIDE VALIDATION ENABLED"); let response = context.response_mut(); let maybe_checksum_headers = check_headers_for_precalculated_checksum( response.headers(), self.response_algorithms, ); + + println!("LNJ maybe_checksum_headers {maybe_checksum_headers:#?}"); if let Some((checksum_algorithm, precalculated_checksum)) = maybe_checksum_headers { + println!("LNJ WRAPPING BODY"); let mut body = SdkBody::taken(); mem::swap(&mut body, response.body_mut()); @@ -134,6 +159,7 @@ pub(crate) fn check_headers_for_precalculated_checksum( headers: &Headers, response_algorithms: &[&str], ) -> Option<(ChecksumAlgorithm, bytes::Bytes)> { + println!("LNJ CHECKING HEADERS: {headers:#?} and ALGOS: {response_algorithms:#?}"); let checksum_algorithms_to_check = aws_smithy_checksums::http::CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER .into_iter() diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 4ff20f7856..b1139834a1 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -142,8 +142,9 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums /** - * Calculate the checksum algorithm based on the trait's `requestAlgorithmMember`. Then instantiate an - * (inlineable) `http_request_checksum` interceptor with that checksum algorithm. + * Calculate the checksum algorithm based on the input member identified by the trait's + * `requestAlgorithmMember`. Then instantiate an (inlineable) `http_request_checksum` + * interceptor with that checksum algorithm. */ class HttpRequestChecksumCustomization( private val codegenContext: ClientCodegenContext, @@ -281,7 +282,7 @@ class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientC /** * Determine if the current service contains any operations with the HttpChecksum trait */ -private fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext) = +fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext) = codegenContext.serviceShape.allOperations .mapNotNull { codegenContext.model.getShape(it).getOrNull() } .any { diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 16a4521679..75783327f1 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -10,17 +10,25 @@ import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Visibility import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rust import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.smithy.customize.AdHocCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.adhocCustomization +import software.amazon.smithy.rust.codegen.core.util.dq import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rust.codegen.core.util.inputShape @@ -42,6 +50,10 @@ private fun RuntimeConfig.awsInlineableHttpResponseChecksum() = ), ) +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ fun HttpChecksumTrait.requestValidationModeMember( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -65,10 +77,39 @@ class HttpResponseChecksumDecorator : ClientCodegenDecorator { baseCustomizations.letIf(applies(operation)) { it + HttpResponseChecksumCustomization(codegenContext, operation) } + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + HttpResponseChecksumConfigCustomization(codegenContext) + + /** + * Copy the `response_checksum_validation` value from the `SdkConfig` to the client config + */ + override fun extraSections(codegenContext: ClientCodegenContext): List = + if (!serviceHasHttpChecksumOperation(codegenContext)) { + listOf() + } else { + listOf( + adhocCustomization { section -> + rust( + """ + ${section.serviceConfigBuilder}.set_response_checksum_validation(${section.sdkConfig}.response_checksum_validation()); + """, + ) + }, + ) + } } // This generator was implemented based on this spec: // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums + +/** + * Calculate the checksum algorithm based on the input member identified by the trait's + * `requestAlgorithmMember`. Then instantiate an (inlineable) `http_request_checksum` + * interceptor with that checksum algorithm. + */ class HttpResponseChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, @@ -93,15 +134,18 @@ class HttpResponseChecksumCustomization( // CRC32, CRC32C, SHA256, SHA1 -> "crc32", "crc32c", "sha256", "sha1" val responseAlgorithms = checksumTrait.responseAlgorithms - .map { algorithm -> algorithm.lowercase() }.joinToString(", ") { algorithm -> "\"$algorithm\"" } + .map { algorithm -> algorithm.lowercase() } + .joinToString(", ") { algorithm -> algorithm.dq() } val runtimeApi = RuntimeType.smithyRuntimeApiClient(codegenContext.runtimeConfig) rustTemplate( """ #{ResponseChecksumInterceptor}::new( [$responseAlgorithms].as_slice(), |input: &#{Input}| { - ${""/* Per [the spec](https://smithy.io/2.0/aws/aws-core.html#http-response-checksums), - we check to see if it's the `ENABLED` variant */} + ${ + ""/* Per [the spec](https://smithy.io/2.0/aws/aws-core.html#http-response-checksums), + we check to see if it's the `ENABLED` variant */ + } let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); matches!(input.$validationModeName(), #{Some}(#{ValidationModeShape}::Enabled)) } @@ -113,7 +157,10 @@ class HttpResponseChecksumCustomization( .resolve("ResponseChecksumInterceptor"), "Input" to runtimeApi.resolve("client::interceptors::context::Input"), "OperationInput" to codegenContext.symbolProvider.toSymbol(inputShape), - "ValidationModeShape" to codegenContext.symbolProvider.toSymbol(requestValidationModeMemberInner), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), ) } } @@ -122,3 +169,86 @@ class HttpResponseChecksumCustomization( } } } + +/** + * Add a `response_checksum_validation;` field to Service config. + */ +class HttpResponseChecksumConfigCustomization(private val codegenContext: ClientCodegenContext) : + NamedCustomization() { + private val rc = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "ResponseChecksumValidation" to + configReexport( + RuntimeType.smithyChecksums(rc) + .resolve("ResponseChecksumValidation"), + ), + ) + + override fun section(section: ServiceConfig): Writable { + // If the service contains no operations with the httpChecksum trait we return early + if (!serviceHasHttpChecksumOperation(codegenContext)) { + return emptySection + } + + // Otherwise we write the necessary sections to the service config + return when (section) { + is ServiceConfig.ConfigImpl -> + writable { + rustTemplate( + """ + /// Return a reference to the response_checksum_validation value contained in this config, if any. + pub fn response_checksum_validation(&self) -> #{Option}<&#{ResponseChecksumValidation}> { + self.config.load::<#{ResponseChecksumValidation}>() + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderImpl -> + writable { + rustTemplate( + """ + /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) + /// to configure checksum behavior. + pub fn response_checksum_validation( + mut self, + response_checksum_validation: #{ResponseChecksumValidation} + ) -> Self { + self.set_response_checksum_validation(#{Some}(response_checksum_validation)); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) + /// to configure checksum behavior. + pub fn set_response_checksum_validation( + &mut self, + response_checksum_validation: #{Option}<#{ResponseChecksumValidation}> + ) -> &mut Self { + self.config.store_or_unset(response_checksum_validation); + self + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_response_checksum_validation(${section.configBag}.load::<#{ResponseChecksumValidation}>().cloned());", + *codegenScope, + ) + } + + else -> emptySection + } + } +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt new file mode 100644 index 0000000000..06ac24943c --- /dev/null +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -0,0 +1,147 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.client.smithy.generators + +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.shapes.MemberShape +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointTypesGenerator +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.util.dq +import software.amazon.smithy.rust.codegen.core.util.expectMember +import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.inputShape +import software.amazon.smithy.rust.codegen.core.util.orNull + +class HttpResponseChecksumMutationInterceptorGenerator( + private val codegenContext: ClientCodegenContext, +) { + private val model = codegenContext.model + private val symbolProvider = codegenContext.symbolProvider + private val codegenScope = + codegenContext.runtimeConfig.let { rc -> + val endpointTypesGenerator = EndpointTypesGenerator.fromContext(codegenContext) + val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() + val interceptors = runtimeApi.resolve("client::interceptors") + val orchestrator = runtimeApi.resolve("client::orchestrator") + + arrayOf( + *preludeScope, + "BoxError" to RuntimeType.boxError(rc), + "ConfigBag" to RuntimeType.configBag(rc), + "ContextAttachedError" to interceptors.resolve("error::ContextAttachedError"), + "HttpRequest" to orchestrator.resolve("HttpRequest"), + "HttpResponse" to orchestrator.resolve("HttpResponse"), + "Intercept" to RuntimeType.intercept(rc), + "InterceptorContext" to RuntimeType.interceptorContext(rc), + "BeforeSerializationInterceptorContextMut" to RuntimeType.beforeSerializationInterceptorContextMut(rc), + "Input" to interceptors.resolve("context::Input"), + "Output" to interceptors.resolve("context::Output"), + "Error" to interceptors.resolve("context::Error"), + "InterceptorError" to interceptors.resolve("error::InterceptorError"), + "Params" to endpointTypesGenerator.paramsStruct(), + "RuntimeComponents" to RuntimeType.runtimeComponents(rc), + "ResponseChecksumValidation" to + CargoDependency.smithyChecksums(rc).toType() + .resolve("ResponseChecksumValidation"), + ) + } + + fun render( + writer: RustWriter, + operationShape: OperationShape, + ) { + // If the operation doesn't have the HttpChecksum trait we return early + val checksumTrait = operationShape.getTrait() ?: return + // Also return early if there is no requestValidationModeMember on the trait + val requestValidationModeMember = + (checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return) + + val requestValidationModeName = symbolProvider.toSymbol(requestValidationModeMember).name + + val operationName = symbolProvider.toSymbol(operationShape).name + val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" + val responseChecksumValidation = + RuntimeType( + "ResponseChecksumValidation", + CargoDependency.smithyChecksums(codegenContext.runtimeConfig), + ).toSymbol().fullName + + val requestValidationModeMemberInner = + if (requestValidationModeMember.isOptional) { + codegenContext.model.expectShape(requestValidationModeMember.target) + } else { + requestValidationModeMember + } + + writer.rustTemplate( + """ + ##[derive(Debug)] + struct $interceptorName; + + impl #{Intercept} for $interceptorName { + fn name(&self) -> &'static str { + ${interceptorName.dq()} + } + + fn modify_before_serialization( + &self, + context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, + _runtime_comps: &#{RuntimeComponents}, + cfg: &mut #{ConfigBag}, + ) -> #{Result}<(), #{BoxError}> { + let input = context + .input_mut() + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + let request_validation_enabled = + matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); + + if !request_validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + let response_checksum_validation = cfg + .load::<#{ResponseChecksumValidation}>() + .expect("set from service config"); + + match response_checksum_validation { + #{ResponseChecksumValidation}::WhenRequired => {} + #{ResponseChecksumValidation}::WhenSupported | _ => { + input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); + } + } + } + + #{Ok}(()) + } + } + """, + *codegenScope, + "OperationInputType" to codegenContext.symbolProvider.toSymbol(operationShape.inputShape(model)), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + ) + } +} + +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestValidationModeMember( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestValidationModeMember = this.requestValidationModeMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestValidationModeMember) +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt index c2200003db..8fccfbd079 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt @@ -93,7 +93,9 @@ open class OperationGenerator( *preludeScope, "Arc" to RuntimeType.Arc, "ConcreteInput" to symbolProvider.toSymbol(operationShape.inputShape(model)), - "Input" to RuntimeType.smithyRuntimeApiClient(runtimeConfig).resolve("client::interceptors::context::Input"), + "Input" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::interceptors::context::Input"), "Operation" to symbolProvider.toSymbol(operationShape), "OperationError" to errorType, "OperationOutput" to outputType, @@ -169,7 +171,9 @@ open class OperationGenerator( } """, *codegenScope, - "Error" to RuntimeType.smithyRuntimeApiClient(runtimeConfig).resolve("client::interceptors::context::Error"), + "Error" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::interceptors::context::Error"), "InterceptorContext" to RuntimeType.interceptorContext(runtimeConfig), "OrchestratorError" to RuntimeType.smithyRuntimeApiClient(runtimeConfig) @@ -211,5 +215,7 @@ open class OperationGenerator( EndpointParamsInterceptorGenerator(codegenContext) .render(operationWriter, operationShape) + + HttpResponseChecksumMutationInterceptorGenerator(codegenContext).render(operationWriter, operationShape) } } From 3d71421465a34dbb119a046ba3d269cba9cfb63e Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 3 Sep 2024 21:37:44 -0700 Subject: [PATCH 07/32] First working test of flexible checksums Note: it doesn't actually work because sigv4 is benig added as a dev dep but is required by one of the inlineable interceptors as a runtime dep so that still needs to be fixed. --- .../src/http_request_checksum.rs | 3 +- .../src/http_response_checksum.rs | 5 +- .../rustsdk/HttpResponseChecksumDecorator.kt | 9 + .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 369 ++++++++++++++++++ ...nseChecksumMutationInterceptorGenerator.kt | 28 +- 5 files changed, 399 insertions(+), 15 deletions(-) create mode 100644 aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index a10d25d981..8125b50138 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -159,9 +159,10 @@ where let request_checksum_required = state.request_checksum_required; // This value is set by the user on the SdkConfig to indicate their preference + // We provide a default here for users that use a client config instead of the SdkConfig let request_checksum_calculation = cfg .load::() - .expect("set from service config"); + .unwrap_or(&RequestChecksumCalculation::WhenSupported); // Determine if we actually calculate the checksum. If the user setting is WhenSupported (the default) // we always calculate it (because this interceptor isn't added if it isn't supported). If it is diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 5f53f5c8d4..9d1a32410a 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -10,8 +10,7 @@ use aws_smithy_checksums::{ChecksumAlgorithm, ResponseChecksumValidation}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, - BeforeSerializationInterceptorContextRef, Input, + BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; @@ -92,7 +91,7 @@ where // This value is set by the user on the SdkConfig to indicate their preference let response_checksum_validation = cfg .load::() - .expect("set from service config"); + .unwrap_or(&ResponseChecksumValidation::WhenSupported); // If validation has not been explicitly enabled we check the ResponseChecksumValidation // from the SdkConfig. If it is WhenSupported (or unknown) we enable validation and if it diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 75783327f1..b0a2ecc006 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -127,6 +127,7 @@ class HttpResponseChecksumCustomization( } val validationModeName = codegenContext.symbolProvider.toMemberName(requestValidationModeMember) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name when (section) { is OperationSection.AdditionalInterceptors -> { @@ -163,6 +164,14 @@ class HttpResponseChecksumCustomization( ), ) } + section.registerInterceptor(codegenContext.runtimeConfig, this) { + val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" + rustTemplate( + """ + $interceptorName + """, + ) + } } else -> {} diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt new file mode 100644 index 0000000000..8b14a2a36d --- /dev/null +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -0,0 +1,369 @@ +package software.amazon.smithy.rustsdk + +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.junit.jupiter.api.Test +import software.amazon.smithy.rust.codegen.client.testutil.testClientCodegenContext +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.DependencyScope +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel +import software.amazon.smithy.rust.codegen.core.testutil.integrationTest + +internal class HttpChecksumTest { + companion object { + private const val PREFIX = "\$version: \"2\"" + private val model = + """ + $PREFIX + namespace test + + use aws.api#service + use aws.auth#sigv4 + use aws.protocols#httpChecksum + use aws.protocols#restJson1 + use smithy.rules#endpointRuleSet + + @service(sdkId: "dontcare") + @restJson1 + @sigv4(name: "dontcare") + @auth([sigv4]) + @endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } + }) + service TestService { + version: "2023-01-01", + operations: [SomeOperation, SomeStreamingOperation] + } + + @http(uri: "/SomeOperation", method: "POST") + @optionalAuth + @httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "SHA1", "SHA256"] + ) + operation SomeOperation { + input: SomeInput, + output: SomeOutput + } + + @input + structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpPayload + @required + body: Blob + } + + @output + structure SomeOutput {} + + @http(uri: "/SomeStreamingOperation", method: "POST") + @optionalAuth + @httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "SHA1", "SHA256"] + ) + operation SomeStreamingOperation { + input: SomeStreamingInput, + output: SomeStreamingOutput + } + + @streaming + blob StreamingBlob + + @input + structure SomeStreamingInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpPayload + @required + body: StreamingBlob + } + + @output + structure SomeStreamingOutput {} + + enum ChecksumAlgorithm { + CRC32 + CRC32C + //CRC64NVME + SHA1 + SHA256 + } + + enum ValidationMode { + ENABLED + } + """.asSmithyModel() + } + + private val codegenContext = testClientCodegenContext(model) +// private val symbolProvider = codegenContext.symbolProvider +// private val operationShape = model.lookup("com.test#TestService") + + // TODO(flexibleChecksums): We can remove the explicit setting of .checksum_algorithm here when modeled defaults for + // enums work. + @Test + fun requestChecksumWorks() { + awsSdkIntegrationTest(model) { context, rustCrate -> + + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + rustCrate.integrationTest("request_checksums") { + rustTemplate( + """ + ##![cfg(feature = "test-util")] + ##![allow(unused_imports)] + + use #{ByteStream}; + use #{Blob}; + use #{Region}; + use #{pretty_assertions}::{assert_eq, assert_ne}; + use #{Sigv4}; + use #{tempfile}; + + ##[#{tokio}::test] + async fn crc32_checksums_work() { + let (http_client, rx) = ::aws_smithy_runtime::client::http::test_util::capture_request(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client.some_operation() + .body(Blob::new(b"Hello world")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::Crc32) + .send() + .await; + let request = rx.expect_request(); + let crc32_header = request.headers() + .get("x-amz-checksum-crc32") + .expect("crc32 header should exist"); + + assert_eq!(crc32_header, "i9aeUg=="); + + let algo_header = request.headers() + .get("x-amz-request-algorithm") + .expect("algo header should exist"); + + assert_eq!(algo_header, "CRC32"); + } + """, + *preludeScope, + "ByteStream" to RuntimeType.smithyTypes(rc).resolve("byte_stream::ByteStream"), + "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), + "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), + "tempfile" to CargoDependency.TempFile.copy(scope = DependencyScope.Compile).toType(), + "Length" to RuntimeType.smithyTypes(rc).resolve("byte_stream::Length"), + "Sigv4" to AwsCargoDependency.awsSigv4(rc).copy(scope = DependencyScope.Build).toType(), +// "AwsConfig" to AwsCargoDependency.awsConfig(rc).toType(), + ) + } + } + } + +// @Test +// fun `generate unnamed enums`() { +// val shape = model.lookup("com.test#UnnamedEnum") +// val sut = ClientInstantiator(codegenContext) +// val data = Node.parse("t2.nano".dq()) +// +// val project = TestWorkspace.testProject(symbolProvider) +// project.moduleFor(shape) { +// ClientEnumGenerator(codegenContext, shape).render(this) +// unitTest("generate_unnamed_enums") { +// withBlock("let result = ", ";") { +// sut.render(this, shape, data) +// } +// rust("""assert_eq!(result, UnnamedEnum("t2.nano".to_owned()));""") +// } +// } +// project.compileAndTest() +// } +} + +// +// internal class ChecksumTestGenerator( +// private val testCases: List, +// private val paramsType: RuntimeType, +// private val resolverType: RuntimeType, +// private val params: Parameters, +// codegenContext: ClientCodegenContext, +// ) { +// private val runtimeConfig = codegenContext.runtimeConfig +// private val types = Types(runtimeConfig) +// private val codegenScope = +// arrayOf( +// "Endpoint" to types.smithyEndpoint, +// "Error" to types.resolveEndpointError, +// "Document" to RuntimeType.document(runtimeConfig), +// "HashMap" to RuntimeType.HashMap, +// "capture_request" to RuntimeType.captureRequest(runtimeConfig), +// ) +// +// private val instantiator = ClientInstantiator(codegenContext) +// +// private fun EndpointTestCase.docs(): Writable { +// val self = this +// return writable { docs(self.documentation.orElse("no docs")) } +// } +// +// private fun generateBaseTest( +// testCase: EndpointTestCase, +// id: Int, +// ): Writable = +// writable { +// rustTemplate( +// """ +// #{docs:W} +// ##[test] +// fn test_$id() { +// let params = #{params:W}; +// let resolver = #{resolver}::new(); +// let endpoint = resolver.resolve_endpoint(¶ms); +// #{assertion:W} +// } +// """, +// *codegenScope, +// "docs" to testCase.docs(), +// "params" to params(testCase), +// "resolver" to resolverType, +// "assertion" to +// writable { +// testCase.expect.endpoint.ifPresent { endpoint -> +// rustTemplate( +// """ +// let endpoint = endpoint.expect("Expected valid endpoint: ${escape(endpoint.url)}"); +// assert_eq!(endpoint, #{expected:W}); +// """, +// *codegenScope, "expected" to generateEndpoint(endpoint), +// ) +// } +// testCase.expect.error.ifPresent { error -> +// val expectedError = +// escape("expected error: $error [${testCase.documentation.orNull() ?: "no docs"}]") +// rustTemplate( +// """ +// let error = endpoint.expect_err(${expectedError.dq()}); +// assert_eq!(format!("{}", error), ${escape(error).dq()}) +// """, +// *codegenScope, +// ) +// } +// }, +// ) +// } +// +// fun generate(): Writable = +// writable { +// var id = 0 +// testCases.forEach { testCase -> +// id += 1 +// generateBaseTest(testCase, id)(this) +// } +// } +// +// private fun params(testCase: EndpointTestCase) = +// writable { +// rust("#T::builder()", paramsType) +// testCase.params.members.forEach { (id, value) -> +// if (params.get(Identifier.of(id)).isPresent) { +// rust(".${Identifier.of(id).rustName()}(#W)", generateValue(Value.fromNode(value))) +// } +// } +// rust(""".build().expect("invalid params")""") +// } +// +// private fun generateValue(value: Value): Writable { +// return { +// when (value) { +// is StringValue -> rust(escape(value.value).dq() + ".to_string()") +// is BooleanValue -> rust(value.toString()) +// is ArrayValue -> { +// rust( +// "vec![#W]", +// value.values.map { member -> +// writable { +// rustTemplate( +// "#{Document}::from(#{value:W})", +// *codegenScope, +// "value" to generateValue(member), +// ) +// } +// }.join(","), +// ) +// } +// +// is IntegerValue -> rust(value.value.toString()) +// +// is RecordValue -> +// rustBlock("") { +// rustTemplate( +// "let mut out = #{HashMap}::::new();", +// *codegenScope, +// ) +// val ids = mutableListOf() +// value.value.forEach { (id, _) -> ids.add(id) } +// ids.forEach { id -> +// val v = value.get(id) +// rust( +// "out.insert(${id.toString().dq()}.to_string(), #W.into());", +// // When writing into the hashmap, it always needs to be an owned type +// generateValue(v), +// ) +// } +// rustTemplate("out") +// } +// +// else -> PANIC("unexpected type: $value") +// } +// } +// } +// +// private fun generateEndpoint(value: ExpectedEndpoint) = +// writable { +// rustTemplate("#{Endpoint}::builder().url(${escape(value.url).dq()})", *codegenScope) +// value.headers.forEach { (headerName, values) -> +// values.forEach { headerValue -> +// rust(".header(${headerName.dq()}, ${headerValue.dq()})") +// } +// } +// value.properties.forEach { (name, value) -> +// rust( +// ".property(${name.dq()}, #W)", +// generateValue(Value.fromNode(value)), +// ) +// } +// rust(".build()") +// } +// } +// +// diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt index 06ac24943c..a4b7251b46 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -21,6 +21,19 @@ import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull +/** + * This class generates a interceptor for operations with the `httpChecksum` trait that support response validations. + * In the `modify_before_serialization` hook the interceptor checks the operation's `requestValidationModeMember`. If + * that member is `ENABLED` then we end early and do nothing. If it is not `ENABLED` then it checks the + * `response_checksum_validation` set by the user on the SdkConfig. If that is `WhenSupported` (or unknown) then we + * update the `requestValidationModeMember` to `ENABLED` and if the value is `WhenRequired` we end without modifying + * anything. + * + * Note that although there is an existing inlineable `ResponseChecksumInterceptor` this logic could not live there. + * Since that interceptor is inlineable it does not have access to the name of the `requestValidationModeMember` on the + * operation's input, and in certain circumstances we need to mutate that member on the input before serializing the + * request and sending it to the service. + */ class HttpResponseChecksumMutationInterceptorGenerator( private val codegenContext: ClientCodegenContext, ) { @@ -64,17 +77,7 @@ class HttpResponseChecksumMutationInterceptorGenerator( // Also return early if there is no requestValidationModeMember on the trait val requestValidationModeMember = (checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return) - val requestValidationModeName = symbolProvider.toSymbol(requestValidationModeMember).name - - val operationName = symbolProvider.toSymbol(operationShape).name - val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" - val responseChecksumValidation = - RuntimeType( - "ResponseChecksumValidation", - CargoDependency.smithyChecksums(codegenContext.runtimeConfig), - ).toSymbol().fullName - val requestValidationModeMemberInner = if (requestValidationModeMember.isOptional) { codegenContext.model.expectShape(requestValidationModeMember.target) @@ -82,6 +85,9 @@ class HttpResponseChecksumMutationInterceptorGenerator( requestValidationModeMember } + val operationName = symbolProvider.toSymbol(operationShape).name + val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" + writer.rustTemplate( """ ##[derive(Debug)] @@ -110,7 +116,7 @@ class HttpResponseChecksumMutationInterceptorGenerator( // This value is set by the user on the SdkConfig to indicate their preference let response_checksum_validation = cfg .load::<#{ResponseChecksumValidation}>() - .expect("set from service config"); + .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); match response_checksum_validation { #{ResponseChecksumValidation}::WhenRequired => {} From ffd530bfda8e7598a57f3b2186ea193c6bd783bf Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 4 Sep 2024 15:07:09 -0700 Subject: [PATCH 08/32] Adding first suite of working checksum tests --- .../rustsdk/HttpRequestChecksumDecorator.kt | 18 + .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 386 +++++++----------- 2 files changed, 168 insertions(+), 236 deletions(-) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index b1139834a1..3144ee380d 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -47,12 +47,15 @@ internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = CargoDependency.smithyHttp(this), CargoDependency.smithyRuntimeApiClient(this), CargoDependency.smithyTypes(this), + AwsCargoDependency.awsSigv4(this), + CargoDependency.TempFile.toDevDependency(), ), ) class HttpRequestChecksumDecorator : ClientCodegenDecorator { override val name: String = "HttpRequestChecksum" override val order: Byte = 0 + private val defaultAlgorithm = "CRC32" override fun operationCustomizations( codegenContext: ClientCodegenContext, @@ -82,6 +85,21 @@ class HttpRequestChecksumDecorator : ClientCodegenDecorator { }, ) } + +// override fun transformModel(service: ServiceShape, model: Model, settings: ClientRustSettings): Model = +// ModelTransformer.create().mapShapes(model) { shape -> +// shape.letIf(isChecksumAlgoRequestMember(shape)) { +// // Update the default on the CreateMPURequest shape +// (shape as MemberShape).toBuilder().addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() +// } +// } +// +// +// private fun isChecksumAlgoRequestMember(service: ServiceShape, currentShape: Shape): Boolean { +// val checksumTrait = currentShape.getTrait() ?: return false +// +// return true +// } } /** diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index 8b14a2a36d..151184564c 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -1,15 +1,18 @@ -package software.amazon.smithy.rustsdk - /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ +package software.amazon.smithy.rustsdk + import org.junit.jupiter.api.Test -import software.amazon.smithy.rust.codegen.client.testutil.testClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency -import software.amazon.smithy.rust.codegen.core.rustlang.DependencyScope +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.join +import software.amazon.smithy.rust.codegen.core.rustlang.plus import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel @@ -120,250 +123,161 @@ internal class HttpChecksumTest { """.asSmithyModel() } - private val codegenContext = testClientCodegenContext(model) +// private val codegenContext = testClientCodegenContext(model) // private val symbolProvider = codegenContext.symbolProvider // private val operationShape = model.lookup("com.test#TestService") - // TODO(flexibleChecksums): We can remove the explicit setting of .checksum_algorithm here when modeled defaults for - // enums work. + // TODO(flexibleChecksums): We can remove the explicit setting of .checksum_algorithm for crc32 when modeled defaults + // for enums work. @Test fun requestChecksumWorks() { + val checksumRequestTests = + listOf( + RequestChecksumCalculationTest( + "CRC32 checksum calculation works.", + "Hello world", + "Crc32", + "CRC32", + "i9aeUg==", + ), + RequestChecksumCalculationTest( + "CRC32C checksum calculation works.", + "Hello world", + "Crc32C", + "CRC32C", + "crUfeA==", + ), + /* We do not yet support Crc64Nvme checksums + RequestChecksumCalculationTest( + "CRC64NVME checksum calculation works.", + "Hello world", + "Crc64Nvme", + "CRC64NVME", + "uc8X9yrZrD4=", + ), + */ + RequestChecksumCalculationTest( + "SHA1 checksum calculation works.", + "Hello world", + "Sha1", + "SHA1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + RequestChecksumCalculationTest( + "SHA256 checksum calculation works.", + "Hello world", + "Sha256", + "SHA256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + awsSdkIntegrationTest(model) { context, rustCrate -> val rc = context.runtimeConfig - val moduleName = context.moduleUseName() + val testWritables = + checksumRequestTests.map { createRequestChecksumCalculationTest(it, context) }.join("\n") + println("LNJ CREATING testBASE") + val testBase = + writable { + rustTemplate( + """ + ##![cfg(feature = "test-util")] + + use #{Blob}; + use #{Region}; + use #{pretty_assertions}::assert_eq; + + """, + *preludeScope, + "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), + "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), + "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), + ) + } rustCrate.integrationTest("request_checksums") { - rustTemplate( - """ - ##![cfg(feature = "test-util")] - ##![allow(unused_imports)] - - use #{ByteStream}; - use #{Blob}; - use #{Region}; - use #{pretty_assertions}::{assert_eq, assert_ne}; - use #{Sigv4}; - use #{tempfile}; - - ##[#{tokio}::test] - async fn crc32_checksums_work() { - let (http_client, rx) = ::aws_smithy_runtime::client::http::test_util::capture_request(None); - let config = $moduleName::Config::builder() - .region(Region::from_static("doesntmatter")) - .with_test_defaults() - .http_client(http_client) - .build(); - - let client = $moduleName::Client::from_conf(config); - let _ = client.some_operation() - .body(Blob::new(b"Hello world")) - .checksum_algorithm($moduleName::types::ChecksumAlgorithm::Crc32) - .send() - .await; - let request = rx.expect_request(); - let crc32_header = request.headers() - .get("x-amz-checksum-crc32") - .expect("crc32 header should exist"); - - assert_eq!(crc32_header, "i9aeUg=="); - - let algo_header = request.headers() - .get("x-amz-request-algorithm") - .expect("algo header should exist"); - - assert_eq!(algo_header, "CRC32"); - } - """, - *preludeScope, - "ByteStream" to RuntimeType.smithyTypes(rc).resolve("byte_stream::ByteStream"), - "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), - "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), - "tokio" to CargoDependency.Tokio.toType(), - "capture_request" to RuntimeType.captureRequest(rc), - "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), - "tempfile" to CargoDependency.TempFile.copy(scope = DependencyScope.Compile).toType(), - "Length" to RuntimeType.smithyTypes(rc).resolve("byte_stream::Length"), - "Sigv4" to AwsCargoDependency.awsSigv4(rc).copy(scope = DependencyScope.Build).toType(), -// "AwsConfig" to AwsCargoDependency.awsConfig(rc).toType(), - ) + testBase.plus(testWritables)() } } } -// @Test -// fun `generate unnamed enums`() { -// val shape = model.lookup("com.test#UnnamedEnum") -// val sut = ClientInstantiator(codegenContext) -// val data = Node.parse("t2.nano".dq()) -// -// val project = TestWorkspace.testProject(symbolProvider) -// project.moduleFor(shape) { -// ClientEnumGenerator(codegenContext, shape).render(this) -// unitTest("generate_unnamed_enums") { -// withBlock("let result = ", ";") { -// sut.render(this, shape, data) -// } -// rust("""assert_eq!(result, UnnamedEnum("t2.nano".to_owned()));""") -// } -// } -// project.compileAndTest() -// } + private fun createRequestChecksumCalculationTest( + testDef: RequestChecksumCalculationTest, + context: ClientCodegenContext, + ): Writable { + println("LNJ TEST") + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn ${algoLower}_checksums_work() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client.some_operation() + .body(Blob::new(b"${testDef.requestPayload}")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .send() + .await; + let request = rx.expect_request(); + let ${algoLower}_header = request.headers() + .get("x-amz-checksum-$algoLower") + .expect("$algoLower header should exist"); + + assert_eq!(${algoLower}_header, "${testDef.checksumHeader}"); + + let algo_header = request.headers() + .get("x-amz-request-algorithm") + .expect("algo header should exist"); + + assert_eq!(algo_header, "${testDef.algoHeader}"); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } } -// -// internal class ChecksumTestGenerator( -// private val testCases: List, -// private val paramsType: RuntimeType, -// private val resolverType: RuntimeType, -// private val params: Parameters, -// codegenContext: ClientCodegenContext, -// ) { -// private val runtimeConfig = codegenContext.runtimeConfig -// private val types = Types(runtimeConfig) -// private val codegenScope = -// arrayOf( -// "Endpoint" to types.smithyEndpoint, -// "Error" to types.resolveEndpointError, -// "Document" to RuntimeType.document(runtimeConfig), -// "HashMap" to RuntimeType.HashMap, -// "capture_request" to RuntimeType.captureRequest(runtimeConfig), -// ) -// -// private val instantiator = ClientInstantiator(codegenContext) -// -// private fun EndpointTestCase.docs(): Writable { -// val self = this -// return writable { docs(self.documentation.orElse("no docs")) } -// } -// -// private fun generateBaseTest( -// testCase: EndpointTestCase, -// id: Int, -// ): Writable = -// writable { -// rustTemplate( -// """ -// #{docs:W} -// ##[test] -// fn test_$id() { -// let params = #{params:W}; -// let resolver = #{resolver}::new(); -// let endpoint = resolver.resolve_endpoint(¶ms); -// #{assertion:W} -// } -// """, -// *codegenScope, -// "docs" to testCase.docs(), -// "params" to params(testCase), -// "resolver" to resolverType, -// "assertion" to -// writable { -// testCase.expect.endpoint.ifPresent { endpoint -> -// rustTemplate( -// """ -// let endpoint = endpoint.expect("Expected valid endpoint: ${escape(endpoint.url)}"); -// assert_eq!(endpoint, #{expected:W}); -// """, -// *codegenScope, "expected" to generateEndpoint(endpoint), -// ) -// } -// testCase.expect.error.ifPresent { error -> -// val expectedError = -// escape("expected error: $error [${testCase.documentation.orNull() ?: "no docs"}]") -// rustTemplate( -// """ -// let error = endpoint.expect_err(${expectedError.dq()}); -// assert_eq!(format!("{}", error), ${escape(error).dq()}) -// """, -// *codegenScope, -// ) -// } -// }, -// ) -// } -// -// fun generate(): Writable = -// writable { -// var id = 0 -// testCases.forEach { testCase -> -// id += 1 -// generateBaseTest(testCase, id)(this) -// } -// } -// -// private fun params(testCase: EndpointTestCase) = -// writable { -// rust("#T::builder()", paramsType) -// testCase.params.members.forEach { (id, value) -> -// if (params.get(Identifier.of(id)).isPresent) { -// rust(".${Identifier.of(id).rustName()}(#W)", generateValue(Value.fromNode(value))) -// } -// } -// rust(""".build().expect("invalid params")""") -// } -// -// private fun generateValue(value: Value): Writable { -// return { -// when (value) { -// is StringValue -> rust(escape(value.value).dq() + ".to_string()") -// is BooleanValue -> rust(value.toString()) -// is ArrayValue -> { -// rust( -// "vec![#W]", -// value.values.map { member -> -// writable { -// rustTemplate( -// "#{Document}::from(#{value:W})", -// *codegenScope, -// "value" to generateValue(member), -// ) -// } -// }.join(","), -// ) -// } -// -// is IntegerValue -> rust(value.value.toString()) -// -// is RecordValue -> -// rustBlock("") { -// rustTemplate( -// "let mut out = #{HashMap}::::new();", -// *codegenScope, -// ) -// val ids = mutableListOf() -// value.value.forEach { (id, _) -> ids.add(id) } -// ids.forEach { id -> -// val v = value.get(id) -// rust( -// "out.insert(${id.toString().dq()}.to_string(), #W.into());", -// // When writing into the hashmap, it always needs to be an owned type -// generateValue(v), -// ) -// } -// rustTemplate("out") -// } -// -// else -> PANIC("unexpected type: $value") -// } -// } -// } -// -// private fun generateEndpoint(value: ExpectedEndpoint) = -// writable { -// rustTemplate("#{Endpoint}::builder().url(${escape(value.url).dq()})", *codegenScope) -// value.headers.forEach { (headerName, values) -> -// values.forEach { headerValue -> -// rust(".header(${headerName.dq()}, ${headerValue.dq()})") -// } -// } -// value.properties.forEach { (name, value) -> -// rust( -// ".property(${name.dq()}, #W)", -// generateValue(Value.fromNode(value)), -// ) -// } -// rust(".build()") -// } -// } -// -// +data class RequestChecksumCalculationTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val algoHeader: String, + val checksumHeader: String, +) + +data class StreamingRequestChecksumCalculationTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val contentHeader: String, + val trailerHeader: String, + val trailerChecksum: String, +) + +data class ResponseChecksumValidationTest( + val docs: String, + val responsePayload: String, + val checksumAlgorithm: String, + val algoHeader: String, + val checksumHeader: String, + val validationType: TestResult, +) + +sealed class TestResult + +object ResponseChecksumValidationTestSuccess : TestResult() + +data class ResponseChecksumValidationTestFailure(val calculatedChecksum: String) : TestResult() From 9881f5aab9dfc8abc6a559f2e0561fe29a237ebc Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 4 Sep 2024 22:21:28 -0700 Subject: [PATCH 09/32] Added tests for successful response checksums --- .../src/http_response_checksum.rs | 1 + .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 201 +++++++++++++----- 2 files changed, 146 insertions(+), 56 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 9d1a32410a..ac5b17b7d8 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -89,6 +89,7 @@ where .expect("set in `read_before_serialization`"); // This value is set by the user on the SdkConfig to indicate their preference + // We provide a default here for users that use a client config instead of the SdkConfig let response_checksum_validation = cfg .load::() .unwrap_or(&ResponseChecksumValidation::WhenSupported); diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index 151184564c..04c59a4587 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -75,7 +75,10 @@ internal class HttpChecksumTest { } @output - structure SomeOutput {} + structure SomeOutput { + @httpPayload + body: Blob + } @http(uri: "/SomeStreamingOperation", method: "POST") @optionalAuth @@ -112,6 +115,7 @@ internal class HttpChecksumTest { enum ChecksumAlgorithm { CRC32 CRC32C + //Value not supported by current smithy version //CRC64NVME SHA1 SHA256 @@ -131,72 +135,38 @@ internal class HttpChecksumTest { // for enums work. @Test fun requestChecksumWorks() { - val checksumRequestTests = - listOf( - RequestChecksumCalculationTest( - "CRC32 checksum calculation works.", - "Hello world", - "Crc32", - "CRC32", - "i9aeUg==", - ), - RequestChecksumCalculationTest( - "CRC32C checksum calculation works.", - "Hello world", - "Crc32C", - "CRC32C", - "crUfeA==", - ), - /* We do not yet support Crc64Nvme checksums - RequestChecksumCalculationTest( - "CRC64NVME checksum calculation works.", - "Hello world", - "Crc64Nvme", - "CRC64NVME", - "uc8X9yrZrD4=", - ), - */ - RequestChecksumCalculationTest( - "SHA1 checksum calculation works.", - "Hello world", - "Sha1", - "SHA1", - "e1AsOh9IyGCa4hLN+2Od7jlnP14=", - ), - RequestChecksumCalculationTest( - "SHA256 checksum calculation works.", - "Hello world", - "Sha256", - "SHA256", - "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", - ), - ) - awsSdkIntegrationTest(model) { context, rustCrate -> val rc = context.runtimeConfig - val testWritables = + val checksumRequestTestWritables = checksumRequestTests.map { createRequestChecksumCalculationTest(it, context) }.join("\n") - println("LNJ CREATING testBASE") + val checksumResponseSuccTestWritables = + checksumResponseSuccTests.map { createResponseChecksumValidationSuccessTest(it, context) }.join("\n") val testBase = writable { rustTemplate( """ ##![cfg(feature = "test-util")] + ##![allow(unused_imports)] use #{Blob}; use #{Region}; use #{pretty_assertions}::assert_eq; - + use #{SdkBody}; """, *preludeScope, "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), + "SdkBody" to RuntimeType.smithyTypes(rc).resolve("body::SdkBody"), ) } rustCrate.integrationTest("request_checksums") { - testBase.plus(testWritables)() + testBase.plus(checksumRequestTestWritables)() + } + + rustCrate.integrationTest("response_succ_checksums") { + testBase.plus(checksumResponseSuccTestWritables)() } } } @@ -205,7 +175,6 @@ internal class HttpChecksumTest { testDef: RequestChecksumCalculationTest, context: ClientCodegenContext, ): Writable { - println("LNJ TEST") val rc = context.runtimeConfig val moduleName = context.moduleUseName() val algoLower = testDef.checksumAlgorithm.lowercase() @@ -214,7 +183,7 @@ internal class HttpChecksumTest { """ //${testDef.docs} ##[#{tokio}::test] - async fn ${algoLower}_checksums_work() { + async fn ${algoLower}_request_checksums_work() { let (http_client, rx) = #{capture_request}(None); let config = $moduleName::Config::builder() .region(Region::from_static("doesntmatter")) @@ -248,6 +217,49 @@ internal class HttpChecksumTest { ) } } + + private fun createResponseChecksumValidationSuccessTest( + testDef: ResponseChecksumValidationSuccessTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[::tokio::test] + async fn ${algoLower}_response_checksums_work() { + let (http_client, _rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}") + .body(SdkBody::from("${testDef.responsePayload}")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let res = client + .some_operation() + .body(Blob::new(b"Doesn't matter.")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .validation_mode($moduleName::types::ValidationMode::Enabled) + .send() + .await; + assert!(res.is_ok()) + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } } data class RequestChecksumCalculationTest( @@ -258,6 +270,47 @@ data class RequestChecksumCalculationTest( val checksumHeader: String, ) +val checksumRequestTests = + listOf( + RequestChecksumCalculationTest( + "CRC32 checksum calculation works.", + "Hello world", + "Crc32", + "CRC32", + "i9aeUg==", + ), + RequestChecksumCalculationTest( + "CRC32C checksum calculation works.", + "Hello world", + "Crc32C", + "CRC32C", + "crUfeA==", + ), + /* We do not yet support Crc64Nvme checksums + RequestChecksumCalculationTest( + "CRC64NVME checksum calculation works.", + "Hello world", + "Crc64Nvme", + "CRC64NVME", + "uc8X9yrZrD4=", + ), + */ + RequestChecksumCalculationTest( + "SHA1 checksum calculation works.", + "Hello world", + "Sha1", + "SHA1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + RequestChecksumCalculationTest( + "SHA256 checksum calculation works.", + "Hello world", + "Sha256", + "SHA256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + data class StreamingRequestChecksumCalculationTest( val docs: String, val requestPayload: String, @@ -267,17 +320,53 @@ data class StreamingRequestChecksumCalculationTest( val trailerChecksum: String, ) -data class ResponseChecksumValidationTest( +data class ResponseChecksumValidationSuccessTest( + val docs: String, + val responsePayload: String, + val checksumAlgorithm: String, + val checksumHeaderValue: String, +) + +val checksumResponseSuccTests = + listOf( + ResponseChecksumValidationSuccessTest( + "Successful payload validation with CRC32 checksum.", + "Hello world", + "Crc32", + "i9aeUg==", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc32C checksum.", + "Hello world", + "Crc32C", + "crUfeA==", + ), + /* + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc64Nvme checksum.", + "Hello world", + "Crc64Nvme", + "uc8X9yrZrD4=", + ),*/ + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Sha1 checksum.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Sha256 checksum.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class ResponseChecksumValidationFailureTest( val docs: String, val responsePayload: String, val checksumAlgorithm: String, val algoHeader: String, val checksumHeader: String, - val validationType: TestResult, + val calculatedChecksum: String, ) - -sealed class TestResult - -object ResponseChecksumValidationTestSuccess : TestResult() - -data class ResponseChecksumValidationTestFailure(val calculatedChecksum: String) : TestResult() From 5557ea38a86bdcd627827bd05461c61c986e4c1c Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 5 Sep 2024 15:24:21 -0700 Subject: [PATCH 10/32] Add tests of failed response checksum validations --- .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 140 ++++++++++++++++-- 1 file changed, 130 insertions(+), 10 deletions(-) diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index 04c59a4587..32f1a7db30 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -138,10 +138,15 @@ internal class HttpChecksumTest { awsSdkIntegrationTest(model) { context, rustCrate -> val rc = context.runtimeConfig + // Create Writables for all test types val checksumRequestTestWritables = checksumRequestTests.map { createRequestChecksumCalculationTest(it, context) }.join("\n") val checksumResponseSuccTestWritables = checksumResponseSuccTests.map { createResponseChecksumValidationSuccessTest(it, context) }.join("\n") + val checksumResponseFailTestWritables = + checksumResponseFailTests.map { createResponseChecksumValidationFailureTest(it, context) }.join("\n") + + // Shared imports for all test types val testBase = writable { rustTemplate( @@ -161,16 +166,25 @@ internal class HttpChecksumTest { "SdkBody" to RuntimeType.smithyTypes(rc).resolve("body::SdkBody"), ) } + + // Create one integ test per test type rustCrate.integrationTest("request_checksums") { testBase.plus(checksumRequestTestWritables)() } - rustCrate.integrationTest("response_succ_checksums") { + rustCrate.integrationTest("response_checksums_success") { testBase.plus(checksumResponseSuccTestWritables)() } + + rustCrate.integrationTest("response_checksums_fail") { + testBase.plus(checksumResponseFailTestWritables)() + } } } + /** + * Generate tests where the request checksum is calculated correctly + */ private fun createRequestChecksumCalculationTest( testDef: RequestChecksumCalculationTest, context: ClientCodegenContext, @@ -218,6 +232,9 @@ internal class HttpChecksumTest { } } + /** + * Generate tests where the response checksum validates successfully + */ private fun createResponseChecksumValidationSuccessTest( testDef: ResponseChecksumValidationSuccessTest, context: ClientCodegenContext, @@ -260,8 +277,72 @@ internal class HttpChecksumTest { ) } } + + /** + * Generate tests where the response checksum fails to validate + */ + private fun createResponseChecksumValidationFailureTest( + testDef: ResponseChecksumValidationFailureTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[::tokio::test] + async fn ${algoLower}_response_checksums_work() { + let (http_client, _rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}") + .body(SdkBody::from("${testDef.responsePayload}")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let res = client + .some_operation() + .body(Blob::new(b"Doesn't matter.")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .validation_mode($moduleName::types::ValidationMode::Enabled) + .send() + .await; + + assert!(res.is_err()); + + let boxed_err = res + .unwrap_err() + .into_source() + .unwrap() + .downcast::(); + let typed_err = boxed_err.as_ref().unwrap().as_ref(); + + match typed_err { + aws_smithy_checksums::body::validate::Error::ChecksumMismatch { actual, .. } => { + let calculated_checksum = aws_smithy_types::base64::encode(actual); + assert_eq!(calculated_checksum, "${testDef.calculatedChecksum}"); + } + _ => panic!("Unknown error type in checksum validation"), + }; + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } } +// Classes and data for test definitions + data class RequestChecksumCalculationTest( val docs: String, val requestPayload: String, @@ -341,13 +422,13 @@ val checksumResponseSuccTests = "Crc32C", "crUfeA==", ), - /* - ResponseChecksumValidationSuccessTest( - "Successful payload validation with Crc64Nvme checksum.", - "Hello world", - "Crc64Nvme", - "uc8X9yrZrD4=", - ),*/ + /* + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc64Nvme checksum.", + "Hello world", + "Crc64Nvme", + "uc8X9yrZrD4=", + ),*/ ResponseChecksumValidationSuccessTest( "Successful payload validation with Sha1 checksum.", "Hello world", @@ -366,7 +447,46 @@ data class ResponseChecksumValidationFailureTest( val docs: String, val responsePayload: String, val checksumAlgorithm: String, - val algoHeader: String, - val checksumHeader: String, + val checksumHeaderValue: String, val calculatedChecksum: String, ) + +val checksumResponseFailTests = + listOf( + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC32 checksum.", + "Hello world", + "Crc32", + "bm90LWEtY2hlY2tzdW0=", + "i9aeUg==", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC32C checksum.", + "Hello world", + "Crc32C", + "bm90LWEtY2hlY2tzdW0=", + "crUfeA==", + ), + /* + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC64NVME checksum.", + "Hello world", + "Crc64Nvme", + "bm90LWEtY2hlY2tzdW0=", + "uc8X9yrZrD4=", + ),*/ + ResponseChecksumValidationFailureTest( + "Failed payload validation with SHA1 checksum.", + "Hello world", + "Sha1", + "bm90LWEtY2hlY2tzdW0=", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with SHA256 checksum.", + "Hello world", + "Sha256", + "bm90LWEtY2hlY2tzdW0=", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) From 738f30580f04eee3a368d41c1617266f5bf3bbbf Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 5 Sep 2024 19:55:58 -0700 Subject: [PATCH 11/32] Add default crc32 to requestAlgorithmMember shapes --- .../rustsdk/HttpRequestChecksumDecorator.kt | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 3144ee380d..4c15506460 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -6,8 +6,15 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.node.Node import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.Shape +import software.amazon.smithy.model.traits.DefaultTrait +import software.amazon.smithy.model.transform.ModelTransformer import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization @@ -33,6 +40,7 @@ import software.amazon.smithy.rust.codegen.core.util.hasTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull import kotlin.jvm.optionals.getOrNull +import kotlin.streams.asSequence internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = RuntimeType.forInlineDependency( @@ -86,20 +94,27 @@ class HttpRequestChecksumDecorator : ClientCodegenDecorator { ) } -// override fun transformModel(service: ServiceShape, model: Model, settings: ClientRustSettings): Model = -// ModelTransformer.create().mapShapes(model) { shape -> -// shape.letIf(isChecksumAlgoRequestMember(shape)) { -// // Update the default on the CreateMPURequest shape -// (shape as MemberShape).toBuilder().addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() -// } -// } -// -// -// private fun isChecksumAlgoRequestMember(service: ServiceShape, currentShape: Shape): Boolean { -// val checksumTrait = currentShape.getTrait() ?: return false -// -// return true -// } + override fun transformModel( + service: ServiceShape, + model: Model, + settings: ClientRustSettings, + ): Model { + val defaultAlgorithm = "CRC32" + val updates = arrayListOf() + + model.shapes() + .asSequence() + .mapNotNull { shape -> + val trait = shape.getTrait() ?: return@mapNotNull null + val requestAlgorithmMember = trait.requestAlgorithmMember.orNull() ?: return@mapNotNull null + + // Shape is operationShape since it has the checksum trait + (shape as OperationShape).inputShape(model).expectMember(requestAlgorithmMember).toBuilder() + .addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() + }.toCollection(updates) + + return ModelTransformer.create().replaceShapes(model, updates) + } } /** From ab6fbf9f5bd512f9cf2de3c552b07ba27ec9443e Mon Sep 17 00:00:00 2001 From: Landon James Date: Fri, 6 Sep 2024 10:03:17 -0700 Subject: [PATCH 12/32] Update docs for checksum config enums --- .../rustsdk/HttpRequestChecksumDecorator.kt | 8 ++++++-- .../rustsdk/HttpResponseChecksumDecorator.kt | 4 ++-- rust-runtime/aws-smithy-checksums/src/lib.rs | 19 +++++++++++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 4c15506460..1dd9ff6171 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -94,6 +94,10 @@ class HttpRequestChecksumDecorator : ClientCodegenDecorator { ) } + /** + * Add default value of `CRC32` to the `requestAlgorithmMember` shape of all operation inputs that have the + * `httpChecksumTrait` with a non-null requestAlgorithmMember` + */ override fun transformModel( service: ServiceShape, model: Model, @@ -271,7 +275,7 @@ class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientC rustTemplate( """ /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) - /// to configure checksum behavior. + /// to determine when a checksum will be calculated for request payloads. pub fn request_checksum_calculation( mut self, request_checksum_calculation: #{RequestChecksumCalculation} @@ -286,7 +290,7 @@ class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientC rustTemplate( """ /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) - /// to configure checksum behavior. + /// to determine when a checksum will be calculated for request payloads. pub fn set_request_checksum_calculation( &mut self, request_checksum_calculation: #{Option}<#{RequestChecksumCalculation}> diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index b0a2ecc006..3446c0c92e 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -221,7 +221,7 @@ class HttpResponseChecksumConfigCustomization(private val codegenContext: Client rustTemplate( """ /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) - /// to configure checksum behavior. + /// to determine when checksum validation will be performed on response payloads. pub fn response_checksum_validation( mut self, response_checksum_validation: #{ResponseChecksumValidation} @@ -236,7 +236,7 @@ class HttpResponseChecksumConfigCustomization(private val codegenContext: Client rustTemplate( """ /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) - /// to configure checksum behavior. + /// to determine when checksum validation will be performed on response payloads. pub fn set_response_checksum_validation( &mut self, response_checksum_validation: #{Option}<#{ResponseChecksumValidation}> diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index 1a598e51bf..1e37892f89 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -105,7 +105,15 @@ impl ChecksumAlgorithm { pub const WHEN_SUPPORTED: &str = "when_supported"; pub const WHEN_REQUIRED: &str = "when_required"; -/// Configure checksum behavior for requests +/// Determines when a checksum will be calculated for request payloads. Values are: +/// * [RequestChecksumCalculation::WhenSupported] - (default) When set, a checksum will be +/// calculated for all request payloads of operations modeled with the +/// `httpChecksum` trait where `requestChecksumRequired` is `true` and/or a +/// `requestAlgorithmMember` is modeled. +/// * [RequestChecksumCalculation::WhenRequired] - When set, a checksum will only be calculated for +/// request payloads of operations modeled with the `httpChecksum` trait where +/// `requestChecksumRequired` is `true` or where a requestAlgorithmMember +/// is modeled and supplied. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum RequestChecksumCalculation { @@ -133,7 +141,14 @@ impl FromStr for RequestChecksumCalculation { } } -/// Configure checksum behavior for responses +/// Determines when checksum validation will be performed on response payloads. Values are: +/// * [ResponseChecksumValidation::WhenSupported] - (default) When set, checksum validation is performed on all +/// response payloads of operations modeled with the `httpChecksum` trait where +/// `responseAlgorithms` is modeled, except when no modeled checksum algorithms +/// are supported. +/// * [ResponseChecksumValidation::WhenRequired] - When set, checksum validation is not performed on +/// response payloads of operations unless the checksum algorithm is supported and +/// the `requestValidationModeMember` member is set to `ENABLED`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum ResponseChecksumValidation { From ea410c48cc35687b776b2d4cbffcd2752766d51e Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 9 Sep 2024 14:37:41 -0700 Subject: [PATCH 13/32] Interceptor to set default request checksum algo --- .../src/http_request_checksum.rs | 12 +- .../src/http_response_checksum.rs | 5 - .../rustsdk/HttpRequestChecksumDecorator.kt | 9 + .../rustsdk/HttpResponseChecksumDecorator.kt | 1 + .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 31 ++-- ...estChecksumMutationInterceptorGenerator.kt | 159 ++++++++++++++++++ ...nseChecksumMutationInterceptorGenerator.kt | 2 +- .../smithy/generators/OperationGenerator.kt | 2 + 8 files changed, 194 insertions(+), 27 deletions(-) create mode 100644 codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 8125b50138..5669714830 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -155,7 +155,7 @@ where .load::() .expect("set in `read_before_serialization`"); - // This value is from the trait, but is needed for runtimem logic + // This value is from the trait, but is needed for runtime logic let request_checksum_required = state.request_checksum_required; // This value is set by the user on the SdkConfig to indicate their preference @@ -172,13 +172,13 @@ where RequestChecksumCalculation::WhenSupported | _ => true, }; - // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) - // If we have made it this far without a checksum being set we set the default as Crc32 - let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg) - .unwrap_or(ChecksumAlgorithm::Crc32); - // Calculate the checksum if necessary if calculate_checksum { + // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) + // If we have made it this far without a checksum being set we set the default as Crc32 + let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg) + .unwrap_or(ChecksumAlgorithm::Crc32); + let request = context.request_mut(); add_checksum_for_request_body(request, checksum_algorithm, cfg)?; } diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index ac5b17b7d8..bdab493a91 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -83,7 +83,6 @@ where _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { - println!("LNJ INSIDE RESPONSE CHECKSUM INTERCEPTOR"); let state = cfg .load::() .expect("set in `read_before_serialization`"); @@ -107,16 +106,13 @@ where }; if validation_enabled { - println!("LNJ INSIDE VALIDATION ENABLED"); let response = context.response_mut(); let maybe_checksum_headers = check_headers_for_precalculated_checksum( response.headers(), self.response_algorithms, ); - println!("LNJ maybe_checksum_headers {maybe_checksum_headers:#?}"); if let Some((checksum_algorithm, precalculated_checksum)) = maybe_checksum_headers { - println!("LNJ WRAPPING BODY"); let mut body = SdkBody::taken(); mem::swap(&mut body, response.body_mut()); @@ -159,7 +155,6 @@ pub(crate) fn check_headers_for_precalculated_checksum( headers: &Headers, response_algorithms: &[&str], ) -> Option<(ChecksumAlgorithm, bytes::Bytes)> { - println!("LNJ CHECKING HEADERS: {headers:#?} and ALGOS: {response_algorithms:#?}"); let checksum_algorithms_to_check = aws_smithy_checksums::http::CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER .into_iter() diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 1dd9ff6171..302401966d 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -196,6 +196,7 @@ class HttpRequestChecksumCustomization( val requestAlgorithmMember = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) val requestChecksumRequired = checksumTrait.isRequestChecksumRequired + val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name when (section) { is OperationSection.AdditionalInterceptors -> { @@ -225,6 +226,14 @@ class HttpRequestChecksumCustomization( ), ) } + section.registerInterceptor(codegenContext.runtimeConfig, this) { + val interceptorName = "${operationName}HttpRequestChecksumMutationInterceptor" + rustTemplate( + """ + $interceptorName + """, + ) + } } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 3446c0c92e..38844fe2f7 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -116,6 +116,7 @@ class HttpResponseChecksumCustomization( ) : OperationCustomization() { override fun section(section: OperationSection): Writable = writable { + // Return early if this operation lacks the `httpChecksum` trait or that trait lacks a `requestValidationModeMember` val checksumTrait = operationShape.getTrait() ?: return@writable val requestValidationModeMember = checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return@writable diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index 32f1a7db30..1b0165ec70 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -127,12 +127,6 @@ internal class HttpChecksumTest { """.asSmithyModel() } -// private val codegenContext = testClientCodegenContext(model) -// private val symbolProvider = codegenContext.symbolProvider -// private val operationShape = model.lookup("com.test#TestService") - - // TODO(flexibleChecksums): We can remove the explicit setting of .checksum_algorithm for crc32 when modeled defaults - // for enums work. @Test fun requestChecksumWorks() { awsSdkIntegrationTest(model) { context, rustCrate -> @@ -192,6 +186,13 @@ internal class HttpChecksumTest { val rc = context.runtimeConfig val moduleName = context.moduleUseName() val algoLower = testDef.checksumAlgorithm.lowercase() + // If the algo is Crc32 don't explicitly set it to test that the default is correctly set + val setChecksumAlgo = + if (testDef.checksumAlgorithm != "Crc32") { + ".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})" + } else { + "" + } return writable { rustTemplate( """ @@ -208,7 +209,7 @@ internal class HttpChecksumTest { let client = $moduleName::Client::from_conf(config); let _ = client.some_operation() .body(Blob::new(b"${testDef.requestPayload}")) - .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + $setChecksumAlgo .send() .await; let request = rx.expect_request(); @@ -467,14 +468,14 @@ val checksumResponseFailTests = "bm90LWEtY2hlY2tzdW0=", "crUfeA==", ), - /* - ResponseChecksumValidationFailureTest( - "Failed payload validation with CRC64NVME checksum.", - "Hello world", - "Crc64Nvme", - "bm90LWEtY2hlY2tzdW0=", - "uc8X9yrZrD4=", - ),*/ + /* + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC64NVME checksum.", + "Hello world", + "Crc64Nvme", + "bm90LWEtY2hlY2tzdW0=", + "uc8X9yrZrD4=", + ),*/ ResponseChecksumValidationFailureTest( "Failed payload validation with SHA1 checksum.", "Hello world", diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt new file mode 100644 index 0000000000..c2c5b5011c --- /dev/null +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt @@ -0,0 +1,159 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.client.smithy.generators + +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.shapes.MemberShape +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointTypesGenerator +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.util.dq +import software.amazon.smithy.rust.codegen.core.util.expectMember +import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.inputShape +import software.amazon.smithy.rust.codegen.core.util.orNull + +/** + * This class generates an interceptor for operations with the `httpChecksum` trait that support request checksums. + * In the `modify_before_serialization` hook the interceptor checks the config's `request_checksum_calculation` value + * and the trait's `requestChecksumRequired` value. If `request_checksum_calculation` is `WhenSupported` or it is + * `WhenRequired` and `requestChecksumRequired` is `true` then we check the operation input's `requestAlgorithmMember`. + * If that is `None` (so has not been explicitly set by the user) we default it to `Crc32`. + * + * Note that although there is an existing inlineable `RequestChecksumInterceptor` this logic could not live there. + * Since that interceptor is inlineable it does not have access to the name of the `requestAlgorithmMember` on the + * operation's input or the `requestChecksumRequired` value from the trait, and in certain circumstances we need to + * mutate that member on the input before serializing the request and sending it to the service. + */ +class HttpRequestChecksumMutationInterceptorGenerator( + private val codegenContext: ClientCodegenContext, +) { + private val model = codegenContext.model + private val symbolProvider = codegenContext.symbolProvider + private val codegenScope = + codegenContext.runtimeConfig.let { rc -> + val endpointTypesGenerator = EndpointTypesGenerator.fromContext(codegenContext) + val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() + val interceptors = runtimeApi.resolve("client::interceptors") + val orchestrator = runtimeApi.resolve("client::orchestrator") + + arrayOf( + *preludeScope, + "BoxError" to RuntimeType.boxError(rc), + "ConfigBag" to RuntimeType.configBag(rc), + "ContextAttachedError" to interceptors.resolve("error::ContextAttachedError"), + "HttpRequest" to orchestrator.resolve("HttpRequest"), + "HttpResponse" to orchestrator.resolve("HttpResponse"), + "Intercept" to RuntimeType.intercept(rc), + "InterceptorContext" to RuntimeType.interceptorContext(rc), + "BeforeSerializationInterceptorContextMut" to RuntimeType.beforeSerializationInterceptorContextMut(rc), + "Input" to interceptors.resolve("context::Input"), + "Output" to interceptors.resolve("context::Output"), + "Error" to interceptors.resolve("context::Error"), + "InterceptorError" to interceptors.resolve("error::InterceptorError"), + "Params" to endpointTypesGenerator.paramsStruct(), + "RuntimeComponents" to RuntimeType.runtimeComponents(rc), + "RequestChecksumCalculation" to + CargoDependency.smithyChecksums(rc).toType() + .resolve("RequestChecksumCalculation"), + ) + } + + fun render( + writer: RustWriter, + operationShape: OperationShape, + ) { + // If the operation doesn't have the HttpChecksum trait we return early + val checksumTrait = operationShape.getTrait() ?: return + // Also return early if there is no requestValidationModeMember on the trait + val requestAlgorithmMember = + (checksumTrait.requestAlgorithmMember(codegenContext, operationShape) ?: return) + val requestAlgorithmMemberName = symbolProvider.toSymbol(requestAlgorithmMember).name + val requestAlgorithmMemberInner = + if (requestAlgorithmMember.isOptional) { + codegenContext.model.expectShape(requestAlgorithmMember.target) + } else { + requestAlgorithmMember + } + + val operationName = symbolProvider.toSymbol(operationShape).name + val interceptorName = "${operationName}HttpRequestChecksumMutationInterceptor" + val requestChecksumRequired = checksumTrait.isRequestChecksumRequired + + writer.rustTemplate( + """ + ##[derive(Debug)] + struct $interceptorName; + + impl #{Intercept} for $interceptorName { + fn name(&self) -> &'static str { + ${interceptorName.dq()} + } + + fn modify_before_serialization( + &self, + context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, + _runtime_comps: &#{RuntimeComponents}, + cfg: &mut #{ConfigBag}, + ) -> #{Result}<(), #{BoxError}> { + let input = context + .input_mut() + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + // This value is set by the user on the SdkConfig to indicate their preference + let request_checksum_calculation = cfg + .load::<#{RequestChecksumCalculation}>() + .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); + + // From the httpChecksum trait + let http_checksum_required = $requestChecksumRequired; + + // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum we + // default to Crc32. If it is WhenRequired and a checksum is required by the trait we also set the + // default. In all other cases we do nothing. + match ( + request_checksum_calculation, + http_checksum_required, + input.checksum_algorithm(), + ) { + (#{RequestChecksumCalculation}::WhenSupported, _, None) + | (#{RequestChecksumCalculation}::WhenRequired, true, None) => { + input.checksum_algorithm = Some(#{ChecksumAlgoShape}::Crc32); + } + _ => {}, + } + + #{Ok}(()) + } + } + """, + *codegenScope, + "OperationInputType" to codegenContext.symbolProvider.toSymbol(operationShape.inputShape(model)), + "ChecksumAlgoShape" to + codegenContext.symbolProvider.toSymbol( + requestAlgorithmMemberInner, + ), + ) + } +} + +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestAlgorithmMember( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt index a4b7251b46..24110a99ba 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -22,7 +22,7 @@ import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull /** - * This class generates a interceptor for operations with the `httpChecksum` trait that support response validations. + * This class generates an interceptor for operations with the `httpChecksum` trait that support response validations. * In the `modify_before_serialization` hook the interceptor checks the operation's `requestValidationModeMember`. If * that member is `ENABLED` then we end early and do nothing. If it is not `ENABLED` then it checks the * `response_checksum_validation` set by the user on the SdkConfig. If that is `WhenSupported` (or unknown) then we diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt index 8fccfbd079..2a33a1f09f 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt @@ -217,5 +217,7 @@ open class OperationGenerator( .render(operationWriter, operationShape) HttpResponseChecksumMutationInterceptorGenerator(codegenContext).render(operationWriter, operationShape) + + HttpRequestChecksumMutationInterceptorGenerator(codegenContext).render(operationWriter, operationShape) } } From 3025611b3c63d4da9c1b42f994b6d1a933794f08 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 9 Sep 2024 15:03:41 -0700 Subject: [PATCH 14/32] Cleaning up some comments --- .../aws-config/src/default_provider.rs | 2 +- .../rustsdk/HttpRequestChecksumDecorator.kt | 37 +------------------ 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/aws/rust-runtime/aws-config/src/default_provider.rs b/aws/rust-runtime/aws-config/src/default_provider.rs index c872bf9a12..42e5a61739 100644 --- a/aws/rust-runtime/aws-config/src/default_provider.rs +++ b/aws/rust-runtime/aws-config/src/default_provider.rs @@ -64,5 +64,5 @@ pub mod disable_request_compression; /// Default "request minimum compression size bytes" provider chain pub mod request_min_compression_size_bytes; -/// Default "request checksum calculation" provider chain +/// Default provider chains for request/response checksum configuration pub mod checksums; diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 302401966d..b1e439d279 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -6,15 +6,8 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.model.Model -import software.amazon.smithy.model.node.Node import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ServiceShape -import software.amazon.smithy.model.shapes.Shape -import software.amazon.smithy.model.traits.DefaultTrait -import software.amazon.smithy.model.transform.ModelTransformer import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization @@ -40,7 +33,6 @@ import software.amazon.smithy.rust.codegen.core.util.hasTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull import kotlin.jvm.optionals.getOrNull -import kotlin.streams.asSequence internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = RuntimeType.forInlineDependency( @@ -93,32 +85,6 @@ class HttpRequestChecksumDecorator : ClientCodegenDecorator { }, ) } - - /** - * Add default value of `CRC32` to the `requestAlgorithmMember` shape of all operation inputs that have the - * `httpChecksumTrait` with a non-null requestAlgorithmMember` - */ - override fun transformModel( - service: ServiceShape, - model: Model, - settings: ClientRustSettings, - ): Model { - val defaultAlgorithm = "CRC32" - val updates = arrayListOf() - - model.shapes() - .asSequence() - .mapNotNull { shape -> - val trait = shape.getTrait() ?: return@mapNotNull null - val requestAlgorithmMember = trait.requestAlgorithmMember.orNull() ?: return@mapNotNull null - - // Shape is operationShape since it has the checksum trait - (shape as OperationShape).inputShape(model).expectMember(requestAlgorithmMember).toBuilder() - .addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() - }.toCollection(updates) - - return ModelTransformer.create().replaceShapes(model, updates) - } } /** @@ -155,7 +121,8 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") } - // Parse the checksum_algorithm to a ChecksumAlgorithm enum + // Parse the checksum_algorithm type from the service's smithy model to a ChecksumAlgorithm enum from the + // aws_smithy_checksums crate. rustTemplate( """ let checksum_algorithm = match checksum_algorithm { From 5c123a5804a2101417a44c5152e8d66496422ac2 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 10 Sep 2024 20:05:38 -0700 Subject: [PATCH 15/32] Add interceptor for defaulting S3 MPU checksum --- .../src/http_response_checksum.rs | 2 - .../s3/S3MultiPartUploadDecorator.kt | 129 +++++++++++++----- 2 files changed, 98 insertions(+), 33 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index bdab493a91..01ebc6a40f 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -72,8 +72,6 @@ where layer.store_put(ResponseChecksumInterceptorState { validation_enabled }); cfg.push_layer(layer); - // let req = context.input_mut().; - Ok(()) } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt index b717e431b4..0ae8b547b7 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt @@ -5,19 +5,20 @@ package software.amazon.smithy.rustsdk.customize.s3 -import software.amazon.smithy.model.Model -import software.amazon.smithy.model.node.Node -import software.amazon.smithy.model.shapes.EnumShape -import software.amazon.smithy.model.shapes.MemberShape -import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.Shape import software.amazon.smithy.model.shapes.ShapeId -import software.amazon.smithy.model.traits.DefaultTrait -import software.amazon.smithy.model.transform.ModelTransformer -import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator -import software.amazon.smithy.rust.codegen.core.util.letIf -import java.util.logging.Logger +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rust +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.util.inputShape +import software.amazon.smithy.rust.codegen.core.util.orNull /** * Add a default value to CreateMultiPartUploadRequest's ChecksumAlgorithm @@ -25,30 +26,96 @@ import java.util.logging.Logger class S3MultiPartUploadDecorator : ClientCodegenDecorator { override val name: String = "S3MultiPartUpload" override val order: Byte = 0 - private val logger: Logger = Logger.getLogger(javaClass.name) private val defaultAlgorithm = "CRC32" - private val targetEnumName = "ChecksumAlgorithm" - private val requestName = "CreateMultipartUploadRequest" + + private val operationName = "CreateMultipartUpload" private val s3Namespace = "com.amazonaws.s3" - private fun isChecksumAlgorithm(shape: Shape): Boolean = - shape is EnumShape && shape.id == ShapeId.from("$s3Namespace#$targetEnumName") - - private fun isChecksumAlgoRequestMember(shape: Shape): Boolean = - shape is MemberShape && shape.id == ShapeId.from("$s3Namespace#$requestName$$targetEnumName") - - override fun transformModel( - service: ServiceShape, - model: Model, - settings: ClientRustSettings, - ): Model = - ModelTransformer.create().mapShapes(model) { shape -> - shape.letIf(isChecksumAlgorithm(shape)) { - // Update root checksum algo enum with a default - (shape as EnumShape).toBuilder().addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() - }.letIf(isChecksumAlgoRequestMember(shape)) { - // Update the default on the CreateMPURequest shape - (shape as MemberShape).toBuilder().addTrait(DefaultTrait(Node.from(defaultAlgorithm))).build() + private fun isS3MPUOperation(shape: Shape): Boolean = + shape is OperationShape && shape.id == ShapeId.from("$s3Namespace#$operationName") + + override fun operationCustomizations( + codegenContext: ClientCodegenContext, + operation: OperationShape, + baseCustomizations: List, + ): List = + if (isS3MPUOperation(operation)) { + baseCustomizations + listOf(S3MPUChecksumCustomization(codegenContext, operation)) + } else { + baseCustomizations + } +} + +/** + * S3 CreateMPU is not modeled with the httpChecksum trait so the checksum_algorithm + * value must be defaulted here in an interceptor. + */ +private class S3MPUChecksumCustomization( + private val codegenContext: ClientCodegenContext, + private val operation: OperationShape, +) : OperationCustomization() { + override fun section(section: OperationSection): Writable = + writable { + val targetEnumName = "ChecksumAlgorithm" + val interceptorName = "S3CreateMultipartUploadInterceptor" + val rc = codegenContext.runtimeConfig + val checksumAlgoShape = + operation.inputShape(codegenContext.model).getMember(targetEnumName).orNull() ?: return@writable + val checksumAlgoInner = codegenContext.model.expectShape(checksumAlgoShape.target) + if (section is OperationSection.AdditionalInterceptors) { + section.registerInterceptor(codegenContext.runtimeConfig, this) { + rust(interceptorName) + } + } + if (section is OperationSection.RuntimePluginSupportingTypes) { +// section.defineType(this) { + rustTemplate( + """ + ##[derive(Debug)] + struct $interceptorName; + + impl #{Intercept} + for $interceptorName + { + fn name(&self) -> &'static str { + "$interceptorName" + } + + fn modify_before_serialization( + &self, + context: &mut #{Context}, + _runtime_components: &#{RuntimeComponents}, + _cfg: &mut #{ConfigBag}, + ) -> Result<(), #{BoxError}> { + let input = context + .input_mut() + .downcast_mut::<#{OperationInputType}>() + .expect("Input should be for S3 CreateMultipartUpload"); + + // S3 CreateMPU is not modeled with the httpChecksum trait so the checksum_algorithm + // value must be defaulted here. + if input.checksum_algorithm.is_none() { + input.checksum_algorithm = Some(#{ChecksumAlgo}::Crc32) + } + + Ok(()) + } + } + """.trimIndent(), + "BoxError" to RuntimeType.boxError(rc), + "ConfigBag" to RuntimeType.configBag(rc), + "RuntimeComponents" to RuntimeType.runtimeComponents(rc), + "Context" to RuntimeType.beforeSerializationInterceptorContextMut(rc), + "Intercept" to RuntimeType.intercept(rc), + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operation.inputShape( + codegenContext.model, + ), + ), + "ChecksumAlgo" to codegenContext.symbolProvider.toSymbol(checksumAlgoInner), + ) +// } } } } From 80f48cabb1e7053ef48c1f02affdb298b9073e4b Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 10 Sep 2024 21:00:37 -0700 Subject: [PATCH 16/32] Refactor http request checksum interceptor --- .../smithy/rustsdk/AwsCodegenDecorator.kt | 1 + ...estChecksumMutationInterceptorGenerator.kt | 190 ++++++++++++++++++ .../s3/S3MultiPartUploadDecorator.kt | 2 - ...estChecksumMutationInterceptorGenerator.kt | 159 --------------- .../smithy/generators/OperationGenerator.kt | 2 - 5 files changed, 191 insertions(+), 163 deletions(-) create mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt delete mode 100644 codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index 5686778c37..cfed272437 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -44,6 +44,7 @@ val DECORATORS: List = UserAgentDecorator(), SigV4AuthDecorator(), HttpRequestChecksumDecorator(), + HttpRequestChecksumMutationInterceptorGenerator(), HttpResponseChecksumDecorator(), IntegrationTestDecorator(), AwsFluentClientDecorator(), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt new file mode 100644 index 0000000000..be5368c688 --- /dev/null +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt @@ -0,0 +1,190 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk + +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.shapes.MemberShape +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator +import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointTypesGenerator +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.util.dq +import software.amazon.smithy.rust.codegen.core.util.expectMember +import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.inputShape +import software.amazon.smithy.rust.codegen.core.util.orNull + +/** + * This class generates an interceptor for operations with the `httpChecksum` trait that support request checksums. + * In the `modify_before_serialization` hook the interceptor checks the config's `request_checksum_calculation` value + * and the trait's `requestChecksumRequired` value. If `request_checksum_calculation` is `WhenSupported` or it is + * `WhenRequired` and `requestChecksumRequired` is `true` then we check the operation input's `requestAlgorithmMember`. + * If that is `None` (so has not been explicitly set by the user) we default it to `Crc32`. + * + * Note that although there is an existing inlineable `RequestChecksumInterceptor` this logic could not live there. + * Since that interceptor is inlineable it does not have access to the name of the `requestAlgorithmMember` on the + * operation's input or the `requestChecksumRequired` value from the trait, and in certain circumstances we need to + * mutate that member on the input before serializing the request and sending it to the service. + */ +class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { + override val name: String = "HttpRequestChecksumMutationInterceptorGenerator" + override val order: Byte = 0 + + override fun operationCustomizations( + codegenContext: ClientCodegenContext, + operation: OperationShape, + baseCustomizations: List, + ): List { + // If the operation doesn't have the HttpChecksum trait we return early + val checksumTrait = operation.getTrait() ?: return baseCustomizations + // Also return early if there is no requestValidationModeMember on the trait + val requestAlgorithmMember = + (checksumTrait.requestAlgorithmMemberShape(codegenContext, operation) ?: return baseCustomizations) + + return baseCustomizations + + listOf( + InterceptorSection( + codegenContext, + operation, + requestAlgorithmMember, + checksumTrait, + ), + ) + } + + private class InterceptorSection( + private val codegenContext: ClientCodegenContext, + private val operation: OperationShape, + private val requestAlgorithmMember: MemberShape, + private val checksumTrait: HttpChecksumTrait, + ) : OperationCustomization() { + override fun section(section: OperationSection): Writable = + writable { + if (section is OperationSection.RuntimePluginSupportingTypes) { + val model = codegenContext.model + val symbolProvider = codegenContext.symbolProvider + val codegenScope = + codegenContext.runtimeConfig.let { rc -> + val endpointTypesGenerator = EndpointTypesGenerator.fromContext(codegenContext) + val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() + val interceptors = runtimeApi.resolve("client::interceptors") + val orchestrator = runtimeApi.resolve("client::orchestrator") + + arrayOf( + *preludeScope, + "BoxError" to RuntimeType.boxError(rc), + "ConfigBag" to RuntimeType.configBag(rc), + "ContextAttachedError" to interceptors.resolve("error::ContextAttachedError"), + "HttpRequest" to orchestrator.resolve("HttpRequest"), + "HttpResponse" to orchestrator.resolve("HttpResponse"), + "Intercept" to RuntimeType.intercept(rc), + "InterceptorContext" to RuntimeType.interceptorContext(rc), + "BeforeSerializationInterceptorContextMut" to + RuntimeType.beforeSerializationInterceptorContextMut( + rc, + ), + "Input" to interceptors.resolve("context::Input"), + "Output" to interceptors.resolve("context::Output"), + "Error" to interceptors.resolve("context::Error"), + "InterceptorError" to interceptors.resolve("error::InterceptorError"), + "Params" to endpointTypesGenerator.paramsStruct(), + "RuntimeComponents" to RuntimeType.runtimeComponents(rc), + "RequestChecksumCalculation" to + CargoDependency.smithyChecksums(rc).toType() + .resolve("RequestChecksumCalculation"), + ) + } + + val requestAlgorithmMemberInner = + if (requestAlgorithmMember.isOptional) { + codegenContext.model.expectShape(requestAlgorithmMember.target) + } else { + requestAlgorithmMember + } + + val operationName = symbolProvider.toSymbol(operation).name + val interceptorName = "${operationName}HttpRequestChecksumMutationInterceptor" + val requestChecksumRequired = checksumTrait.isRequestChecksumRequired + + rustTemplate( + """ + ##[derive(Debug)] + struct $interceptorName; + + impl #{Intercept} for $interceptorName { + fn name(&self) -> &'static str { + ${interceptorName.dq()} + } + + fn modify_before_serialization( + &self, + context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, + _runtime_comps: &#{RuntimeComponents}, + cfg: &mut #{ConfigBag}, + ) -> #{Result}<(), #{BoxError}> { + let input = context + .input_mut() + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + // This value is set by the user on the SdkConfig to indicate their preference + let request_checksum_calculation = cfg + .load::<#{RequestChecksumCalculation}>() + .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); + + // From the httpChecksum trait + let http_checksum_required = $requestChecksumRequired; + + // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum we + // default to Crc32. If it is WhenRequired and a checksum is required by the trait we also set the + // default. In all other cases we do nothing. + match ( + request_checksum_calculation, + http_checksum_required, + input.checksum_algorithm(), + ) { + (#{RequestChecksumCalculation}::WhenSupported, _, None) + | (#{RequestChecksumCalculation}::WhenRequired, true, None) => { + input.checksum_algorithm = Some(#{ChecksumAlgoShape}::Crc32); + } + _ => {}, + } + + #{Ok}(()) + } + } + """, + *codegenScope, + "OperationInputType" to codegenContext.symbolProvider.toSymbol(operation.inputShape(model)), + "ChecksumAlgoShape" to + codegenContext.symbolProvider.toSymbol( + requestAlgorithmMemberInner, + ), + ) + } + } + } +} + +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestAlgorithmMemberShape( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt index 0ae8b547b7..6c662bceee 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt @@ -68,7 +68,6 @@ private class S3MPUChecksumCustomization( } } if (section is OperationSection.RuntimePluginSupportingTypes) { -// section.defineType(this) { rustTemplate( """ ##[derive(Debug)] @@ -115,7 +114,6 @@ private class S3MPUChecksumCustomization( ), "ChecksumAlgo" to codegenContext.symbolProvider.toSymbol(checksumAlgoInner), ) -// } } } } diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt deleted file mode 100644 index c2c5b5011c..0000000000 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpRequestChecksumMutationInterceptorGenerator.kt +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.smithy.rust.codegen.client.smithy.generators - -import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.model.shapes.MemberShape -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointTypesGenerator -import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency -import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter -import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.util.dq -import software.amazon.smithy.rust.codegen.core.util.expectMember -import software.amazon.smithy.rust.codegen.core.util.getTrait -import software.amazon.smithy.rust.codegen.core.util.inputShape -import software.amazon.smithy.rust.codegen.core.util.orNull - -/** - * This class generates an interceptor for operations with the `httpChecksum` trait that support request checksums. - * In the `modify_before_serialization` hook the interceptor checks the config's `request_checksum_calculation` value - * and the trait's `requestChecksumRequired` value. If `request_checksum_calculation` is `WhenSupported` or it is - * `WhenRequired` and `requestChecksumRequired` is `true` then we check the operation input's `requestAlgorithmMember`. - * If that is `None` (so has not been explicitly set by the user) we default it to `Crc32`. - * - * Note that although there is an existing inlineable `RequestChecksumInterceptor` this logic could not live there. - * Since that interceptor is inlineable it does not have access to the name of the `requestAlgorithmMember` on the - * operation's input or the `requestChecksumRequired` value from the trait, and in certain circumstances we need to - * mutate that member on the input before serializing the request and sending it to the service. - */ -class HttpRequestChecksumMutationInterceptorGenerator( - private val codegenContext: ClientCodegenContext, -) { - private val model = codegenContext.model - private val symbolProvider = codegenContext.symbolProvider - private val codegenScope = - codegenContext.runtimeConfig.let { rc -> - val endpointTypesGenerator = EndpointTypesGenerator.fromContext(codegenContext) - val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() - val interceptors = runtimeApi.resolve("client::interceptors") - val orchestrator = runtimeApi.resolve("client::orchestrator") - - arrayOf( - *preludeScope, - "BoxError" to RuntimeType.boxError(rc), - "ConfigBag" to RuntimeType.configBag(rc), - "ContextAttachedError" to interceptors.resolve("error::ContextAttachedError"), - "HttpRequest" to orchestrator.resolve("HttpRequest"), - "HttpResponse" to orchestrator.resolve("HttpResponse"), - "Intercept" to RuntimeType.intercept(rc), - "InterceptorContext" to RuntimeType.interceptorContext(rc), - "BeforeSerializationInterceptorContextMut" to RuntimeType.beforeSerializationInterceptorContextMut(rc), - "Input" to interceptors.resolve("context::Input"), - "Output" to interceptors.resolve("context::Output"), - "Error" to interceptors.resolve("context::Error"), - "InterceptorError" to interceptors.resolve("error::InterceptorError"), - "Params" to endpointTypesGenerator.paramsStruct(), - "RuntimeComponents" to RuntimeType.runtimeComponents(rc), - "RequestChecksumCalculation" to - CargoDependency.smithyChecksums(rc).toType() - .resolve("RequestChecksumCalculation"), - ) - } - - fun render( - writer: RustWriter, - operationShape: OperationShape, - ) { - // If the operation doesn't have the HttpChecksum trait we return early - val checksumTrait = operationShape.getTrait() ?: return - // Also return early if there is no requestValidationModeMember on the trait - val requestAlgorithmMember = - (checksumTrait.requestAlgorithmMember(codegenContext, operationShape) ?: return) - val requestAlgorithmMemberName = symbolProvider.toSymbol(requestAlgorithmMember).name - val requestAlgorithmMemberInner = - if (requestAlgorithmMember.isOptional) { - codegenContext.model.expectShape(requestAlgorithmMember.target) - } else { - requestAlgorithmMember - } - - val operationName = symbolProvider.toSymbol(operationShape).name - val interceptorName = "${operationName}HttpRequestChecksumMutationInterceptor" - val requestChecksumRequired = checksumTrait.isRequestChecksumRequired - - writer.rustTemplate( - """ - ##[derive(Debug)] - struct $interceptorName; - - impl #{Intercept} for $interceptorName { - fn name(&self) -> &'static str { - ${interceptorName.dq()} - } - - fn modify_before_serialization( - &self, - context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, - _runtime_comps: &#{RuntimeComponents}, - cfg: &mut #{ConfigBag}, - ) -> #{Result}<(), #{BoxError}> { - let input = context - .input_mut() - .downcast_mut::<#{OperationInputType}>() - .ok_or("failed to downcast to #{OperationInputType}")?; - - // This value is set by the user on the SdkConfig to indicate their preference - let request_checksum_calculation = cfg - .load::<#{RequestChecksumCalculation}>() - .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); - - // From the httpChecksum trait - let http_checksum_required = $requestChecksumRequired; - - // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum we - // default to Crc32. If it is WhenRequired and a checksum is required by the trait we also set the - // default. In all other cases we do nothing. - match ( - request_checksum_calculation, - http_checksum_required, - input.checksum_algorithm(), - ) { - (#{RequestChecksumCalculation}::WhenSupported, _, None) - | (#{RequestChecksumCalculation}::WhenRequired, true, None) => { - input.checksum_algorithm = Some(#{ChecksumAlgoShape}::Crc32); - } - _ => {}, - } - - #{Ok}(()) - } - } - """, - *codegenScope, - "OperationInputType" to codegenContext.symbolProvider.toSymbol(operationShape.inputShape(model)), - "ChecksumAlgoShape" to - codegenContext.symbolProvider.toSymbol( - requestAlgorithmMemberInner, - ), - ) - } -} - -/** - * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in - * the HTTP response of the operation. - */ -fun HttpChecksumTrait.requestAlgorithmMember( - codegenContext: ClientCodegenContext, - operationShape: OperationShape, -): MemberShape? { - val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null - return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) -} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt index 2a33a1f09f..8fccfbd079 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt @@ -217,7 +217,5 @@ open class OperationGenerator( .render(operationWriter, operationShape) HttpResponseChecksumMutationInterceptorGenerator(codegenContext).render(operationWriter, operationShape) - - HttpRequestChecksumMutationInterceptorGenerator(codegenContext).render(operationWriter, operationShape) } } From 24f0024a50d75abb26642e09da336710ac613561 Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 11 Sep 2024 13:43:05 -0700 Subject: [PATCH 17/32] Refactor http response checksum mutation intercept --- .../smithy/rustsdk/AwsCodegenDecorator.kt | 1 + ...estChecksumMutationInterceptorGenerator.kt | 8 - ...nseChecksumMutationInterceptorGenerator.kt | 180 ++++++++++++++++++ ...nseChecksumMutationInterceptorGenerator.kt | 153 --------------- .../smithy/generators/OperationGenerator.kt | 2 - 5 files changed, 181 insertions(+), 163 deletions(-) create mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt delete mode 100644 codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index cfed272437..8797a64cc7 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -46,6 +46,7 @@ val DECORATORS: List = HttpRequestChecksumDecorator(), HttpRequestChecksumMutationInterceptorGenerator(), HttpResponseChecksumDecorator(), + HttpResponseChecksumMutationInterceptorGenerator(), IntegrationTestDecorator(), AwsFluentClientDecorator(), CrateLicenseDecorator(), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt index be5368c688..015505c012 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt @@ -10,7 +10,6 @@ import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator -import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointTypesGenerator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency @@ -76,7 +75,6 @@ class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { val symbolProvider = codegenContext.symbolProvider val codegenScope = codegenContext.runtimeConfig.let { rc -> - val endpointTypesGenerator = EndpointTypesGenerator.fromContext(codegenContext) val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() val interceptors = runtimeApi.resolve("client::interceptors") val orchestrator = runtimeApi.resolve("client::orchestrator") @@ -85,11 +83,7 @@ class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { *preludeScope, "BoxError" to RuntimeType.boxError(rc), "ConfigBag" to RuntimeType.configBag(rc), - "ContextAttachedError" to interceptors.resolve("error::ContextAttachedError"), - "HttpRequest" to orchestrator.resolve("HttpRequest"), - "HttpResponse" to orchestrator.resolve("HttpResponse"), "Intercept" to RuntimeType.intercept(rc), - "InterceptorContext" to RuntimeType.interceptorContext(rc), "BeforeSerializationInterceptorContextMut" to RuntimeType.beforeSerializationInterceptorContextMut( rc, @@ -97,8 +91,6 @@ class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { "Input" to interceptors.resolve("context::Input"), "Output" to interceptors.resolve("context::Output"), "Error" to interceptors.resolve("context::Error"), - "InterceptorError" to interceptors.resolve("error::InterceptorError"), - "Params" to endpointTypesGenerator.paramsStruct(), "RuntimeComponents" to RuntimeType.runtimeComponents(rc), "RequestChecksumCalculation" to CargoDependency.smithyChecksums(rc).toType() diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt new file mode 100644 index 0000000000..8ccce4ba5f --- /dev/null +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -0,0 +1,180 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk + +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.shapes.MemberShape +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.util.dq +import software.amazon.smithy.rust.codegen.core.util.expectMember +import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.inputShape +import software.amazon.smithy.rust.codegen.core.util.orNull + +/** + * This class generates an interceptor for operations with the `httpChecksum` trait that support response validations. + * In the `modify_before_serialization` hook the interceptor checks the operation's `requestValidationModeMember`. If + * that member is `ENABLED` then we end early and do nothing. If it is not `ENABLED` then it checks the + * `response_checksum_validation` set by the user on the SdkConfig. If that is `WhenSupported` (or unknown) then we + * update the `requestValidationModeMember` to `ENABLED` and if the value is `WhenRequired` we end without modifying + * anything since there is no way to indicate that a response checksum is required. + * + * Note that although there is an existing inlineable `ResponseChecksumInterceptor` this logic could not live there. + * Since that interceptor is inlineable it does not have access to the name of the `requestValidationModeMember` on the + * operation's input, and in certain circumstances we need to mutate that member on the input before serializing the + * request and sending it to the service. + */ +class HttpResponseChecksumMutationInterceptorGenerator : ClientCodegenDecorator { + override val name: String = "HttpResponseChecksumMutationInterceptorGenerator" + override val order: Byte = 0 + + override fun operationCustomizations( + codegenContext: ClientCodegenContext, + operation: OperationShape, + baseCustomizations: List, + ): List { + // If the operation doesn't have the HttpChecksum trait we return early + val checksumTrait = operation.getTrait() ?: return baseCustomizations + // Also return early if there is no requestValidationModeMember on the trait + val requestValidationModeMember = + (checksumTrait.requestValidationModeMemberShape(codegenContext, operation) ?: return baseCustomizations) + + return baseCustomizations + + listOf( + InterceptorSection( + codegenContext, + operation, + requestValidationModeMember, + checksumTrait, + ), + ) + } + + private class InterceptorSection( + private val codegenContext: ClientCodegenContext, + private val operation: OperationShape, + private val requestValidationModeMember: MemberShape, + private val checksumTrait: HttpChecksumTrait, + ) : OperationCustomization() { + override fun section(section: OperationSection): Writable = + writable { + if (section is OperationSection.RuntimePluginSupportingTypes) { + val model = codegenContext.model + val symbolProvider = codegenContext.symbolProvider + val codegenScope = + codegenContext.runtimeConfig.let { rc -> + val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() + val interceptors = runtimeApi.resolve("client::interceptors") + val orchestrator = runtimeApi.resolve("client::orchestrator") + + arrayOf( + *preludeScope, + "BoxError" to RuntimeType.boxError(rc), + "ConfigBag" to RuntimeType.configBag(rc), + "Intercept" to RuntimeType.intercept(rc), + "BeforeSerializationInterceptorContextMut" to + RuntimeType.beforeSerializationInterceptorContextMut( + rc, + ), + "Input" to interceptors.resolve("context::Input"), + "Output" to interceptors.resolve("context::Output"), + "Error" to interceptors.resolve("context::Error"), + "RuntimeComponents" to RuntimeType.runtimeComponents(rc), + "ResponseChecksumValidation" to + CargoDependency.smithyChecksums(rc).toType() + .resolve("ResponseChecksumValidation"), + ) + } + + val requestValidationModeName = symbolProvider.toSymbol(requestValidationModeMember).name + val requestValidationModeMemberInner = + if (requestValidationModeMember.isOptional) { + codegenContext.model.expectShape(requestValidationModeMember.target) + } else { + requestValidationModeMember + } + + val operationName = symbolProvider.toSymbol(operation).name + val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" + + rustTemplate( + """ + ##[derive(Debug)] + struct $interceptorName; + + impl #{Intercept} for $interceptorName { + fn name(&self) -> &'static str { + ${interceptorName.dq()} + } + + fn modify_before_serialization( + &self, + context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, + _runtime_comps: &#{RuntimeComponents}, + cfg: &mut #{ConfigBag}, + ) -> #{Result}<(), #{BoxError}> { + let input = context + .input_mut() + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + let request_validation_enabled = + matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); + + if !request_validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + let response_checksum_validation = cfg + .load::<#{ResponseChecksumValidation}>() + .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); + + // If validation setting is WhenSupported (or unknown) we enable response checksum + // validation. If it is WhenRequired we do not enable (since there is no way to + // indicate that a response checksum is required). + match response_checksum_validation { + #{ResponseChecksumValidation}::WhenRequired => {} + #{ResponseChecksumValidation}::WhenSupported | _ => { + input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); + } + } + } + + #{Ok}(()) + } + } + """, + *codegenScope, + "OperationInputType" to codegenContext.symbolProvider.toSymbol(operation.inputShape(model)), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + ) + } + } + } +} + +/** + * Get the top-level operation input member used to opt in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestValidationModeMemberShape( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestValidationModeMember = this.requestValidationModeMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestValidationModeMember) +} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt deleted file mode 100644 index 24110a99ba..0000000000 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/HttpResponseChecksumMutationInterceptorGenerator.kt +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.smithy.rust.codegen.client.smithy.generators - -import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.model.shapes.MemberShape -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.endpoint.EndpointTypesGenerator -import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency -import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter -import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.util.dq -import software.amazon.smithy.rust.codegen.core.util.expectMember -import software.amazon.smithy.rust.codegen.core.util.getTrait -import software.amazon.smithy.rust.codegen.core.util.inputShape -import software.amazon.smithy.rust.codegen.core.util.orNull - -/** - * This class generates an interceptor for operations with the `httpChecksum` trait that support response validations. - * In the `modify_before_serialization` hook the interceptor checks the operation's `requestValidationModeMember`. If - * that member is `ENABLED` then we end early and do nothing. If it is not `ENABLED` then it checks the - * `response_checksum_validation` set by the user on the SdkConfig. If that is `WhenSupported` (or unknown) then we - * update the `requestValidationModeMember` to `ENABLED` and if the value is `WhenRequired` we end without modifying - * anything. - * - * Note that although there is an existing inlineable `ResponseChecksumInterceptor` this logic could not live there. - * Since that interceptor is inlineable it does not have access to the name of the `requestValidationModeMember` on the - * operation's input, and in certain circumstances we need to mutate that member on the input before serializing the - * request and sending it to the service. - */ -class HttpResponseChecksumMutationInterceptorGenerator( - private val codegenContext: ClientCodegenContext, -) { - private val model = codegenContext.model - private val symbolProvider = codegenContext.symbolProvider - private val codegenScope = - codegenContext.runtimeConfig.let { rc -> - val endpointTypesGenerator = EndpointTypesGenerator.fromContext(codegenContext) - val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() - val interceptors = runtimeApi.resolve("client::interceptors") - val orchestrator = runtimeApi.resolve("client::orchestrator") - - arrayOf( - *preludeScope, - "BoxError" to RuntimeType.boxError(rc), - "ConfigBag" to RuntimeType.configBag(rc), - "ContextAttachedError" to interceptors.resolve("error::ContextAttachedError"), - "HttpRequest" to orchestrator.resolve("HttpRequest"), - "HttpResponse" to orchestrator.resolve("HttpResponse"), - "Intercept" to RuntimeType.intercept(rc), - "InterceptorContext" to RuntimeType.interceptorContext(rc), - "BeforeSerializationInterceptorContextMut" to RuntimeType.beforeSerializationInterceptorContextMut(rc), - "Input" to interceptors.resolve("context::Input"), - "Output" to interceptors.resolve("context::Output"), - "Error" to interceptors.resolve("context::Error"), - "InterceptorError" to interceptors.resolve("error::InterceptorError"), - "Params" to endpointTypesGenerator.paramsStruct(), - "RuntimeComponents" to RuntimeType.runtimeComponents(rc), - "ResponseChecksumValidation" to - CargoDependency.smithyChecksums(rc).toType() - .resolve("ResponseChecksumValidation"), - ) - } - - fun render( - writer: RustWriter, - operationShape: OperationShape, - ) { - // If the operation doesn't have the HttpChecksum trait we return early - val checksumTrait = operationShape.getTrait() ?: return - // Also return early if there is no requestValidationModeMember on the trait - val requestValidationModeMember = - (checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return) - val requestValidationModeName = symbolProvider.toSymbol(requestValidationModeMember).name - val requestValidationModeMemberInner = - if (requestValidationModeMember.isOptional) { - codegenContext.model.expectShape(requestValidationModeMember.target) - } else { - requestValidationModeMember - } - - val operationName = symbolProvider.toSymbol(operationShape).name - val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" - - writer.rustTemplate( - """ - ##[derive(Debug)] - struct $interceptorName; - - impl #{Intercept} for $interceptorName { - fn name(&self) -> &'static str { - ${interceptorName.dq()} - } - - fn modify_before_serialization( - &self, - context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, - _runtime_comps: &#{RuntimeComponents}, - cfg: &mut #{ConfigBag}, - ) -> #{Result}<(), #{BoxError}> { - let input = context - .input_mut() - .downcast_mut::<#{OperationInputType}>() - .ok_or("failed to downcast to #{OperationInputType}")?; - - let request_validation_enabled = - matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); - - if !request_validation_enabled { - // This value is set by the user on the SdkConfig to indicate their preference - let response_checksum_validation = cfg - .load::<#{ResponseChecksumValidation}>() - .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); - - match response_checksum_validation { - #{ResponseChecksumValidation}::WhenRequired => {} - #{ResponseChecksumValidation}::WhenSupported | _ => { - input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); - } - } - } - - #{Ok}(()) - } - } - """, - *codegenScope, - "OperationInputType" to codegenContext.symbolProvider.toSymbol(operationShape.inputShape(model)), - "ValidationModeShape" to - codegenContext.symbolProvider.toSymbol( - requestValidationModeMemberInner, - ), - ) - } -} - -/** - * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in - * the HTTP response of the operation. - */ -fun HttpChecksumTrait.requestValidationModeMember( - codegenContext: ClientCodegenContext, - operationShape: OperationShape, -): MemberShape? { - val requestValidationModeMember = this.requestValidationModeMember.orNull() ?: return null - return operationShape.inputShape(codegenContext.model).expectMember(requestValidationModeMember) -} diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt index 8fccfbd079..bb4c69e29f 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt @@ -215,7 +215,5 @@ open class OperationGenerator( EndpointParamsInterceptorGenerator(codegenContext) .render(operationWriter, operationShape) - - HttpResponseChecksumMutationInterceptorGenerator(codegenContext).render(operationWriter, operationShape) } } From 50728c40b44759d27d0833559f773502135abcb7 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 21 Sep 2024 17:48:40 -0700 Subject: [PATCH 18/32] Add streaming checksum tests --- .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 123 +++++++++++++++++- 1 file changed, 121 insertions(+), 2 deletions(-) diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index 1b0165ec70..feefbb50a6 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -139,6 +139,8 @@ internal class HttpChecksumTest { checksumResponseSuccTests.map { createResponseChecksumValidationSuccessTest(it, context) }.join("\n") val checksumResponseFailTestWritables = checksumResponseFailTests.map { createResponseChecksumValidationFailureTest(it, context) }.join("\n") + val checksumStreamingRequestTestWritables = + streamingRequestTests.map { createStreamingRequestChecksumCalculationTest(it, context) }.join("\n") // Shared imports for all test types val testBase = @@ -152,6 +154,8 @@ internal class HttpChecksumTest { use #{Region}; use #{pretty_assertions}::assert_eq; use #{SdkBody}; + use std::io::Write; + use http_body::Body; """, *preludeScope, "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), @@ -173,6 +177,10 @@ internal class HttpChecksumTest { rustCrate.integrationTest("response_checksums_fail") { testBase.plus(checksumResponseFailTestWritables)() } + + rustCrate.integrationTest("streaming_request_checksums") { + testBase.plus(checksumStreamingRequestTestWritables)() + } } } @@ -233,6 +241,85 @@ internal class HttpChecksumTest { } } + /** + * Generate tests where the request is streaming and checksum is calculated correctly + */ + private fun createStreamingRequestChecksumCalculationTest( + testDef: StreamingRequestChecksumCalculationTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // If the algo is Crc32 don't explicitly set it to test that the default is correctly set + val setChecksumAlgo = + if (testDef.checksumAlgorithm != "Crc32") { + ".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})" + } else { + "" + } + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn ${algoLower}_request_checksums_work() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.as_file_mut() + .write_all("${testDef.requestPayload}".as_bytes()) + .unwrap(); + + let streaming_body = aws_smithy_types::byte_stream::ByteStream::read_from() + .path(&file) + .build() + .await + .unwrap(); + + let _operation = client + .some_streaming_operation() + .body(streaming_body) + $setChecksumAlgo + .send() + .await; + + + let request = rx.expect_request(); + + let headers = request.headers(); + + assert_eq!( + headers.get("x-amz-trailer").unwrap(), + "x-amz-checksum-$algoLower", + ); + assert_eq!(headers.get("content-encoding").unwrap(), "aws-chunked"); + + let mut body = request.body().try_clone().expect("body is retryable"); + + let mut body_data = bytes::BytesMut::new(); + while let Some(data) = body.data().await { + body_data.extend_from_slice(&data.unwrap()) + } + + let body_string = std::str::from_utf8(&body_data).unwrap(); + assert!(body_string.contains("x-amz-checksum-$algoLower:${testDef.trailerChecksum}")); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + /** * Generate tests where the response checksum validates successfully */ @@ -397,11 +484,43 @@ data class StreamingRequestChecksumCalculationTest( val docs: String, val requestPayload: String, val checksumAlgorithm: String, - val contentHeader: String, - val trailerHeader: String, val trailerChecksum: String, ) +val streamingRequestTests = + listOf( + StreamingRequestChecksumCalculationTest( + "CRC32 streaming checksum calculation works.", + "Hello world", + "Crc32", + "i9aeUg==", + ), + StreamingRequestChecksumCalculationTest( + "CRC32C streaming checksum calculation works.", + "Hello world", + "Crc32C", + "crUfeA==", + ), +// StreamingRequestChecksumCalculationTest( +// "CRC64NVME streaming checksum calculation works.", +// "Hello world", +// "Crc64Nvme", +// "uc8X9yrZrD4=", +// ), + StreamingRequestChecksumCalculationTest( + "SHA1 streaming checksum calculation works.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + StreamingRequestChecksumCalculationTest( + "SHA256 streaming checksum calculation works.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + data class ResponseChecksumValidationSuccessTest( val docs: String, val responsePayload: String, From e9d4c89b19f0c0d139cb92aba71c5e56d6d778a9 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 21 Sep 2024 18:03:08 -0700 Subject: [PATCH 19/32] Remove S3MPU Decorator --- .../src/http_response_checksum.rs | 3 +- .../smithy/rustsdk/AwsCodegenDecorator.kt | 2 -- .../s3/S3MultiPartUploadDecorator.kt | 29 ------------------- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 01ebc6a40f..b657de6e23 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -93,7 +93,8 @@ where // If validation has not been explicitly enabled we check the ResponseChecksumValidation // from the SdkConfig. If it is WhenSupported (or unknown) we enable validation and if it - // is WhenRequired we leave it disabled. + // is WhenRequired we leave it disabled since there is no way to indicate that a response + // checksum is required. let validation_enabled = if !state.validation_enabled { match response_checksum_validation { ResponseChecksumValidation::WhenRequired => false, diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index 8797a64cc7..45f71eee5c 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -24,7 +24,6 @@ import software.amazon.smithy.rustsdk.customize.s3.S3Decorator import software.amazon.smithy.rustsdk.customize.s3.S3ExpiresDecorator import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator -import software.amazon.smithy.rustsdk.customize.s3.S3MultiPartUploadDecorator import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator import software.amazon.smithy.rustsdk.customize.sso.SSODecorator import software.amazon.smithy.rustsdk.customize.sts.STSDecorator @@ -86,7 +85,6 @@ val DECORATORS: List = S3ExtendedRequestIdDecorator(), IsTruncatedPaginatorDecorator(), S3ExpiresDecorator(), - S3MultiPartUploadDecorator(), ), S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"), STSDecorator().onlyApplyTo("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt index 6c662bceee..4ca44ac588 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt @@ -6,10 +6,7 @@ package software.amazon.smithy.rustsdk.customize.s3 import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.Shape -import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection import software.amazon.smithy.rust.codegen.core.rustlang.Writable @@ -20,32 +17,6 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull -/** - * Add a default value to CreateMultiPartUploadRequest's ChecksumAlgorithm - */ -class S3MultiPartUploadDecorator : ClientCodegenDecorator { - override val name: String = "S3MultiPartUpload" - override val order: Byte = 0 - private val defaultAlgorithm = "CRC32" - - private val operationName = "CreateMultipartUpload" - private val s3Namespace = "com.amazonaws.s3" - - private fun isS3MPUOperation(shape: Shape): Boolean = - shape is OperationShape && shape.id == ShapeId.from("$s3Namespace#$operationName") - - override fun operationCustomizations( - codegenContext: ClientCodegenContext, - operation: OperationShape, - baseCustomizations: List, - ): List = - if (isS3MPUOperation(operation)) { - baseCustomizations + listOf(S3MPUChecksumCustomization(codegenContext, operation)) - } else { - baseCustomizations - } -} - /** * S3 CreateMPU is not modeled with the httpChecksum trait so the checksum_algorithm * value must be defaulted here in an interceptor. From 6ae5120ddf3b0c74eaa18b7963737dbc2ecb3958 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 21 Sep 2024 18:33:21 -0700 Subject: [PATCH 20/32] Add changelog --- .changelog/flexible-checksums.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changelog/flexible-checksums.md diff --git a/.changelog/flexible-checksums.md b/.changelog/flexible-checksums.md new file mode 100644 index 0000000000..de76f7a8d3 --- /dev/null +++ b/.changelog/flexible-checksums.md @@ -0,0 +1,10 @@ +--- +applies_to: ["client", "aws-sdk-rust"] +authors: ["landonxjames"] +references: ["smithy-rs#3845"] +breaking: false +new_feature: true +bug_fix: true +--- + +Updating the implementation of flexible checksums to match the updated spec. From ae0a7b90c81af9cd7b401245e4d6068e6c465e9a Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 21 Sep 2024 18:35:57 -0700 Subject: [PATCH 21/32] Cleaning up --- aws/rust-runtime/Cargo.lock | 104 +++--- aws/rust-runtime/aws-config/Cargo.lock | 285 +++------------- .../aws-config/external-types.toml | 2 + .../src/http_request_checksum.rs | 7 +- .../src/http_response_checksum.rs | 3 +- .../aws-types/external-types.toml | 2 + ...nseChecksumMutationInterceptorGenerator.kt | 1 + .../s3/S3MultiPartUploadDecorator.kt | 90 ------ rust-runtime/Cargo.lock | 306 ++++++++---------- .../aws-smithy-checksums/external-types.toml | 2 + 10 files changed, 256 insertions(+), 546 deletions(-) delete mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index 4e773efdd1..7196bf17b1 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "ahash" @@ -386,17 +386,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -480,9 +480,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bytes-utils" @@ -521,9 +521,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.16" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", @@ -574,18 +574,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstyle", "clap_lex", @@ -630,9 +630,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -955,9 +955,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "group" @@ -1314,11 +1314,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -1502,9 +1502,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -1515,15 +1515,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -1545,9 +1545,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -1682,9 +1682,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags", ] @@ -1791,9 +1791,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1874,11 +1874,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1926,9 +1926,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -1948,9 +1948,9 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -1967,9 +1967,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -2377,24 +2377,24 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "untrusted" @@ -2663,9 +2663,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" diff --git a/aws/rust-runtime/aws-config/Cargo.lock b/aws/rust-runtime/aws-config/Cargo.lock index 990c977d1b..1e31aa09d7 100644 --- a/aws/rust-runtime/aws-config/Cargo.lock +++ b/aws/rust-runtime/aws-config/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aho-corasick" @@ -45,8 +45,7 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "1.5.6" -version = "1.5.6" +version = "1.5.7" dependencies = [ "aws-credential-types", "aws-runtime", @@ -82,7 +81,6 @@ dependencies = [ [[package]] name = "aws-credential-types" version = "1.2.1" -version = "1.2.1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -99,7 +97,6 @@ dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-runtime", - "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -108,8 +105,6 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "once_cell", - "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -201,7 +196,6 @@ dependencies = [ [[package]] name = "aws-smithy-async" version = "1.2.1" -version = "1.2.1" dependencies = [ "futures-util", "pin-project-lite", @@ -210,7 +204,7 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.60.12" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -238,7 +232,6 @@ dependencies = [ "futures-core", "http 0.2.12", "http-body 0.4.6", - "http-body 0.4.6", "once_cell", "percent-encoding", "pin-project-lite", @@ -249,7 +242,6 @@ dependencies = [ [[package]] name = "aws-smithy-json" version = "0.60.7" -version = "0.60.7" dependencies = [ "aws-smithy-types", ] @@ -257,20 +249,16 @@ dependencies = [ [[package]] name = "aws-smithy-protocol-test" version = "0.62.0" -version = "0.62.0" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", "base64-simd", "cbor-diag", - "base64-simd", - "cbor-diag", "http 0.2.12", "pretty_assertions", "regex-lite", "roxmltree", "serde_cbor", - "serde_cbor", "serde_json", "thiserror", ] @@ -278,7 +266,6 @@ dependencies = [ [[package]] name = "aws-smithy-query" version = "0.60.7" -version = "0.60.7" dependencies = [ "aws-smithy-types", "urlencoding", @@ -287,7 +274,6 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" version = "1.7.1" -version = "1.7.1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -301,13 +287,9 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "http-body 0.4.6", - "http-body 1.0.1", - "httparse", "hyper", "hyper-rustls", "indexmap", - "indexmap", "once_cell", "pin-project-lite", "pin-utils", @@ -322,7 +304,6 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" version = "1.7.2" -version = "1.7.2" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -347,10 +328,6 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "http-body-util", - "http 1.1.0", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", "itoa", "num-integer", "pin-project-lite", @@ -370,11 +347,9 @@ dependencies = [ [[package]] name = "aws-types" version = "1.3.3" -version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-checksums", "aws-smithy-runtime-api", "aws-smithy-types", "rustc_version", @@ -383,17 +358,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -436,20 +411,11 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bytes-utils" @@ -480,30 +446,11 @@ dependencies = [ "uuid", ] -[[package]] -name = "cbor-diag" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" -dependencies = [ - "bs58", - "chrono", - "data-encoding", - "half 2.4.1", - "nom", - "num-bigint", - "num-rational", - "num-traits", - "separator", - "url", - "uuid", -] - [[package]] name = "cc" -version = "1.1.16" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "shlex", ] @@ -523,15 +470,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "chrono" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" -dependencies = [ - "num-traits", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -550,19 +488,13 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - [[package]] name = "crc32c" version = "0.6.8" @@ -603,12 +535,6 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" -[[package]] -name = "data-encoding" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" - [[package]] name = "deranged" version = "0.3.11" @@ -742,9 +668,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "h2" @@ -781,22 +707,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - -[[package]] -name = "half" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" -dependencies = [ - "cfg-if", - "crunchy", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -880,29 +790,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http 1.1.0", -] - -[[package]] -name = "http-body-util" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" -dependencies = [ - "bytes", - "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "pin-project-lite", -] - [[package]] name = "httparse" version = "1.9.4" @@ -928,7 +815,6 @@ dependencies = [ "h2", "http 0.2.12", "http-body 0.4.6", - "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -976,7 +862,6 @@ dependencies = [ "equivalent", "hashbrown", "serde", - "serde", ] [[package]] @@ -1038,13 +923,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1053,11 +931,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -1069,17 +947,7 @@ dependencies = [ "hermit-abi", "libc", "wasi", - "windows-sys", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", + "windows-sys 0.52.0", ] [[package]] @@ -1112,16 +980,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -1148,17 +1006,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1250,9 +1097,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -1302,9 +1149,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags", ] @@ -1371,7 +1218,7 @@ dependencies = [ "libc", "spin", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1449,11 +1296,11 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1487,9 +1334,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -1507,17 +1354,11 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" -[[package]] -name = "separator" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" - [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -1532,21 +1373,11 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half 1.8.3", - "serde", -] - [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -1559,7 +1390,6 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap", "indexmap", "itoa", "memchr", @@ -1635,7 +1465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1741,24 +1571,6 @@ dependencies = [ "tinyvec_macros", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" -dependencies = [ - "tinyvec_macros", -] - [[package]] name = "tinyvec_macros" version = "0.1.1" @@ -1780,7 +1592,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1938,15 +1750,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -2050,6 +1862,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2122,9 +1943,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zeroize" diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml index 9c8d3c1b24..06a63b208b 100644 --- a/aws/rust-runtime/aws-config/external-types.toml +++ b/aws/rust-runtime/aws-config/external-types.toml @@ -20,6 +20,8 @@ allowed_external_types = [ "aws_smithy_async::rt::sleep::SharedAsyncSleep", "aws_smithy_async::time::SharedTimeSource", "aws_smithy_async::time::TimeSource", + "aws_smithy_checksums::RequestChecksumCalculation", + "aws_smithy_checksums::ResponseChecksumValidation", "aws_smithy_runtime::client::identity::cache::IdentityCache", "aws_smithy_runtime::client::identity::cache::lazy::LazyCacheBuilder", "aws_smithy_runtime_api::box_error::BoxError", diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 5669714830..28ad4a6520 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -169,7 +169,8 @@ where // WhenRequired we only calculate it if the checksum is marked required on the trait. let calculate_checksum = match request_checksum_calculation { RequestChecksumCalculation::WhenRequired => request_checksum_required, - RequestChecksumCalculation::WhenSupported | _ => true, + RequestChecksumCalculation::WhenSupported => true, + _ => true, }; // Calculate the checksum if necessary @@ -209,7 +210,7 @@ fn add_checksum_for_request_body( // If the header has not already been set we set it. If it was already set by the user // we do nothing and maintain their set value. - if let None = request.headers().get(checksum.header_name()) { + if request.headers().get(checksum.header_name()).is_none() { tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); checksum.update(data); @@ -239,7 +240,7 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( let checksum = checksum_algorithm.into_impl(); // If the user already set the header value then do nothing and return early - if let Some(_) = request.headers().get(checksum.header_name()) { + if request.headers().get(checksum.header_name()).is_some() { return Ok(()); } diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index b657de6e23..fca126c1f2 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -98,7 +98,8 @@ where let validation_enabled = if !state.validation_enabled { match response_checksum_validation { ResponseChecksumValidation::WhenRequired => false, - ResponseChecksumValidation::WhenSupported | _ => true, + ResponseChecksumValidation::WhenSupported => true, + _ => true, } } else { true diff --git a/aws/rust-runtime/aws-types/external-types.toml b/aws/rust-runtime/aws-types/external-types.toml index 5dce405f84..af506bc4be 100644 --- a/aws/rust-runtime/aws-types/external-types.toml +++ b/aws/rust-runtime/aws-types/external-types.toml @@ -6,6 +6,8 @@ allowed_external_types = [ "aws_smithy_async::rt::sleep::SharedAsyncSleep", "aws_smithy_async::time::SharedTimeSource", "aws_smithy_async::time::TimeSource", + "aws_smithy_checksums::ResponseChecksumValidation", + "aws_smithy_checksums::RequestChecksumCalculation", "aws_smithy_runtime_api::client::behavior_version::BehaviorVersion", "aws_smithy_runtime_api::client::http::HttpClient", "aws_smithy_runtime_api::client::http::SharedHttpClient", diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt index 8ccce4ba5f..30f69e2c46 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -143,6 +143,7 @@ class HttpResponseChecksumMutationInterceptorGenerator : ClientCodegenDecorator // If validation setting is WhenSupported (or unknown) we enable response checksum // validation. If it is WhenRequired we do not enable (since there is no way to // indicate that a response checksum is required). + ##[allow(clippy::wildcard_in_or_patterns)] match response_checksum_validation { #{ResponseChecksumValidation}::WhenRequired => {} #{ResponseChecksumValidation}::WhenSupported | _ => { diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt deleted file mode 100644 index 4ca44ac588..0000000000 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3MultiPartUploadDecorator.kt +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.smithy.rustsdk.customize.s3 - -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization -import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection -import software.amazon.smithy.rust.codegen.core.rustlang.Writable -import software.amazon.smithy.rust.codegen.core.rustlang.rust -import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate -import software.amazon.smithy.rust.codegen.core.rustlang.writable -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType -import software.amazon.smithy.rust.codegen.core.util.inputShape -import software.amazon.smithy.rust.codegen.core.util.orNull - -/** - * S3 CreateMPU is not modeled with the httpChecksum trait so the checksum_algorithm - * value must be defaulted here in an interceptor. - */ -private class S3MPUChecksumCustomization( - private val codegenContext: ClientCodegenContext, - private val operation: OperationShape, -) : OperationCustomization() { - override fun section(section: OperationSection): Writable = - writable { - val targetEnumName = "ChecksumAlgorithm" - val interceptorName = "S3CreateMultipartUploadInterceptor" - val rc = codegenContext.runtimeConfig - val checksumAlgoShape = - operation.inputShape(codegenContext.model).getMember(targetEnumName).orNull() ?: return@writable - val checksumAlgoInner = codegenContext.model.expectShape(checksumAlgoShape.target) - if (section is OperationSection.AdditionalInterceptors) { - section.registerInterceptor(codegenContext.runtimeConfig, this) { - rust(interceptorName) - } - } - if (section is OperationSection.RuntimePluginSupportingTypes) { - rustTemplate( - """ - ##[derive(Debug)] - struct $interceptorName; - - impl #{Intercept} - for $interceptorName - { - fn name(&self) -> &'static str { - "$interceptorName" - } - - fn modify_before_serialization( - &self, - context: &mut #{Context}, - _runtime_components: &#{RuntimeComponents}, - _cfg: &mut #{ConfigBag}, - ) -> Result<(), #{BoxError}> { - let input = context - .input_mut() - .downcast_mut::<#{OperationInputType}>() - .expect("Input should be for S3 CreateMultipartUpload"); - - // S3 CreateMPU is not modeled with the httpChecksum trait so the checksum_algorithm - // value must be defaulted here. - if input.checksum_algorithm.is_none() { - input.checksum_algorithm = Some(#{ChecksumAlgo}::Crc32) - } - - Ok(()) - } - } - """.trimIndent(), - "BoxError" to RuntimeType.boxError(rc), - "ConfigBag" to RuntimeType.configBag(rc), - "RuntimeComponents" to RuntimeType.runtimeComponents(rc), - "Context" to RuntimeType.beforeSerializationInterceptorContextMut(rc), - "Intercept" to RuntimeType.intercept(rc), - "OperationInputType" to - codegenContext.symbolProvider.toSymbol( - operation.inputShape( - codegenContext.model, - ), - ), - "ChecksumAlgo" to codegenContext.symbolProvider.toSymbol(checksumAlgoInner), - ) - } - } -} diff --git a/rust-runtime/Cargo.lock b/rust-runtime/Cargo.lock index 71d7b2804a..b78c57ace0 100644 --- a/rust-runtime/Cargo.lock +++ b/rust-runtime/Cargo.lock @@ -4,19 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -152,7 +146,7 @@ checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", "zeroize", ] @@ -172,9 +166,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", @@ -185,9 +179,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.20.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62" dependencies = [ "bindgen", "cc", @@ -200,18 +194,18 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2424565416eef55906f9f8cece2072b6b6a76075e3ff81483ebe938a89a4c05f" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-eventstream 0.60.4", - "aws-smithy-http 0.60.10", + "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", "aws-types", "bytes", "fastrand", @@ -226,23 +220,23 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.48.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00a545b16c05af9302b0b4b38a7584f6f323749e407169aa3e9b210e7c0a808d" +checksum = "c09fd4b5c7ed75f52b913b4f3ff0501dae7f8cb9125f6d45db4553980cbc0528" dependencies = [ "ahash", "aws-credential-types", "aws-runtime", "aws-sigv4", "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-checksums 0.60.12 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-eventstream 0.60.4", - "aws-smithy-http 0.60.10", + "aws-smithy-checksums 0.60.12", + "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-json 0.60.7 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", - "aws-smithy-xml 0.60.8", + "aws-smithy-types 1.2.6", + "aws-smithy-xml 0.60.9 (registry+https://github.com/rust-lang/crates.io-index)", "aws-types", "bytes", "fastrand", @@ -261,15 +255,15 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" dependencies = [ "aws-credential-types", - "aws-smithy-eventstream 0.60.4", - "aws-smithy-http 0.60.10", + "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", "bytes", "crypto-bigint 0.5.5", "form_urlencoded", @@ -314,7 +308,7 @@ dependencies = [ name = "aws-smithy-cbor" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "criterion", "minicbor", ] @@ -322,11 +316,12 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" version = "0.60.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.6", "bytes", - "bytes-utils", "crc32c", "crc32fast", "hex", @@ -334,24 +329,19 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "pretty_assertions", "sha1", "sha2", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-smithy-checksums" version = "0.60.13" -version = "0.60.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" dependencies = [ "aws-smithy-http 0.60.11", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "bytes", + "bytes-utils", "crc32c", "crc32fast", "hex", @@ -359,9 +349,12 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", + "pretty_assertions", "sha1", "sha2", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -373,7 +366,7 @@ name = "aws-smithy-compression" version = "0.0.2" dependencies = [ "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "bytes", "bytes-utils", "flate2", @@ -391,25 +384,25 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +version = "0.60.5" dependencies = [ - "aws-smithy-types 1.2.4", + "arbitrary", + "aws-smithy-types 1.2.7", "bytes", + "bytes-utils", "crc32fast", + "derive_arbitrary", ] [[package]] name = "aws-smithy-eventstream" version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ - "arbitrary", "aws-smithy-types 1.2.6", "bytes", - "bytes-utils", "crc32fast", - "derive_arbitrary", ] [[package]] @@ -419,7 +412,7 @@ dependencies = [ "aws-smithy-async 1.2.1", "aws-smithy-runtime 1.7.1", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "h2 0.4.6", "http 1.1.0", "hyper 1.4.1", @@ -427,7 +420,7 @@ dependencies = [ "hyper-util", "once_cell", "pin-project-lite", - "rustls 0.23.12", + "rustls 0.23.13", "tokio", "tower", "tracing", @@ -435,46 +428,46 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01dbcb6e2588fd64cfb6d7529661b06466419e4c54ed1c62d6510d2d0350a728" +version = "0.60.11" dependencies = [ - "aws-smithy-eventstream 0.60.4", - "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "async-stream", + "aws-smithy-eventstream 0.60.5", + "aws-smithy-runtime-api 1.7.2", + "aws-smithy-types 1.2.7", "bytes", "bytes-utils", "futures-core", + "futures-util", "http 0.2.12", "http-body 0.4.6", + "hyper 0.14.30", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", + "proptest", + "tokio", "tracing", ] [[package]] name = "aws-smithy-http" version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ - "async-stream", - "aws-smithy-eventstream 0.60.5", - "aws-smithy-runtime-api 1.7.2", + "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.6", "bytes", "bytes-utils", "futures-core", - "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.30", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "proptest", - "tokio", "tracing", ] @@ -490,7 +483,7 @@ dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-json 0.60.7", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "aws-smithy-xml 0.60.9", "bytes", "futures-util", @@ -520,7 +513,7 @@ dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-http-server", "aws-smithy-json 0.60.7", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "aws-smithy-xml 0.60.9", "bytes", "futures", @@ -560,7 +553,7 @@ version = "0.60.3" name = "aws-smithy-json" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "proptest", "serde_json", ] @@ -571,7 +564,7 @@ version = "0.60.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" dependencies = [ - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", ] [[package]] @@ -580,7 +573,7 @@ version = "0.2.1" dependencies = [ "aws-sdk-s3", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "tokio", ] @@ -624,7 +617,7 @@ dependencies = [ name = "aws-smithy-query" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "urlencoding", ] @@ -637,7 +630,7 @@ dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-protocol-test 0.62.0", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "bytes", "fastrand", "futures-util", @@ -670,10 +663,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" dependencies = [ "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-http 0.60.10", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-protocol-test 0.62.0 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", "bytes", "fastrand", "h2 0.3.26", @@ -700,7 +693,7 @@ name = "aws-smithy-runtime-api" version = "1.7.2" dependencies = [ "aws-smithy-async 1.2.1", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "bytes", "http 0.2.12", "http 1.1.0", @@ -718,7 +711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" dependencies = [ "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", "bytes", "http 0.2.12", "http 1.1.0", @@ -730,9 +723,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.4" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273dcdfd762fae3e1650b8024624e7cd50e484e37abdab73a7a706188ad34543" +checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" dependencies = [ "base64-simd", "bytes", @@ -756,7 +749,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" dependencies = [ "base64 0.13.1", "base64-simd", @@ -793,7 +786,7 @@ name = "aws-smithy-types-convert" version = "0.60.8" dependencies = [ "aws-smithy-async 1.2.1", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "chrono", "futures-core", "time", @@ -805,7 +798,7 @@ version = "0.1.3" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "bytes", "http 1.1.0", "tracing", @@ -814,20 +807,20 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +version = "0.60.9" dependencies = [ + "aws-smithy-protocol-test 0.62.0", + "base64 0.13.1", + "proptest", "xmlparser", ] [[package]] name = "aws-smithy-xml" version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ - "aws-smithy-protocol-test 0.62.0", - "base64 0.13.1", - "proptest", "xmlparser", ] @@ -840,7 +833,7 @@ dependencies = [ "aws-credential-types", "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.4", + "aws-smithy-types 1.2.6", "rustc_version", "tracing", ] @@ -863,17 +856,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -1004,9 +997,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" dependencies = [ "serde", ] @@ -1048,9 +1041,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.16" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", @@ -1136,18 +1129,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstyle", "clap_lex 0.7.2", @@ -1210,9 +1203,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1244,7 +1237,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.17", + "clap 4.5.18", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1506,7 +1499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -1642,9 +1635,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -1932,7 +1925,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -1942,9 +1935,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -2007,7 +2000,7 @@ dependencies = [ "aws-smithy-json 0.60.7", "aws-smithy-runtime 1.7.1", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.6", + "aws-smithy-types 1.2.7", "aws-smithy-xml 0.60.9", "bytes", "fastrand", @@ -2251,9 +2244,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.24.2" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f8e213c36148d828083ae01948eed271d03f95f7e72571fa242d78184029af2" +checksum = "29be4f60e41fde478b36998b88821946aafac540e53591e76db53921a0cc225b" dependencies = [ "half 2.4.1", "minicbor-derive", @@ -2261,9 +2254,9 @@ dependencies = [ [[package]] name = "minicbor-derive" -version = "0.15.1" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297ad6022c004a6c54f0fb28bbd2306314d5a4edc767e56b4b4cc48fc2371654" +checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2", "quote", @@ -2276,15 +2269,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2532,9 +2516,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -2545,15 +2529,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -2575,9 +2559,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -2833,9 +2817,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags 2.6.0", ] @@ -2963,9 +2947,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -2988,15 +2972,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "aws-lc-rs", "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -3063,9 +3047,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "aws-lc-rs", "ring 0.17.8", @@ -3102,11 +3086,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3154,9 +3138,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -3176,9 +3160,9 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -3195,9 +3179,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -3566,7 +3550,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "tokio", ] @@ -3802,15 +3786,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -4128,9 +4112,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yasna" @@ -4167,17 +4151,3 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.77", -] diff --git a/rust-runtime/aws-smithy-checksums/external-types.toml b/rust-runtime/aws-smithy-checksums/external-types.toml index dbd4b4eee2..ddd93c5af1 100644 --- a/rust-runtime/aws-smithy-checksums/external-types.toml +++ b/rust-runtime/aws-smithy-checksums/external-types.toml @@ -1,5 +1,7 @@ allowed_external_types = [ "aws_smithy_types::body::SdkBody", + "aws_smithy_types::config_bag::storable::StoreReplace", + "aws_smithy_types::config_bag::storable::Storable", "bytes::bytes::Bytes", "http::header::map::HeaderMap", "http::header::name::HeaderName", From d1c6aadec519b84dd7528067c973f8c28695b4d9 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 22 Sep 2024 13:12:24 -0700 Subject: [PATCH 22/32] Move checksum configs to aws_smithy_types Update codegen to point to new checksum config --- aws/rust-runtime/Cargo.lock | 3 +- aws/rust-runtime/aws-config/Cargo.lock | 63 +------ aws/rust-runtime/aws-config/Cargo.toml | 1 - .../aws-config/external-types.toml | 4 +- .../src/default_provider/checksums.rs | 5 +- aws/rust-runtime/aws-config/src/lib.rs | 4 +- .../src/http_request_checksum.rs | 3 +- .../src/http_response_checksum.rs | 3 +- aws/rust-runtime/aws-types/Cargo.toml | 1 - .../aws-types/external-types.toml | 4 +- aws/rust-runtime/aws-types/src/sdk_config.rs | 8 +- .../rustsdk/HttpRequestChecksumDecorator.kt | 4 +- ...estChecksumMutationInterceptorGenerator.kt | 4 +- .../rustsdk/HttpResponseChecksumDecorator.kt | 4 +- ...nseChecksumMutationInterceptorGenerator.kt | 4 +- rust-runtime/Cargo.lock | 2 +- rust-runtime/aws-smithy-checksums/Cargo.toml | 2 +- .../aws-smithy-checksums/external-types.toml | 2 - .../aws-smithy-checksums/src/error.rs | 62 ------- rust-runtime/aws-smithy-checksums/src/lib.rs | 80 +-------- .../aws-smithy-types/src/checksum_config.rs | 155 ++++++++++++++++++ rust-runtime/aws-smithy-types/src/lib.rs | 1 + 22 files changed, 189 insertions(+), 230 deletions(-) create mode 100644 rust-runtime/aws-smithy-types/src/checksum_config.rs diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index 7196bf17b1..1ca6cbfe50 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -240,7 +240,7 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -371,7 +371,6 @@ version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-checksums", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", diff --git a/aws/rust-runtime/aws-config/Cargo.lock b/aws/rust-runtime/aws-config/Cargo.lock index 1e31aa09d7..0b5ccfc6d4 100644 --- a/aws/rust-runtime/aws-config/Cargo.lock +++ b/aws/rust-runtime/aws-config/Cargo.lock @@ -53,7 +53,6 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-checksums", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", @@ -202,25 +201,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "aws-smithy-checksums" -version = "0.60.12" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - [[package]] name = "aws-smithy-http" version = "0.60.11" @@ -318,7 +298,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" dependencies = [ "base64-simd", "bytes", @@ -346,7 +326,7 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -495,24 +475,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - [[package]] name = "crunchy" version = "0.2.2" @@ -907,16 +869,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - [[package]] name = "memchr" version = "2.7.4" @@ -1397,17 +1349,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.8" diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 86c290cea9..bedb7d0bcc 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -33,7 +33,6 @@ aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" } aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client"] } aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] } aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" } -aws-smithy-checksums = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-checksums" } aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" } bytes = "1.1.0" http = "0.2.4" diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml index 06a63b208b..3e2c6726d1 100644 --- a/aws/rust-runtime/aws-config/external-types.toml +++ b/aws/rust-runtime/aws-config/external-types.toml @@ -20,8 +20,6 @@ allowed_external_types = [ "aws_smithy_async::rt::sleep::SharedAsyncSleep", "aws_smithy_async::time::SharedTimeSource", "aws_smithy_async::time::TimeSource", - "aws_smithy_checksums::RequestChecksumCalculation", - "aws_smithy_checksums::ResponseChecksumValidation", "aws_smithy_runtime::client::identity::cache::IdentityCache", "aws_smithy_runtime::client::identity::cache::lazy::LazyCacheBuilder", "aws_smithy_runtime_api::box_error::BoxError", @@ -36,6 +34,8 @@ allowed_external_types = [ "aws_smithy_runtime_api::client::result::SdkError", "aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig", "aws_smithy_types::body::SdkBody", + "aws_smithy_types::checksum_config::RequestChecksumCalculation", + "aws_smithy_types::checksum_config::ResponseChecksumValidation", "aws_smithy_types::retry", "aws_smithy_types::retry::*", "aws_smithy_types::timeout", diff --git a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs index 605d8ce015..3e9298b8e8 100644 --- a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs +++ b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs @@ -81,9 +81,10 @@ mod test { #[allow(deprecated)] use crate::profile::profile_file::{ProfileFileKind, ProfileFiles}; use crate::provider_config::ProviderConfig; - use aws_smithy_checksums::ResponseChecksumValidation; + use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, + }; use aws_types::os_shim_internal::{Env, Fs}; - use aws_types::sdk_config::RequestChecksumCalculation; use tracing_test::traced_test; #[tokio::test] diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index 7c753501d1..241c8c7549 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -222,13 +222,15 @@ mod loader { use aws_credential_types::Credentials; use aws_smithy_async::rt::sleep::{default_async_sleep, AsyncSleep, SharedAsyncSleep}; use aws_smithy_async::time::{SharedTimeSource, TimeSource}; - use aws_smithy_checksums::{RequestChecksumCalculation, ResponseChecksumValidation}; use aws_smithy_runtime::client::identity::IdentityCache; use aws_smithy_runtime_api::client::behavior_version::BehaviorVersion; use aws_smithy_runtime_api::client::http::HttpClient; use aws_smithy_runtime_api::client::identity::{ResolveCachedIdentity, SharedIdentityCache}; use aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig; use aws_smithy_runtime_api::shared::IntoShared; + use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, + }; use aws_smithy_types::retry::RetryConfig; use aws_smithy_types::timeout::TimeoutConfig; use aws_types::app_name::AppName; diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 28ad4a6520..35e1e435f5 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -10,8 +10,8 @@ use aws_runtime::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions}; use aws_runtime::{auth::SigV4OperationSigningConfig, content_encoding::header_value::AWS_CHUNKED}; use aws_sigv4::http_request::SignableBody; +use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; -use aws_smithy_checksums::{ChecksumAlgorithm, RequestChecksumCalculation}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ BeforeSerializationInterceptorContextRef, BeforeTransmitInterceptorContextMut, Input, @@ -20,6 +20,7 @@ use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::checksum_config::RequestChecksumCalculation; use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use aws_smithy_types::error::operation::BuildError; use http::HeaderValue; diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index fca126c1f2..90754dbceb 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -7,7 +7,7 @@ //! Interceptor for handling Smithy `@httpChecksum` response checksumming -use aws_smithy_checksums::{ChecksumAlgorithm, ResponseChecksumValidation}; +use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, @@ -16,6 +16,7 @@ use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_runtime_api::http::Headers; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::checksum_config::ResponseChecksumValidation; use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use std::{fmt, mem}; diff --git a/aws/rust-runtime/aws-types/Cargo.toml b/aws/rust-runtime/aws-types/Cargo.toml index fc4a514b2b..e13428e4ba 100644 --- a/aws/rust-runtime/aws-types/Cargo.toml +++ b/aws/rust-runtime/aws-types/Cargo.toml @@ -17,7 +17,6 @@ aws-smithy-async = { path = "../../../rust-runtime/aws-smithy-async" } aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" } aws-smithy-runtime = { path = "../../../rust-runtime/aws-smithy-runtime", optional = true } aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] } -aws-smithy-checksums = {path = "../../../rust-runtime/aws-smithy-checksums"} tracing = "0.1" # cargo does not support optional test dependencies, so to completely disable rustls # we need to add the webpki-roots feature here. diff --git a/aws/rust-runtime/aws-types/external-types.toml b/aws/rust-runtime/aws-types/external-types.toml index af506bc4be..ee28a1732a 100644 --- a/aws/rust-runtime/aws-types/external-types.toml +++ b/aws/rust-runtime/aws-types/external-types.toml @@ -6,8 +6,6 @@ allowed_external_types = [ "aws_smithy_async::rt::sleep::SharedAsyncSleep", "aws_smithy_async::time::SharedTimeSource", "aws_smithy_async::time::TimeSource", - "aws_smithy_checksums::ResponseChecksumValidation", - "aws_smithy_checksums::RequestChecksumCalculation", "aws_smithy_runtime_api::client::behavior_version::BehaviorVersion", "aws_smithy_runtime_api::client::http::HttpClient", "aws_smithy_runtime_api::client::http::SharedHttpClient", @@ -15,6 +13,8 @@ allowed_external_types = [ "aws_smithy_runtime_api::client::identity::SharedIdentityCache", "aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig", "aws_smithy_runtime_api::http::headers::Headers", + "aws_smithy_types::checksum_config::ResponseChecksumValidation", + "aws_smithy_types::checksum_config::RequestChecksumCalculation", "aws_smithy_types::config_bag::storable::Storable", "aws_smithy_types::config_bag::storable::StoreReplace", "aws_smithy_types::config_bag::storable::Storer", diff --git a/aws/rust-runtime/aws-types/src/sdk_config.rs b/aws/rust-runtime/aws-types/src/sdk_config.rs index 923cb47be0..19220fc526 100644 --- a/aws/rust-runtime/aws-types/src/sdk_config.rs +++ b/aws/rust-runtime/aws-types/src/sdk_config.rs @@ -19,13 +19,15 @@ pub use aws_credential_types::provider::SharedCredentialsProvider; use aws_smithy_async::rt::sleep::AsyncSleep; pub use aws_smithy_async::rt::sleep::SharedAsyncSleep; pub use aws_smithy_async::time::{SharedTimeSource, TimeSource}; -pub use aws_smithy_checksums::{RequestChecksumCalculation, ResponseChecksumValidation}; use aws_smithy_runtime_api::client::behavior_version::BehaviorVersion; use aws_smithy_runtime_api::client::http::HttpClient; pub use aws_smithy_runtime_api::client::http::SharedHttpClient; use aws_smithy_runtime_api::client::identity::{ResolveCachedIdentity, SharedIdentityCache}; pub use aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig; use aws_smithy_runtime_api::shared::IntoShared; +pub use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, +}; pub use aws_smithy_types::retry::RetryConfig; pub use aws_smithy_types::timeout::TimeoutConfig; use std::collections::HashMap; @@ -183,7 +185,7 @@ impl Builder { /// # Examples /// ``` /// use aws_types::SdkConfig; - /// use aws_smithy_checksums::RequestChecksumCalculation; + /// use aws_smithy_types::checksum_config::RequestChecksumCalculation; /// let config = SdkConfig::builder().request_checksum_calculation(RequestChecksumCalculation::WhenSupported).build(); /// ``` pub fn request_checksum_calculation( @@ -207,7 +209,7 @@ impl Builder { /// # Examples /// ``` /// use aws_types::SdkConfig; - /// use aws_smithy_checksums::ResponseChecksumValidation; + /// use aws_smithy_types::checksum_config::ResponseChecksumValidation; /// let config = SdkConfig::builder().response_checksum_validation(ResponseChecksumValidation::WhenSupported).build(); /// ``` pub fn response_checksum_validation( diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index b1e439d279..84f53209b3 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -220,8 +220,8 @@ class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientC *preludeScope, "RequestChecksumCalculation" to configReexport( - RuntimeType.smithyChecksums(rc) - .resolve("RequestChecksumCalculation"), + RuntimeType.smithyTypes(rc) + .resolve("checksum_config::RequestChecksumCalculation"), ), ) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt index 015505c012..f692891065 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt @@ -93,8 +93,8 @@ class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { "Error" to interceptors.resolve("context::Error"), "RuntimeComponents" to RuntimeType.runtimeComponents(rc), "RequestChecksumCalculation" to - CargoDependency.smithyChecksums(rc).toType() - .resolve("RequestChecksumCalculation"), + CargoDependency.smithyTypes(rc).toType() + .resolve("checksum_config::RequestChecksumCalculation"), ) } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 38844fe2f7..98c4b37886 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -191,8 +191,8 @@ class HttpResponseChecksumConfigCustomization(private val codegenContext: Client *preludeScope, "ResponseChecksumValidation" to configReexport( - RuntimeType.smithyChecksums(rc) - .resolve("ResponseChecksumValidation"), + RuntimeType.smithyTypes(rc) + .resolve("checksum_config::ResponseChecksumValidation"), ), ) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt index 30f69e2c46..f3a181533d 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -94,8 +94,8 @@ class HttpResponseChecksumMutationInterceptorGenerator : ClientCodegenDecorator "Error" to interceptors.resolve("context::Error"), "RuntimeComponents" to RuntimeType.runtimeComponents(rc), "ResponseChecksumValidation" to - CargoDependency.smithyChecksums(rc).toType() - .resolve("ResponseChecksumValidation"), + CargoDependency.smithyTypes(rc).toType() + .resolve("checksum_config::ResponseChecksumValidation"), ) } diff --git a/rust-runtime/Cargo.lock b/rust-runtime/Cargo.lock index b78c57ace0..5acf1f25bf 100644 --- a/rust-runtime/Cargo.lock +++ b/rust-runtime/Cargo.lock @@ -336,7 +336,7 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-types 1.2.7", diff --git a/rust-runtime/aws-smithy-checksums/Cargo.toml b/rust-runtime/aws-smithy-checksums/Cargo.toml index 0888246992..97b295130c 100644 --- a/rust-runtime/aws-smithy-checksums/Cargo.toml +++ b/rust-runtime/aws-smithy-checksums/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" authors = [ "AWS Rust SDK Team ", "Zelda Hessler ", diff --git a/rust-runtime/aws-smithy-checksums/external-types.toml b/rust-runtime/aws-smithy-checksums/external-types.toml index ddd93c5af1..dbd4b4eee2 100644 --- a/rust-runtime/aws-smithy-checksums/external-types.toml +++ b/rust-runtime/aws-smithy-checksums/external-types.toml @@ -1,7 +1,5 @@ allowed_external_types = [ "aws_smithy_types::body::SdkBody", - "aws_smithy_types::config_bag::storable::StoreReplace", - "aws_smithy_types::config_bag::storable::Storable", "bytes::bytes::Bytes", "http::header::map::HeaderMap", "http::header::name::HeaderName", diff --git a/rust-runtime/aws-smithy-checksums/src/error.rs b/rust-runtime/aws-smithy-checksums/src/error.rs index f184001829..6a93cddbb7 100644 --- a/rust-runtime/aws-smithy-checksums/src/error.rs +++ b/rust-runtime/aws-smithy-checksums/src/error.rs @@ -36,65 +36,3 @@ impl fmt::Display for UnknownChecksumAlgorithmError { } impl Error for UnknownChecksumAlgorithmError {} - -/// Unknown setting for `request_checksum_calculation` -#[derive(Debug)] -pub struct UnknownRequestChecksumCalculationError { - request_checksum_calculation: String, -} - -impl UnknownRequestChecksumCalculationError { - pub(crate) fn new(request_checksum_calculation: impl Into) -> Self { - Self { - request_checksum_calculation: request_checksum_calculation.into(), - } - } - - /// The unknown value - pub fn request_checksum_calculation(&self) -> &str { - &self.request_checksum_calculation - } -} - -impl fmt::Display for UnknownRequestChecksumCalculationError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - r#"unknown request_checksum_calculation value "{}", please pass a known name ("when_supported", "when_required")"#, - self.request_checksum_calculation - ) - } -} - -impl Error for UnknownRequestChecksumCalculationError {} - -/// Unknown setting for `response_checksum_validation` -#[derive(Debug)] -pub struct UnknownResponseChecksumValidationError { - response_checksum_validation: String, -} - -impl UnknownResponseChecksumValidationError { - pub(crate) fn new(response_checksum_validation: impl Into) -> Self { - Self { - response_checksum_validation: response_checksum_validation.into(), - } - } - - /// The unknown value - pub fn response_checksum_validation(&self) -> &str { - &self.response_checksum_validation - } -} - -impl fmt::Display for UnknownResponseChecksumValidationError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - r#"unknown response_checksum_validation value "{}", please pass a known name ("when_supported", "when_required")"#, - self.response_checksum_validation - ) - } -} - -impl Error for UnknownResponseChecksumValidationError {} diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index 1e37892f89..b900bac5fa 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -17,10 +17,7 @@ //! Checksum calculation and verification callbacks. use crate::error::UnknownChecksumAlgorithmError; -use crate::error::{ - UnknownRequestChecksumCalculationError, UnknownResponseChecksumValidationError, -}; -use aws_smithy_types::config_bag::{Storable, StoreReplace}; + use bytes::Bytes; use std::str::FromStr; @@ -101,81 +98,6 @@ impl ChecksumAlgorithm { } } -// Valid names for RequestChecksumCalculation and ResponseChecksumValidation -pub const WHEN_SUPPORTED: &str = "when_supported"; -pub const WHEN_REQUIRED: &str = "when_required"; - -/// Determines when a checksum will be calculated for request payloads. Values are: -/// * [RequestChecksumCalculation::WhenSupported] - (default) When set, a checksum will be -/// calculated for all request payloads of operations modeled with the -/// `httpChecksum` trait where `requestChecksumRequired` is `true` and/or a -/// `requestAlgorithmMember` is modeled. -/// * [RequestChecksumCalculation::WhenRequired] - When set, a checksum will only be calculated for -/// request payloads of operations modeled with the `httpChecksum` trait where -/// `requestChecksumRequired` is `true` or where a requestAlgorithmMember -/// is modeled and supplied. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum RequestChecksumCalculation { - WhenSupported, - WhenRequired, -} - -impl Storable for RequestChecksumCalculation { - type Storer = StoreReplace; -} - -impl FromStr for RequestChecksumCalculation { - type Err = UnknownRequestChecksumCalculationError; - - fn from_str(request_checksum_calculation: &str) -> Result { - if request_checksum_calculation.eq_ignore_ascii_case(WHEN_SUPPORTED) { - Ok(Self::WhenSupported) - } else if request_checksum_calculation.eq_ignore_ascii_case(WHEN_REQUIRED) { - Ok(Self::WhenRequired) - } else { - Err(UnknownRequestChecksumCalculationError::new( - request_checksum_calculation, - )) - } - } -} - -/// Determines when checksum validation will be performed on response payloads. Values are: -/// * [ResponseChecksumValidation::WhenSupported] - (default) When set, checksum validation is performed on all -/// response payloads of operations modeled with the `httpChecksum` trait where -/// `responseAlgorithms` is modeled, except when no modeled checksum algorithms -/// are supported. -/// * [ResponseChecksumValidation::WhenRequired] - When set, checksum validation is not performed on -/// response payloads of operations unless the checksum algorithm is supported and -/// the `requestValidationModeMember` member is set to `ENABLED`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum ResponseChecksumValidation { - WhenSupported, - WhenRequired, -} - -impl Storable for ResponseChecksumValidation { - type Storer = StoreReplace; -} - -impl FromStr for ResponseChecksumValidation { - type Err = UnknownResponseChecksumValidationError; - - fn from_str(response_checksum_validation: &str) -> Result { - if response_checksum_validation.eq_ignore_ascii_case(WHEN_SUPPORTED) { - Ok(Self::WhenSupported) - } else if response_checksum_validation.eq_ignore_ascii_case(WHEN_REQUIRED) { - Ok(Self::WhenRequired) - } else { - Err(UnknownResponseChecksumValidationError::new( - response_checksum_validation, - )) - } - } -} - /// Types implementing this trait can calculate checksums. /// /// Checksum algorithms are used to validate the integrity of data. Structs that implement this trait diff --git a/rust-runtime/aws-smithy-types/src/checksum_config.rs b/rust-runtime/aws-smithy-types/src/checksum_config.rs new file mode 100644 index 0000000000..f5b412bb79 --- /dev/null +++ b/rust-runtime/aws-smithy-types/src/checksum_config.rs @@ -0,0 +1,155 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Types that allow users to indicate their preferences for checksum calculation and validation + +use std::error::Error; +use std::fmt; +use std::str::FromStr; + +use crate::config_bag::{Storable, StoreReplace}; + +// Valid names for RequestChecksumCalculation and ResponseChecksumValidation +const WHEN_SUPPORTED: &str = "when_supported"; +const WHEN_REQUIRED: &str = "when_required"; + +/// Determines when a checksum will be calculated for request payloads. Values are: +/// * [RequestChecksumCalculation::WhenSupported] - (default) When set, a checksum will be +/// calculated for all request payloads of operations modeled with the +/// `httpChecksum` trait where `requestChecksumRequired` is `true` and/or a +/// `requestAlgorithmMember` is modeled. +/// * [RequestChecksumCalculation::WhenRequired] - When set, a checksum will only be calculated for +/// request payloads of operations modeled with the `httpChecksum` trait where +/// `requestChecksumRequired` is `true` or where a requestAlgorithmMember +/// is modeled and supplied. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RequestChecksumCalculation { + /// Calculate request checksums when they are supported. + WhenSupported, + /// Caulculate request checksums only when they are required. + WhenRequired, +} + +impl Storable for RequestChecksumCalculation { + type Storer = StoreReplace; +} + +impl FromStr for RequestChecksumCalculation { + type Err = UnknownRequestChecksumCalculationError; + + fn from_str(request_checksum_calculation: &str) -> Result { + if request_checksum_calculation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if request_checksum_calculation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownRequestChecksumCalculationError::new( + request_checksum_calculation, + )) + } + } +} + +/// Determines when checksum validation will be performed on response payloads. Values are: +/// * [ResponseChecksumValidation::WhenSupported] - (default) When set, checksum validation is performed on all +/// response payloads of operations modeled with the `httpChecksum` trait where +/// `responseAlgorithms` is modeled, except when no modeled checksum algorithms +/// are supported. +/// * [ResponseChecksumValidation::WhenRequired] - When set, checksum validation is not performed on +/// response payloads of operations unless the checksum algorithm is supported and +/// the `requestValidationModeMember` member is set to `ENABLED`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ResponseChecksumValidation { + /// Validate response checksums when they are supported. + WhenSupported, + /// Validate response checksums only when they are required. + WhenRequired, +} + +impl Storable for ResponseChecksumValidation { + type Storer = StoreReplace; +} + +impl FromStr for ResponseChecksumValidation { + type Err = UnknownResponseChecksumValidationError; + + fn from_str(response_checksum_validation: &str) -> Result { + if response_checksum_validation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if response_checksum_validation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownResponseChecksumValidationError::new( + response_checksum_validation, + )) + } + } +} + +/// Unknown setting for `request_checksum_calculation` +#[derive(Debug)] +#[non_exhaustive] +pub struct UnknownRequestChecksumCalculationError { + request_checksum_calculation: String, +} + +impl UnknownRequestChecksumCalculationError { + pub(crate) fn new(request_checksum_calculation: impl Into) -> Self { + Self { + request_checksum_calculation: request_checksum_calculation.into(), + } + } + + /// The unknown value + pub fn request_checksum_calculation(&self) -> &str { + &self.request_checksum_calculation + } +} + +impl fmt::Display for UnknownRequestChecksumCalculationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown request_checksum_calculation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.request_checksum_calculation + ) + } +} + +impl Error for UnknownRequestChecksumCalculationError {} + +/// Unknown setting for `response_checksum_validation` +#[derive(Debug)] +#[non_exhaustive] +pub struct UnknownResponseChecksumValidationError { + response_checksum_validation: String, +} + +impl UnknownResponseChecksumValidationError { + pub(crate) fn new(response_checksum_validation: impl Into) -> Self { + Self { + response_checksum_validation: response_checksum_validation.into(), + } + } + + /// The unknown value + pub fn response_checksum_validation(&self) -> &str { + &self.response_checksum_validation + } +} + +impl fmt::Display for UnknownResponseChecksumValidationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown response_checksum_validation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.response_checksum_validation + ) + } +} + +impl Error for UnknownResponseChecksumValidationError {} diff --git a/rust-runtime/aws-smithy-types/src/lib.rs b/rust-runtime/aws-smithy-types/src/lib.rs index ed53de191b..5af73e1fb1 100644 --- a/rust-runtime/aws-smithy-types/src/lib.rs +++ b/rust-runtime/aws-smithy-types/src/lib.rs @@ -20,6 +20,7 @@ pub mod base64; pub mod body; pub mod byte_stream; +pub mod checksum_config; /// A typemap for storing configuration. pub mod config_bag; pub mod date_time; From b9d6ee0e8aedd740e85ab27a643d2b73f51e59a7 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 22 Sep 2024 15:48:11 -0700 Subject: [PATCH 23/32] Exclude x-amz-checksum-mode header from signing --- aws/rust-runtime/Cargo.lock | 2 +- aws/rust-runtime/aws-sigv4/Cargo.toml | 2 +- .../aws-sigv4/src/http_request/canonical_request.rs | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index 1ca6cbfe50..d0de775e90 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -195,7 +195,7 @@ version = "0.60.3" [[package]] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.5" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", diff --git a/aws/rust-runtime/aws-sigv4/Cargo.toml b/aws/rust-runtime/aws-sigv4/Cargo.toml index 49af10cc21..0d247d5436 100644 --- a/aws/rust-runtime/aws-sigv4/Cargo.toml +++ b/aws/rust-runtime/aws-sigv4/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.5" authors = ["AWS Rust SDK Team ", "David Barsky "] description = "SigV4 signer for HTTP requests and Event Stream messages." edition = "2021" diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs index 03404276d9..4622f841d9 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs @@ -31,6 +31,7 @@ pub(crate) mod header { pub(crate) const X_AMZ_DATE: &str = "x-amz-date"; pub(crate) const X_AMZ_SECURITY_TOKEN: &str = "x-amz-security-token"; pub(crate) const X_AMZ_USER_AGENT: &str = "x-amz-user-agent"; + pub(crate) const X_AMZ_CHECKSUM_MODE: &str = "x-amz-checksum-mode"; } pub(crate) mod param { @@ -293,8 +294,10 @@ impl<'a> CanonicalRequest<'a> { } if params.settings().signature_location == SignatureLocation::QueryParams { - // The X-Amz-User-Agent header should not be signed if this is for a presigned URL - if name == HeaderName::from_static(header::X_AMZ_USER_AGENT) { + // The X-Amz-User-Agent and x-amz-checksum-mode headers should not be signed if this is for a presigned URL + if name == HeaderName::from_static(header::X_AMZ_USER_AGENT) + || name == HeaderName::from_static(header::X_AMZ_CHECKSUM_MODE) + { continue; } } From 06b384ffa8e672ccae6b3ec22164bb5f6f69e38a Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 23 Sep 2024 19:52:11 -0700 Subject: [PATCH 24/32] PR feedback --- .../src/default_provider/checksums.rs | 20 +++++++++++++++++++ .../src/http_response_checksum.rs | 6 +++--- .../smithy/rustsdk/AwsCodegenDecorator.kt | 4 ++-- ...estChecksumMutationInterceptorGenerator.kt | 3 +-- ...nseChecksumMutationInterceptorGenerator.kt | 3 +-- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs index 3e9298b8e8..25da664a39 100644 --- a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs +++ b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs @@ -162,6 +162,16 @@ mod test { ); } + #[tokio::test] + #[traced_test] + async fn default_works_request() { + let conf = ProviderConfig::empty(); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + } + #[tokio::test] #[traced_test] async fn log_error_on_invalid_value_response() { @@ -236,4 +246,14 @@ mod test { Some(ResponseChecksumValidation::WhenRequired) ); } + + #[tokio::test] + #[traced_test] + async fn default_works_response() { + let conf = ProviderConfig::empty(); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + } } diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 90754dbceb..2b3c622f24 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -10,7 +10,7 @@ use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, + BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextRef, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; @@ -61,9 +61,9 @@ where "ResponseChecksumInterceptor" } - fn modify_before_serialization( + fn read_before_serialization( &self, - context: &mut BeforeSerializationInterceptorContextMut<'_>, + context: &BeforeSerializationInterceptorContextRef<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index 45f71eee5c..46fb7f71fc 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -43,9 +43,9 @@ val DECORATORS: List = UserAgentDecorator(), SigV4AuthDecorator(), HttpRequestChecksumDecorator(), - HttpRequestChecksumMutationInterceptorGenerator(), + HttpRequestChecksumMutationInterceptorDecorator(), HttpResponseChecksumDecorator(), - HttpResponseChecksumMutationInterceptorGenerator(), + HttpResponseChecksumMutationInterceptorDecorator(), IntegrationTestDecorator(), AwsFluentClientDecorator(), CrateLicenseDecorator(), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt index f692891065..ac007671c3 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt @@ -36,7 +36,7 @@ import software.amazon.smithy.rust.codegen.core.util.orNull * operation's input or the `requestChecksumRequired` value from the trait, and in certain circumstances we need to * mutate that member on the input before serializing the request and sending it to the service. */ -class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { +class HttpRequestChecksumMutationInterceptorDecorator : ClientCodegenDecorator { override val name: String = "HttpRequestChecksumMutationInterceptorGenerator" override val order: Byte = 0 @@ -77,7 +77,6 @@ class HttpRequestChecksumMutationInterceptorGenerator : ClientCodegenDecorator { codegenContext.runtimeConfig.let { rc -> val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() val interceptors = runtimeApi.resolve("client::interceptors") - val orchestrator = runtimeApi.resolve("client::orchestrator") arrayOf( *preludeScope, diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt index f3a181533d..413421fa06 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt @@ -37,7 +37,7 @@ import software.amazon.smithy.rust.codegen.core.util.orNull * operation's input, and in certain circumstances we need to mutate that member on the input before serializing the * request and sending it to the service. */ -class HttpResponseChecksumMutationInterceptorGenerator : ClientCodegenDecorator { +class HttpResponseChecksumMutationInterceptorDecorator : ClientCodegenDecorator { override val name: String = "HttpResponseChecksumMutationInterceptorGenerator" override val order: Byte = 0 @@ -78,7 +78,6 @@ class HttpResponseChecksumMutationInterceptorGenerator : ClientCodegenDecorator codegenContext.runtimeConfig.let { rc -> val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() val interceptors = runtimeApi.resolve("client::interceptors") - val orchestrator = runtimeApi.resolve("client::orchestrator") arrayOf( *preludeScope, From c4b41bc5c2a0b228a540d0558f24c0fd5778d695 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 24 Sep 2024 10:15:43 -0700 Subject: [PATCH 25/32] Add comment requested in PR feedback --- rust-runtime/aws-smithy-types/src/checksum_config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust-runtime/aws-smithy-types/src/checksum_config.rs b/rust-runtime/aws-smithy-types/src/checksum_config.rs index f5b412bb79..555939b8c1 100644 --- a/rust-runtime/aws-smithy-types/src/checksum_config.rs +++ b/rust-runtime/aws-smithy-types/src/checksum_config.rs @@ -5,6 +5,12 @@ //! Types that allow users to indicate their preferences for checksum calculation and validation +// Note: These types would likely make more sense in `aws-smithy-checksums` and were originally +// added there. But we have lints protecting against exporting non-stable types from stable crates +// and the checksums crate is not yet 1.0, so these types cannot live there for now. In the future +// if we do decide to 1.0 the checksums crate we can move these types there and re-export them here +// to maintain the current behavior. + use std::error::Error; use std::fmt; use std::str::FromStr; From 13d16264732b0864bc4bc89cefc052428ee8481f Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 25 Sep 2024 12:24:29 -0700 Subject: [PATCH 26/32] Update serviceHasHttpChecksumOperation To traverse the model via a knowledge index --- .../smithy/rustsdk/HttpRequestChecksumDecorator.kt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index 84f53209b3..f9b7b9c9d0 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -6,6 +6,7 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.knowledge.TopDownIndex import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.configReexport @@ -32,7 +33,6 @@ import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rust.codegen.core.util.hasTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull -import kotlin.jvm.optionals.getOrNull internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = RuntimeType.forInlineDependency( @@ -295,9 +295,8 @@ class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientC /** * Determine if the current service contains any operations with the HttpChecksum trait */ -fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext) = - codegenContext.serviceShape.allOperations - .mapNotNull { codegenContext.model.getShape(it).getOrNull() } - .any { - it.hasTrait() - } +fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext): Boolean { + val index = TopDownIndex.of(codegenContext.model) + val ops = index.getContainedOperations(codegenContext.serviceShape.id) + return ops.any { it.hasTrait() } +} From 1f25cb4f6c9b78b28db8d13ff27b08018a2fa0c0 Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 25 Sep 2024 13:18:28 -0700 Subject: [PATCH 27/32] Update some s3 integ and protocol tests Update PutBucketLifecycleConfiguration test it now checks the Crc32 checksum header instead of Md5 --- aws/sdk/aws-models-extra/s3-tests.smithy | 4 +--- aws/sdk/integration-tests/s3/tests/endpoints.rs | 2 +- aws/sdk/integration-tests/s3/tests/express.rs | 7 ++++++- aws/sdk/integration-tests/s3/tests/presigning.rs | 14 +++++++++----- .../s3/tests/stalled-stream-protection.rs | 3 +++ 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/aws/sdk/aws-models-extra/s3-tests.smithy b/aws/sdk/aws-models-extra/s3-tests.smithy index 59ced8bc10..bee98811f6 100644 --- a/aws/sdk/aws-models-extra/s3-tests.smithy +++ b/aws/sdk/aws-models-extra/s3-tests.smithy @@ -111,9 +111,7 @@ apply PutBucketLifecycleConfiguration @httpRequestTests([ protocol: "aws.protocols#restXml", uri: "/", headers: { - // we can assert this, but when this test is promoted, it can't assert - // on the exact contents - "content-md5": "JP8DTuCSH6yDC8wNGg4+mA==", + "x-amz-checksum-crc32": "11+f3g==", }, bodyMediaType: "application/xml", body: """ diff --git a/aws/sdk/integration-tests/s3/tests/endpoints.rs b/aws/sdk/integration-tests/s3/tests/endpoints.rs index 6b4f88c8f8..11b370ecc3 100644 --- a/aws/sdk/integration-tests/s3/tests/endpoints.rs +++ b/aws/sdk/integration-tests/s3/tests/endpoints.rs @@ -80,7 +80,7 @@ async fn multi_region_access_points() { let auth_header = captured_request.headers().get("AUTHORIZATION").unwrap(); // Verifies that the sigv4a signing algorithm was used, that the signing scope doesn't include a region, and that the x-amz-region-set header was signed. let expected_start = - "AWS4-ECDSA-P256-SHA256 Credential=ANOTREAL/20090213/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-region-set;x-amz-user-agent, Signature="; + "AWS4-ECDSA-P256-SHA256 Credential=ANOTREAL/20090213/s3/aws4_request, SignedHeaders=host;x-amz-checksum-mode;x-amz-content-sha256;x-amz-date;x-amz-region-set;x-amz-user-agent, Signature="; assert!( auth_header.starts_with(expected_start), diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index 8621b6c108..44ae4c7671 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -183,7 +183,12 @@ async fn presigning() { ][..], &query_params ); - assert_eq!(presigned.headers().count(), 0); + // Presigned request has one header and that is the x-amz-checksum-mode + // header with default value ENABLED + assert_eq!(presigned.headers().count(), 1); + let headers = presigned.headers().collect::>(); + assert_eq!(headers.get(0).unwrap().0, "x-amz-checksum-mode"); + assert_eq!(headers.get(0).unwrap().1, "ENABLED"); } fn operation_request_with_checksum( diff --git a/aws/sdk/integration-tests/s3/tests/presigning.rs b/aws/sdk/integration-tests/s3/tests/presigning.rs index 768dd563bb..f887b0643d 100644 --- a/aws/sdk/integration-tests/s3/tests/presigning.rs +++ b/aws/sdk/integration-tests/s3/tests/presigning.rs @@ -100,7 +100,9 @@ async fn test_presigning() { ][..], &query_params ); - assert_eq!(presigned.headers().count(), 0); + assert_eq!(presigned.headers().count(), 1); + let headers = presigned.headers().collect::>(); + assert_eq!(headers.get("x-amz-checksum-mode").unwrap(), &"ENABLED"); } #[tokio::test] @@ -135,8 +137,8 @@ async fn test_presigning_with_payload_headers() { "X-Amz-Date=20090213T233131Z", "X-Amz-Expires=30", "X-Amz-Security-Token=notarealsessiontoken", - "X-Amz-Signature=be1d41dc392f7019750e4f5e577234fb9059dd20d15f6a99734196becce55e52", - "X-Amz-SignedHeaders=content-length%3Bcontent-type%3Bhost", + "X-Amz-Signature=9403eb5961c8af066f2473f88cb9248b39fd61f8eedc33af14ec9c2d26c22974", + "X-Amz-SignedHeaders=content-length%3Bcontent-type%3Bhost%3Bx-amz-checksum-crc32%3Bx-amz-sdk-checksum-algorithm", "x-id=PutObject" ][..], &query_params @@ -148,7 +150,9 @@ async fn test_presigning_with_payload_headers() { Some(&"application/x-test") ); assert_eq!(headers.get(CONTENT_LENGTH.as_str()), Some(&"12345")); - assert_eq!(headers.len(), 2); + assert_eq!(headers.get("x-amz-sdk-checksum-algorithm"), Some(&"CRC32")); + assert_eq!(headers.get("x-amz-checksum-crc32"), Some(&"AAAAAA==")); + assert_eq!(headers.len(), 4); } #[tokio::test] @@ -164,7 +168,7 @@ async fn test_presigned_upload_part() { }) .await; pretty_assertions::assert_eq!( - "https://bucket.s3.us-east-1.amazonaws.com/key?x-id=UploadPart&partNumber=0&uploadId=upload-id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ANOTREAL%2F20090213%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20090213T233131Z&X-Amz-Expires=30&X-Amz-SignedHeaders=content-length%3Bhost&X-Amz-Signature=a702867244f0bd1fb4d161e2a062520dcbefae3b9992d2e5366bcd61a60c6ddd&X-Amz-Security-Token=notarealsessiontoken", + "https://bucket.s3.us-east-1.amazonaws.com/key?x-id=UploadPart&partNumber=0&uploadId=upload-id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ANOTREAL%2F20090213%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20090213T233131Z&X-Amz-Expires=30&X-Amz-SignedHeaders=content-length%3Bhost%3Bx-amz-checksum-crc32%3Bx-amz-sdk-checksum-algorithm&X-Amz-Signature=0b5835e056c463d6c0963326966f6cf42c75b7a218057836274d38288e055d36&X-Amz-Security-Token=notarealsessiontoken", presigned.uri().to_string(), ); } diff --git a/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs b/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs index 283999d60e..920c95c1d6 100644 --- a/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs +++ b/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs @@ -96,6 +96,9 @@ async fn test_stalled_stream_protection_defaults_for_upload() { .credentials_provider(Credentials::for_tests()) .region(Region::new("us-east-1")) .endpoint_url(format!("http://{server_addr}")) + // The Body used here is odd and fails the body.size_hint().exact() check in the streaming branch of + // the `RequestChecksumInterceptor` + .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired) .build(); let client = Client::from_conf(conf); From d682c8df6ebfbf170e2ef1afb29c4e95b001223d Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 29 Sep 2024 17:00:46 -0700 Subject: [PATCH 28/32] Moving request checksum mutator to a closure --- aws/rust-runtime/aws-config/Cargo.toml | 2 +- .../src/http_request_checksum.rs | 24 ++- .../smithy/rustsdk/AwsCodegenDecorator.kt | 1 - .../rustsdk/HttpRequestChecksumDecorator.kt | 73 +++++-- ...estChecksumMutationInterceptorGenerator.kt | 181 ------------------ rust-runtime/aws-smithy-types/Cargo.toml | 2 +- 6 files changed, 77 insertions(+), 206 deletions(-) delete mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index bedb7d0bcc..a583475971 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-config" -version = "1.5.7" +version = "1.5.8" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 35e1e435f5..90a10398eb 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -14,7 +14,7 @@ use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeSerializationInterceptorContextRef, BeforeTransmitInterceptorContextMut, Input, + BeforeSerializationInterceptorContextMut, BeforeTransmitInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; @@ -102,36 +102,42 @@ impl DefaultRequestChecksumOverride { } } -pub(crate) struct RequestChecksumInterceptor { +pub(crate) struct RequestChecksumInterceptor { algorithm_provider: AP, + checksum_mutator: CM, } -impl fmt::Debug for RequestChecksumInterceptor { +impl fmt::Debug for RequestChecksumInterceptor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RequestChecksumInterceptor").finish() } } -impl RequestChecksumInterceptor { - pub(crate) fn new(algorithm_provider: AP) -> Self { - Self { algorithm_provider } +impl RequestChecksumInterceptor { + pub(crate) fn new(algorithm_provider: AP, checksum_mutator: CM) -> Self { + Self { + algorithm_provider, + checksum_mutator, + } } } -impl Intercept for RequestChecksumInterceptor +impl Intercept for RequestChecksumInterceptor where AP: Fn(&Input) -> Result<(Option, bool), BoxError> + Send + Sync, + CM: Fn(&mut Input, &ConfigBag) -> Result<(), BoxError> + Send + Sync, { fn name(&self) -> &'static str { "RequestChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { + let _ = (self.checksum_mutator)(context.input_mut(), cfg); let checksum_algorithm = (self.algorithm_provider)(context.input())?; let mut layer = Layer::new("RequestChecksumInterceptor"); layer.store_put(RequestChecksumInterceptorState { diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index 46fb7f71fc..ed47f1a452 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -43,7 +43,6 @@ val DECORATORS: List = UserAgentDecorator(), SigV4AuthDecorator(), HttpRequestChecksumDecorator(), - HttpRequestChecksumMutationInterceptorDecorator(), HttpResponseChecksumDecorator(), HttpResponseChecksumMutationInterceptorDecorator(), IntegrationTestDecorator(), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index f9b7b9c9d0..ffd7b81062 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -160,29 +160,71 @@ class HttpRequestChecksumCustomization( writable { // Get the `HttpChecksumTrait`, returning early if this `OperationShape` doesn't have one val checksumTrait = operationShape.getTrait() ?: return@writable - val requestAlgorithmMember = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) + val requestAlgorithmMember = + checksumTrait.requestAlgorithmMemberShape(codegenContext, operationShape) ?: return@writable + val requestAlgorithmMemberName = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) val requestChecksumRequired = checksumTrait.isRequestChecksumRequired val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name + val requestAlgorithmMemberInner = + if (requestAlgorithmMember.isOptional) { + codegenContext.model.expectShape(requestAlgorithmMember.target) + } else { + requestAlgorithmMember + } when (section) { is OperationSection.AdditionalInterceptors -> { - if (requestAlgorithmMember != null) { + if (requestAlgorithmMemberName != null) { section.registerInterceptor(runtimeConfig, this) { val runtimeApi = RuntimeType.smithyRuntimeApiClient(runtimeConfig) rustTemplate( """ - #{RequestChecksumInterceptor}::new(|input: &#{Input}| { + #{RequestChecksumInterceptor}::new( + |input: &#{Input}| { let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); - let checksum_algorithm = input.$requestAlgorithmMember(); + let checksum_algorithm = input.$requestAlgorithmMemberName(); #{checksum_algorithm_to_str} #{Result}::<_, #{BoxError}>::Ok((checksum_algorithm, $requestChecksumRequired)) - }) + }, + |input: &mut #{Input}, cfg: &#{ConfigBag}| { + let input = input + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + // This value is set by the user on the SdkConfig to indicate their preference + let request_checksum_calculation = cfg + .load::<#{RequestChecksumCalculation}>() + .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); + + // From the httpChecksum trait + let http_checksum_required = $requestChecksumRequired; + + // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum we + // default to Crc32. If it is WhenRequired and a checksum is required by the trait we also set the + // default. In all other cases we do nothing. + match ( + request_checksum_calculation, + http_checksum_required, + input.checksum_algorithm(), + ) { + (#{RequestChecksumCalculation}::WhenSupported, _, None) + | (#{RequestChecksumCalculation}::WhenRequired, true, None) => { + input.checksum_algorithm = Some(#{ChecksumAlgoShape}::Crc32); + } + _ => {}, + } + + Ok(()) + } + + ) """, *preludeScope, "BoxError" to RuntimeType.boxError(runtimeConfig), "Input" to runtimeApi.resolve("client::interceptors::context::Input"), "OperationInput" to codegenContext.symbolProvider.toSymbol(inputShape), + "ConfigBag" to RuntimeType.configBag(runtimeConfig), "RequestChecksumInterceptor" to runtimeConfig.awsInlineableHttpRequestChecksum() .resolve("RequestChecksumInterceptor"), @@ -191,14 +233,19 @@ class HttpRequestChecksumCustomization( codegenContext, operationShape, ), - ) - } - section.registerInterceptor(codegenContext.runtimeConfig, this) { - val interceptorName = "${operationName}HttpRequestChecksumMutationInterceptor" - rustTemplate( - """ - $interceptorName - """, + "RequestChecksumCalculation" to + CargoDependency.smithyTypes(runtimeConfig).toType() + .resolve("checksum_config::RequestChecksumCalculation"), + "ChecksumAlgoShape" to + codegenContext.symbolProvider.toSymbol( + requestAlgorithmMemberInner, + ), + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operationShape.inputShape( + codegenContext.model, + ), + ), ) } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt deleted file mode 100644 index ac007671c3..0000000000 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumMutationInterceptorGenerator.kt +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.smithy.rustsdk - -import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.model.shapes.MemberShape -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator -import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization -import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection -import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency -import software.amazon.smithy.rust.codegen.core.rustlang.Writable -import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate -import software.amazon.smithy.rust.codegen.core.rustlang.writable -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.util.dq -import software.amazon.smithy.rust.codegen.core.util.expectMember -import software.amazon.smithy.rust.codegen.core.util.getTrait -import software.amazon.smithy.rust.codegen.core.util.inputShape -import software.amazon.smithy.rust.codegen.core.util.orNull - -/** - * This class generates an interceptor for operations with the `httpChecksum` trait that support request checksums. - * In the `modify_before_serialization` hook the interceptor checks the config's `request_checksum_calculation` value - * and the trait's `requestChecksumRequired` value. If `request_checksum_calculation` is `WhenSupported` or it is - * `WhenRequired` and `requestChecksumRequired` is `true` then we check the operation input's `requestAlgorithmMember`. - * If that is `None` (so has not been explicitly set by the user) we default it to `Crc32`. - * - * Note that although there is an existing inlineable `RequestChecksumInterceptor` this logic could not live there. - * Since that interceptor is inlineable it does not have access to the name of the `requestAlgorithmMember` on the - * operation's input or the `requestChecksumRequired` value from the trait, and in certain circumstances we need to - * mutate that member on the input before serializing the request and sending it to the service. - */ -class HttpRequestChecksumMutationInterceptorDecorator : ClientCodegenDecorator { - override val name: String = "HttpRequestChecksumMutationInterceptorGenerator" - override val order: Byte = 0 - - override fun operationCustomizations( - codegenContext: ClientCodegenContext, - operation: OperationShape, - baseCustomizations: List, - ): List { - // If the operation doesn't have the HttpChecksum trait we return early - val checksumTrait = operation.getTrait() ?: return baseCustomizations - // Also return early if there is no requestValidationModeMember on the trait - val requestAlgorithmMember = - (checksumTrait.requestAlgorithmMemberShape(codegenContext, operation) ?: return baseCustomizations) - - return baseCustomizations + - listOf( - InterceptorSection( - codegenContext, - operation, - requestAlgorithmMember, - checksumTrait, - ), - ) - } - - private class InterceptorSection( - private val codegenContext: ClientCodegenContext, - private val operation: OperationShape, - private val requestAlgorithmMember: MemberShape, - private val checksumTrait: HttpChecksumTrait, - ) : OperationCustomization() { - override fun section(section: OperationSection): Writable = - writable { - if (section is OperationSection.RuntimePluginSupportingTypes) { - val model = codegenContext.model - val symbolProvider = codegenContext.symbolProvider - val codegenScope = - codegenContext.runtimeConfig.let { rc -> - val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() - val interceptors = runtimeApi.resolve("client::interceptors") - - arrayOf( - *preludeScope, - "BoxError" to RuntimeType.boxError(rc), - "ConfigBag" to RuntimeType.configBag(rc), - "Intercept" to RuntimeType.intercept(rc), - "BeforeSerializationInterceptorContextMut" to - RuntimeType.beforeSerializationInterceptorContextMut( - rc, - ), - "Input" to interceptors.resolve("context::Input"), - "Output" to interceptors.resolve("context::Output"), - "Error" to interceptors.resolve("context::Error"), - "RuntimeComponents" to RuntimeType.runtimeComponents(rc), - "RequestChecksumCalculation" to - CargoDependency.smithyTypes(rc).toType() - .resolve("checksum_config::RequestChecksumCalculation"), - ) - } - - val requestAlgorithmMemberInner = - if (requestAlgorithmMember.isOptional) { - codegenContext.model.expectShape(requestAlgorithmMember.target) - } else { - requestAlgorithmMember - } - - val operationName = symbolProvider.toSymbol(operation).name - val interceptorName = "${operationName}HttpRequestChecksumMutationInterceptor" - val requestChecksumRequired = checksumTrait.isRequestChecksumRequired - - rustTemplate( - """ - ##[derive(Debug)] - struct $interceptorName; - - impl #{Intercept} for $interceptorName { - fn name(&self) -> &'static str { - ${interceptorName.dq()} - } - - fn modify_before_serialization( - &self, - context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, - _runtime_comps: &#{RuntimeComponents}, - cfg: &mut #{ConfigBag}, - ) -> #{Result}<(), #{BoxError}> { - let input = context - .input_mut() - .downcast_mut::<#{OperationInputType}>() - .ok_or("failed to downcast to #{OperationInputType}")?; - - // This value is set by the user on the SdkConfig to indicate their preference - let request_checksum_calculation = cfg - .load::<#{RequestChecksumCalculation}>() - .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); - - // From the httpChecksum trait - let http_checksum_required = $requestChecksumRequired; - - // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum we - // default to Crc32. If it is WhenRequired and a checksum is required by the trait we also set the - // default. In all other cases we do nothing. - match ( - request_checksum_calculation, - http_checksum_required, - input.checksum_algorithm(), - ) { - (#{RequestChecksumCalculation}::WhenSupported, _, None) - | (#{RequestChecksumCalculation}::WhenRequired, true, None) => { - input.checksum_algorithm = Some(#{ChecksumAlgoShape}::Crc32); - } - _ => {}, - } - - #{Ok}(()) - } - } - """, - *codegenScope, - "OperationInputType" to codegenContext.symbolProvider.toSymbol(operation.inputShape(model)), - "ChecksumAlgoShape" to - codegenContext.symbolProvider.toSymbol( - requestAlgorithmMemberInner, - ), - ) - } - } - } -} - -/** - * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in - * the HTTP response of the operation. - */ -fun HttpChecksumTrait.requestAlgorithmMemberShape( - codegenContext: ClientCodegenContext, - operationShape: OperationShape, -): MemberShape? { - val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null - return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) -} diff --git a/rust-runtime/aws-smithy-types/Cargo.toml b/rust-runtime/aws-smithy-types/Cargo.toml index 7b52b8b6d8..3fd9186d0f 100644 --- a/rust-runtime/aws-smithy-types/Cargo.toml +++ b/rust-runtime/aws-smithy-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.8" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", From 6c9c3da7632f14697c7499317adffbea75f62afa Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 29 Sep 2024 17:33:16 -0700 Subject: [PATCH 29/32] Move response mutator to closure --- .../src/http_response_checksum.rs | 19 +- .../smithy/rustsdk/AwsCodegenDecorator.kt | 1 - .../rustsdk/HttpRequestChecksumDecorator.kt | 13 ++ .../rustsdk/HttpResponseChecksumDecorator.kt | 51 ++++- ...nseChecksumMutationInterceptorGenerator.kt | 180 ------------------ 5 files changed, 68 insertions(+), 196 deletions(-) delete mode 100644 aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 2b3c622f24..7c59b6899f 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -10,7 +10,7 @@ use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextRef, Input, + BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; @@ -28,12 +28,13 @@ impl Storable for ResponseChecksumInterceptorState { type Storer = StoreReplace; } -pub(crate) struct ResponseChecksumInterceptor { +pub(crate) struct ResponseChecksumInterceptor { response_algorithms: &'static [&'static str], validation_enabled: VE, + checksum_mutator: CM, } -impl fmt::Debug for ResponseChecksumInterceptor { +impl fmt::Debug for ResponseChecksumInterceptor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ResponseChecksumInterceptor") .field("response_algorithms", &self.response_algorithms) @@ -41,32 +42,36 @@ impl fmt::Debug for ResponseChecksumInterceptor { } } -impl ResponseChecksumInterceptor { +impl ResponseChecksumInterceptor { pub(crate) fn new( response_algorithms: &'static [&'static str], validation_enabled: VE, + checksum_mutator: CM, ) -> Self { Self { response_algorithms, validation_enabled, + checksum_mutator, } } } -impl Intercept for ResponseChecksumInterceptor +impl Intercept for ResponseChecksumInterceptor where VE: Fn(&Input) -> bool + Send + Sync, + CM: Fn(&mut Input, &ConfigBag) -> Result<(), BoxError> + Send + Sync, { fn name(&self) -> &'static str { "ResponseChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { + let _ = (self.checksum_mutator)(context.input_mut(), cfg); let validation_enabled = (self.validation_enabled)(context.input()); let mut layer = Layer::new("ResponseChecksumInterceptor"); diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt index ed47f1a452..f27ca2c45b 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt @@ -44,7 +44,6 @@ val DECORATORS: List = SigV4AuthDecorator(), HttpRequestChecksumDecorator(), HttpResponseChecksumDecorator(), - HttpResponseChecksumMutationInterceptorDecorator(), IntegrationTestDecorator(), AwsFluentClientDecorator(), CrateLicenseDecorator(), diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index ffd7b81062..ee14bde53f 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -7,6 +7,7 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.model.knowledge.TopDownIndex +import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.client.smithy.configReexport @@ -347,3 +348,15 @@ fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext): Boole val ops = index.getContainedOperations(codegenContext.serviceShape.id) return ops.any { it.hasTrait() } } + +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestAlgorithmMemberShape( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 98c4b37886..0f0ef550f9 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -120,6 +120,7 @@ class HttpResponseChecksumCustomization( val checksumTrait = operationShape.getTrait() ?: return@writable val requestValidationModeMember = checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return@writable + val requestValidationModeName = codegenContext.symbolProvider.toSymbol(requestValidationModeMember).name val requestValidationModeMemberInner = if (requestValidationModeMember.isOptional) { codegenContext.model.expectShape(requestValidationModeMember.target) @@ -150,6 +151,34 @@ class HttpResponseChecksumCustomization( } let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); matches!(input.$validationModeName(), #{Some}(#{ValidationModeShape}::Enabled)) + }, + |input: &mut #{Input}, cfg: &#{ConfigBag}| { + let input = input + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + let request_validation_enabled = + matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); + + if !request_validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + let response_checksum_validation = cfg + .load::<#{ResponseChecksumValidation}>() + .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); + + // If validation setting is WhenSupported (or unknown) we enable response checksum + // validation. If it is WhenRequired we do not enable (since there is no way to + // indicate that a response checksum is required). + ##[allow(clippy::wildcard_in_or_patterns)] + match response_checksum_validation { + #{ResponseChecksumValidation}::WhenRequired => {} + #{ResponseChecksumValidation}::WhenSupported | _ => { + input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); + } + } + } + + #{Ok}(()) } ) """, @@ -163,14 +192,20 @@ class HttpResponseChecksumCustomization( codegenContext.symbolProvider.toSymbol( requestValidationModeMemberInner, ), - ) - } - section.registerInterceptor(codegenContext.runtimeConfig, this) { - val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" - rustTemplate( - """ - $interceptorName - """, + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operationShape.inputShape( + codegenContext.model, + ), + ), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + "ResponseChecksumValidation" to + CargoDependency.smithyTypes(codegenContext.runtimeConfig).toType() + .resolve("checksum_config::ResponseChecksumValidation"), + "ConfigBag" to RuntimeType.configBag(codegenContext.runtimeConfig), ) } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt deleted file mode 100644 index 413421fa06..0000000000 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumMutationInterceptorGenerator.kt +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.smithy.rustsdk - -import software.amazon.smithy.aws.traits.HttpChecksumTrait -import software.amazon.smithy.model.shapes.MemberShape -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext -import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator -import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization -import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection -import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency -import software.amazon.smithy.rust.codegen.core.rustlang.Writable -import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate -import software.amazon.smithy.rust.codegen.core.rustlang.writable -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType -import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.util.dq -import software.amazon.smithy.rust.codegen.core.util.expectMember -import software.amazon.smithy.rust.codegen.core.util.getTrait -import software.amazon.smithy.rust.codegen.core.util.inputShape -import software.amazon.smithy.rust.codegen.core.util.orNull - -/** - * This class generates an interceptor for operations with the `httpChecksum` trait that support response validations. - * In the `modify_before_serialization` hook the interceptor checks the operation's `requestValidationModeMember`. If - * that member is `ENABLED` then we end early and do nothing. If it is not `ENABLED` then it checks the - * `response_checksum_validation` set by the user on the SdkConfig. If that is `WhenSupported` (or unknown) then we - * update the `requestValidationModeMember` to `ENABLED` and if the value is `WhenRequired` we end without modifying - * anything since there is no way to indicate that a response checksum is required. - * - * Note that although there is an existing inlineable `ResponseChecksumInterceptor` this logic could not live there. - * Since that interceptor is inlineable it does not have access to the name of the `requestValidationModeMember` on the - * operation's input, and in certain circumstances we need to mutate that member on the input before serializing the - * request and sending it to the service. - */ -class HttpResponseChecksumMutationInterceptorDecorator : ClientCodegenDecorator { - override val name: String = "HttpResponseChecksumMutationInterceptorGenerator" - override val order: Byte = 0 - - override fun operationCustomizations( - codegenContext: ClientCodegenContext, - operation: OperationShape, - baseCustomizations: List, - ): List { - // If the operation doesn't have the HttpChecksum trait we return early - val checksumTrait = operation.getTrait() ?: return baseCustomizations - // Also return early if there is no requestValidationModeMember on the trait - val requestValidationModeMember = - (checksumTrait.requestValidationModeMemberShape(codegenContext, operation) ?: return baseCustomizations) - - return baseCustomizations + - listOf( - InterceptorSection( - codegenContext, - operation, - requestValidationModeMember, - checksumTrait, - ), - ) - } - - private class InterceptorSection( - private val codegenContext: ClientCodegenContext, - private val operation: OperationShape, - private val requestValidationModeMember: MemberShape, - private val checksumTrait: HttpChecksumTrait, - ) : OperationCustomization() { - override fun section(section: OperationSection): Writable = - writable { - if (section is OperationSection.RuntimePluginSupportingTypes) { - val model = codegenContext.model - val symbolProvider = codegenContext.symbolProvider - val codegenScope = - codegenContext.runtimeConfig.let { rc -> - val runtimeApi = CargoDependency.smithyRuntimeApiClient(rc).toType() - val interceptors = runtimeApi.resolve("client::interceptors") - - arrayOf( - *preludeScope, - "BoxError" to RuntimeType.boxError(rc), - "ConfigBag" to RuntimeType.configBag(rc), - "Intercept" to RuntimeType.intercept(rc), - "BeforeSerializationInterceptorContextMut" to - RuntimeType.beforeSerializationInterceptorContextMut( - rc, - ), - "Input" to interceptors.resolve("context::Input"), - "Output" to interceptors.resolve("context::Output"), - "Error" to interceptors.resolve("context::Error"), - "RuntimeComponents" to RuntimeType.runtimeComponents(rc), - "ResponseChecksumValidation" to - CargoDependency.smithyTypes(rc).toType() - .resolve("checksum_config::ResponseChecksumValidation"), - ) - } - - val requestValidationModeName = symbolProvider.toSymbol(requestValidationModeMember).name - val requestValidationModeMemberInner = - if (requestValidationModeMember.isOptional) { - codegenContext.model.expectShape(requestValidationModeMember.target) - } else { - requestValidationModeMember - } - - val operationName = symbolProvider.toSymbol(operation).name - val interceptorName = "${operationName}HttpResponseChecksumMutationInterceptor" - - rustTemplate( - """ - ##[derive(Debug)] - struct $interceptorName; - - impl #{Intercept} for $interceptorName { - fn name(&self) -> &'static str { - ${interceptorName.dq()} - } - - fn modify_before_serialization( - &self, - context: &mut #{BeforeSerializationInterceptorContextMut}<'_, #{Input}, #{Output}, #{Error}>, - _runtime_comps: &#{RuntimeComponents}, - cfg: &mut #{ConfigBag}, - ) -> #{Result}<(), #{BoxError}> { - let input = context - .input_mut() - .downcast_mut::<#{OperationInputType}>() - .ok_or("failed to downcast to #{OperationInputType}")?; - - let request_validation_enabled = - matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); - - if !request_validation_enabled { - // This value is set by the user on the SdkConfig to indicate their preference - let response_checksum_validation = cfg - .load::<#{ResponseChecksumValidation}>() - .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); - - // If validation setting is WhenSupported (or unknown) we enable response checksum - // validation. If it is WhenRequired we do not enable (since there is no way to - // indicate that a response checksum is required). - ##[allow(clippy::wildcard_in_or_patterns)] - match response_checksum_validation { - #{ResponseChecksumValidation}::WhenRequired => {} - #{ResponseChecksumValidation}::WhenSupported | _ => { - input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); - } - } - } - - #{Ok}(()) - } - } - """, - *codegenScope, - "OperationInputType" to codegenContext.symbolProvider.toSymbol(operation.inputShape(model)), - "ValidationModeShape" to - codegenContext.symbolProvider.toSymbol( - requestValidationModeMemberInner, - ), - ) - } - } - } -} - -/** - * Get the top-level operation input member used to opt in to best-effort validation of a checksum returned in - * the HTTP response of the operation. - */ -fun HttpChecksumTrait.requestValidationModeMemberShape( - codegenContext: ClientCodegenContext, - operationShape: OperationShape, -): MemberShape? { - val requestValidationModeMember = this.requestValidationModeMember.orNull() ?: return null - return operationShape.inputShape(codegenContext.model).expectMember(requestValidationModeMember) -} From 2027715e8ba75e3f23020e8d188d4036de72e593 Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 26 Sep 2024 23:05:59 -0700 Subject: [PATCH 30/32] Adding user-agent metrics for flexible checksums --- .../src/http_request_checksum.rs | 56 +++++++++++++++++++ .../src/http_response_checksum.rs | 16 ++++++ aws/rust-runtime/aws-runtime/Cargo.toml | 2 +- .../aws-runtime/src/user_agent/metrics.rs | 41 +++++++++++++- rust-runtime/aws-smithy-runtime/Cargo.toml | 2 +- .../src/client/sdk_feature.rs | 9 +++ 6 files changed, 122 insertions(+), 4 deletions(-) diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 90a10398eb..8c5b0ea675 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -12,6 +12,7 @@ use aws_runtime::{auth::SigV4OperationSigningConfig, content_encoding::header_va use aws_sigv4::http_request::SignableBody; use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; +use aws_smithy_runtime::client::sdk_feature::SmithySdkFeature; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ BeforeSerializationInterceptorContextMut, BeforeTransmitInterceptorContextMut, Input, @@ -187,12 +188,67 @@ where let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg) .unwrap_or(ChecksumAlgorithm::Crc32); + // Set the user-agent metric for the selected checksum algorithm + match checksum_algorithm { + ChecksumAlgorithm::Crc32 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32); + } + ChecksumAlgorithm::Crc32c => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32c); + } + ChecksumAlgorithm::Md5 => { + tracing::warn!(more_info = "Unsupported ChecksumAlgorithm MD5 set"); + } + ChecksumAlgorithm::Sha1 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqSha1); + } + ChecksumAlgorithm::Sha256 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqSha256); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ChecksumAlgorithm detected when setting user-agent metrics", + unsupported = ?unsupported), + } + let request = context.request_mut(); add_checksum_for_request_body(request, checksum_algorithm, cfg)?; } Ok(()) } + + /// Set the user-agent metrics for `RequestChecksumCalculation` here to avoid ownership issues + /// with the mutable borrow of cfg in `modify_before_signing` + fn read_after_serialization( + &self, + _context: &aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextRef<'_>, + _runtime_components: &RuntimeComponents, + cfg: &mut ConfigBag, + ) -> Result<(), BoxError> { + let request_checksum_calculation = cfg + .load::() + .unwrap_or(&RequestChecksumCalculation::WhenSupported); + + match request_checksum_calculation { + RequestChecksumCalculation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqWhenSupported); + } + RequestChecksumCalculation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of RequestChecksumCalculation when setting user-agent metrics", + unsupported = ?unsupported), + }; + + Ok(()) + } } fn incorporate_custom_default( diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 7c59b6899f..3fc428f1b8 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -8,6 +8,7 @@ //! Interceptor for handling Smithy `@httpChecksum` response checksumming use aws_smithy_checksums::ChecksumAlgorithm; +use aws_smithy_runtime::client::sdk_feature::SmithySdkFeature; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, @@ -131,6 +132,21 @@ where } } + // Set the user-agent feature metric for the response checksum config + match response_checksum_validation { + ResponseChecksumValidation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenSupported); + } + ResponseChecksumValidation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ResponseChecksumValidation when setting user-agent metrics", + unsupported = ?unsupported), + }; + Ok(()) } } diff --git a/aws/rust-runtime/aws-runtime/Cargo.toml b/aws/rust-runtime/aws-runtime/Cargo.toml index 2265a7f3b4..9656489047 100644 --- a/aws/rust-runtime/aws-runtime/Cargo.toml +++ b/aws/rust-runtime/aws-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-runtime" -version = "1.4.3" +version = "1.4.4" authors = ["AWS Rust SDK Team "] description = "Runtime support code for the AWS SDK. This crate isn't intended to be used directly." edition = "2021" diff --git a/aws/rust-runtime/aws-runtime/src/user_agent/metrics.rs b/aws/rust-runtime/aws-runtime/src/user_agent/metrics.rs index 1acaf86fc9..f625fdd3b6 100644 --- a/aws/rust-runtime/aws-runtime/src/user_agent/metrics.rs +++ b/aws/rust-runtime/aws-runtime/src/user_agent/metrics.rs @@ -126,7 +126,17 @@ iterable_enum!( AccountIdModeDisabled, AccountIdModeRequired, Sigv4aSigning, - ResolvedAccountId + ResolvedAccountId, + FlexibleChecksumsReqCrc32, + FlexibleChecksumsReqCrc32c, + FlexibleChecksumsReqCrc64, + FlexibleChecksumsReqSha1, + FlexibleChecksumsReqSha256, + FlexibleChecksumsReqWhenSupported, + FlexibleChecksumsReqWhenRequired, + FlexibleChecksumsResWhenSupported, + FlexibleChecksumsResWhenRequired, + DdbMapper ); pub(crate) trait ProvideBusinessMetric { @@ -141,6 +151,23 @@ impl ProvideBusinessMetric for SmithySdkFeature { Paginator => Some(BusinessMetric::Paginator), GzipRequestCompression => Some(BusinessMetric::GzipRequestCompression), ProtocolRpcV2Cbor => Some(BusinessMetric::ProtocolRpcV2Cbor), + FlexibleChecksumsReqCrc32 => Some(BusinessMetric::FlexibleChecksumsReqCrc32), + FlexibleChecksumsReqCrc32c => Some(BusinessMetric::FlexibleChecksumsReqCrc32c), + FlexibleChecksumsReqCrc64 => Some(BusinessMetric::FlexibleChecksumsReqCrc64), + FlexibleChecksumsReqSha1 => Some(BusinessMetric::FlexibleChecksumsReqSha1), + FlexibleChecksumsReqSha256 => Some(BusinessMetric::FlexibleChecksumsReqSha256), + FlexibleChecksumsReqWhenSupported => { + Some(BusinessMetric::FlexibleChecksumsReqWhenSupported) + } + FlexibleChecksumsReqWhenRequired => { + Some(BusinessMetric::FlexibleChecksumsReqWhenRequired) + } + FlexibleChecksumsResWhenSupported => { + Some(BusinessMetric::FlexibleChecksumsResWhenSupported) + } + FlexibleChecksumsResWhenRequired => { + Some(BusinessMetric::FlexibleChecksumsResWhenRequired) + } otherwise => { // This may occur if a customer upgrades only the `aws-smithy-runtime-api` crate // while continuing to use an outdated version of an SDK crate or the `aws-runtime` @@ -250,7 +277,17 @@ mod tests { "ACCOUNT_ID_MODE_DISABLED": "Q", "ACCOUNT_ID_MODE_REQUIRED": "R", "SIGV4A_SIGNING": "S", - "RESOLVED_ACCOUNT_ID": "T" + "RESOLVED_ACCOUNT_ID": "T", + "FLEXIBLE_CHECKSUMS_REQ_CRC32": "U", + "FLEXIBLE_CHECKSUMS_REQ_CRC32C": "V", + "FLEXIBLE_CHECKSUMS_REQ_CRC64": "W", + "FLEXIBLE_CHECKSUMS_REQ_SHA1": "X", + "FLEXIBLE_CHECKSUMS_REQ_SHA256": "Y", + "FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED": "Z", + "FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED": "a", + "FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED": "b", + "FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED": "c", + "DDB_MAPPER": "d" } "#; diff --git a/rust-runtime/aws-smithy-runtime/Cargo.toml b/rust-runtime/aws-smithy-runtime/Cargo.toml index de05b3ea38..b53f1d7805 100644 --- a/rust-runtime/aws-smithy-runtime/Cargo.toml +++ b/rust-runtime/aws-smithy-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.2" authors = ["AWS Rust SDK Team ", "Zelda Hessler "] description = "The new smithy runtime crate" edition = "2021" diff --git a/rust-runtime/aws-smithy-runtime/src/client/sdk_feature.rs b/rust-runtime/aws-smithy-runtime/src/client/sdk_feature.rs index 5a8ab95a13..a6c4f04ad5 100644 --- a/rust-runtime/aws-smithy-runtime/src/client/sdk_feature.rs +++ b/rust-runtime/aws-smithy-runtime/src/client/sdk_feature.rs @@ -12,6 +12,15 @@ pub enum SmithySdkFeature { Paginator, GzipRequestCompression, ProtocolRpcV2Cbor, + FlexibleChecksumsReqCrc32, + FlexibleChecksumsReqCrc32c, + FlexibleChecksumsReqCrc64, + FlexibleChecksumsReqSha1, + FlexibleChecksumsReqSha256, + FlexibleChecksumsReqWhenSupported, + FlexibleChecksumsReqWhenRequired, + FlexibleChecksumsResWhenSupported, + FlexibleChecksumsResWhenRequired, } impl Storable for SmithySdkFeature { From aec21809b9ecf70676535442e79235834fd54e08 Mon Sep 17 00:00:00 2001 From: Landon James Date: Fri, 27 Sep 2024 19:24:20 -0700 Subject: [PATCH 31/32] Add tests for checksum user-agent business metrics --- aws/rust-runtime/Cargo.lock | 6 +- .../src/http_request_checksum.rs | 3 +- .../src/http_response_checksum.rs | 34 ++-- .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 163 +++++++++++++++++- rust-runtime/Cargo.lock | 76 ++++---- 5 files changed, 224 insertions(+), 58 deletions(-) diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index d0de775e90..fc1de1db37 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -149,7 +149,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.4.4" dependencies = [ "arbitrary", "aws-credential-types", @@ -303,7 +303,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.2" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.8" dependencies = [ "base64-simd", "bytes", diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 8c5b0ea675..a045c1b003 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -153,7 +153,7 @@ where /// Calculate a checksum and modify the request to include the checksum as a header /// (for in-memory request bodies) or a trailer (for streaming request bodies). /// Streaming bodies must be sized or this will return an error. - fn modify_before_signing( + fn modify_before_retry_loop( &self, context: &mut BeforeTransmitInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, @@ -198,6 +198,7 @@ where cfg.interceptor_state() .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32c); } + #[allow(deprecated)] ChecksumAlgorithm::Md5 => { tracing::warn!(more_info = "Unsupported ChecksumAlgorithm MD5 set"); } diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 3fc428f1b8..e757d667f8 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -79,6 +79,25 @@ where layer.store_put(ResponseChecksumInterceptorState { validation_enabled }); cfg.push_layer(layer); + let response_checksum_validation = cfg + .load::() + .unwrap_or(&ResponseChecksumValidation::WhenSupported); + + // Set the user-agent feature metric for the response checksum config + match response_checksum_validation { + ResponseChecksumValidation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenSupported); + } + ResponseChecksumValidation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ResponseChecksumValidation when setting user-agent metrics", + unsupported = ?unsupported), + }; + Ok(()) } @@ -132,21 +151,6 @@ where } } - // Set the user-agent feature metric for the response checksum config - match response_checksum_validation { - ResponseChecksumValidation::WhenSupported => { - cfg.interceptor_state() - .store_append(SmithySdkFeature::FlexibleChecksumsResWhenSupported); - } - ResponseChecksumValidation::WhenRequired => { - cfg.interceptor_state() - .store_append(SmithySdkFeature::FlexibleChecksumsResWhenRequired); - } - unsupported => tracing::warn!( - more_info = "Unsupported value of ResponseChecksumValidation when setting user-agent metrics", - unsupported = ?unsupported), - }; - Ok(()) } } diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index feefbb50a6..e7b2c897d5 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -141,6 +141,7 @@ internal class HttpChecksumTest { checksumResponseFailTests.map { createResponseChecksumValidationFailureTest(it, context) }.join("\n") val checksumStreamingRequestTestWritables = streamingRequestTests.map { createStreamingRequestChecksumCalculationTest(it, context) }.join("\n") + val miscTests = createMiscellaneousTests(context) // Shared imports for all test types val testBase = @@ -156,18 +157,42 @@ internal class HttpChecksumTest { use #{SdkBody}; use std::io::Write; use http_body::Body; + use #{HttpRequest}; """, *preludeScope, "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), "SdkBody" to RuntimeType.smithyTypes(rc).resolve("body::SdkBody"), + "HttpRequest" to RuntimeType.smithyRuntimeApi(rc).resolve("client::orchestrator::HttpRequest"), + ) + } + + val uaExtractor = + writable { + rustTemplate( + """ + fn get_sdk_metric_str(req: &HttpRequest) -> &str { + req.headers() + .get("x-amz-user-agent") + .unwrap() + .split(" ") + .filter_map(|sec| { + if sec.starts_with("m/") { + Some(&sec[2..]) + } else { + None + } + }) + .collect::>()[0] + } + """.trimIndent(), ) } // Create one integ test per test type rustCrate.integrationTest("request_checksums") { - testBase.plus(checksumRequestTestWritables)() + testBase.plus(checksumRequestTestWritables).plus(uaExtractor)() } rustCrate.integrationTest("response_checksums_success") { @@ -181,6 +206,10 @@ internal class HttpChecksumTest { rustCrate.integrationTest("streaming_request_checksums") { testBase.plus(checksumStreamingRequestTestWritables)() } + + rustCrate.integrationTest("misc_tests") { + testBase.plus(uaExtractor).plus(miscTests)() + } } } @@ -232,6 +261,10 @@ internal class HttpChecksumTest { .expect("algo header should exist"); assert_eq!(algo_header, "${testDef.algoHeader}"); + + // Check the user-agent metrics for the selected algo + let sdk_metrics = get_sdk_metric_str(&request); + assert!(sdk_metrics.contains("${testDef.algoFeatureId}")); } """, *preludeScope, @@ -427,6 +460,128 @@ internal class HttpChecksumTest { ) } } + + /** + * Generate miscellaneous tests + */ + private fun createMiscellaneousTests(context: ClientCodegenContext): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + return writable { + rustTemplate( + """ + // The following tests confirm that the user-agent business metrics header is correctly + // set for the request/response checksum config values + ##[::tokio::test] + async fn request_config_ua_required() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .request_checksum_calculation( + aws_types::sdk_config::RequestChecksumCalculation::WhenRequired, + ) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .some_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = get_sdk_metric_str(&request); + assert!(sdk_metrics.contains("a")); + assert!(!sdk_metrics.contains("Z")); + } + + ##[::tokio::test] + async fn request_config_ua_supported() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .some_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = get_sdk_metric_str(&request); + assert!(sdk_metrics.contains("Z")); + assert!(!sdk_metrics.contains("a")); + } + + ##[::tokio::test] + async fn response_config_ua_supported() { + let (http_client, rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-crc32", "i9aeUg==") + .body(SdkBody::from("Hello world")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .some_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = get_sdk_metric_str(&request); + assert!(sdk_metrics.contains("b")); + assert!(!sdk_metrics.contains("c")); + } + + ##[::tokio::test] + async fn response_config_ua_required() { + let (http_client, rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-crc32", "i9aeUg==") + .body(SdkBody::from("Hello world")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .response_checksum_validation( + aws_types::sdk_config::ResponseChecksumValidation::WhenRequired, + ) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .some_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = get_sdk_metric_str(&request); + assert!(sdk_metrics.contains("c")); + assert!(!sdk_metrics.contains("b")); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } } // Classes and data for test definitions @@ -437,6 +592,7 @@ data class RequestChecksumCalculationTest( val checksumAlgorithm: String, val algoHeader: String, val checksumHeader: String, + val algoFeatureId: String, ) val checksumRequestTests = @@ -447,6 +603,7 @@ val checksumRequestTests = "Crc32", "CRC32", "i9aeUg==", + "U", ), RequestChecksumCalculationTest( "CRC32C checksum calculation works.", @@ -454,6 +611,7 @@ val checksumRequestTests = "Crc32C", "CRC32C", "crUfeA==", + "V", ), /* We do not yet support Crc64Nvme checksums RequestChecksumCalculationTest( @@ -462,6 +620,7 @@ val checksumRequestTests = "Crc64Nvme", "CRC64NVME", "uc8X9yrZrD4=", + "W", ), */ RequestChecksumCalculationTest( @@ -470,6 +629,7 @@ val checksumRequestTests = "Sha1", "SHA1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "X", ), RequestChecksumCalculationTest( "SHA256 checksum calculation works.", @@ -477,6 +637,7 @@ val checksumRequestTests = "Sha256", "SHA256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "Y", ), ) diff --git a/rust-runtime/Cargo.lock b/rust-runtime/Cargo.lock index 5acf1f25bf..82ae4896e7 100644 --- a/rust-runtime/Cargo.lock +++ b/rust-runtime/Cargo.lock @@ -203,7 +203,7 @@ dependencies = [ "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-runtime 1.7.1", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.6", "aws-types", @@ -233,7 +233,7 @@ dependencies = [ "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-json 0.60.7 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-runtime 1.7.1", "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.6", "aws-smithy-xml 0.60.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -308,7 +308,7 @@ dependencies = [ name = "aws-smithy-cbor" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "criterion", "minicbor", ] @@ -339,7 +339,7 @@ name = "aws-smithy-checksums" version = "0.62.0" dependencies = [ "aws-smithy-http 0.60.11", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "bytes", "bytes-utils", "crc32c", @@ -366,7 +366,7 @@ name = "aws-smithy-compression" version = "0.0.2" dependencies = [ "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "bytes", "bytes-utils", "flate2", @@ -387,7 +387,7 @@ name = "aws-smithy-eventstream" version = "0.60.5" dependencies = [ "arbitrary", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "bytes", "bytes-utils", "crc32fast", @@ -410,9 +410,9 @@ name = "aws-smithy-experimental" version = "0.1.4" dependencies = [ "aws-smithy-async 1.2.1", - "aws-smithy-runtime 1.7.1", + "aws-smithy-runtime 1.7.2", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "h2 0.4.6", "http 1.1.0", "hyper 1.4.1", @@ -433,7 +433,7 @@ dependencies = [ "async-stream", "aws-smithy-eventstream 0.60.5", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "bytes", "bytes-utils", "futures-core", @@ -483,7 +483,7 @@ dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-json 0.60.7", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "aws-smithy-xml 0.60.9", "bytes", "futures-util", @@ -513,7 +513,7 @@ dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-http-server", "aws-smithy-json 0.60.7", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "aws-smithy-xml 0.60.9", "bytes", "futures", @@ -553,7 +553,7 @@ version = "0.60.3" name = "aws-smithy-json" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "proptest", "serde_json", ] @@ -573,7 +573,7 @@ version = "0.2.1" dependencies = [ "aws-sdk-s3", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "tokio", ] @@ -617,26 +617,25 @@ dependencies = [ name = "aws-smithy-query" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "urlencoding", ] [[package]] name = "aws-smithy-runtime" version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" dependencies = [ - "approx", - "aws-smithy-async 1.2.1", - "aws-smithy-http 0.60.11", - "aws-smithy-protocol-test 0.62.0", - "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-protocol-test 0.62.0 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.6", "bytes", "fastrand", - "futures-util", "h2 0.3.26", "http 0.2.12", - "http 1.1.0", "http-body 0.4.6", "http-body 1.0.1", "httparse", @@ -646,31 +645,30 @@ dependencies = [ "once_cell", "pin-project-lite", "pin-utils", - "pretty_assertions", "rustls 0.21.12", "serde", "serde_json", "tokio", "tracing", "tracing-subscriber", - "tracing-test", ] [[package]] name = "aws-smithy-runtime" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +version = "1.7.2" dependencies = [ - "aws-smithy-async 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-protocol-test 0.62.0 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.6", + "approx", + "aws-smithy-async 1.2.1", + "aws-smithy-http 0.60.11", + "aws-smithy-protocol-test 0.62.0", + "aws-smithy-runtime-api 1.7.2", + "aws-smithy-types 1.2.8", "bytes", "fastrand", + "futures-util", "h2 0.3.26", "http 0.2.12", + "http 1.1.0", "http-body 0.4.6", "http-body 1.0.1", "httparse", @@ -680,12 +678,14 @@ dependencies = [ "once_cell", "pin-project-lite", "pin-utils", + "pretty_assertions", "rustls 0.21.12", "serde", "serde_json", "tokio", "tracing", "tracing-subscriber", + "tracing-test", ] [[package]] @@ -693,7 +693,7 @@ name = "aws-smithy-runtime-api" version = "1.7.2" dependencies = [ "aws-smithy-async 1.2.1", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "bytes", "http 0.2.12", "http 1.1.0", @@ -749,7 +749,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.8" dependencies = [ "base64 0.13.1", "base64-simd", @@ -786,7 +786,7 @@ name = "aws-smithy-types-convert" version = "0.60.8" dependencies = [ "aws-smithy-async 1.2.1", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "chrono", "futures-core", "time", @@ -798,7 +798,7 @@ version = "0.1.3" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "bytes", "http 1.1.0", "tracing", @@ -1998,9 +1998,9 @@ dependencies = [ "aws-smithy-compression", "aws-smithy-http 0.60.11", "aws-smithy-json 0.60.7", - "aws-smithy-runtime 1.7.1", + "aws-smithy-runtime 1.7.2", "aws-smithy-runtime-api 1.7.2", - "aws-smithy-types 1.2.7", + "aws-smithy-types 1.2.8", "aws-smithy-xml 0.60.9", "bytes", "fastrand", From d714eaa67f02859192622eefa309e5facfca81b0 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 1 Oct 2024 13:47:58 -0700 Subject: [PATCH 32/32] Update HttpChecksumTest to use UA test utils --- .../aws-runtime/src/user_agent/test_util.rs | 21 ++-- .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 95 +++++++++++-------- 2 files changed, 69 insertions(+), 47 deletions(-) diff --git a/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs b/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs index a0a5679b63..de8013454f 100644 --- a/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs +++ b/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs @@ -17,13 +17,8 @@ static RE: Lazy = Lazy::new(|| Regex::new(r"m/([A-Za-z0-9+/=_,-]+)").unwr /// Refer to the end of the parent module file `user_agent.rs` for the complete ABNF specification /// of `business-metrics`. pub fn assert_ua_contains_metric_values(user_agent: &str, values: &[&str]) { - match RE.find(user_agent) { - Some(matched) => { - let csv = matched - .as_str() - .strip_prefix("m/") - .expect("prefix `m/` is guaranteed to exist by regex match"); - let metrics: Vec<&str> = csv.split(',').collect(); + match extract_ua_values(user_agent) { + Some(metrics) => { let mut missed = vec![]; for value in values.iter() { @@ -43,6 +38,18 @@ pub fn assert_ua_contains_metric_values(user_agent: &str, values: &[&str]) { } } +/// Extract the metric values from the `user_agent` string +pub fn extract_ua_values(user_agent: &str) -> Option> { + RE.find(user_agent).map(|matched| { + matched + .as_str() + .strip_prefix("m/") + .expect("prefix `m/` is guaranteed to exist by regex match") + .split(',') + .collect() + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt index e7b2c897d5..b0fc6265ed 100644 --- a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -8,6 +8,7 @@ package software.amazon.smithy.rustsdk import org.junit.jupiter.api.Test import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.Feature import software.amazon.smithy.rust.codegen.core.rustlang.Writable import software.amazon.smithy.rust.codegen.core.rustlang.join import software.amazon.smithy.rust.codegen.core.rustlang.plus @@ -130,8 +131,10 @@ internal class HttpChecksumTest { @Test fun requestChecksumWorks() { awsSdkIntegrationTest(model) { context, rustCrate -> - + // Allows us to use the user-agent test-utils in aws-runtime + rustCrate.mergeFeature(Feature("test-util", true, listOf("aws-runtime/test-util"))) val rc = context.runtimeConfig + // Create Writables for all test types val checksumRequestTestWritables = checksumRequestTests.map { createRequestChecksumCalculationTest(it, context) }.join("\n") @@ -158,6 +161,8 @@ internal class HttpChecksumTest { use std::io::Write; use http_body::Body; use #{HttpRequest}; + use #{UaAssert}; + use #{UaExtract}; """, *preludeScope, "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), @@ -165,34 +170,18 @@ internal class HttpChecksumTest { "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), "SdkBody" to RuntimeType.smithyTypes(rc).resolve("body::SdkBody"), "HttpRequest" to RuntimeType.smithyRuntimeApi(rc).resolve("client::orchestrator::HttpRequest"), - ) - } - - val uaExtractor = - writable { - rustTemplate( - """ - fn get_sdk_metric_str(req: &HttpRequest) -> &str { - req.headers() - .get("x-amz-user-agent") - .unwrap() - .split(" ") - .filter_map(|sec| { - if sec.starts_with("m/") { - Some(&sec[2..]) - } else { - None - } - }) - .collect::>()[0] - } - """.trimIndent(), + "UaAssert" to + AwsRuntimeType.awsRuntime(rc) + .resolve("user_agent::test_util::assert_ua_contains_metric_values"), + "UaExtract" to + AwsRuntimeType.awsRuntime(rc) + .resolve("user_agent::test_util::extract_ua_values"), ) } // Create one integ test per test type rustCrate.integrationTest("request_checksums") { - testBase.plus(checksumRequestTestWritables).plus(uaExtractor)() + testBase.plus(checksumRequestTestWritables)() } rustCrate.integrationTest("response_checksums_success") { @@ -208,7 +197,7 @@ internal class HttpChecksumTest { } rustCrate.integrationTest("misc_tests") { - testBase.plus(uaExtractor).plus(miscTests)() + testBase.plus(miscTests)() } } } @@ -263,8 +252,13 @@ internal class HttpChecksumTest { assert_eq!(algo_header, "${testDef.algoHeader}"); // Check the user-agent metrics for the selected algo - let sdk_metrics = get_sdk_metric_str(&request); - assert!(sdk_metrics.contains("${testDef.algoFeatureId}")); + assert_ua_contains_metric_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + &["${testDef.algoFeatureId}"], + ); } """, *preludeScope, @@ -462,7 +456,8 @@ internal class HttpChecksumTest { } /** - * Generate miscellaneous tests + * Generate miscellaneous tests, currently mostly focused on the inclusion of the checksum config metrics in the + * user-agent header */ private fun createMiscellaneousTests(context: ClientCodegenContext): Writable { val rc = context.runtimeConfig @@ -492,9 +487,14 @@ internal class HttpChecksumTest { .await; let request = rx.expect_request(); - let sdk_metrics = get_sdk_metric_str(&request); - assert!(sdk_metrics.contains("a")); - assert!(!sdk_metrics.contains("Z")); + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"a")); + assert!(!sdk_metrics.contains(&"Z")); } ##[::tokio::test] @@ -514,9 +514,14 @@ internal class HttpChecksumTest { .await; let request = rx.expect_request(); - let sdk_metrics = get_sdk_metric_str(&request); - assert!(sdk_metrics.contains("Z")); - assert!(!sdk_metrics.contains("a")); + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"Z")); + assert!(!sdk_metrics.contains(&"a")); } ##[::tokio::test] @@ -541,9 +546,14 @@ internal class HttpChecksumTest { .await; let request = rx.expect_request(); - let sdk_metrics = get_sdk_metric_str(&request); - assert!(sdk_metrics.contains("b")); - assert!(!sdk_metrics.contains("c")); + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"b")); + assert!(!sdk_metrics.contains(&"c")); } ##[::tokio::test] @@ -571,9 +581,14 @@ internal class HttpChecksumTest { .await; let request = rx.expect_request(); - let sdk_metrics = get_sdk_metric_str(&request); - assert!(sdk_metrics.contains("c")); - assert!(!sdk_metrics.contains("b")); + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"c")); + assert!(!sdk_metrics.contains(&"b")); } """, *preludeScope,