Replace serde_cbor with ciborium due to security vulnerability (#…

…3855)

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt

@@ -310,7 +310,7 @@ data class CargoDependency(
  val Hound: CargoDependency = CargoDependency("hound", CratesIo("3.4.0"), DependencyScope.Dev)
  val PrettyAssertions: CargoDependency =
  CargoDependency("pretty_assertions", CratesIo("1.3.0"), DependencyScope.Dev)
- val SerdeCbor: CargoDependency = CargoDependency("serde_cbor", CratesIo("0.11"), DependencyScope.Dev)
+ val Ciborium: CargoDependency = CargoDependency("ciborium", CratesIo("0.2"), DependencyScope.Dev)
  val SerdeJson: CargoDependency = CargoDependency("serde_json", CratesIo("1.0.0"), DependencyScope.Dev)
  val Smol: CargoDependency = CargoDependency("smol", CratesIo("1.2.0"), DependencyScope.Dev)
  val TempFile: CargoDependency = CargoDependency("tempfile", CratesIo("3.2.0"), DependencyScope.Dev)

codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt

@@ -341,7 +341,7 @@ class SerdeDecoratorTest {
  arrayOf(
  "crate" to RustType.Opaque(ctx.moduleUseName()),
  "serde_json" to CargoDependency("serde_json", CratesIo("1")).toDevDependency().toType(),
- "serde_cbor" to CargoDependency("serde_cbor", CratesIo("0.11.2")).toDevDependency().toType(),
+ "ciborium" to CargoDependency.Ciborium.toType(),
  // we need the derive feature
  "serde" to CargoDependency.Serde.toDevDependency().toType(),
  )
@@ -437,7 +437,9 @@ class SerdeDecoratorTest {
  use #{crate}::serde::*;
  let input = StreamingInput::builder().data(ByteStream::from_static(b"123")).build().unwrap();
  let settings = SerializationSettings::default();
- let serialized = #{serde_cbor}::to_vec(&input.serialize_ref(&settings)).expect("failed to serialize");
+ let mut serialized = Vec::new();
+ #{ciborium}::ser::into_writer(&input.serialize_ref(&settings), &mut serialized)
+ .expect("failed to serialize input into CBOR format using `ciborium`");
  assert_eq!(serialized, b"\xa1ddataC123");
  """,
  *codegenScope,

codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/serialize/CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest.kt

@@ -62,7 +62,7 @@ import java.util.logging.Logger
 /**
  * This lives in `codegen-server` because we want to run a full integration test for convenience,
  * but there's really nothing server-specific here. We're just testing that the CBOR (de)serializers work like
- * the ones generated by `serde_cbor`. This is a good exhaustive litmus test for correctness, since `serde_cbor`
+ * the ones generated by `ciborium`. This is a good exhaustive litmus test for correctness, since `ciborium`
  * is battle-tested.
  */
 internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
@@ -175,7 +175,7 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
  }
 
  @Test
- fun `serde_cbor round trip`() {
+ fun `ciborium round trip`() {
  val addDeriveSerdeSerializeDeserializeDecorator =
  object : ServerCodegenDecorator {
  override val name: String = "Add `#[derive(serde::Serialize, serde::Deserialize)]`"
@@ -240,7 +240,7 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
  val codegenScope =
  arrayOf(
  "AssertEq" to RuntimeType.PrettyAssertions.resolve("assert_eq!"),
- "SerdeCbor" to CargoDependency.SerdeCbor.toType(),
+ "ciborium" to CargoDependency.Ciborium.toType(),
  )
 
  val instantiator = ServerInstantiator(codegenContext, ignoreMissingMembers = true, withinTest = true)
@@ -278,14 +278,14 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
  if (expectFail.contains(test.id)) {
  writeWithNoFormatting("#[should_panic]")
  }
- unitTest("we_serialize_and_serde_cbor_deserializes_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
+ unitTest("we_serialize_and_ciborium_deserializes_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
  rustTemplate(
  """
  let expected = #{InstantiateShape:W};
  let bytes = #{SerializeFn}(&expected)
  .expect("our generated CBOR serializer failed");
- let actual = #{SerdeCbor}::from_slice(&bytes)
- .expect("serde_cbor failed deserializing from bytes");
+ let actual = #{ciborium}::from_reader(::std::io::Cursor::new(&bytes))
+ .expect("failed to deserialize bytes with `ciborium`");
  #{AssertEq}(expected, actual);
  """,
  "InstantiateShape" to instantiator.generate(targetShape, params),
@@ -330,12 +330,13 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
  if (expectFail.contains(test.id)) {
  writeWithNoFormatting("#[should_panic]")
  }
- unitTest("serde_cbor_serializes_and_we_deserialize_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
+ unitTest("ciborium_serializes_and_we_deserialize_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
  rustTemplate(
  """
  let expected = #{InstantiateShape:W};
- let bytes: Vec<u8> = #{SerdeCbor}::to_vec(&expected)
- .expect("serde_cbor failed serializing to `Vec<u8>`");
+ let mut bytes = Vec::new();
+ #{ciborium}::into_writer(&expected, &mut bytes)
+ .expect("failed to serialize to `Vec<u8>` with `ciborium`");
  let input = #{InputBuilder}::default();
  let input = #{DeserializeFn}(&bytes, input)
  .expect("our generated CBOR deserializer failed");

rust-runtime/aws-smithy-protocol-test/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aws-smithy-protocol-test"
-version = "0.62.0"
+version = "0.63.0"
 authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>", "Russell Cohen <rcoh@amazon.com>"]
 description = "A collection of library functions to validate HTTP requests against Smithy protocol tests."
 edition = "2021"
@@ -12,7 +12,7 @@ repository = "https://github.com/smithy-lang/smithy-rs"
 assert-json-diff = "1.1"
 base64-simd = "0.8"
 cbor-diag = "0.1.12"
-serde_cbor = "0.11"
+ciborium = "0.2"
 http = "0.2.1"
 pretty_assertions = "1.3"
 regex-lite = "0.1.5"

rust-runtime/aws-smithy-protocol-test/src/lib.rs

@@ -415,23 +415,105 @@ fn try_json_eq(expected: &str, actual: &str) -> Result<(), ProtocolTestFailure>
  }
 }
 
+/// Compares two `ciborium::value::Value` instances for semantic equality.
+///
+/// This function recursively compares two CBOR values, correctly handling arrays and maps
+/// according to the CBOR specification. Arrays are compared element-wise in order,
+/// while maps are compared without considering the order of key-value pairs.
+fn cbor_values_equal(
+ a: &ciborium::value::Value,
+ b: &ciborium::value::Value,
+) -> Result<bool, ProtocolTestFailure> {
+ match (a, b) {
+ (ciborium::value::Value::Array(a_array), ciborium::value::Value::Array(b_array)) => {
+ // Both arrays should be equal in size.
+ if a_array.len() != b_array.len() {
+ return Ok(false);
+ }
+ // Compare arrays element-wise.
+ for (a_elem, b_elem) in a_array.iter().zip(b_array.iter()) {
+ if !cbor_values_equal(a_elem, b_elem)? {
+ return Ok(false);
+ }
+ }
+ Ok(true)
+ }
+
+ // Convert `ciborium::value::Value::Map` to a `HashMap`, and then compare the values of
+ // each key in `a` with those in `b`.
+ (ciborium::value::Value::Map(a_map), ciborium::value::Value::Map(b_map)) => {
+ if a_map.len() != b_map.len() {
+ return Ok(false);
+ }
+
+ let b_hashmap = ciborium_map_to_hashmap(b_map)?;
+ // Each key in `a` should exist in `b`, and the values should match.
+ for a_key_value in a_map.iter() {
+ let (a_key, a_value) = get_text_key_value(a_key_value)?;
+ match b_hashmap.get(a_key) {
+ Some(b_value) => {
+ if !cbor_values_equal(a_value, b_value)? {
+ return Ok(false);
+ }
+ }
+ None => return Ok(false),
+ }
+ }
+ Ok(true)
+ }
+
+ (ciborium::value::Value::Float(a_float), ciborium::value::Value::Float(b_float)) => {
+ Ok(a_float == b_float || (a_float.is_nan() && b_float.is_nan()))
+ }
+
+ _ => Ok(a == b),
+ }
+}
+
+/// Converts a `ciborium::value::Value::Map` into a `HashMap<&String, &ciborium::value::Value>`.
+///
+/// CBOR maps (`Value::Map`) are internally represented as vectors of key-value pairs,
+/// and direct comparison is affected by the order of these pairs.
+/// Since the CBOR specification treats maps as unordered collections,
+/// this function transforms the vector into a `HashMap`, for order-independent comparisons
+/// between maps.
+fn ciborium_map_to_hashmap(
+ cbor_map: &[(ciborium::value::Value, ciborium::value::Value)],
+) -> Result<std::collections::HashMap<&String, &ciborium::value::Value>, ProtocolTestFailure> {
+ cbor_map.iter().map(get_text_key_value).collect()
+}
+
+/// Extracts a string key and its associated value from a CBOR key-value pair.
+/// Returns a `ProtocolTestFailure::InvalidBodyFormat` error if the key is not a text value.
+fn get_text_key_value(
+ (key, value): &(ciborium::value::Value, ciborium::value::Value),
+) -> Result<(&String, &ciborium::value::Value), ProtocolTestFailure> {
+ match key {
+ ciborium::value::Value::Text(key_str) => Ok((key_str, value)),
+ _ => Err(ProtocolTestFailure::InvalidBodyFormat {
+ expected: "a text key as map entry".to_string(),
+ found: format!("{:?}", key),
+ }),
+ }
+}
+
 fn try_cbor_eq<T: AsRef<[u8]> + Debug>(
  actual_body: T,
  expected_body: &str,
 ) -> Result<(), ProtocolTestFailure> {
  let decoded = base64_simd::STANDARD
  .decode_to_vec(expected_body)
  .expect("smithy protocol test `body` property is not properly base64 encoded");
- let expected_cbor_value: serde_cbor::Value =
- serde_cbor::from_slice(decoded.as_slice()).expect("expected value must be valid CBOR");
- let actual_cbor_value: serde_cbor::Value = serde_cbor::from_slice(actual_body.as_ref())
+ let expected_cbor_value: ciborium::value::Value =
+ ciborium::de::from_reader(decoded.as_slice()).expect("expected value must be valid CBOR");
+ let actual_cbor_value: ciborium::value::Value = ciborium::de::from_reader(actual_body.as_ref())
  .map_err(|e| ProtocolTestFailure::InvalidBodyFormat {
  expected: "cbor".to_owned(),
  found: format!("{} {:?}", e, actual_body),
  })?;
  let actual_body_base64 = base64_simd::STANDARD.encode_to_string(&actual_body);
 
- if expected_cbor_value != actual_cbor_value {
+ if !cbor_values_equal(&expected_cbor_value, &actual_cbor_value)? {
  let expected_body_annotated_hex: String = cbor_diag::parse_bytes(&decoded)
  .expect("smithy protocol test `body` property is not valid CBOR")
  .to_hex();
@@ -599,6 +681,46 @@ mod tests {
  validate_body(actual, expected, MediaType::Json).expect_err("bodies do not match");
  }
 
+ #[test]
+ fn test_validate_cbor_body() {
+ let base64_encode = |v: &[u8]| base64_simd::STANDARD.encode_to_string(v);
+
+ // The following is the CBOR representation of `{"abc": 5 }`.
+ let actual = [0xbf, 0x63, 0x61, 0x62, 0x63, 0x05, 0xff];
+ // The following is the base64-encoded CBOR representation of `{"abc": 5 }` using a definite length map.
+ let expected_base64 = base64_encode(&[0xA1, 0x63, 0x61, 0x62, 0x63, 0x05]);
+
+ validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
+ .expect("unexpected mismatch between CBOR definite and indefinite map encodings");
+
+ // The following is the CBOR representation of `{"a":1, "b":2}`.
+ let actual = [0xBF, 0x61, 0x61, 0x01, 0x61, 0x62, 0x02, 0xFF];
+ // The following is the base64-encoded CBOR representation of `{"b":2, "a":1}`.
+ let expected_base64 = base64_encode(&[0xBF, 0x61, 0x62, 0x02, 0x61, 0x61, 0x01, 0xFF]);
+ validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
+ .expect("different ordering in CBOR decoded maps do not match");
+
+ // The following is the CBOR representation of `{"a":[1,2,{"b":3, "c":4}]}`.
+ let actual = [
+ 0xBF, 0x61, 0x61, 0x9F, 0x01, 0x02, 0xBF, 0x61, 0x62, 0x03, 0x61, 0x63, 0x04, 0xFF,
+ 0xFF, 0xFF,
+ ];
+ // The following is the base64-encoded CBOR representation of `{"a":[1,2,{"c":4, "b":3}]}`.
+ let expected_base64 = base64_encode(&[
+ 0xBF, 0x61, 0x61, 0x9F, 0x01, 0x02, 0xBF, 0x61, 0x63, 0x04, 0x61, 0x62, 0x03, 0xFF,
+ 0xFF, 0xFF,
+ ]);
+ validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
+ .expect("different ordering in CBOR decoded maps do not match");
+
+ // The following is the CBOR representation of `{"a":[1,2]}`.
+ let actual = [0xBF, 0x61, 0x61, 0x9F, 0x01, 0x02, 0xFF, 0xFF];
+ // The following is the CBOR representation of `{"a":[2,1]}`.
+ let expected_base64 = base64_encode(&[0xBF, 0x61, 0x61, 0x9F, 0x02, 0x01, 0xFF, 0xFF]);
+ validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
+ .expect_err("arrays in CBOR should follow strict ordering");
+ }
+
  #[test]
  fn test_validate_xml_body() {
  let expected = r#"<a>