From 5b5d19568efaaaa3722e361c33125b30d13577a0 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Sun, 24 Jan 2021 23:31:33 +0100 Subject: [PATCH 01/12] Added ability to parse a list of schemas, which may have cross dependencies. Added tests --- src/schema.rs | 180 ++++++++++++++++++++--------- tests/schema.rs | 294 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+), 52 deletions(-) diff --git a/src/schema.rs b/src/schema.rs index cfe1c1e..1770abe 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -275,11 +275,11 @@ pub enum RecordFieldOrder { impl RecordField { /// Parse a `serde_json::Value` into a `RecordField`. - fn parse(field: &Map, position: usize) -> AvroResult { + fn parse(field: &Map, position: usize, parser: &mut Parser) -> AvroResult { let name = field.name().ok_or(Error::GetNameFieldFromRecord)?; // TODO: "type" = "" - let schema = Schema::parse_complex(field)?; + let schema = parser.parse_complex(field)?; let default = field.get("default").cloned(); @@ -378,25 +378,13 @@ fn parse_json_integer_for_decimal(value: &serde_json::Number) -> Result Result { - // TODO: (#82) this should be a ParseSchemaError wrapping the JSON error - let value = serde_json::from_str(input).map_err(Error::ParseSchemaJson)?; - Self::parse(&value) - } - - /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro - /// schema. - pub fn parse(value: &Value) -> AvroResult { - match *value { - Value::String(ref t) => Schema::parse_primitive(t.as_str()), - Value::Object(ref data) => Schema::parse_complex(data), - Value::Array(ref data) => Schema::parse_union(data), - _ => Err(Error::ParseSchemaFromValidJson), - } - } +#[derive(Default)] +struct Parser { + input_schemas: HashMap, + parsed_schemas: HashMap +} +impl Schema { /// Converts `self` into its [Parsing Canonical Form]. /// /// [Parsing Canonical Form]: @@ -421,10 +409,85 @@ impl Schema { } } - /// Parse a `serde_json::Value` representing a primitive Avro type into a - /// `Schema`. - fn parse_primitive(primitive: &str) -> AvroResult { - match primitive { + /// Create a `Schema` from a string representing a JSON Avro schema. + pub fn parse_str(input: &str) -> Result { + let mut parser = Parser::default(); + parser.parse_str(input) + } + + /// Create a array of `Schema`'s from a list of JSON Avro schemas. It is allowed that + /// the schemas have cross-dependencies; these will be resolved during parsing. + pub fn parse_list(input: &[&str]) -> Result, Error>{ + let mut input_schemas: HashMap = HashMap::with_capacity(input.len()); + for js in input { + let schema: Value = serde_json::from_str(js).map_err(Error::ParseSchemaJson)?; + if let Value::Object(inner) = &schema { + let fullname = Name::parse(&inner)?.fullname(None); + let _ = input_schemas.insert(fullname, schema); + } else { + return Err(Error::GetNameField); + } + } + let mut parser = Parser { + input_schemas, + parsed_schemas: HashMap::with_capacity(input.len()) + }; + parser.parse_list() + } + + pub fn parse(value: &Value) -> AvroResult { + let mut parser = Parser::default(); + parser.parse(value) + } +} + +impl Parser { + + /// Create a `Schema` from a string representing a JSON Avro schema. + fn parse_str(&mut self, input: &str) -> Result { + // TODO: (#82) this should be a ParseSchemaError wrapping the JSON error + let value = serde_json::from_str(input).map_err(Error::ParseSchemaJson)?; + self.parse(&value) + } + + /// Create a array of `Schema`'s from a an iterator of JSON Avro schemas. It is allowed that + /// the schemas have cross-dependencies; these will be resolved during parsing. + fn parse_list(&mut self) -> Result, Error> { + + while !self.input_schemas.is_empty() { + let next_name = self.input_schemas.keys().next() + .expect("Input schemas unexpectedly empty") + .to_owned(); + let (name, value) = self.input_schemas.remove_entry(&next_name) + .expect("Key unexpectedly missing"); + let parsed = self.parse(&value)?; + self.parsed_schemas.insert(name, parsed); + } + + let mut parsed_schemas = Vec::new(); + for (_, parsed) in self.parsed_schemas.drain() { + parsed_schemas.push(parsed); + } + Ok(parsed_schemas) + + } + + /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro + /// schema. + fn parse(&mut self, value: &Value) -> AvroResult { + match *value { + Value::String(ref t) => self.parse_known_schema(t.as_str()), + Value::Object(ref data) => self.parse_complex(data), + Value::Array(ref data) => self.parse_union(data), + _ => Err(Error::ParseSchemaFromValidJson), + } + } + + /// Parse a `serde_json::Value` representing an Avro type whose Schema is known into a + /// `Schema`. A Schema for a `serde_json::Value` is known if it is primitive or has + /// been parsed previously by the parsed and stored in its map of parsed_schemas. + fn parse_known_schema(&mut self, name: &str) -> AvroResult { + match name { "null" => Ok(Schema::Null), "boolean" => Ok(Schema::Boolean), "int" => Ok(Schema::Int), @@ -433,8 +496,19 @@ impl Schema { "float" => Ok(Schema::Float), "bytes" => Ok(Schema::Bytes), "string" => Ok(Schema::String), - other => Err(Error::ParsePrimitive(other.into())), + _ => self.fetch_schema(name), + } + } + + fn fetch_schema(&mut self, name: &str) -> AvroResult { + if let Some(parsed) = self.parsed_schemas.get(name) { + return Ok(parsed.clone()); } + let value= self.input_schemas.remove(name) + .ok_or(Error::ParsePrimitive(name.into()))?; + let parsed = self.parse(&value)?; + self.parsed_schemas.insert(name.to_string(), parsed.clone()); + Ok(parsed) } fn parse_precision_and_scale( @@ -463,14 +537,15 @@ impl Schema { /// /// Avro supports "recursive" definition of types. /// e.g: {"type": {"type": "string"}} - fn parse_complex(complex: &Map) -> AvroResult { + fn parse_complex(&mut self, complex: &Map) -> AvroResult { fn logical_verify_type( complex: &Map, kinds: &[SchemaKind], + parser: &mut Parser ) -> AvroResult { match complex.get("type") { Some(value) => { - let ty = Schema::parse(value)?; + let ty = parser.parse(value)?; if kinds .iter() .any(|&kind| SchemaKind::from(ty.clone()) == kind) @@ -489,6 +564,7 @@ impl Schema { let inner = Box::new(logical_verify_type( complex, &[SchemaKind::Fixed, SchemaKind::Bytes], + self )?); let (precision, scale) = Self::parse_precision_and_scale(complex)?; @@ -500,31 +576,31 @@ impl Schema { }); } "uuid" => { - logical_verify_type(complex, &[SchemaKind::String])?; + logical_verify_type(complex, &[SchemaKind::String],self)?; return Ok(Schema::Uuid); } "date" => { - logical_verify_type(complex, &[SchemaKind::Int])?; + logical_verify_type(complex, &[SchemaKind::Int], self)?; return Ok(Schema::Date); } "time-millis" => { - logical_verify_type(complex, &[SchemaKind::Int])?; + logical_verify_type(complex, &[SchemaKind::Int], self)?; return Ok(Schema::TimeMillis); } "time-micros" => { - logical_verify_type(complex, &[SchemaKind::Long])?; + logical_verify_type(complex, &[SchemaKind::Long], self)?; return Ok(Schema::TimeMicros); } "timestamp-millis" => { - logical_verify_type(complex, &[SchemaKind::Long])?; + logical_verify_type(complex, &[SchemaKind::Long], self)?; return Ok(Schema::TimestampMillis); } "timestamp-micros" => { - logical_verify_type(complex, &[SchemaKind::Long])?; + logical_verify_type(complex, &[SchemaKind::Long], self)?; return Ok(Schema::TimestampMicros); } "duration" => { - logical_verify_type(complex, &[SchemaKind::Fixed])?; + logical_verify_type(complex, &[SchemaKind::Fixed], self)?; return Ok(Schema::Duration); } // In this case, of an unknown logical type, we just pass through to the underlying @@ -539,15 +615,15 @@ impl Schema { } match complex.get("type") { Some(&Value::String(ref t)) => match t.as_str() { - "record" => Schema::parse_record(complex), - "enum" => Schema::parse_enum(complex), - "array" => Schema::parse_array(complex), - "map" => Schema::parse_map(complex), - "fixed" => Schema::parse_fixed(complex), - other => Schema::parse_primitive(other), + "record" => self.parse_record(complex), + "enum" => Self::parse_enum(complex), + "array" => self.parse_array(complex), + "map" => self.parse_map(complex), + "fixed" => Self::parse_fixed(complex), + other => self.parse_known_schema(other), }, - Some(&Value::Object(ref data)) => Schema::parse_complex(data), - Some(&Value::Array(ref variants)) => Schema::parse_union(variants), + Some(&Value::Object(ref data)) => self.parse_complex(data), + Some(&Value::Array(ref variants)) => self.parse_union(variants), Some(unknown) => Err(Error::GetComplexType(unknown.clone())), None => Err(Error::GetComplexTypeField), } @@ -555,7 +631,7 @@ impl Schema { /// Parse a `serde_json::Value` representing a Avro record type into a /// `Schema`. - fn parse_record(complex: &Map) -> AvroResult { + fn parse_record(&mut self, complex: &Map) -> AvroResult { let name = Name::parse(complex)?; let mut lookup = HashMap::new(); @@ -569,7 +645,7 @@ impl Schema { .iter() .filter_map(|field| field.as_object()) .enumerate() - .map(|(position, field)| RecordField::parse(field, position)) + .map(|(position, field)| RecordField::parse(field, position, self)) .collect::>() })?; @@ -587,7 +663,7 @@ impl Schema { /// Parse a `serde_json::Value` representing a Avro enum type into a /// `Schema`. - fn parse_enum(complex: &Map) -> AvroResult { + fn parse_enum(complex: &Map) -> AvroResult { let name = Name::parse(complex)?; let symbols = complex @@ -611,37 +687,37 @@ impl Schema { /// Parse a `serde_json::Value` representing a Avro array type into a /// `Schema`. - fn parse_array(complex: &Map) -> AvroResult { + fn parse_array(&mut self, complex: &Map) -> AvroResult { complex .get("items") .ok_or(Error::GetArrayItemsField) - .and_then(|items| Schema::parse(items)) + .and_then(|items| self.parse(items)) .map(|schema| Schema::Array(Box::new(schema))) } /// Parse a `serde_json::Value` representing a Avro map type into a /// `Schema`. - fn parse_map(complex: &Map) -> AvroResult { + fn parse_map(&mut self, complex: &Map) -> AvroResult { complex .get("values") .ok_or(Error::GetMapValuesField) - .and_then(|items| Schema::parse(items)) + .and_then(|items| self.parse(items)) .map(|schema| Schema::Map(Box::new(schema))) } /// Parse a `serde_json::Value` representing a Avro union type into a /// `Schema`. - fn parse_union(items: &[Value]) -> AvroResult { + fn parse_union(&mut self, items: &[Value]) -> AvroResult { items .iter() - .map(Schema::parse) + .map(|v| self.parse(v)) .collect::, _>>() .and_then(|schemas| Ok(Schema::Union(UnionSchema::new(schemas)?))) } /// Parse a `serde_json::Value` representing a Avro fixed type into a /// `Schema`. - fn parse_fixed(complex: &Map) -> AvroResult { + fn parse_fixed(complex: &Map) -> AvroResult { let name = Name::parse(complex)?; let size = complex diff --git a/tests/schema.rs b/tests/schema.rs index 7f5eabf..92d5260 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -624,6 +624,300 @@ fn test_equivalence_after_round_trip() { } } +#[test] +/// Test that a list of schemas whose definitions do not depend on each other produces the same +/// result as parsing each element of the list individually +fn test_parse_list_without_cross_deps() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "map", + "values": "double" + }"#; + let schema_strs = [schema_str_1, schema_str_2]; + let schemas = Schema::parse_list(&schema_strs).expect("Test failed"); + + for schema_str in &schema_strs { + let parsed = Schema::parse_str(schema_str).expect("Test failed"); + assert!(schemas.contains(&parsed)); + } +} + +#[test] +/// Test that the parsing of a list of schemas, whose definitions do depend on each other, can +/// perform the necessary schema composition. This should work regardless of the order in which +/// the schemas are input. +fn test_parse_list_with_cross_deps_basic() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "map", + "values": "A" + }"#; + let schema_composite = r#"{ + "name": "B", + "type": "map", + "values": { + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + } + }"#; + let schema_strs_first = [schema_str_1, schema_str_2]; + let schema_strs_second = [schema_str_2, schema_str_1]; + let schemas_first = Schema::parse_list(&schema_strs_first).expect("Test failed"); + let schemas_second = Schema::parse_list(&schema_strs_second).expect("Test failed"); + + for schema_str in &[schema_str_1, schema_composite] { + let parsed = Schema::parse_str(&schema_str).expect("Test failed"); + assert!(schemas_first.contains(&parsed)); + assert!(schemas_second.contains(&parsed)); + assert_eq!(schemas_first.len(), 2); + assert_eq!(schemas_second.len(), 2); + } +} + + +#[test] +/// Test that schema composition resolves namespaces. +fn test_parse_list_with_cross_deps_and_namespaces() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "namespace": "namespace", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "map", + "values": "namespace.A" + }"#; + let schema_composite = r#"{ + "name": "B", + "type": "map", + "values": { + "name": "A", + "type": "record", + "namespace": "namespace", + "fields": [ + {"name": "field_one", "type": "float"} + ] + } + }"#; + let schema_strs_first = [schema_str_1, schema_str_2]; + let schema_strs_second = [schema_str_2, schema_str_1]; + let schemas_first = Schema::parse_list(&schema_strs_first).expect("Test failed"); + let schemas_second = Schema::parse_list(&schema_strs_second).expect("Test failed"); + + for schema_str in &[schema_str_1, schema_composite] { + let parsed = Schema::parse_str(&schema_str).expect("Test failed"); + assert!(schemas_first.contains(&parsed)); + assert!(schemas_second.contains(&parsed)); + assert_eq!(schemas_first.len(), 2); + assert_eq!(schemas_second.len(), 2); + } +} + +#[test] +/// Test that schema composition fails on namespace errors. +fn test_parse_list_with_cross_deps_and_namespaces_error() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "namespace": "namespace", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "map", + "values": "A" + }"#; + + let schema_strs_first = [schema_str_1, schema_str_2]; + let schema_strs_second = [schema_str_2, schema_str_1]; + let _ = Schema::parse_list(&schema_strs_first).expect_err("Test failed"); + let _ = Schema::parse_list(&schema_strs_second).expect_err("Test failed"); +} + +/// Return all permutations of an input slice +fn permutations(list: &[T]) -> Vec> { + let size = list.len(); + let indices = permutation_indices((0..size).collect()); + let mut perms = Vec::new(); + for perm_map in &indices { + let mut perm = Vec::new(); + for ix in perm_map { + perm.push(&list[*ix]); + } + perms.push(perm) + } + perms +} + +/// Return all permutations of the indices of a vector +fn permutation_indices<'a>(indices: Vec) -> Vec>{ + let size = indices.len(); + let mut perms: Vec> = Vec::new(); + if size == 1 { + perms.push(indices); + return perms; + } + for index in 0..size { + let (head, tail) = indices.split_at(index); + let (first, rest) = tail.split_at(1); + let mut head = head.to_vec(); + head.extend_from_slice(rest); + for mut sub_index in permutation_indices(head) { + sub_index.insert(0, first[0]); + perms.push(sub_index); + } + + } + return perms +} + +#[test] +/// Test that a type that depends on more than one other type is parsed correctly when all +/// definitions are passed in as a list. This should work regardless of the ordering of the list. +fn test_parse_list_multiple_dependencies() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": ["null", "B", "C"]} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "array", + "items": "double" + }"#; + let schema_str_3 = r#"{ + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + }"#; + let schema_composite = r#"{ + "name": "A", + "type": "record", + "fields": [ + { + "name": "field_one", + "type": [ + "null", + { + "name": "B", + "type": "array", + "items": "double" + }, + { + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + } + ] + } + ] + }"#; + let parsed = vec!( + Schema::parse_str(schema_str_2).expect("Test failed"), + Schema::parse_str(schema_str_3).expect("Test failed"), + Schema::parse_str(schema_composite).expect("Test failed") + ); + let schema_strs = vec!(schema_str_1, schema_str_2, schema_str_3); + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemas = Schema::parse_list(&schema_str_perm).expect("Test failed"); + assert_eq!(schemas.len(), 3); + for parsed_schema in &parsed { + assert!(schemas.contains(parsed_schema)); + } + } +} + +#[test] +/// Test that a type that is depended on by more than one other type is parsed correctly when all +/// definitions are passed in as a list. This should work regardless of the ordering of the list. +fn test_parse_list_shared_dependency() { + let schema_str_1 = r#"{ + "name": "A", + "type": "map", + "values": "C" + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "array", + "items": "C" + }"#; + let schema_str_3 = r#"{ + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + }"#; + let schema_composite_1 = r#"{ + "name": "A", + "type": "map", + "values": { + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + } + }"#; + let schema_composite_2 = r#"{ + "name": "B", + "type": "array", + "items": { + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + } + }"#; + let parsed = vec!( + Schema::parse_str(schema_str_3).expect("Test failed"), + Schema::parse_str(schema_composite_1).expect("Test failed"), + Schema::parse_str(schema_composite_2).expect("Test failed") + ); + let schema_strs = vec!(schema_str_1, schema_str_2, schema_str_3); + for schema_str_perm in permutations(&schema_strs) { + let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); + let schemas = Schema::parse_list(&schema_str_perm).expect("Test failed"); + assert_eq!(schemas.len(), 3); + for parsed_schema in &parsed { + assert!(schemas.contains(parsed_schema)); + } + } +} + + + // The fullname is determined in one of the following ways: // * A name and namespace are both specified. For example, // one might use "name": "X", "namespace": "org.foo" From 68f78972044586ae4527f8a52eb519a5a4b1ce04 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Sun, 24 Jan 2021 23:56:57 +0100 Subject: [PATCH 02/12] Fixing the formatting --- src/schema.rs | 27 +++++++++++++++------------ tests/schema.rs | 21 ++++++++++----------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/schema.rs b/src/schema.rs index 1770abe..3770194 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -381,7 +381,7 @@ fn parse_json_integer_for_decimal(value: &serde_json::Number) -> Result, - parsed_schemas: HashMap + parsed_schemas: HashMap, } impl Schema { @@ -417,7 +417,7 @@ impl Schema { /// Create a array of `Schema`'s from a list of JSON Avro schemas. It is allowed that /// the schemas have cross-dependencies; these will be resolved during parsing. - pub fn parse_list(input: &[&str]) -> Result, Error>{ + pub fn parse_list(input: &[&str]) -> Result, Error> { let mut input_schemas: HashMap = HashMap::with_capacity(input.len()); for js in input { let schema: Value = serde_json::from_str(js).map_err(Error::ParseSchemaJson)?; @@ -430,7 +430,7 @@ impl Schema { } let mut parser = Parser { input_schemas, - parsed_schemas: HashMap::with_capacity(input.len()) + parsed_schemas: HashMap::with_capacity(input.len()), }; parser.parse_list() } @@ -442,7 +442,6 @@ impl Schema { } impl Parser { - /// Create a `Schema` from a string representing a JSON Avro schema. fn parse_str(&mut self, input: &str) -> Result { // TODO: (#82) this should be a ParseSchemaError wrapping the JSON error @@ -453,12 +452,15 @@ impl Parser { /// Create a array of `Schema`'s from a an iterator of JSON Avro schemas. It is allowed that /// the schemas have cross-dependencies; these will be resolved during parsing. fn parse_list(&mut self) -> Result, Error> { - while !self.input_schemas.is_empty() { - let next_name = self.input_schemas.keys().next() + let next_name = self.input_schemas + .keys() + .next() .expect("Input schemas unexpectedly empty") .to_owned(); - let (name, value) = self.input_schemas.remove_entry(&next_name) + let (name, value) = self + .input_schemas + .remove_entry(&next_name) .expect("Key unexpectedly missing"); let parsed = self.parse(&value)?; self.parsed_schemas.insert(name, parsed); @@ -469,7 +471,6 @@ impl Parser { parsed_schemas.push(parsed); } Ok(parsed_schemas) - } /// Create a `Schema` from a `serde_json::Value` representing a JSON Avro @@ -504,7 +505,9 @@ impl Parser { if let Some(parsed) = self.parsed_schemas.get(name) { return Ok(parsed.clone()); } - let value= self.input_schemas.remove(name) + let value= self + .input_schemas + .remove(name) .ok_or(Error::ParsePrimitive(name.into()))?; let parsed = self.parse(&value)?; self.parsed_schemas.insert(name.to_string(), parsed.clone()); @@ -541,7 +544,7 @@ impl Parser { fn logical_verify_type( complex: &Map, kinds: &[SchemaKind], - parser: &mut Parser + parser: &mut Parser, ) -> AvroResult { match complex.get("type") { Some(value) => { @@ -564,7 +567,7 @@ impl Parser { let inner = Box::new(logical_verify_type( complex, &[SchemaKind::Fixed, SchemaKind::Bytes], - self + self, )?); let (precision, scale) = Self::parse_precision_and_scale(complex)?; @@ -576,7 +579,7 @@ impl Parser { }); } "uuid" => { - logical_verify_type(complex, &[SchemaKind::String],self)?; + logical_verify_type(complex, &[SchemaKind::String], self)?; return Ok(Schema::Uuid); } "date" => { diff --git a/tests/schema.rs b/tests/schema.rs index 92d5260..d9d6083 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -691,7 +691,6 @@ fn test_parse_list_with_cross_deps_basic() { } } - #[test] /// Test that schema composition resolves namespaces. fn test_parse_list_with_cross_deps_and_namespaces() { @@ -773,7 +772,7 @@ fn permutations(list: &[T]) -> Vec> { } /// Return all permutations of the indices of a vector -fn permutation_indices<'a>(indices: Vec) -> Vec>{ +fn permutation_indices(indices: Vec) -> Vec> { let size = indices.len(); let mut perms: Vec> = Vec::new(); if size == 1 { @@ -789,8 +788,8 @@ fn permutation_indices<'a>(indices: Vec) -> Vec>{ sub_index.insert(0, first[0]); perms.push(sub_index); } - } + return perms } @@ -841,12 +840,13 @@ fn test_parse_list_multiple_dependencies() { } ] }"#; - let parsed = vec!( + + let parsed = vec![ Schema::parse_str(schema_str_2).expect("Test failed"), Schema::parse_str(schema_str_3).expect("Test failed"), Schema::parse_str(schema_composite).expect("Test failed") - ); - let schema_strs = vec!(schema_str_1, schema_str_2, schema_str_3); + ]; + let schema_strs = vec![schema_str_1, schema_str_2, schema_str_3]; for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); let schemas = Schema::parse_list(&schema_str_perm).expect("Test failed"); @@ -900,12 +900,13 @@ fn test_parse_list_shared_dependency() { ] } }"#; - let parsed = vec!( + + let parsed = vec![ Schema::parse_str(schema_str_3).expect("Test failed"), Schema::parse_str(schema_composite_1).expect("Test failed"), Schema::parse_str(schema_composite_2).expect("Test failed") - ); - let schema_strs = vec!(schema_str_1, schema_str_2, schema_str_3); + ]; + let schema_strs = vec![schema_str_1, schema_str_2, schema_str_3]; for schema_str_perm in permutations(&schema_strs) { let schema_str_perm: Vec<&str> = schema_str_perm.iter().map(|s| **s).collect(); let schemas = Schema::parse_list(&schema_str_perm).expect("Test failed"); @@ -916,8 +917,6 @@ fn test_parse_list_shared_dependency() { } } - - // The fullname is determined in one of the following ways: // * A name and namespace are both specified. For example, // one might use "name": "X", "namespace": "org.foo" From 41d1fe751b602f8e940752c2e15b4ae1decde7e2 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Mon, 25 Jan 2021 00:01:08 +0100 Subject: [PATCH 03/12] A bit more formatting --- src/schema.rs | 5 +++-- tests/schema.rs | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/schema.rs b/src/schema.rs index 3770194..0151bc9 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -453,7 +453,8 @@ impl Parser { /// the schemas have cross-dependencies; these will be resolved during parsing. fn parse_list(&mut self) -> Result, Error> { while !self.input_schemas.is_empty() { - let next_name = self.input_schemas + let next_name = self + .input_schemas .keys() .next() .expect("Input schemas unexpectedly empty") @@ -505,7 +506,7 @@ impl Parser { if let Some(parsed) = self.parsed_schemas.get(name) { return Ok(parsed.clone()); } - let value= self + let value = self .input_schemas .remove(name) .ok_or(Error::ParsePrimitive(name.into()))?; diff --git a/tests/schema.rs b/tests/schema.rs index d9d6083..37a3448 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -790,7 +790,7 @@ fn permutation_indices(indices: Vec) -> Vec> { } } - return perms + perms } #[test] @@ -900,11 +900,11 @@ fn test_parse_list_shared_dependency() { ] } }"#; - + let parsed = vec![ Schema::parse_str(schema_str_3).expect("Test failed"), Schema::parse_str(schema_composite_1).expect("Test failed"), - Schema::parse_str(schema_composite_2).expect("Test failed") + Schema::parse_str(schema_composite_2).expect("Test failed"), ]; let schema_strs = vec![schema_str_1, schema_str_2, schema_str_3]; for schema_str_perm in permutations(&schema_strs) { From 0eac1f02a411ba5d170aa093faebcfd61a812e45 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Mon, 25 Jan 2021 00:03:13 +0100 Subject: [PATCH 04/12] Added a comma to please clippy --- tests/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/schema.rs b/tests/schema.rs index 37a3448..c6c107d 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -844,7 +844,7 @@ fn test_parse_list_multiple_dependencies() { let parsed = vec![ Schema::parse_str(schema_str_2).expect("Test failed"), Schema::parse_str(schema_str_3).expect("Test failed"), - Schema::parse_str(schema_composite).expect("Test failed") + Schema::parse_str(schema_composite).expect("Test failed"), ]; let schema_strs = vec![schema_str_1, schema_str_2, schema_str_3]; for schema_str_perm in permutations(&schema_strs) { From ae1bf2e2fea54a96ffea4a0e959aac17e7a2fa96 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Mon, 25 Jan 2021 00:07:38 +0100 Subject: [PATCH 05/12] Switched an ok_or to an ok_or_else at the behest of clippy --- src/schema.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema.rs b/src/schema.rs index 0151bc9..023bb66 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -509,7 +509,7 @@ impl Parser { let value = self .input_schemas .remove(name) - .ok_or(Error::ParsePrimitive(name.into()))?; + .ok_or_else(|| Error::ParsePrimitive(name.into()))?; let parsed = self.parse(&value)?; self.parsed_schemas.insert(name.to_string(), parsed.clone()); Ok(parsed) From 88c9f004c32406fb1f8588bcf44bb3f6d93f4ba7 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 00:36:27 +0100 Subject: [PATCH 06/12] Following fixes: Fixed some docstrings, a new error for name collisions when using parse_list, guarantees that input order matches output order --- src/error.rs | 4 ++ src/schema.rs | 33 +++++++-- tests/schema.rs | 175 +++++++++++++++++++++++++++++++----------------- 3 files changed, 144 insertions(+), 68 deletions(-) diff --git a/src/error.rs b/src/error.rs index d693bb1..bed9579 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,6 @@ use crate::{schema::SchemaKind, types::ValueKind}; use std::fmt; +use crate::types::Value; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -18,6 +19,9 @@ pub enum Error { #[error("Not a string value, required for uuid: {0:?}")] GetUuidFromStringValue(ValueKind), + #[error("Two schemas with the same fullname were given: {0:?}")] + NameCollision(String), + #[error("Not a fixed or bytes type, required for decimal schema, got: {0:?}")] ResolveDecimalSchema(SchemaKind), diff --git a/src/schema.rs b/src/schema.rs index 023bb66..cb993d1 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -381,6 +381,7 @@ fn parse_json_integer_for_decimal(value: &serde_json::Number) -> Result, + input_order: Vec, parsed_schemas: HashMap, } @@ -415,21 +416,32 @@ impl Schema { parser.parse_str(input) } - /// Create a array of `Schema`'s from a list of JSON Avro schemas. It is allowed that - /// the schemas have cross-dependencies; these will be resolved during parsing. + /// Create a array of `Schema`'s from a list of named JSON Avro schemas (Record, Enum, and + /// Fixed). + /// + /// It is allowed that the schemas have cross-dependencies; these will be resolved + /// during parsing. + /// + /// If two of the input schemas have the same fullname, an Error will be returned. pub fn parse_list(input: &[&str]) -> Result, Error> { let mut input_schemas: HashMap = HashMap::with_capacity(input.len()); + let mut input_order: Vec = Vec::with_capacity(input.len()); for js in input { let schema: Value = serde_json::from_str(js).map_err(Error::ParseSchemaJson)?; if let Value::Object(inner) = &schema { let fullname = Name::parse(&inner)?.fullname(None); - let _ = input_schemas.insert(fullname, schema); + let previous_value = input_schemas.insert(fullname.clone(), schema); + if previous_value.is_some() { + return Err(Error::NameCollision(fullname)) + } + let _ = input_order.push(fullname); } else { return Err(Error::GetNameField); } } let mut parser = Parser { input_schemas, + input_order, parsed_schemas: HashMap::with_capacity(input.len()), }; parser.parse_list() @@ -449,7 +461,7 @@ impl Parser { self.parse(&value) } - /// Create a array of `Schema`'s from a an iterator of JSON Avro schemas. It is allowed that + /// Create an array of `Schema`'s from an iterator of JSON Avro schemas. It is allowed that /// the schemas have cross-dependencies; these will be resolved during parsing. fn parse_list(&mut self) -> Result, Error> { while !self.input_schemas.is_empty() { @@ -467,8 +479,10 @@ impl Parser { self.parsed_schemas.insert(name, parsed); } - let mut parsed_schemas = Vec::new(); - for (_, parsed) in self.parsed_schemas.drain() { + let mut parsed_schemas = Vec::with_capacity(self.parsed_schemas.len()); + for name in self.input_order.drain(0..) { + let parsed = self.parsed_schemas.remove(&name) + .expect("One of the input schemas was unexpectedly not parsed"); parsed_schemas.push(parsed); } Ok(parsed_schemas) @@ -502,6 +516,13 @@ impl Parser { } } + /// Given a name, tries to retrieve the parsed schema from `parsed_schemas`. + /// If a parsed schema is not found, it checks if a json with that name exists + /// in `input_schemas` and then parses it (removing it from `input_schemas`) + /// and adds the parsed schema to `parsed_schemas` + /// + /// This method allows schemas definitions that depend on other types to + /// parse their dependencies (or look them up if already parsed). fn fetch_schema(&mut self, name: &str) -> AvroResult { if let Some(parsed) = self.parsed_schemas.get(name) { return Ok(parsed.clone()); diff --git a/tests/schema.rs b/tests/schema.rs index c6c107d..11b9ffa 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -637,8 +637,8 @@ fn test_parse_list_without_cross_deps() { }"#; let schema_str_2 = r#"{ "name": "B", - "type": "map", - "values": "double" + "type": "fixed", + "size": 16 }"#; let schema_strs = [schema_str_1, schema_str_2]; let schemas = Schema::parse_list(&schema_strs).expect("Test failed"); @@ -653,6 +653,7 @@ fn test_parse_list_without_cross_deps() { /// Test that the parsing of a list of schemas, whose definitions do depend on each other, can /// perform the necessary schema composition. This should work regardless of the order in which /// the schemas are input. +/// However, the output order is guaranteed to be the same as the input order. fn test_parse_list_with_cross_deps_basic() { let schema_str_1 = r#"{ "name": "A", @@ -663,32 +664,37 @@ fn test_parse_list_with_cross_deps_basic() { }"#; let schema_str_2 = r#"{ "name": "B", - "type": "map", - "values": "A" + "type": "record", + "fields": [ + {"name": "field_one", "type": "A"} + ] }"#; let schema_composite = r#"{ "name": "B", - "type": "map", - "values": { - "name": "A", - "type": "record", - "fields": [ - {"name": "field_one", "type": "float"} - ] - } + "type": "record", + "fields": [ + { "name": "field_one", + "type": { + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + } + } + ] + }"#; let schema_strs_first = [schema_str_1, schema_str_2]; let schema_strs_second = [schema_str_2, schema_str_1]; let schemas_first = Schema::parse_list(&schema_strs_first).expect("Test failed"); let schemas_second = Schema::parse_list(&schema_strs_second).expect("Test failed"); - for schema_str in &[schema_str_1, schema_composite] { - let parsed = Schema::parse_str(&schema_str).expect("Test failed"); - assert!(schemas_first.contains(&parsed)); - assert!(schemas_second.contains(&parsed)); - assert_eq!(schemas_first.len(), 2); - assert_eq!(schemas_second.len(), 2); - } + + let parsed_1 = Schema::parse_str(&schema_str_1).expect("Test failed"); + let parsed_2 = Schema::parse_str(&schema_composite).expect("Test failed"); + assert_eq!(schemas_first, vec!(parsed_1.clone(), parsed_2.clone())); + assert_eq!(schemas_second, vec!(parsed_2, parsed_1)); } #[test] @@ -704,33 +710,37 @@ fn test_parse_list_with_cross_deps_and_namespaces() { }"#; let schema_str_2 = r#"{ "name": "B", - "type": "map", - "values": "namespace.A" + "type": "record", + "fields": [ + {"name": "field_one", "type": "namespace.A"} + ] }"#; let schema_composite = r#"{ "name": "B", - "type": "map", - "values": { - "name": "A", - "type": "record", - "namespace": "namespace", - "fields": [ - {"name": "field_one", "type": "float"} - ] - } + "type": "record", + "fields": [ + { "name": "field_one", + "type": { + "name": "A", + "type": "record", + "namespace": "namespace", + "fields": [ + {"name": "field_one", "type": "float"} + ] + } + } + ] }"#; let schema_strs_first = [schema_str_1, schema_str_2]; let schema_strs_second = [schema_str_2, schema_str_1]; let schemas_first = Schema::parse_list(&schema_strs_first).expect("Test failed"); let schemas_second = Schema::parse_list(&schema_strs_second).expect("Test failed"); - for schema_str in &[schema_str_1, schema_composite] { - let parsed = Schema::parse_str(&schema_str).expect("Test failed"); - assert!(schemas_first.contains(&parsed)); - assert!(schemas_second.contains(&parsed)); - assert_eq!(schemas_first.len(), 2); - assert_eq!(schemas_second.len(), 2); - } + + let parsed_1 = Schema::parse_str(&schema_str_1).expect("Test failed"); + let parsed_2 = Schema::parse_str(&schema_composite).expect("Test failed"); + assert_eq!(schemas_first, vec!(parsed_1.clone(), parsed_2.clone())); + assert_eq!(schemas_second, vec!(parsed_2, parsed_1)); } #[test] @@ -746,8 +756,10 @@ fn test_parse_list_with_cross_deps_and_namespaces_error() { }"#; let schema_str_2 = r#"{ "name": "B", - "type": "map", - "values": "A" + "type": "record", + "fields": [ + {"name": "field_one", "type": "A"} + ] }"#; let schema_strs_first = [schema_str_1, schema_str_2]; @@ -806,8 +818,8 @@ fn test_parse_list_multiple_dependencies() { }"#; let schema_str_2 = r#"{ "name": "B", - "type": "array", - "items": "double" + "type": "fixed", + "size": 16 }"#; let schema_str_3 = r#"{ "name": "C", @@ -826,8 +838,8 @@ fn test_parse_list_multiple_dependencies() { "null", { "name": "B", - "type": "array", - "items": "double" + "type": "fixed", + "size": 16 }, { "name": "C", @@ -863,13 +875,17 @@ fn test_parse_list_multiple_dependencies() { fn test_parse_list_shared_dependency() { let schema_str_1 = r#"{ "name": "A", - "type": "map", - "values": "C" + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "array", "items": "C"}} + ] }"#; let schema_str_2 = r#"{ "name": "B", - "type": "array", - "items": "C" + "type": "record", + "fields": [ + {"name": "field_one", "type": {"type": "map", "values": "C"}} + ] }"#; let schema_str_3 = r#"{ "name": "C", @@ -880,25 +896,37 @@ fn test_parse_list_shared_dependency() { }"#; let schema_composite_1 = r#"{ "name": "A", - "type": "map", - "values": { - "name": "C", - "type": "record", - "fields": [ - {"name": "field_one", "type": "string"} - ] - } + "type": "record", + "fields": [ + { "name": "field_one", + "type": {"type": "array", + "items": { + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + } + } + } + ] }"#; let schema_composite_2 = r#"{ "name": "B", - "type": "array", - "items": { - "name": "C", - "type": "record", - "fields": [ - {"name": "field_one", "type": "string"} + "type": "record", + "fields": [ + { "name": "field_one", + "type": {"type": "map", + "values": { + "name": "C", + "type": "record", + "fields": [ + {"name": "field_one", "type": "string"} + ] + } + } + } ] - } }"#; let parsed = vec![ @@ -917,6 +945,29 @@ fn test_parse_list_shared_dependency() { } } +#[test] +/// Test that trying to parse two schemas with the same fullname returns an Error +fn test_name_collision_error() { + let schema_str_1 = r#"{ + "name": "foo.A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "double"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "A", + "type": "record", + "namespace": "foo", + "fields": [ + {"name": "field_two", "type": "string"} + ] + }"#; + + let _ = Schema::parse_list(&[schema_str_1, schema_str_2]).expect_err("Test failed"); + +} + // The fullname is determined in one of the following ways: // * A name and namespace are both specified. For example, // one might use "name": "X", "namespace": "org.foo" From e0b711683d50a20e5ae8f696cca21ad67a0333d4 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 09:30:14 +0100 Subject: [PATCH 07/12] Removed an unused import --- src/error.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index bed9579..4de9b80 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,5 @@ use crate::{schema::SchemaKind, types::ValueKind}; use std::fmt; -use crate::types::Value; #[derive(thiserror::Error, Debug)] pub enum Error { From 38382aa67997d0ca179ec802357c708ba8040e98 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 09:36:52 +0100 Subject: [PATCH 08/12] Small formatting fixes --- src/schema.rs | 6 ++++-- tests/schema.rs | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/schema.rs b/src/schema.rs index cb993d1..77d5537 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -432,7 +432,7 @@ impl Schema { let fullname = Name::parse(&inner)?.fullname(None); let previous_value = input_schemas.insert(fullname.clone(), schema); if previous_value.is_some() { - return Err(Error::NameCollision(fullname)) + return Err(Error::NameCollision(fullname)); } let _ = input_order.push(fullname); } else { @@ -481,7 +481,9 @@ impl Parser { let mut parsed_schemas = Vec::with_capacity(self.parsed_schemas.len()); for name in self.input_order.drain(0..) { - let parsed = self.parsed_schemas.remove(&name) + let parsed = self + .parsed_schemas + .remove(&name) .expect("One of the input schemas was unexpectedly not parsed"); parsed_schemas.push(parsed); } diff --git a/tests/schema.rs b/tests/schema.rs index 11b9ffa..8444c18 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -690,7 +690,6 @@ fn test_parse_list_with_cross_deps_basic() { let schemas_first = Schema::parse_list(&schema_strs_first).expect("Test failed"); let schemas_second = Schema::parse_list(&schema_strs_second).expect("Test failed"); - let parsed_1 = Schema::parse_str(&schema_str_1).expect("Test failed"); let parsed_2 = Schema::parse_str(&schema_composite).expect("Test failed"); assert_eq!(schemas_first, vec!(parsed_1.clone(), parsed_2.clone())); @@ -736,7 +735,6 @@ fn test_parse_list_with_cross_deps_and_namespaces() { let schemas_first = Schema::parse_list(&schema_strs_first).expect("Test failed"); let schemas_second = Schema::parse_list(&schema_strs_second).expect("Test failed"); - let parsed_1 = Schema::parse_str(&schema_str_1).expect("Test failed"); let parsed_2 = Schema::parse_str(&schema_composite).expect("Test failed"); assert_eq!(schemas_first, vec!(parsed_1.clone(), parsed_2.clone())); From 46f0cd529f69c929f878ce85eb4119072d3c8bb3 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 09:45:52 +0100 Subject: [PATCH 09/12] Added another test involving fullnames --- tests/schema.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/schema.rs b/tests/schema.rs index 8444c18..e7c62d8 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -963,7 +963,31 @@ fn test_name_collision_error() { }"#; let _ = Schema::parse_list(&[schema_str_1, schema_str_2]).expect_err("Test failed"); +} +#[test] +/// Test that having the same name but different fullnames does not return an error +fn test_namespace_prevents_collisions() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "double"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "A", + "type": "record", + "namespace": "foo", + "fields": [ + {"name": "field_two", "type": "string"} + ] + }"#; + + let parsed = Schema::parse_list(&[schema_str_1, schema_str_2]).expect("Test failed"); + let parsed_1 = Schema::parse_str(&schema_str_1).expect("Test failed"); + let parsed_2 = Schema::parse_str(&schema_str_2).expect("Test failed"); + assert_eq!(parsed, vec!(parsed_1, parsed_2)); } // The fullname is determined in one of the following ways: From 5f60f56c0c6fc8e790268320d4bbebc923594b52 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 13:02:53 +0100 Subject: [PATCH 10/12] Added test that ensures when a recursive type is put in, the algorithm does blow the top off the stack but returns an error --- tests/schema.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/schema.rs b/tests/schema.rs index e7c62d8..fbc8534 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -696,6 +696,47 @@ fn test_parse_list_with_cross_deps_basic() { assert_eq!(schemas_second, vec!(parsed_2, parsed_1)); } +#[test] +/// Test that if a cycle of dependencies occurs in the input schema jsons, the algorithm terminates +/// and returns an error. N.B. In the future, when recursive types are supported, this should be +/// revisited. +fn test_parse_list_recursive_type_error() { + let schema_str_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "B"} + ] + }"#; + let schema_str_2 = r#"{ + "name": "B", + "type": "record", + "fields": [ + {"name": "field_one", "type": "A"} + ] + }"#; + let schema_composite = r#"{ + "name": "B", + "type": "record", + "fields": [ + { "name": "field_one", + "type": { + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + } + } + ] + + }"#; + let schema_strs_first = [schema_str_1, schema_str_2]; + let schema_strs_second = [schema_str_2, schema_str_1]; + let _ = Schema::parse_list(&schema_strs_first).expect_err("Test failed"); + let _ = Schema::parse_list(&schema_strs_second).expect_err("Test failed"); +} + #[test] /// Test that schema composition resolves namespaces. fn test_parse_list_with_cross_deps_and_namespaces() { From 064c61459346def884149858315556fe017e8732 Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 13:11:28 +0100 Subject: [PATCH 11/12] removed unused variable --- tests/schema.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/schema.rs b/tests/schema.rs index fbc8534..7d38616 100644 --- a/tests/schema.rs +++ b/tests/schema.rs @@ -715,22 +715,6 @@ fn test_parse_list_recursive_type_error() { {"name": "field_one", "type": "A"} ] }"#; - let schema_composite = r#"{ - "name": "B", - "type": "record", - "fields": [ - { "name": "field_one", - "type": { - "name": "A", - "type": "record", - "fields": [ - {"name": "field_one", "type": "float"} - ] - } - } - ] - - }"#; let schema_strs_first = [schema_str_1, schema_str_2]; let schema_strs_second = [schema_str_2, schema_str_1]; let _ = Schema::parse_list(&schema_strs_first).expect_err("Test failed"); From b4bc76794a44a40542392465f836b53be2b828fb Mon Sep 17 00:00:00 2001 From: R2D2 Date: Fri, 29 Jan 2021 20:09:47 +0100 Subject: [PATCH 12/12] Updated readme and one tiny fix --- README.md | 32 ++++++++++++++++++++++++++++++++ src/lib.rs | 32 ++++++++++++++++++++++++++++++++ src/schema.rs | 2 +- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b1296b8..04051b9 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,38 @@ let schema = Schema::parse_str(raw_schema).unwrap(); println!("{:?}", schema); ``` +Additionally, a list of of definitions (which may depend on each other) can be given and all of +them will be parsed into the corresponding schemas. + +```rust +use avro_rs::Schema; + +let raw_schema_1 = r#"{ + "name": "A", + "type": "record", + "fields": [ + {"name": "field_one", "type": "float"} + ] + }"#; + +// This definition depends on the definition of A above +let raw_schema_2 = r#"{ + "name": "B", + "type": "record", + "fields": [ + {"name": "field_one", "type": "A"} + ] + }"#; + +// if the schemas are not valid, this function will return an error +let schemas = Schema::parse_list(&[raw_schema_1, raw_schema_2]).unwrap(); + +// schemas can be printed for debugging +println!("{:?}", schemas); +``` +*N.B.* It is important to note that the composition of schema definitions requires schemas with names. +For this reason, only schemas of type Record, Enum, and Fixed should be input into this function. + The library provides also a programmatic interface to define schemas without encoding them in JSON (for advanced use), but we highly recommend the JSON interface. Please read the API reference in case you are interested. diff --git a/src/lib.rs b/src/lib.rs index 215c294..d9164f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -79,6 +79,38 @@ //! println!("{:?}", schema); //! ``` //! +//! Additionally, a list of of definitions (which may depend on each other) can be given and all of +//! them will be parsed into the corresponding schemas. +//! +//! ``` +//! use avro_rs::Schema; +//! +//! let raw_schema_1 = r#"{ +//! "name": "A", +//! "type": "record", +//! "fields": [ +//! {"name": "field_one", "type": "float"} +//! ] +//! }"#; +//! +//! // This definition depends on the definition of A above +//! let raw_schema_2 = r#"{ +//! "name": "B", +//! "type": "record", +//! "fields": [ +//! {"name": "field_one", "type": "A"} +//! ] +//! }"#; +//! +//! // if the schemas are not valid, this function will return an error +//! let schemas = Schema::parse_list(&[raw_schema_1, raw_schema_2]).unwrap(); +//! +//! // schemas can be printed for debugging +//! println!("{:?}", schemas); +//! ``` +//! *N.B.* It is important to note that the composition of schema definitions requires schemas with names. +//! For this reason, only schemas of type Record, Enum, and Fixed should be input into this function. +//! //! The library provides also a programmatic interface to define schemas without encoding them in //! JSON (for advanced use), but we highly recommend the JSON interface. Please read the API //! reference in case you are interested. diff --git a/src/schema.rs b/src/schema.rs index 77d5537..cacfbda 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -434,7 +434,7 @@ impl Schema { if previous_value.is_some() { return Err(Error::NameCollision(fullname)); } - let _ = input_order.push(fullname); + input_order.push(fullname); } else { return Err(Error::GetNameField); }