From 99721a9ad55d5b418c7c7fc8816cce26820d83b3 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 29 Oct 2020 19:27:35 +0100 Subject: [PATCH] Refactor spec file (#75) * Refactor spec file * 197 Co-authored-by: Cameron Taggart --- codegen/src/codegen.rs | 38 ++-- codegen/src/spec.rs | 244 ++++++++++++-------------- codegen/tests/azure_rest_api_specs.rs | 20 +-- 3 files changed, 145 insertions(+), 157 deletions(-) diff --git a/codegen/src/codegen.rs b/codegen/src/codegen.rs index 9d625b9..dc3283c 100644 --- a/codegen/src/codegen.rs +++ b/codegen/src/codegen.rs @@ -57,23 +57,6 @@ impl CodeGen { self.config.api_version.as_deref() } - // For create_models. Recursively adds schema refs. - fn add_schema_refs(&self, schemas: &mut IndexMap, doc_file: &Path, schema_ref: &str) -> Result<()> { - let schema = self.spec.resolve_schema_ref(doc_file, schema_ref).context(SpecError)?; - if let Some(ref_key) = schema.ref_key.clone() { - if !schemas.contains_key(&ref_key) { - if !self.spec.is_input_file(&ref_key.file) { - let refs = get_schema_schema_refs(&schema.schema); - schemas.insert(ref_key.clone(), schema); - for rf in refs { - self.add_schema_refs(schemas, &ref_key.file, &rf)?; - } - } - } - } - Ok(()) - } - pub fn create_models(&self) -> Result { let mut file = TokenStream::new(); file.extend(create_generated_by_header()); @@ -104,7 +87,7 @@ impl CodeGen { for (doc_file, doc) in &self.spec.docs { if self.spec.is_input_file(doc_file) { for rf in get_api_schema_refs(doc) { - self.add_schema_refs(&mut all_schemas, doc_file, &rf)?; + self.add_schema_refs(&mut all_schemas, doc_file, Reference::parse(&rf))?; } } } @@ -152,7 +135,7 @@ impl CodeGen { let paths = self.spec.resolve_path_map(doc_file, &doc.paths).context(SpecError)?; for (path, item) in &paths { // println!("{}", path); - for op in spec::pathitem_operations(item) { + for op in spec::path_item_operations(item) { // println!("{:?}", op.operation_id); let (module_name, function_name) = op.function_name(path); let function = create_function(self, doc_file, path, item, &op, ¶m_re, &function_name)?; @@ -191,6 +174,23 @@ impl CodeGen { Ok(file) } + // For create_models. Recursively adds schema refs. + fn add_schema_refs(&self, schemas: &mut IndexMap, doc_file: &Path, schema_ref: Reference) -> Result<()> { + let schema = self.spec.resolve_schema_ref(doc_file, schema_ref).context(SpecError)?; + if let Some(ref_key) = schema.ref_key.clone() { + if !schemas.contains_key(&ref_key) { + if !self.spec.is_input_file(&ref_key.file) { + let refs = get_schema_schema_refs(&schema.schema); + schemas.insert(ref_key.clone(), schema); + for rf in refs { + self.add_schema_refs(schemas, &ref_key.file, Reference::parse(&rf))?; + } + } + } + } + Ok(()) + } + fn create_vec_alias(&self, doc_file: &Path, alias_name: &str, schema: &ResolvedSchema) -> Result { let items = get_schema_array_items(&schema.schema.common)?; let typ = ident(&alias_name.to_camel_case()); diff --git a/codegen/src/spec.rs b/codegen/src/spec.rs index 0f515fc..001d141 100644 --- a/codegen/src/spec.rs +++ b/codegen/src/spec.rs @@ -10,6 +10,7 @@ use std::{ }; pub type Result = std::result::Result; + #[derive(Debug, Snafu)] pub enum Error { PathJoin { source: path::Error }, @@ -25,40 +26,25 @@ pub enum Error { /// An API specification #[derive(Clone, Debug)] pub struct Spec { - /// Documents for an API specification where the first one is the root + /// A store of all the documents for an API specification keyed on their file paths where the first one is the root document pub docs: IndexMap, schemas: IndexMap, parameters: IndexMap, - input_files: IndexSet, -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct RefKey { - pub file: PathBuf, - pub name: String, -} - -pub struct ResolvedSchema { - pub ref_key: Option, - pub schema: Schema, + input_files_paths: IndexSet, } impl Spec { - pub fn is_input_file>(&self, path: P) -> bool { - self.input_files.contains(path.as_ref()) - } - - pub fn read_files>(input_files: &[P]) -> Result { + pub fn read_files>(input_files_paths: &[P]) -> Result { let mut docs: IndexMap = IndexMap::new(); - for input_file in input_files { - let doc = read_api_file(&input_file)?; - let files = get_ref_files(&doc)?; + for input_file in input_files_paths { + let doc = openapi::parse(&input_file)?; + let ref_files = openapi::get_ref_files(&doc); docs.insert(input_file.as_ref().to_owned(), doc); - for file in files { - let doc_path = path::join(&input_file, &file).context(PathJoin)?; + for ref_file in ref_files { + let doc_path = path::join(&input_file, &ref_file).context(PathJoin)?; if !docs.contains_key(&doc_path) { - let doc = read_api_file(&doc_path)?; + let doc = openapi::parse(&doc_path)?; docs.insert(doc_path, doc); } } @@ -66,15 +52,14 @@ impl Spec { let mut schemas: IndexMap = IndexMap::new(); let mut parameters: IndexMap = IndexMap::new(); - for (file, doc) in &docs { + for (path, doc) in &docs { for (name, schema) in &doc.definitions { match schema { ReferenceOr::Reference { .. } => {} ReferenceOr::Item(schema) => { - // println!("insert schema {} {}", &file, &name); schemas.insert( RefKey { - file: file.clone(), + file: path.clone(), name: name.clone(), }, schema.clone(), @@ -82,11 +67,11 @@ impl Spec { } } } + for (name, param) in &doc.parameters { - // println!("insert parameter {} {}", &file, name); parameters.insert( RefKey { - file: file.clone(), + file: path.clone(), name: name.clone(), }, param.clone(), @@ -94,103 +79,92 @@ impl Spec { } } - Ok(Spec { + Ok(Self { docs, schemas, parameters, - input_files: input_files.iter().map(|f| f.as_ref().to_owned()).collect(), + input_files_paths: input_files_paths.iter().map(|f| f.as_ref().to_owned()).collect(), }) } - pub fn resolve_schema_ref>(&self, doc_file: P, reference: &str) -> Result { - let doc_file: PathBuf = doc_file.into(); - let rf = Reference::parse(reference); - let file = match rf.file { - None => doc_file.to_owned(), - Some(file) => path::join(doc_file, &file).context(PathJoin)?, + pub fn is_input_file>(&self, path: P) -> bool { + self.input_files_paths.contains(path.as_ref()) + } + + /// Find the schema for a given doc path and reference + pub fn resolve_schema_ref>(&self, doc_path: P, reference: Reference) -> Result { + let doc_path: PathBuf = doc_path.into(); + let full_path = match reference.file { + None => doc_path, + Some(file) => path::join(doc_path, &file).context(PathJoin)?, }; - match rf.name { - // None => Err(format!("no name in reference {}", &reference))?, - None => NoNameInReference.fail(), - Some(nm) => { - let ref_key = RefKey { - file: file.clone(), - name: nm.clone(), - }; - let schema = self - .schemas - .get(&ref_key) - // .ok_or_else(|| format!("schema not found {} {}", &file.display(), &nm))? - .context(SchemaNotFound)? - .clone(); - Ok(ResolvedSchema { - ref_key: Some(ref_key), - schema, - }) - } - } + + let name = reference.name.ok_or_else(|| Error::NoNameInReference)?; + let ref_key = RefKey { file: full_path, name }; + let schema = self.schemas.get(&ref_key).context(SchemaNotFound)?.clone(); + Ok(ResolvedSchema { + ref_key: Some(ref_key), + schema, + }) } - pub fn resolve_parameter_ref(&self, doc_file: &Path, reference: &str) -> Result { - let rf = Reference::parse(reference); - let file = match rf.file { - None => doc_file.to_owned(), - Some(file) => path::join(doc_file, &file).context(PathJoin)?, + /// Find the parameter for a given doc path and reference + pub fn resolve_parameter_ref>(&self, doc_path: P, reference: Reference) -> Result { + let doc_path: PathBuf = doc_path.into(); + let full_path = match reference.file { + None => doc_path, + Some(file) => path::join(doc_path, &file).context(PathJoin)?, }; - match rf.name { - // None => Err(format!("no name in reference {}", &reference))?, - None => NoNameInReference.fail(), - Some(nm) => Ok(self - .parameters - .get(&RefKey { - file: file.clone(), - name: nm.clone(), - }) - // .ok_or_else(|| format!("parameter not found {} {}", &file.display(), &nm))? - .context(ParameterNotFound)? - .clone()), - } + let name = reference.name.ok_or_else(|| Error::NoNameInReference)?; + Ok(self + .parameters + .get(&RefKey { file: full_path, name }) + .context(ParameterNotFound)? + .clone()) } - pub fn resolve_schema(&self, doc_file: &Path, schema: &ReferenceOr) -> Result { - match schema { + /// Resolve a reference or schema to a resolved schema + pub fn resolve_schema>(&self, doc_path: P, ref_or_schema: &ReferenceOr) -> Result { + match ref_or_schema { ReferenceOr::Item(schema) => Ok(ResolvedSchema { ref_key: None, schema: schema.clone(), }), - ReferenceOr::Reference { reference, .. } => self.resolve_schema_ref(doc_file, reference), + ReferenceOr::Reference { reference, .. } => self.resolve_schema_ref(doc_path.as_ref(), Reference::parse(reference)), } } - pub fn resolve_schemas(&self, doc_file: &Path, schemas: &Vec>) -> Result> { + /// Resolve a collection of references or schemas to a collection of resolved schemas + pub fn resolve_schemas>(&self, doc_path: P, ref_or_schemas: &[ReferenceOr]) -> Result> { let mut resolved = Vec::new(); - for schema in schemas { - resolved.push(self.resolve_schema(doc_file, schema)?); + for schema in ref_or_schemas { + resolved.push(self.resolve_schema(&doc_path, schema)?); } Ok(resolved) } - pub fn resolve_schema_map( + /// Resolve a collection of references or schemas to a collection of resolved schemas + pub fn resolve_schema_map>( &self, - doc_file: &Path, - schemas: &IndexMap>, + doc_path: P, + ref_or_schemas: &IndexMap>, ) -> Result> { let mut resolved = IndexMap::new(); - for (name, schema) in schemas { - resolved.insert(name.clone(), self.resolve_schema(doc_file, schema)?); + for (name, schema) in ref_or_schemas { + resolved.insert(name.clone(), self.resolve_schema(&doc_path, schema)?); } Ok(resolved) } - pub fn resolve_path(&self, _doc_file: &Path, path: &ReferenceOr) -> Result { + pub fn resolve_path>(&self, _doc_path: P, path: &ReferenceOr) -> Result { match path { ReferenceOr::Item(path) => Ok(path.clone()), ReferenceOr::Reference { .. } => // self.resolve_path_ref(doc_file, reference), { - // Err("path references not implemented")? + // TODO NotImplemented.fail() - } // TODO + } } } @@ -205,7 +179,7 @@ impl Spec { pub fn resolve_parameter(&self, doc_file: &Path, parameter: &ReferenceOr) -> Result { match parameter { ReferenceOr::Item(param) => Ok(param.clone()), - ReferenceOr::Reference { reference, .. } => self.resolve_parameter_ref(doc_file, reference), + ReferenceOr::Reference { reference, .. } => self.resolve_parameter_ref(doc_file, Reference::parse(reference)), } } @@ -218,16 +192,43 @@ impl Spec { } } -pub fn read_api_file>(path: P) -> Result { - let path = path.as_ref(); - let bytes = fs::read(path).context(ReadFile)?; - let api = if path.extension() == Some(OsStr::new("yaml")) || path.extension() == Some(OsStr::new("yml")) { - serde_yaml::from_slice(&bytes).context(DeserializeYaml)? - } else { - serde_json::from_slice(&bytes).context(DeserializeJson)? - }; +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RefKey { + pub file: PathBuf, + pub name: String, +} + +pub struct ResolvedSchema { + pub ref_key: Option, + pub schema: Schema, +} - Ok(api) +pub mod openapi { + use super::*; + + /// Parse an OpenAPI object from a file located at `path` + pub fn parse>(path: P) -> Result { + let path = path.as_ref(); + let bytes = fs::read(path).context(ReadFile)?; + let api = if path.extension() == Some(OsStr::new("yaml")) || path.extension() == Some(OsStr::new("yml")) { + serde_yaml::from_slice(&bytes).context(DeserializeYaml)? + } else { + serde_json::from_slice(&bytes).context(DeserializeJson)? + }; + + Ok(api) + } + + /// Returns a set of referenced relative file paths + pub fn get_ref_files(api: &OpenAPI) -> IndexSet { + get_refs(api) + .iter() + .filter_map(|rf| match rf { + RefString::Example(_) => None, + rs => Reference::parse(rs.as_str()).file, + }) + .collect() + } } pub enum OperationVerb<'a> { @@ -294,7 +295,7 @@ fn create_function_name(path: &str, verb_name: &str) -> String { path.join("_") } -pub fn pathitem_operations(item: &PathItem) -> impl Iterator { +pub fn path_item_operations(item: &PathItem) -> impl Iterator { vec![ item.get.as_ref().map(OperationVerb::Get), item.post.as_ref().map(OperationVerb::Post), @@ -317,17 +318,23 @@ pub enum RefString { Example(String), } -impl ToString for RefString { - fn to_string(&self) -> String { +impl RefString { + fn as_str(&self) -> &str { match self { - RefString::PathItem(s) => s.to_owned(), - RefString::Parameter(s) => s.to_owned(), - RefString::Schema(s) => s.to_owned(), - RefString::Example(s) => s.to_owned(), + RefString::PathItem(s) => s, + RefString::Parameter(s) => s, + RefString::Schema(s) => s, + RefString::Example(s) => s, } } } +impl ToString for RefString { + fn to_string(&self) -> String { + self.as_str().to_owned() + } +} + fn add_refs_for_schema(list: &mut Vec, schema: &Schema) { for (_, schema) in &schema.properties { match schema { @@ -369,11 +376,11 @@ pub fn get_refs(api: &OpenAPI) -> Vec { match item { ReferenceOr::Reference { reference, .. } => list.push(RefString::PathItem(reference.clone())), ReferenceOr::Item(item) => { - for verb in pathitem_operations(&item) { + for verb in path_item_operations(&item) { let op = verb.operation(); // parameters - for prm in &op.parameters { - match prm { + for param in &op.parameters { + match param { ReferenceOr::Reference { reference, .. } => list.push(RefString::Parameter(reference.clone())), ReferenceOr::Item(parameter) => match ¶meter.schema { Some(ReferenceOr::Reference { reference, .. }) => list.push(RefString::Schema(reference.clone())), @@ -415,25 +422,6 @@ pub fn get_refs(api: &OpenAPI) -> Vec { list } -/// returns a set of referenced files -pub fn get_ref_files(api: &OpenAPI) -> Result> { - let ref_strings: IndexSet<_> = get_refs(api) - .iter() - .filter_map(|rf| match rf { - RefString::Example(_) => None, - rs => Some(rs.to_string()), - }) - .collect(); - - let mut set = IndexSet::new(); - for s in &ref_strings { - if let Some(file) = Reference::parse(s).file { - set.insert(file); - } - } - Ok(set) -} - pub fn get_api_schema_refs(api: &OpenAPI) -> Vec { get_refs(api) .iter() diff --git a/codegen/tests/azure_rest_api_specs.rs b/codegen/tests/azure_rest_api_specs.rs index 56925f8..6511274 100644 --- a/codegen/tests/azure_rest_api_specs.rs +++ b/codegen/tests/azure_rest_api_specs.rs @@ -11,7 +11,7 @@ pub type Result = std::result::Result; #[test] fn refs_count_security_common() -> Result<()> { - let api = &spec::read_api_file("../../azure-rest-api-specs/specification/security/resource-manager/common/v1/types.json")?; + let api = &spec::openapi::parse("../../azure-rest-api-specs/specification/security/resource-manager/common/v1/types.json")?; let refs = spec::get_refs(api); assert_eq!(13, refs.len()); Ok(()) @@ -19,20 +19,20 @@ fn refs_count_security_common() -> Result<()> { #[test] fn refs_count_avs() -> Result<()> { - let api = &spec::read_api_file( + let api = &spec::openapi::parse( "../../azure-rest-api-specs/specification/vmware/resource-manager/Microsoft.AVS/stable/2020-03-20/vmware.json", )?; let refs = spec::get_refs(api); - assert_eq!(190, refs.len()); + assert_eq!(197, refs.len()); Ok(()) } #[test] fn ref_files() -> Result<()> { - let api = &spec::read_api_file( + let api = &spec::openapi::parse( "../../azure-rest-api-specs/specification/vmware/resource-manager/Microsoft.AVS/stable/2020-03-20/vmware.json", )?; - let files = spec::get_ref_files(api)?; + let files = spec::openapi::get_ref_files(api); assert_eq!(1, files.len()); assert!(files.contains("../../../../../common-types/resource-management/v1/types.json")); Ok(()) @@ -55,10 +55,10 @@ fn test_resolve_schema_ref() -> Result<()> { let file = PathBuf::from("../../azure-rest-api-specs/specification/vmware/resource-manager/Microsoft.AVS/stable/2020-03-20/vmware.json"); let spec = &Spec::read_files(&[&file])?; - spec.resolve_schema_ref(&file, "#/definitions/OperationList")?; + spec.resolve_schema_ref(&file, Reference::parse("#/definitions/OperationList"))?; spec.resolve_schema_ref( &file, - "../../../../../common-types/resource-management/v1/types.json#/definitions/ErrorResponse", + Reference::parse("../../../../../common-types/resource-management/v1/types.json#/definitions/ErrorResponse"), )?; Ok(()) } @@ -70,7 +70,7 @@ fn test_resolve_parameter_ref() -> Result<()> { let spec = &Spec::read_files(&[&file])?; spec.resolve_parameter_ref( &file, - "../../../../../common-types/resource-management/v1/types.json#/parameters/ApiVersionParameter", + Reference::parse("../../../../../common-types/resource-management/v1/types.json#/parameters/ApiVersionParameter"), )?; Ok(()) } @@ -87,10 +87,10 @@ fn test_resolve_all_refs() -> Result<()> { RefString::PathItem(_) => {} RefString::Example(_) => {} RefString::Parameter(reference) => { - spec.resolve_parameter_ref(&doc_file, &reference)?; + spec.resolve_parameter_ref(&doc_file, Reference::parse(&reference))?; } RefString::Schema(reference) => { - spec.resolve_schema_ref(&doc_file, &reference)?; + spec.resolve_schema_ref(&doc_file, Reference::parse(&reference))?; } } }