From 30b65c223faf8ca7df2c8ba70cb26495e4b96f6f Mon Sep 17 00:00:00 2001 From: Yuji Mise Date: Mon, 10 Feb 2025 09:45:56 +0900 Subject: [PATCH] feat(engine): implement AttributeFlattener processor for city GML attributes transformation (#841) * feat(engine): implement AttributeFlattener processor for city GML attributes transformation * feat(engine): add AttributeValue processing for nusamai city GML attributes --- .../src/plateau4/attribute_flattener.rs | 593 +----------------- .../plateau4/attribute_flattener/constants.rs | 288 +++++++++ .../plateau4/attribute_flattener/flattener.rs | 297 +++++++++ .../plateau4/attribute_flattener/processor.rs | 148 +++++ .../src/plateau4/mapping.rs | 4 +- .../src/feature/reader/citygml.rs | 16 +- .../action-source/src/file/reader/citygml.rs | 15 +- .../plateau4/01-bldg/workflow.yml | 61 +- engine/runtime/types/src/attribute.rs | 105 ++++ .../runtime/types/src/conversion/nusamai.rs | 157 ++++- 10 files changed, 1086 insertions(+), 598 deletions(-) create mode 100644 engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/constants.rs create mode 100644 engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/flattener.rs create mode 100644 engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/processor.rs diff --git a/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener.rs b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener.rs index 524a1183d..34239b3f4 100644 --- a/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener.rs +++ b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener.rs @@ -1,590 +1,3 @@ -use std::collections::{HashMap, HashSet}; - -use itertools::Itertools; -use once_cell::sync::Lazy; -use reearth_flow_runtime::{ - channels::ProcessorChannelForwarder, - errors::BoxedError, - event::EventHub, - executor_operation::{ExecutorContext, NodeContext}, - node::{Port, Processor, ProcessorFactory, DEFAULT_PORT}, -}; -use reearth_flow_types::{Attribute, AttributeValue}; -use serde_json::Value; - -use super::errors::PlateauProcessorError; - -const DELIM: &str = "_"; - -static FLATTEN_PREFIXES: Lazy> = Lazy::new(|| { - let flatten_attributes = vec![ - ("gml:name", ""), - ("bldg:class", ""), - ("bldg:usage", ""), - ("bldg:yearOfConstruction", ""), - ("bldg:measuredHeight", ""), - ("bldg:storeysAboveGround", ""), - ("bldg:storeysBelowGround", ""), - ("bldg:address", ""), - ("uro:buildingIDAttribute", "uro:buildingID"), - ("uro:buildingIDAttribute", "uro:branchID"), - ("uro:buildingIDAttribute", "uro:partID"), - ("uro:buildingIDAttribute", "uro:prefecture"), - ("uro:buildingIDAttribute", "uro:city"), - ( - "uro:buildingDetailAttribute", - "uro:serialNumberOfBuildingCertification", - ), - ("uro:buildingDetailAttribute", "uro:siteArea"), - ("uro:buildingDetailAttribute", "uro:totalFloorArea"), - ("uro:buildingDetailAttribute", "uro:buildingFootprintArea"), - ("uro:buildingDetailAttribute", "uro:buildingRoofEdgeArea"), - ("uro:buildingDetailAttribute", "uro:buildingStructureType"), - ("uro:buildingDetailAttribute", "uro:fireproofStructureType"), - ("uro:buildingDetailAttribute", "uro:urbanPlanType"), - ("uro:buildingDetailAttribute", "uro:areaClassificationType"), - ("uro:buildingDetailAttribute", "uro:districtsAndZonesType"), - ("uro:buildingDetailAttribute", "uro:landUseType"), - ("uro:buildingDetailAttribute", "uro:vacancy"), - ("uro:buildingDetailAttribute", "uro:buildingCoverageRate"), - ("uro:buildingDetailAttribute", "uro:floorAreaRate"), - ( - "uro:buildingDetailAttribute", - "uro:specifiedBuildingCoverageRate", - ), - ("uro:buildingDetailAttribute", "uro:specifiedFloorAreaRate"), - ("uro:buildingDetailAttribute", "uro:standardFloorAreaRate"), - ("uro:buildingDetailAttribute", "uro:buildingHeight"), - ("uro:buildingDetailAttribute", "uro:eaveHeight"), - ("uro:buildingDetailAttribute", "uro:surveyYear"), - ("uro:largeCustomerFacilityAttribute", "uro:class"), - ("uro:largeCustomerFacilityAttribute", "uro:name"), - ("uro:largeCustomerFacilityAttribute", "uro:capacity"), - ("uro:largeCustomerFacilityAttribute", "uro:totalFloorArea"), - ("uro:largeCustomerFacilityAttribute", "uro:inauguralDate"), - ("uro:largeCustomerFacilityAttribute", "uro:yearOpened"), - ("uro:largeCustomerFacilityAttribute", "uro:yearClosed"), - ("uro:largeCustomerFacilityAttribute", "uro:urbanPlanType"), - ( - "uro:largeCustomerFacilityAttribute", - "uro:areaClassificationType", - ), - ( - "uro:largeCustomerFacilityAttribute", - "uro:districtsAndZonesType", - ), - ("uro:largeCustomerFacilityAttribute", "uro:landUseType"), - ("uro:largeCustomerFacilityAttribute", "uro:surveyYear"), - ("uro:buildingDataQualityAttribute", "uro:lod1HeightType"), - ("uro:realEstateIDAttribute", "uro:realEstateIDOfBuilding"), - ("uro:realEstateIDAttribute", "uro:matchingScore"), - ]; - let mut flatten_prefixes = HashSet::new(); - flatten_attributes.iter().for_each(|(k1, k2)| { - flatten_prefixes.insert(k1.to_string()); - if !k2.is_empty() { - flatten_prefixes.insert(format!("{}{}{}", k1, DELIM, k2)); - } - }); - flatten_prefixes -}); - -#[derive(Debug, Clone, Default)] -struct Flattener { - risk_to_attr_defs: HashMap>, -} - -impl Flattener { - fn new() -> Self { - Flattener { - risk_to_attr_defs: HashMap::new(), - } - } - - fn extract_fld_risk_attribute( - &mut self, - city_gml_attribute: &HashMap, - ) -> HashMap { - let disaster_risks = match city_gml_attribute.get("uro:buildingDisasterRiskAttribute") { - Some(AttributeValue::Array(disaster_risks)) => disaster_risks, - _ => return HashMap::new(), - }; - let mut result = HashMap::new(); - for risk in disaster_risks { - let risk_map = match risk { - AttributeValue::Map(risk_map) => risk_map, - _ => continue, - }; - let desc = risk_map.get("uro:description").map(|v| v.to_string()); - let admin = risk_map.get("uro:adminType").map(|v| v.to_string()); - let scale = risk_map.get("uro:scale").map(|v| v.to_string()); - - if desc.is_none() || admin.is_none() || scale.is_none() { - continue; - } - - let basename = format!( - "{}（{}管理区間）_{}", - desc.unwrap(), - admin.unwrap(), - scale.unwrap() - ); - - let rank = risk_map - .get("uro:rank") - .or_else(|| risk_map.get("uro:rank")); - let depth = risk_map.get("uro:depth"); - let duration = risk_map.get("uro:duration"); - - let attrib = vec![ - ("浸水ランク", rank), - ("浸水深", depth), - ("浸水継続時間", duration), - ]; - - for (k, v) in attrib { - let value = match v { - Some(value) => value, - None => continue, - }; - let name = format!("{}{}{}", basename, DELIM, k); - let attribute_name = Attribute::new(name.clone()); - result.insert(attribute_name, value.clone()); - self.risk_to_attr_defs.entry("fld".to_string()).or_default(); - } - } - result - } - - pub fn extract_tnm_htd_ifld_risk_attribute( - &mut self, - city_gml_attribute: &HashMap, - ) -> HashMap { - let mut result = HashMap::new(); - let src = vec![ - ("uro:buildingTsunamiRiskAttribute", "津波浸水想定", "tnm"), - ("uro:buildingHighTideRiskAttribute", "高潮浸水想定", "htd"), - ( - "uro:buildingInlandFloodingRiskAttribute", - "内水浸水想定", - "ifld", - ), - ]; - - for (tag, title, package) in src { - let disaster_risks = match city_gml_attribute.get(tag) { - Some(AttributeValue::Array(disaster_risks)) => disaster_risks, - _ => continue, - }; - - for risk in disaster_risks { - let risk_map = match risk { - AttributeValue::Map(risk_map) => risk_map, - _ => continue, - }; - - let desc = risk_map.get("uro:description").map(|v| v.to_string()); - if desc.is_none() { - continue; - } - - let basename = format!("{}_{}", title, desc.unwrap()); - - let rank = risk_map - .get("uro:rank") - .or_else(|| risk_map.get("uro:rankOrg")); - let rank_code = risk_map - .get("uro:rank_code") - .or_else(|| risk_map.get("uro:rankOrg_code")); - let depth = risk_map.get("uro:depth"); - - let attrib = vec![ - ("浸水ランク", rank), - ("浸水ランクコード", rank_code), - ("浸水深", depth), - ]; - - for (k, v) in attrib { - let value = match v { - Some(value) => value, - None => continue, - }; - let name = format!("{}_{}", basename, k); - let attribute_name = Attribute::new(name.clone()); - result.insert(attribute_name, value.clone()); - self.risk_to_attr_defs - .entry(package.to_string()) - .or_default(); - } - } - } - result - } - - pub fn extract_lsld_risk_attribute( - &mut self, - city_gml_attribute: &HashMap, - ) -> HashMap { - let mut result = HashMap::new(); - let disaster_risks = match city_gml_attribute.get("uro:buildingLandSlideRiskAttribute") { - Some(AttributeValue::Array(disaster_risks)) => disaster_risks, - _ => return HashMap::new(), - }; - for risk in disaster_risks { - let risk_map = match risk { - AttributeValue::Map(risk_map) => risk_map, - _ => continue, - }; - let desc = risk_map.get("uro:description"); - let area_type = risk_map.get("uro:areaType"); - let area_type_code = risk_map.get("uro:areaType_code"); - let area_type_code_count = match area_type_code { - Some(AttributeValue::Number(n)) => n.as_i64(), - _ => None, - }; - if desc.is_none() || area_type_code.is_none() || area_type_code_count.unwrap() > 2 { - continue; - } - - let attrib = vec![ - ( - format!("土砂災害リスク_{}_区域区分", desc.unwrap()), - area_type, - ), - ( - format!("土砂災害リスク_{}_区域区分コード", desc.unwrap()), - area_type_code, - ), - ]; - - for (k, v) in attrib { - let value = match v { - Some(value) => value, - None => continue, - }; - let attribute_name = Attribute::new(k.clone()); - result.insert(attribute_name, value.clone()); - self.risk_to_attr_defs - .entry("lsld".to_string()) - .or_default(); - } - } - result - } -} - -#[derive(Debug, Clone)] -struct CommonAttributeProcessor { - max_lod: i64, - gml_path_to_max_lod: HashMap, -} - -impl CommonAttributeProcessor { - fn flatten_attribute(key: &str, attribute: &AttributeValue) -> HashMap { - if !FLATTEN_PREFIXES.contains(key) { - return HashMap::from([(key.to_string(), attribute.clone())]); - } - match attribute { - AttributeValue::Array(value) => { - let mut result = HashMap::new(); - for (i, v) in value.iter().enumerate() { - let new_key = if value.len() == 1 { - key.to_string() - } else { - format!("{}{}{}", key, DELIM, i) - }; - let new_value = Self::flatten_attribute(new_key.as_str(), v); - result.extend(new_value); - } - result - } - AttributeValue::Map(value) => { - let mut result = HashMap::new(); - for (k, v) in value { - let new_key = format!("{}{}{}", key, DELIM, k); - let new_value = Self::flatten_attribute(new_key.as_str(), v); - result.extend(new_value); - } - result - } - v => HashMap::from([(key.to_string(), v.clone())]), - } - } - - fn flatten_generic_attributes( - &mut self, - attrib: &HashMap, - ) -> HashMap { - if let Some(AttributeValue::Map(obj_map)) = attrib.get("gen:genericAttribute") { - obj_map - .iter() - .filter(|(k, _)| k.as_str() != "type") - .map(|(k, v)| (Attribute::new(k.clone()), v.clone())) - .collect() - } else { - HashMap::new() - } - } - - fn update_max_lod(&mut self, attributes: &HashMap) { - let gml_path = match attributes.get(&Attribute::new("cityGmlPath")) { - Some(AttributeValue::String(gml_path)) => gml_path.clone(), - _ => return, - }; - let mut gml_max_lod = *self.gml_path_to_max_lod.get(&gml_path).unwrap_or(&0); - for lod in 0..5 { - let key = format!("numLod{}", lod); - let attribute_name = Attribute::new(key.clone()); - let num_lod = match attributes.get(&attribute_name) { - Some(AttributeValue::Number(num_lod)) => num_lod, - _ => continue, - }; - if num_lod.as_i64().unwrap() > 0 { - if self.max_lod < lod { - self.max_lod = lod; - } - if gml_max_lod < lod { - gml_max_lod = lod; - self.gml_path_to_max_lod - .insert(gml_path.clone(), gml_max_lod); - } - } - } - } - - fn extract_lod_types( - &self, - attrib: &HashMap, - parent_tag: &str, - ) -> HashMap { - let mut result = HashMap::new(); - let parent_attr = match attrib.get(parent_tag) { - Some(AttributeValue::Array(parent_attr)) => parent_attr, - _ => return result, - }; - if parent_attr.is_empty() { - return result; - } - let first_element = match &parent_attr[0] { - AttributeValue::Map(first_element) => first_element, - _ => return result, - }; - let lod_types = match first_element.get("uro:lodType") { - Some(AttributeValue::Array(lod_types)) => lod_types, - _ => return result, - }; - for lod_type in lod_types { - let s = match lod_type.to_string().chars().next() { - Some(s) => s, - None => continue, - }; - if s.to_string() == "2" || s.to_string() == "3" || s.to_string() == "4" { - let key = format!("lod_type_{}", s); - let attribute_name = Attribute::new(key.clone()); - result.insert(attribute_name, lod_type.clone()); - } - } - result - } -} - -#[derive(Debug, Clone, Default)] -pub struct AttributeFlattenerFactory; - -impl ProcessorFactory for AttributeFlattenerFactory { - fn name(&self) -> &str { - "PLATEAU4.AttributeFlattener" - } - - fn description(&self) -> &str { - "Flatten attributes for building feature" - } - - fn parameter_schema(&self) -> Option { - None - } - - fn categories(&self) -> &[&'static str] { - &["PLATEAU"] - } - - fn get_input_ports(&self) -> Vec { - vec![DEFAULT_PORT.clone()] - } - - fn get_output_ports(&self) -> Vec { - vec![DEFAULT_PORT.clone()] - } - - fn build( - &self, - _ctx: NodeContext, - _event_hub: EventHub, - _action: String, - _with: Option>, - ) -> Result, BoxedError> { - let flattener = Flattener::new(); - let common_processor = CommonAttributeProcessor { - max_lod: 0, - gml_path_to_max_lod: HashMap::new(), - }; - - let process = AttributeFlattener { - flattener, - common_processor, - }; - Ok(Box::new(process)) - } -} - -#[derive(Debug, Clone)] -pub struct AttributeFlattener { - flattener: Flattener, - common_processor: CommonAttributeProcessor, -} - -impl Processor for AttributeFlattener { - fn process( - &mut self, - ctx: ExecutorContext, - fw: &mut dyn ProcessorChannelForwarder, - ) -> Result<(), BoxedError> { - let feature = &ctx.feature; - let Some(AttributeValue::Map(city_gml_attribute)) = feature.get(&"cityGmlAttributes") - else { - return Err(PlateauProcessorError::AttributeFlattener(format!( - "No cityGmlAttributes found with feature id = {:?}", - feature.id - )) - .into()); - }; - let Some(AttributeValue::String(ftype)) = city_gml_attribute.get("type") else { - return Err(PlateauProcessorError::AttributeFlattener(format!( - "No type found with feature id = {:?}", - feature.id - )) - .into()); - }; - let mut flattened = HashMap::new(); - if ftype.as_str() == "bldg:Building" { - flattened.extend( - self.flattener - .extract_fld_risk_attribute(city_gml_attribute), - ); - flattened.extend( - self.flattener - .extract_tnm_htd_ifld_risk_attribute(city_gml_attribute), - ); - flattened.extend( - self.flattener - .extract_lsld_risk_attribute(city_gml_attribute), - ); - flattened.extend( - self.common_processor - .flatten_generic_attributes(city_gml_attribute), - ); - flattened.extend( - self.common_processor - .extract_lod_types(city_gml_attribute, "uro:buildingDataQualityAttribute"), - ); - } else { - // For child elements, the attributes of the root element (Building) are extracted and merged.。 - let root_city_gml_attribute = match city_gml_attribute.get("ancestors") { - Some(AttributeValue::Map(attributes)) => attributes.clone(), - _ => - // default to empty - { - HashMap::new() - } - }; - for (name, vroot) in &root_city_gml_attribute { - let v = flattened.get(&Attribute::new(name)); - - let value = if v.is_none() || v == Some(vroot) { - vroot.clone() - } else { - AttributeValue::String(format!( - "{} {}", - vroot.as_string().unwrap_or_default(), - v.and_then(|v| v.as_string()).unwrap_or_default() - )) - }; - flattened.insert(Attribute::new(name), value.clone()); - } - - if ftype == "bldg:BuildingPart" { - flattened.extend( - self.flattener - .extract_fld_risk_attribute(&root_city_gml_attribute), - ); - flattened.extend( - self.flattener - .extract_tnm_htd_ifld_risk_attribute(&root_city_gml_attribute), - ); - flattened.extend( - self.flattener - .extract_lsld_risk_attribute(&root_city_gml_attribute), - ); - - flattened.extend( - self.flattener - .extract_fld_risk_attribute(city_gml_attribute), - ); - flattened.extend( - self.flattener - .extract_tnm_htd_ifld_risk_attribute(city_gml_attribute), - ); - flattened.extend( - self.flattener - .extract_lsld_risk_attribute(city_gml_attribute), - ); - } - self.common_processor.update_max_lod(&feature.attributes); - } - // Set attributes to be flattened - let mut new_city_gml_attribute = HashMap::new(); - for (k, v) in city_gml_attribute.iter() { - let new_value = CommonAttributeProcessor::flatten_attribute(k, v); - new_city_gml_attribute.extend(new_value); - } - let mut feature = feature.clone(); - feature.attributes.extend( - new_city_gml_attribute - .iter() - .map(|(k, v)| (Attribute::new(k.clone()), v.clone())) - .collect::>(), - ); - feature - .attributes - .remove(&Attribute::new("cityGmlAttributes")); - feature.attributes.extend(flattened); - let keys = feature.attributes.keys().cloned().collect_vec(); - let attributes = &mut feature.attributes; - for key in keys.iter() { - if (key.to_string().starts_with("uro:") || key.to_string().starts_with("bldg:")) - && key.to_string().ends_with("_type") - { - attributes.remove(key); - } - if ["gen:genericAttribute", "uro:buildingDisasterRiskAttribute"] - .contains(&key.to_string().as_str()) - { - attributes.remove(key); - } - } - fw.send(ctx.new_with_feature_and_port(feature, DEFAULT_PORT.clone())); - Ok(()) - } - - fn finish( - &self, - _ctx: NodeContext, - _fw: &mut dyn ProcessorChannelForwarder, - ) -> Result<(), BoxedError> { - Ok(()) - } - - fn name(&self) -> &str { - "AttributeFlattener" - } -} +pub(super) mod constants; +pub(super) mod flattener; +pub(super) mod processor; diff --git a/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/constants.rs b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/constants.rs new file mode 100644 index 000000000..20169f2d6 --- /dev/null +++ b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/constants.rs @@ -0,0 +1,288 @@ +use std::collections::HashMap; + +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(super) struct AttributePath { + pub(super) attribute: String, + pub(super) json_path: String, +} + +pub(super) static FLATTEN_ATTRIBUTES: Lazy>> = Lazy::new(|| { + let value = json!( + { + "bldg/bldg:Building": [ + { + "attribute": "gml:name", + "data_type": "buffer", + "json_path": "gml:name" + }, + { + "attribute": "core:creationDate", + "data_type": "date", + "json_path": "core:creationDate" + }, + { + "attribute": "bldg:class", + "data_type": "buffer", + "json_path": "bldg:class" + }, + { + "attribute": "bldg:usage", + "data_type": "buffer", + "json_path": "bldg:usage" + }, + { + "attribute": "bldg:yearOfConstruction", + "data_type": "int16", + "json_path": "bldg:yearOfConstruction" + }, + { + "attribute": "bldg:measuredHeight", + "data_type": "real64", + "json_path": "bldg:measuredHeight" + }, + { + "attribute": "bldg:storeysAboveGround", + "data_type": "int16", + "json_path": "bldg:storeysAboveGround" + }, + { + "attribute": "bldg:storeysBelowGround", + "data_type": "int16", + "json_path": "bldg:storeysBelowGround" + }, + { + "attribute": "bldg:address", + "data_type": "buffer", + "json_path": "bldg:address" + }, + { + "attribute": "uro:buildingID", + "data_type": "buffer", + "json_path": "uro:BuildingIDAttribute uro:buildingID" + }, + { + "attribute": "uro:branchID", + "data_type": "int16", + "json_path": "uro:BuildingIDAttribute uro:branchID" + }, + { + "attribute": "uro:partID", + "data_type": "int16", + "json_path": "uro:BuildingIDAttribute uro:partID" + }, + { + "attribute": "uro:prefecture", + "data_type": "buffer", + "json_path": "uro:BuildingIDAttribute uro:prefecture" + }, + { + "attribute": "uro:city", + "data_type": "buffer", + "json_path": "uro:BuildingIDAttribute uro:city" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:serialNumberOfBuildingCertification", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:serialNumberOfBuildingCertification" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:siteArea", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:siteArea" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:totalFloorArea", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:totalFloorArea" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:buildingFootprintArea", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:buildingFootprintArea" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:buildingRoofEdgeArea", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:buildingRoofEdgeArea" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:buildingStructureType", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:buildingStructureType" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:fireproofStructureType", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:fireproofStructureType" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:urbanPlanType", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:urbanPlanType" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:areaClassificationType", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:areaClassificationType" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:districtsAndZonesType", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:districtsAndZonesType" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:landUseType", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:landUseType" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:vacancy", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:vacancy" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:buildingCoverageRate", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:buildingCoverageRate" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:floorAreaRate", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:floorAreaRate" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:specifiedBuildingCoverageRate", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:specifiedBuildingCoverageRate" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:specifiedFloorAreaRate", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:specifiedFloorAreaRate" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:standardFloorAreaRate", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:standardFloorAreaRate" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:buildingHeight", + "data_type": "real64", + "json_path": "uro:BuildingDetailAttribute uro:buildingHeight" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:eaveHeight", + "data_type": "int16", + "json_path": "uro:BuildingDetailAttribute uro:eaveHeight" + }, + { + "attribute": "uro:BuildingDetailAttribute_uro:surveyYear", + "data_type": "buffer", + "json_path": "uro:BuildingDetailAttribute uro:surveyYear" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:class", + "data_type": "buffer", + "json_path": "uro:LargeCustomerFacilityAttribute uro:class" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:name", + "data_type": "buffer", + "json_path": "uro:LargeCustomerFacilityAttribute uro:name" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:capacity", + "data_type": "int16", + "json_path": "uro:LargeCustomerFacilityAttribute uro:capacity" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:totalFloorArea", + "data_type": "real64", + "json_path": "uro:LargeCustomerFacilityAttribute uro:totalFloorArea" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:inauguralDate", + "data_type": "date", + "json_path": "uro:LargeCustomerFacilityAttribute uro:inauguralDate" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:yearOpened", + "data_type": "int16", + "json_path": "uro:LargeCustomerFacilityAttribute uro:yearOpened" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:yearClosed", + "data_type": "int16", + "json_path": "uro:LargeCustomerFacilityAttribute uro:yearClosed" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:urbanPlanType", + "data_type": "buffer", + "json_path": "uro:LargeCustomerFacilityAttribute uro:urbanPlanType" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:areaClassificationType", + "data_type": "buffer", + "json_path": "uro:LargeCustomerFacilityAttribute uro:areaClassificationType" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:districtsAndZonesType", + "data_type": "buffer", + "json_path": "uro:LargeCustomerFacilityAttribute uro:districtsAndZonesType" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:landUseType", + "data_type": "buffer", + "json_path": "uro:LargeCustomerFacilityAttribute uro:landUseType" + }, + { + "attribute": "uro:LargeCustomerFacilityAttribute_uro:surveyYear", + "data_type": "int16", + "json_path": "uro:LargeCustomerFacilityAttribute uro:surveyYear" + }, + { + "attribute": "uro:RealEstateIDAttribute_uro:realEstateIDOfBuilding", + "data_type": "buffer", + "json_path": "uro:RealEstateIDAttribute uro:realEstateIDOfBuilding" + }, + { + "attribute": "uro:RealEstateIDAttribute_uro:matchingScore", + "data_type": "int16", + "json_path": "uro:RealEstateIDAttribute uro:matchingScore" + }, + { + "attribute": "uro:geometrySrcDescLod1", + "data_type": "buffer", + "json_path": "uro:publicSurveyDataQualityAttribute uro:geometrySrcDescLod1" + }, + { + "attribute": "uro:lodType", + "data_type": "buffer", + "json_path": "uro:buildingDataQualityAttribute uro:lodType" + }, + { + "attribute": "uro:lod1HeightType", + "data_type": "buffer", + "json_path": "uro:buildingDataQualityAttribute uro:lod1HeightType" + } + ] + } + ); + serde_json::from_value(value).unwrap() +}); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_flatten_attributes() { + let flatten_attributes = FLATTEN_ATTRIBUTES.clone(); + assert_eq!(flatten_attributes.len(), 1); + assert!(flatten_attributes.contains_key("bldg/bldg:Building")); + } +} diff --git a/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/flattener.rs b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/flattener.rs new file mode 100644 index 000000000..d15b73a6a --- /dev/null +++ b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/flattener.rs @@ -0,0 +1,297 @@ +use std::collections::HashMap; + +use reearth_flow_types::{Attribute, AttributeValue}; + +#[derive(Debug, Clone, Default)] +pub(super) struct Flattener; + +impl Flattener { + pub(super) fn extract_fld_risk_attribute( + &mut self, + attributes: &HashMap, + ) -> HashMap { + let Some(disaster_risks) = attributes.get("uro:RiverFloodingRiskAttribute") else { + return HashMap::new(); + }; + let disaster_risks = match disaster_risks { + AttributeValue::Array(disaster_risks) => disaster_risks, + AttributeValue::Map(disaster_risks) => { + &vec![AttributeValue::Map(disaster_risks.clone())] + } + _ => return HashMap::new(), + }; + let mut result = HashMap::new(); + for risk in disaster_risks { + let risk_obj = match risk.as_map() { + Some(obj) => obj, + None => continue, + }; + let desc = risk_obj.get("uro:description").map(|v| v.to_string()); + let admin = risk_obj.get("uro:adminType").map(|v| v.to_string()); + let scale = risk_obj.get("uro:scale").map(|v| v.to_string()); + + if desc.is_none() || admin.is_none() || scale.is_none() { + continue; + } + let desc = desc.unwrap(); + let admin = admin.unwrap(); + let scale = scale.unwrap(); + + let basename = format!("{}（{}管理区間）_{}", desc, admin, scale); + + let mut rank = risk_obj.get("uro:rank").map(|v| v.to_string()); + let mut rank_code = risk_obj.get("uro:rank_code").map(|v| v.to_string()); + if rank.is_none() { + rank = risk_obj.get("uro:rankOrg").map(|v| v.to_string()); + rank_code = risk_obj.get("uro:rankOrg_code").map(|v| v.to_string()); + } + + let depth = risk_obj.get("uro:depth").map(|v| v.to_string()); + let duration = risk_obj.get("uro:duration").map(|v| v.to_string()); + + let attribs = vec![ + ("浸水ランク", rank), + ("浸水ランクコード", rank_code), + ("浸水深", depth), + ("浸水継続時間", duration), + ]; + + for (label, value_opt) in attribs { + if let Some(value_str) = value_opt { + let name = format!("{}_{}", basename, label); + result.insert(Attribute::new(name), AttributeValue::String(value_str)); + } + } + } + result + } + + pub(super) fn extract_tnm_htd_ifld_risk_attribute( + &mut self, + attributes: &HashMap, + ) -> HashMap { + let src = [ + ("uro:TsunamiRiskAttribute", "津波浸水想定"), + ("uro:HighTideRiskAttribute", "高潮浸水想定"), + ("uro:InlandFloodingRiskAttribute", "内水浸水想定"), + ("uro:ReservoirFloodingRiskAttribute", "ため池浸水想定"), + ]; + + let mut result = HashMap::new(); + for (tag, title) in src.iter() { + let Some(disaster_risks) = attributes.get(*tag) else { + continue; + }; + let disaster_risks = match disaster_risks { + AttributeValue::Array(disaster_risks) => disaster_risks, + AttributeValue::Map(disaster_risks) => { + &vec![AttributeValue::Map(disaster_risks.clone())] + } + _ => return HashMap::new(), + }; + + for risk_value in disaster_risks { + let risk_obj = match risk_value.as_map() { + Some(obj) => obj, + None => continue, + }; + + let desc_opt = risk_obj.get("uro:description").map(|v| v.to_string()); + if desc_opt.is_none() { + continue; + } + let desc = desc_opt.unwrap(); + let basename = format!("{}_{}", title, desc); + + let mut rank_opt = risk_obj.get("uro:rank").map(|v| v.to_string()); + let mut rank_code_opt = risk_obj.get("uro:rank_code").map(|v| v.to_string()); + if rank_opt.is_none() { + rank_opt = risk_obj.get("uro:rankOrg").map(|v| v.to_string()); + rank_code_opt = risk_obj.get("uro:rankOrg_code").map(|v| v.to_string()); + } + + let depth_opt = risk_obj.get("uro:depth").map(|v| v.to_string()); + + let attribs = vec![ + ("浸水ランク", rank_opt), + ("浸水ランクコード", rank_code_opt), + ("浸水深", depth_opt), + ]; + + for (label, value_str_opt) in attribs { + if let Some(value_str) = value_str_opt { + let name = format!("{}_{}", basename, label); + result.insert(Attribute::new(name), AttributeValue::String(value_str)); + } + } + } + } + result + } + + pub(super) fn extract_lsld_risk_attribute( + &mut self, + attributes: &HashMap, + ) -> HashMap { + let Some(disaster_risks) = attributes.get("uro:LandSlideRiskAttribute") else { + return HashMap::new(); + }; + let disaster_risks = match disaster_risks { + AttributeValue::Array(disaster_risks) => disaster_risks, + AttributeValue::Map(disaster_risks) => { + &vec![AttributeValue::Map(disaster_risks.clone())] + } + _ => return HashMap::new(), + }; + let mut result = HashMap::new(); + for risk_value in disaster_risks { + let risk_obj = match risk_value.as_map() { + Some(obj) => obj, + None => continue, + }; + + let desc_opt = risk_obj.get("uro:description").map(|v| v.to_string()); + let area_type_opt = risk_obj.get("uro:areaType").map(|v| v.to_string()); + let type_code_opt = risk_obj.get("uro:areaType_code").map(|v| v.to_string()); + + if desc_opt.is_none() || type_code_opt.is_none() { + continue; + } + let desc = desc_opt.unwrap(); + let type_code_str = type_code_opt.unwrap(); + + let type_code_int: i32 = match type_code_str.parse() { + Ok(n) => n, + Err(_) => continue, + }; + if type_code_int > 2 { + continue; + } + + let entries = vec![ + ( + format!("土砂災害リスク_{}_区域区分", desc), + area_type_opt.unwrap_or("".to_string()), + ), + ( + format!("土砂災害リスク_{}_区域区分コード", desc), + type_code_str, + ), + ]; + + for (attr_key, attr_value) in entries { + if attr_value.is_empty() { + continue; + } + result.insert( + Attribute::new(attr_key.clone()), + AttributeValue::String(attr_value.clone()), + ); + } + } + result + } +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub(super) struct CommonAttributeProcessor { + max_lod: i64, + gml_path_to_max_lod: HashMap, +} + +impl CommonAttributeProcessor { + #[allow(dead_code)] + fn flatten_generic_attributes( + &mut self, + attrib: &HashMap, + ) -> HashMap { + if let Some(AttributeValue::Map(obj_map)) = attrib.get("gen:genericAttribute") { + obj_map + .iter() + .filter(|(k, _)| k.as_str() != "type") + .map(|(k, v)| (Attribute::new(k.clone()), v.clone())) + .collect() + } else { + HashMap::new() + } + } + + #[allow(dead_code)] + fn update_max_lod(&mut self, attributes: &HashMap) { + let gml_path = match attributes.get(&Attribute::new("cityGmlPath")) { + Some(AttributeValue::String(gml_path)) => gml_path.clone(), + _ => return, + }; + let mut gml_max_lod = *self.gml_path_to_max_lod.get(&gml_path).unwrap_or(&0); + for lod in 0..5 { + let key = format!("numLod{}", lod); + let attribute_name = Attribute::new(key.clone()); + let num_lod = match attributes.get(&attribute_name) { + Some(AttributeValue::Number(num_lod)) => num_lod, + _ => continue, + }; + if num_lod.as_i64().unwrap() > 0 { + if self.max_lod < lod { + self.max_lod = lod; + } + if gml_max_lod < lod { + gml_max_lod = lod; + self.gml_path_to_max_lod + .insert(gml_path.clone(), gml_max_lod); + } + } + } + } + + #[allow(dead_code)] + fn extract_lod_types( + &self, + attrib: &HashMap, + parent_tag: &str, + ) -> HashMap { + let mut result = HashMap::new(); + let parent_attr = match attrib.get(parent_tag) { + Some(AttributeValue::Array(parent_attr)) => parent_attr, + _ => return result, + }; + if parent_attr.is_empty() { + return result; + } + let first_element = match &parent_attr[0] { + AttributeValue::Map(first_element) => first_element, + _ => return result, + }; + let lod_types = match first_element.get("uro:lodType") { + Some(AttributeValue::Array(lod_types)) => lod_types, + _ => return result, + }; + for lod_type in lod_types { + let s = match lod_type.to_string().chars().next() { + Some(s) => s, + None => continue, + }; + if s.to_string() == "2" || s.to_string() == "3" || s.to_string() == "4" { + let key = format!("lod_type_{}", s); + let attribute_name = Attribute::new(key.clone()); + result.insert(attribute_name, lod_type.clone()); + } + } + result + } +} + +pub(super) fn get_value_from_json_path( + paths: &[&str], + attrib: &HashMap, +) -> Option { + let key = paths.first()?; + let value = attrib.get(*key)?; + if let AttributeValue::Map(map) = &value { + get_value_from_json_path(&paths[1..], map) + } else if *key == "uro:lodType" { + Some(AttributeValue::String(value.to_string())) + } else { + Some(value.clone()) + } +} diff --git a/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/processor.rs b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/processor.rs new file mode 100644 index 000000000..19dbfa91b --- /dev/null +++ b/engine/runtime/action-plateau-processor/src/plateau4/attribute_flattener/processor.rs @@ -0,0 +1,148 @@ +use std::collections::HashMap; + +use itertools::Itertools; +use reearth_flow_runtime::{ + channels::ProcessorChannelForwarder, + errors::BoxedError, + event::EventHub, + executor_operation::{ExecutorContext, NodeContext}, + node::{Port, Processor, ProcessorFactory, DEFAULT_PORT}, +}; +use reearth_flow_types::{Attribute, AttributeValue}; +use serde_json::Value; + +use crate::plateau4::errors::PlateauProcessorError; + +#[derive(Debug, Clone, Default)] +pub struct AttributeFlattenerFactory; + +impl ProcessorFactory for AttributeFlattenerFactory { + fn name(&self) -> &str { + "PLATEAU4.AttributeFlattener" + } + + fn description(&self) -> &str { + "Flatten attributes for building feature" + } + + fn parameter_schema(&self) -> Option { + None + } + + fn categories(&self) -> &[&'static str] { + &["PLATEAU"] + } + + fn get_input_ports(&self) -> Vec { + vec![DEFAULT_PORT.clone()] + } + + fn get_output_ports(&self) -> Vec { + vec![DEFAULT_PORT.clone()] + } + + fn build( + &self, + _ctx: NodeContext, + _event_hub: EventHub, + _action: String, + _with: Option>, + ) -> Result, BoxedError> { + let flattener = super::flattener::Flattener; + let process = AttributeFlattener { flattener }; + Ok(Box::new(process)) + } +} + +#[derive(Debug, Clone)] +pub struct AttributeFlattener { + flattener: super::flattener::Flattener, +} + +impl Processor for AttributeFlattener { + fn process( + &mut self, + ctx: ExecutorContext, + fw: &mut dyn ProcessorChannelForwarder, + ) -> Result<(), BoxedError> { + let mut feature = ctx.feature.clone(); + let Some(AttributeValue::Map(city_gml_attribute)) = feature.get(&"cityGmlAttributes") + else { + return Err(PlateauProcessorError::AttributeFlattener(format!( + "No cityGmlAttributes found with feature id = {:?}", + feature.id + )) + .into()); + }; + let mut new_city_gml_attribute = HashMap::new(); + if let Some(flatten_attributes) = + super::constants::FLATTEN_ATTRIBUTES.get("bldg/bldg:Building") + { + for attribute in flatten_attributes { + let mut json_path: Vec<&str> = vec![]; + json_path.extend(attribute.json_path.split(" ")); + let Some(new_attribute) = + super::flattener::get_value_from_json_path(&json_path, city_gml_attribute) + else { + continue; + }; + new_city_gml_attribute + .insert(Attribute::new(attribute.attribute.clone()), new_attribute); + } + } + let edit_city_gml_attribute = city_gml_attribute + .clone() + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect::>(); + new_city_gml_attribute.extend( + self.flattener + .extract_fld_risk_attribute(&edit_city_gml_attribute), + ); + + new_city_gml_attribute.extend( + self.flattener + .extract_tnm_htd_ifld_risk_attribute(&edit_city_gml_attribute), + ); + new_city_gml_attribute.extend( + self.flattener + .extract_lsld_risk_attribute(&edit_city_gml_attribute), + ); + + feature.attributes.extend( + new_city_gml_attribute + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(), + ); + feature.remove(&"cityGmlAttributes"); + let keys = feature.attributes.keys().cloned().collect_vec(); + let attributes = &mut feature.attributes; + for key in keys.iter() { + if (key.to_string().starts_with("uro:") || key.to_string().starts_with("bldg:")) + && key.to_string().ends_with("_type") + { + attributes.remove(key); + } + if ["gen:genericAttribute", "uro:buildingDisasterRiskAttribute"] + .contains(&key.to_string().as_str()) + { + attributes.remove(key); + } + } + fw.send(ctx.new_with_feature_and_port(feature, DEFAULT_PORT.clone())); + Ok(()) + } + + fn finish( + &self, + _ctx: NodeContext, + _fw: &mut dyn ProcessorChannelForwarder, + ) -> Result<(), BoxedError> { + Ok(()) + } + + fn name(&self) -> &str { + "AttributeFlattener" + } +} diff --git a/engine/runtime/action-plateau-processor/src/plateau4/mapping.rs b/engine/runtime/action-plateau-processor/src/plateau4/mapping.rs index ed2dd202d..9d34a1da9 100644 --- a/engine/runtime/action-plateau-processor/src/plateau4/mapping.rs +++ b/engine/runtime/action-plateau-processor/src/plateau4/mapping.rs @@ -4,8 +4,8 @@ use once_cell::sync::Lazy; use reearth_flow_runtime::node::{NodeKind, ProcessorFactory}; use super::{ - attribute_flattener::AttributeFlattenerFactory, city_code_extractor::CityCodeExtractorFactory, - max_lod_extractor::MaxLodExtractorFactory, + attribute_flattener::processor::AttributeFlattenerFactory, + city_code_extractor::CityCodeExtractorFactory, max_lod_extractor::MaxLodExtractorFactory, missing_attribute_detector::MissingAttributeDetectorFactory, object_list_extractor::ObjectListExtractorFactory, udx_folder_extractor::UDXFolderExtractorFactory, diff --git a/engine/runtime/action-processor/src/feature/reader/citygml.rs b/engine/runtime/action-processor/src/feature/reader/citygml.rs index 7660d4f1b..35bc30252 100644 --- a/engine/runtime/action-processor/src/feature/reader/citygml.rs +++ b/engine/runtime/action-processor/src/feature/reader/citygml.rs @@ -154,7 +154,19 @@ fn parse_tree_reader( (v[0], v[1], v[2]) = (v[1], v[0], v[2]); }); } - let attributes = entity.root.to_attribute_json(); + let attributes = AttributeValue::from_nusamai_cityml_value(&entity.root); + let attributes = AttributeValue::convert_array_attributes(&attributes); + let city_gml_attributes = match attributes.len() { + 0 => AttributeValue::Null, + 1 => attributes.values().next().unwrap().clone(), + _ => AttributeValue::Map(attributes), + }; + let city_gml_attributes = city_gml_attributes.flatten(); + let city_gml_attributes = if let AttributeValue::Map(map) = &city_gml_attributes { + AttributeValue::Map(AttributeValue::convert_array_attributes(map)) + } else { + city_gml_attributes + }; let gml_id = entity.root.id(); let name = entity.root.typename(); let lod = LodMask::find_lods_by_citygml_value(&entity.root); @@ -164,7 +176,7 @@ fn parse_tree_reader( lod: Some(lod), }; let mut attributes = HashMap::::from([ - (Attribute::new("cityGmlAttributes"), attributes.into()), + (Attribute::new("cityGmlAttributes"), city_gml_attributes), ( Attribute::new("gmlName"), name.map(|s| AttributeValue::String(s.to_string())) diff --git a/engine/runtime/action-source/src/file/reader/citygml.rs b/engine/runtime/action-source/src/file/reader/citygml.rs index 6552b77dd..ca2b1efc8 100644 --- a/engine/runtime/action-source/src/file/reader/citygml.rs +++ b/engine/runtime/action-source/src/file/reader/citygml.rs @@ -129,11 +129,22 @@ async fn parse_tree_reader( (v[0], v[1], v[2]) = (v[1], v[0], v[2]); }); } - let attributes = entity.root.to_attribute_json(); + let attributes = AttributeValue::from_nusamai_cityml_value(&entity.root); + let city_gml_attributes = match attributes.len() { + 0 => AttributeValue::Null, + 1 => attributes.values().next().unwrap().clone(), + _ => AttributeValue::Map(attributes), + }; + let city_gml_attributes = city_gml_attributes.flatten(); + let city_gml_attributes = if let AttributeValue::Map(map) = &city_gml_attributes { + AttributeValue::Map(AttributeValue::convert_array_attributes(map)) + } else { + city_gml_attributes + }; let gml_id = entity.root.id(); let name = entity.root.typename(); let attributes = HashMap::::from([ - (Attribute::new("cityGmlAttributes"), attributes.into()), + (Attribute::new("cityGmlAttributes"), city_gml_attributes), ( Attribute::new("gmlName"), name.map(|s| AttributeValue::String(s.to_string())) diff --git a/engine/runtime/examples/plateau/testdata/workflow/data-convert/plateau4/01-bldg/workflow.yml b/engine/runtime/examples/plateau/testdata/workflow/data-convert/plateau4/01-bldg/workflow.yml index c3e6f9c49..d29b1ce2d 100644 --- a/engine/runtime/examples/plateau/testdata/workflow/data-convert/plateau4/01-bldg/workflow.yml +++ b/engine/runtime/examples/plateau/testdata/workflow/data-convert/plateau4/01-bldg/workflow.yml @@ -111,12 +111,66 @@ graphs: with: reprojectorType: jgd2011ToWgs84 + - id: 3e637368-48dc-48c2-aade-253d5b0cfeeb + name: AttributeManagerRemoveUnnecessaryAttributes + type: action + action: AttributeManager + with: + operations: + - attribute: meshcode + method: create + value: | + file::extract_filename_without_ext(env.get("__value")["path"]).split("_")[0] + - attribute: type + method: rename + value: "feature_type" + - attribute: path + method: rename + value: "_path" + - attribute: cityCode + method: rename + value: "city_code" + - attribute: cityName + method: rename + value: "city_name" + - attribute: gmlId + method: rename + value: "gml_id" + - attribute: package + method: remove + - attribute: fileIndex + method: remove + - attribute: extension + method: remove + - attribute: root + method: remove + - attribute: gmlRootId + method: remove + - attribute: udxDirs + method: remove + - attribute: cityGmlPath + method: remove + - attribute: schemas + method: remove + - attribute: dirCodelists + method: remove + - attribute: dirSchemas + method: remove + - attribute: gmlName + method: remove + - attribute: codelists + method: remove + - attribute: maxLod + method: remove + - attribute: dirRoot + method: remove + - id: c32a279d-97be-4584-b282-4d65627b1132 name: FeatureLodFilter type: action action: FeatureLodFilter with: - filterKey: path + filterKey: _path - id: 41d69f1f-2a4d-4bc1-a5d0-175698273571 name: cesium3DTilesWriterByLod1 @@ -252,6 +306,11 @@ graphs: toPort: default - id: fddd597c-3e9c-400c-abd9-02b6c2214459 from: 8b05f9d4-1cb2-4071-a1da-c968431bc0ec + to: 3e637368-48dc-48c2-aade-253d5b0cfeeb + fromPort: default + toPort: default + - id: 64540b5c-066e-449f-ace1-4af29c0e3b3c + from: 3e637368-48dc-48c2-aade-253d5b0cfeeb to: c32a279d-97be-4584-b282-4d65627b1132 fromPort: default toPort: default diff --git a/engine/runtime/types/src/attribute.rs b/engine/runtime/types/src/attribute.rs index 4c08546c1..47f5f74d3 100644 --- a/engine/runtime/types/src/attribute.rs +++ b/engine/runtime/types/src/attribute.rs @@ -178,6 +178,21 @@ impl AttributeValue { _ => Err(error::Error::internal_runtime("Cannot extend")), } } + + pub fn flatten(self) -> Self { + let mut result = HashMap::new(); + match self { + AttributeValue::Array(map) => { + for value in map { + if let AttributeValue::Map(v) = value { + result.extend(v); + } + } + } + _ => return self, + } + AttributeValue::Map(result) + } } impl Eq for AttributeValue {} @@ -476,6 +491,77 @@ pub(crate) fn all_attribute_keys(items: &HashMap) -> Vec keys } +impl AttributeValue { + pub fn get_recursive>( + key: T, + items: &HashMap, + ) -> Vec { + let mut values = Vec::new(); + for (k, v) in items { + if k.as_str() == key.as_ref() { + values.push(v.clone()); + } + if let AttributeValue::Array(array) = v { + for item in array { + if let AttributeValue::Map(map) = item { + values.extend(Self::get_recursive(key.as_ref(), map)); + } + } + } + if let AttributeValue::Map(map) = v { + values.extend(Self::get_recursive(key.as_ref(), map)); + } + } + values + } + + pub fn convert_array_attributes( + attributes: &HashMap, + ) -> HashMap { + let mut result = HashMap::new(); + for (k, v) in attributes.iter() { + match v { + AttributeValue::Array(arr) if arr.len() == 1 => { + let value = arr.first().cloned().unwrap_or(AttributeValue::Null); + match value { + AttributeValue::Map(map) => { + result.insert( + k.clone(), + AttributeValue::Map(Self::convert_array_attributes(&map)), + ); + } + _ => { + result.insert(k.clone(), value); + } + } + } + AttributeValue::Array(arr) => { + let mut new_arr = Vec::new(); + for item in arr.iter() { + new_arr.push(match item { + AttributeValue::Map(map) => { + AttributeValue::Map(Self::convert_array_attributes(map)) + } + _ => item.clone(), + }); + } + result.insert(k.clone(), AttributeValue::Array(new_arr)); + } + AttributeValue::Map(map) => { + result.insert( + k.clone(), + AttributeValue::Map(Self::convert_array_attributes(map)), + ); + } + _ => { + result.insert(k.clone(), v.clone()); + } + } + } + result + } +} + #[cfg(test)] mod tests { use super::*; @@ -573,4 +659,23 @@ mod tests { vec!["key1".to_string(), "key2".to_string(), "nested".to_string()] ); } + + // generate get_recursive test + #[test] + fn test_get_recursive() { + let mut map = HashMap::new(); + map.insert( + "key1".to_string(), + AttributeValue::String("value1".to_string()), + ); + let mut nested_map = HashMap::new(); + nested_map.insert( + "key2".to_string(), + AttributeValue::String("value2".to_string()), + ); + map.insert("nested".to_string(), AttributeValue::Map(nested_map)); + + let values = AttributeValue::get_recursive("key2", &map); + assert_eq!(values, vec![AttributeValue::String("value2".to_string())]); + } } diff --git a/engine/runtime/types/src/conversion/nusamai.rs b/engine/runtime/types/src/conversion/nusamai.rs index 94b6cc4ad..050e96311 100644 --- a/engine/runtime/types/src/conversion/nusamai.rs +++ b/engine/runtime/types/src/conversion/nusamai.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use nusamai_citygml::GeometryRef; use nusamai_citygml::{object::ObjectStereotype, GeometryType, Value}; use nusamai_plateau::Entity; @@ -5,7 +7,7 @@ use reearth_flow_geometry::types::line_string::LineString3D; use reearth_flow_geometry::types::polygon::Polygon3D; use crate::error::Error; -use crate::{CityGmlGeometry, Geometry, GeometryValue, GmlGeometry}; +use crate::{AttributeValue, CityGmlGeometry, Geometry, GeometryValue, GmlGeometry}; impl TryFrom for Geometry { type Error = Error; @@ -158,3 +160,156 @@ impl TryFrom for Geometry { )) } } + +impl AttributeValue { + pub fn from_nusamai_cityml_value( + value: &nusamai_citygml::object::Value, + ) -> HashMap { + Self::from_key_and_nusamai_cityml_value(None, value) + } + + pub(crate) fn from_key_and_nusamai_cityml_value( + key: Option, + value: &nusamai_citygml::object::Value, + ) -> HashMap { + match value { + nusamai_citygml::object::Value::Object(obj) => Self::handle_object(obj), + nusamai_citygml::object::Value::Array(arr) => Self::handle_array(arr), + nusamai_citygml::object::Value::Code(code) => Self::handle_code(key, code), + _ => Self::handle_simple_value(key, value), + } + } + + fn handle_object(obj: &nusamai_citygml::object::Object) -> HashMap { + let mut result = HashMap::new(); + let value = Self::process_object_attributes(&obj.attributes); + Self::merge_into_result(&mut result, &obj.typename, AttributeValue::Map(value)); + result + } + + fn process_object_attributes( + attributes: &nusamai_citygml::object::Map, + ) -> HashMap { + attributes + .iter() + .map(|(k, v)| Self::process_attribute(k, v)) + .fold(HashMap::new(), Self::merge_attribute_maps) + } + + fn process_attribute( + key: &str, + value: &nusamai_citygml::Value, + ) -> HashMap { + let mut result = HashMap::new(); + match value { + nusamai_citygml::Value::Code(v) => { + result.insert( + key.to_string(), + AttributeValue::String(v.value().to_owned()), + ); + result.insert( + format!("{}_code", key), + AttributeValue::String(v.code().to_owned()), + ); + } + nusamai_citygml::Value::Array(arr) => { + Self::process_array_attribute(&mut result, key, arr); + } + _ => { + result.insert(key.to_string(), AttributeValue::from(value.clone())); + } + } + result + } + + fn process_array_attribute( + result: &mut HashMap, + key: &str, + arr: &[nusamai_citygml::object::Value], + ) { + for value in arr { + let target = Self::from_key_and_nusamai_cityml_value(Some(key.to_string()), value); + for (key, value) in target { + Self::merge_into_result(result, &key, value); + } + } + } + + fn handle_array(arr: &[nusamai_citygml::object::Value]) -> HashMap { + arr.iter() + .map(Self::from_nusamai_cityml_value) + .fold(HashMap::new(), Self::merge_attribute_maps) + } + + fn handle_code( + key: Option, + code: &nusamai_citygml::Code, + ) -> HashMap { + let mut result = HashMap::new(); + if let Some(key) = key { + result.insert(key.clone(), AttributeValue::String(code.value().to_owned())); + result.insert( + format!("{}_code", key), + AttributeValue::String(code.code().to_owned()), + ); + } + result + } + + fn handle_simple_value( + key: Option, + value: &nusamai_citygml::object::Value, + ) -> HashMap { + let mut result = HashMap::new(); + if let Some(key) = key { + result.insert(key, AttributeValue::from(value.clone())); + } + result + } + + fn merge_attribute_maps( + mut result: HashMap, + new_map: HashMap, + ) -> HashMap { + for (key, value) in new_map { + Self::merge_into_result(&mut result, &key, value); + } + result + } + + fn merge_into_result( + result: &mut HashMap, + key: &str, + value: AttributeValue, + ) { + match result.get(key) { + Some(AttributeValue::Array(existing_arr)) => { + let mut new_arr = existing_arr.clone(); + match value { + AttributeValue::Array(arr) => new_arr.extend(arr), + _ => new_arr.push(value), + } + result.insert(key.to_string(), AttributeValue::Array(new_arr)); + } + Some(AttributeValue::Map(existing_map)) => match value { + AttributeValue::Map(_) => { + result.insert( + key.to_string(), + AttributeValue::Array(vec![ + AttributeValue::Map(existing_map.clone()), + value, + ]), + ); + } + _ => { + let mut new_map = existing_map.clone(); + new_map.insert(key.to_string(), value); + result.insert(key.to_string(), AttributeValue::Map(new_map)); + } + }, + _ => { + result.insert(key.to_string(), value); + } + } + } +}