From 631c04f19a90a5b7eb3a046dda7920deb15f0497 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Fri, 15 Sep 2023 12:59:23 +0000 Subject: [PATCH 01/13] C8y mapping of entity registration messages --- .../src/tedge_config_cli/tedge_config.rs | 2 +- crates/core/c8y_api/src/json_c8y.rs | 8 +- .../core/c8y_api/src/smartrest/inventory.rs | 35 ++ crates/core/c8y_api/src/smartrest/mod.rs | 1 + crates/core/c8y_api/src/smartrest/topic.rs | 39 +- crates/core/tedge_api/src/entity_store.rs | 492 ++++++++++++++---- crates/core/tedge_api/src/event.rs | 2 +- crates/core/tedge_api/src/mqtt_topics.rs | 4 + .../extensions/c8y_mapper_ext/src/config.rs | 2 + .../c8y_mapper_ext/src/converter.rs | 169 +++++- crates/extensions/c8y_mapper_ext/src/error.rs | 6 + .../c8y_mapper_ext/src/log_upload.rs | 13 +- .../c8y_mapper_ext/src/serializer.rs | 2 +- crates/extensions/c8y_mapper_ext/src/tests.rs | 166 +++++- .../contribute/design/mqtt-topic-design.md | 2 +- docs/src/references/mappers/c8y-mapper.md | 10 +- docs/src/references/mqtt-api.md | 12 +- .../registration/device_registration.robot | 77 ++- .../tests/tedge/call_tedge_config_list.robot | 2 +- 19 files changed, 887 insertions(+), 157 deletions(-) create mode 100644 crates/core/c8y_api/src/smartrest/inventory.rs diff --git a/crates/common/tedge_config/src/tedge_config_cli/tedge_config.rs b/crates/common/tedge_config/src/tedge_config_cli/tedge_config.rs index b6388cf1cf5..b5bf352a104 100644 --- a/crates/common/tedge_config/src/tedge_config_cli/tedge_config.rs +++ b/crates/common/tedge_config/src/tedge_config_cli/tedge_config.rs @@ -369,7 +369,7 @@ define_tedge_config! { /// Set of MQTT topics the Cumulocity mapper should subscribe to #[tedge_config(example = "te/+/+/+/+/a/+,te/+/+/+/+/m/+,te/+/+/+/+/e/+")] - #[tedge_config(default(value = "te/+/+/+/+/m/+,te/+/+/+/+/e/+,te/+/+/+/+/a/+,tedge/health/+,tedge/health/+/+"))] + #[tedge_config(default(value = "te/+/+/+/+,te/+/+/+/+/m/+,te/+/+/+/+/e/+,te/+/+/+/+/a/+,tedge/health/+,tedge/health/+/+"))] topics: TemplatesSet, enable: { diff --git a/crates/core/c8y_api/src/json_c8y.rs b/crates/core/c8y_api/src/json_c8y.rs index b5a88ab0e24..c69cbf97bbc 100644 --- a/crates/core/c8y_api/src/json_c8y.rs +++ b/crates/core/c8y_api/src/json_c8y.rs @@ -319,8 +319,8 @@ impl C8yAlarm { fn convert_source(entity: &EntityMetadata) -> Option { match entity.r#type { EntityType::MainDevice => None, - EntityType::ChildDevice => Some(make_c8y_source_fragment(&entity.entity_id.clone())), - EntityType::Service => Some(make_c8y_source_fragment(&entity.entity_id.clone())), + EntityType::ChildDevice => Some(make_c8y_source_fragment(entity.entity_id.as_ref())), + EntityType::Service => Some(make_c8y_source_fragment(entity.entity_id.as_ref())), } } @@ -415,7 +415,7 @@ mod tests { use serde_json::json; use tedge_api::alarm::ThinEdgeAlarm; use tedge_api::alarm::ThinEdgeAlarmData; - use tedge_api::entity_store::EntityRegistrationMessage; + use tedge_api::entity_store::{EntityExternalId, EntityRegistrationMessage}; use tedge_api::event::ThinEdgeEventData; use tedge_api::mqtt_topics::EntityTopicId; use test_case::test_case; @@ -781,7 +781,7 @@ mod tests { r#"{"@id": "external_source", "@type": "child-device"}"#, )) .unwrap(); - entity_store.update(child_registration).unwrap(); + entity_store.update(child_registration, EntityExternalId::from("external_source")).unwrap(); let actual_c8y_alarm = C8yAlarm::try_from(&tedge_alarm, &entity_store).unwrap(); assert_eq!(actual_c8y_alarm, expected_c8y_alarm); diff --git a/crates/core/c8y_api/src/smartrest/inventory.rs b/crates/core/c8y_api/src/smartrest/inventory.rs new file mode 100644 index 00000000000..fe89b224f05 --- /dev/null +++ b/crates/core/c8y_api/src/smartrest/inventory.rs @@ -0,0 +1,35 @@ +use crate::smartrest::topic::publish_topic_from_ancestors; +use mqtt_channel::Message; + +pub fn child_device_creation_message( + child_id: &str, + device_name: Option<&str>, + device_type: Option<&str>, + ancestors: &[String], +) -> Message { + Message::new( + &publish_topic_from_ancestors(ancestors), + format!( + "101,{},{},{}", + child_id, + device_name.unwrap_or(child_id), + device_type.unwrap_or("thin-edge.io-child") + ), + ) +} + +pub fn service_creation_message( + service_id: &str, + service_name: &str, + service_type: &str, + service_status: &str, + ancestors: &[String], +) -> Message { + Message::new( + &publish_topic_from_ancestors(ancestors), + format!( + "102,{},{},{},{}", + service_id, service_type, service_name, service_status + ), + ) +} diff --git a/crates/core/c8y_api/src/smartrest/mod.rs b/crates/core/c8y_api/src/smartrest/mod.rs index 2feb5e991eb..5e9cf03e9fc 100644 --- a/crates/core/c8y_api/src/smartrest/mod.rs +++ b/crates/core/c8y_api/src/smartrest/mod.rs @@ -1,5 +1,6 @@ pub mod alarm; pub mod error; +pub mod inventory; pub mod message; pub mod operations; pub mod smartrest_deserializer; diff --git a/crates/core/c8y_api/src/smartrest/topic.rs b/crates/core/c8y_api/src/smartrest/topic.rs index f3c36f8b5f4..18c47af9813 100644 --- a/crates/core/c8y_api/src/smartrest/topic.rs +++ b/crates/core/c8y_api/src/smartrest/topic.rs @@ -24,7 +24,7 @@ impl C8yTopic { match entity.r#type { EntityType::MainDevice => Some(C8yTopic::upstream_topic()), EntityType::ChildDevice | EntityType::Service => { - Self::ChildSmartRestResponse(entity.entity_id.clone()) + Self::ChildSmartRestResponse(entity.entity_id.clone().into()) .to_topic() .ok() } @@ -108,7 +108,7 @@ impl From<&EntityMetadata> for C8yTopic { fn from(value: &EntityMetadata) -> Self { match value.r#type { EntityType::MainDevice => Self::SmartRestResponse, - EntityType::ChildDevice => Self::ChildSmartRestResponse(value.entity_id.clone()), + EntityType::ChildDevice => Self::ChildSmartRestResponse(value.entity_id.clone().into()), EntityType::Service => Self::SmartRestResponse, // TODO how services are handled by c8y? } } @@ -164,10 +164,35 @@ impl TryFrom for MapperSubscribeTopic { } } +/// Generates the SmartREST topic to publish to, for a given managed object +/// from the list of external IDs of itself and all its parents. +/// +/// The parents are appended in the reverse order, +/// starting from the main device at the end of the list. +/// The main device itself is represented by the root topic c8y/s/us, +/// with the rest of the children appended to it at each topic level. +/// +/// # Examples +/// +/// - `["main"]` -> `c8y/s/us` +/// - `["child1", "main"]` -> `c8y/s/us/child1` +/// - `["child2", "child1", "main"]` -> `c8y/s/us/child1/child2` +pub fn publish_topic_from_ancestors(ancestors: &[String]) -> Topic { + let mut target_topic = SMARTREST_PUBLISH_TOPIC.to_string(); + for ancestor in ancestors.iter().rev().skip(1) { + // Skipping the last ancestor as it is the main device represented by the root topic itself + target_topic.push('/'); + target_topic.push_str(ancestor); + } + + Topic::new_unchecked(&target_topic) +} + #[cfg(test)] mod tests { use super::*; use std::convert::TryInto; + use test_case::test_case; #[test] fn convert_c8y_topic_to_str() { @@ -211,4 +236,14 @@ mod tests { let error: Result = Topic::new("test").unwrap().try_into(); assert!(error.is_err()); } + + #[test_case(&["main"], "c8y/s/us")] + #[test_case(&["foo"], "c8y/s/us")] + #[test_case(&["child1", "main"], "c8y/s/us/child1")] + #[test_case(&["child3", "child2", "child1", "main"], "c8y/s/us/child1/child2/child3")] + fn topic_from_ancestors(ancestors: &[&str], topic: &str) { + let ancestors: Vec = ancestors.iter().map(|v| v.to_string()).collect(); + let nested_child_topic = publish_topic_from_ancestors(&ancestors); + assert_eq!(nested_child_topic, Topic::new_unchecked(topic)); + } } diff --git a/crates/core/tedge_api/src/entity_store.rs b/crates/core/tedge_api/src/entity_store.rs index a05c2480f82..6c72ab70521 100644 --- a/crates/core/tedge_api/src/entity_store.rs +++ b/crates/core/tedge_api/src/entity_store.rs @@ -26,6 +26,36 @@ use mqtt_channel::Topic; // In the future, root will be read from config const MQTT_ROOT: &str = "te"; +/// Represents externally provided unique ID of an entity. +/// Although this struct doesn't enforce any restrictions for the values, +/// the consumers may impose restrictions on the accepted values. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct EntityExternalId(String); + +impl AsRef for EntityExternalId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl From<&str> for EntityExternalId { + fn from(val: &str) -> Self { + Self(val.to_string()) + } +} + +impl From for EntityExternalId { + fn from(val: String) -> Self { + Self(val) + } +} + +impl From for String { + fn from(value: EntityExternalId) -> Self { + value.0 + } +} + /// A store for topic-based entity metadata lookup. /// /// This object is a hashmap from MQTT identifiers to entities (devices or @@ -58,7 +88,7 @@ const MQTT_ROOT: &str = "te"; pub struct EntityStore { main_device: EntityTopicId, entities: HashMap, - entity_id_index: HashMap, + entity_id_index: HashMap, } impl EntityStore { @@ -69,7 +99,7 @@ impl EntityStore { return None; } - let entity_id = main_device.entity_id?; + let entity_id: EntityExternalId = main_device.entity_id?.into(); let metadata = EntityMetadata { topic_id: main_device.topic_id.clone(), entity_id: entity_id.clone(), @@ -92,7 +122,7 @@ impl EntityStore { /// Returns information for an entity under a given device/service id . pub fn get_by_id(&self, entity_id: &str) -> Option<&EntityMetadata> { - let topic_id = self.entity_id_index.get(entity_id)?; + let topic_id = self.entity_id_index.get(&entity_id.into())?; self.get(topic_id) } @@ -103,9 +133,54 @@ impl EntityStore { &self.main_device } - /// Returns the name of main device. - pub fn main_device_name(&self) -> &str { - self.get(&self.main_device).unwrap().entity_id.as_str() + /// Returns the external id of the main device. + pub fn main_device_external_id(&self) -> EntityExternalId { + self.get(&self.main_device).unwrap().entity_id.clone() + } + + /// Returns an ordered list of ancestors of the given entity + /// starting from the immediate parent all the way till the root main device. + /// The last parent in the list for any entity would always be the main device. + /// The list would be empty for the main device as it has no further parents. + pub fn ancestors(&self, entity_topic_id: &EntityTopicId) -> Result, Error> { + if self.entities.get(entity_topic_id).is_none() { + return Err(Error::UnknownEntity(entity_topic_id.to_string())); + } + + let mut ancestors = vec![]; + + let mut current_entity_id = entity_topic_id; + while let Some(entity) = self.entities.get(current_entity_id) { + if let Some(parent_id) = &entity.parent { + ancestors.push(parent_id); + current_entity_id = parent_id; + } else { + break; // No more parents + } + } + + Ok(ancestors) + } + + /// Returns an ordered list of ancestors' external ids of the given entity + /// starting from the immediate parent all the way till the root main device. + /// The last parent in the list for any entity would always be the main device id. + pub fn ancestors_external_ids( + &self, + entity_topic_id: &EntityTopicId, + ) -> Result, Error> { + let mapped_ancestors = self + .ancestors(entity_topic_id)? + .iter() + .map(|tid| { + self.entities + .get(tid) + .map(|e| e.entity_id.clone().into()) + .unwrap() + }) + .collect(); + + Ok(mapped_ancestors) } /// Returns MQTT identifiers of child devices of a given device. @@ -144,6 +219,7 @@ impl EntityStore { pub fn update( &mut self, message: EntityRegistrationMessage, + external_id: EntityExternalId, ) -> Result, Error> { if message.r#type == EntityType::MainDevice && message.topic_id != self.main_device { return Err(Error::MainDeviceAlreadyRegistered( @@ -153,10 +229,13 @@ impl EntityStore { let mut affected_entities = vec![]; - let parent = if message.r#type == EntityType::MainDevice { - None - } else { - message.parent.or(Some(self.main_device.clone())) + let parent = match message.r#type { + EntityType::MainDevice => None, + EntityType::ChildDevice => message.parent.or_else(|| Some(self.main_device.clone())), + EntityType::Service => message + .parent + .or_else(|| message.topic_id.default_parent_identifier()) + .or_else(|| Some(self.main_device.clone())), }; // parent device is affected if new device is its child @@ -168,13 +247,10 @@ impl EntityStore { affected_entities.push(parent.clone()); } - let entity_id = message - .entity_id - .unwrap_or_else(|| self.derive_entity_id(&message.topic_id)); let entity_metadata = EntityMetadata { topic_id: message.topic_id.clone(), r#type: message.r#type, - entity_id: entity_id.clone(), + entity_id: external_id.clone(), parent, other: message.payload, }; @@ -187,7 +263,7 @@ impl EntityStore { if previous.is_some() { affected_entities.push(message.topic_id); } else { - self.entity_id_index.insert(entity_id, message.topic_id); + self.entity_id_index.insert(external_id, message.topic_id); } Ok(affected_entities) @@ -198,30 +274,6 @@ impl EntityStore { self.entities.iter() } - /// Generate child device external ID. - /// - /// The external id is generated by prefixing the id with main device name - /// (device_common_name) and then appending the MQTT entity topic with `/` - /// characters replaced by `:`. - /// - /// # Examples - /// - `device/main//` => `DEVICE_COMMON_NAME` - /// - `device/child001//` => `DEVICE_COMMON_NAME:device:child001` - /// - `device/child001/service/service001` => `DEVICE_COMMON_NAME:device:child001:service:service001` - /// - `factory01/hallA/packaging/belt001` => `DEVICE_COMMON_NAME:factory01:hallA:packaging:belt001` - fn derive_entity_id(&self, entity_topic: &EntityTopicId) -> String { - let main_device_entity_id = &self.get(&self.main_device).unwrap().entity_id; - - if entity_topic == &self.main_device { - main_device_entity_id.to_string() - } else { - let entity_id_suffix = entity_topic.to_string().replace('/', ":"); - let entity_id_suffix = entity_id_suffix.trim_matches(':'); - - format!("{main_device_entity_id}:{entity_id_suffix}") - } - } - /// Performs auto-registration process for an entity under a given /// identifier. /// @@ -233,29 +285,35 @@ impl EntityStore { /// being registered. pub fn auto_register_entity( &mut self, - entity_id: &EntityTopicId, + entity_topic_id: &EntityTopicId, + entity_external_id: EntityExternalId, + parent_external_id: Option, ) -> Result, entity_store::Error> { - let device_id = match entity_id.default_device_name() { + let device_id = match entity_topic_id.default_device_name() { Some(device_id) => device_id, None => return Ok(vec![]), }; let mut register_messages = vec![]; - let parent_id = entity_id.default_parent_identifier().unwrap(); + let parent_id = entity_topic_id.default_parent_identifier().unwrap(); if self.get(&parent_id).is_none() { let device_type = if device_id == "main" { "device" } else { "child-device" }; - let device_name = if device_id == "main" { - self.main_device_name() + let device_external_id = if device_id == "main" { + self.main_device_external_id() } else { - device_id + parent_external_id + .clone() + .ok_or_else(|| Error::ExternalIdNotGiven(parent_id.clone()))? }; - let device_register_payload = - format!("{{ \"@type\":\"{device_type}\", \"@id\":\"{device_name}\"}}"); + let device_register_payload = format!( + "{{ \"@type\":\"{device_type}\", \"@id\":\"{}\"}}", + device_external_id.as_ref() + ); // FIXME: The root prefix should not be added this way. // The simple fix is to change the signature of the method, @@ -264,11 +322,14 @@ impl EntityStore { let device_register_message = Message::new(&topic, device_register_payload).with_retain(); register_messages.push(device_register_message.clone()); - self.update(EntityRegistrationMessage::try_from(&device_register_message).unwrap())?; + self.update( + EntityRegistrationMessage::try_from(&device_register_message).unwrap(), + parent_external_id.ok_or_else(|| Error::ExternalIdNotGiven(parent_id.clone()))?, + )?; } // register service itself - if let Some(service_id) = entity_id.default_service_name() { + if let Some(service_id) = entity_topic_id.default_service_name() { let service_topic = format!("{MQTT_ROOT}/device/{device_id}/service/{service_id}"); let service_register_payload = r#"{"@type": "service", "type": "systemd"}"#.to_string(); let service_register_message = Message::new( @@ -277,7 +338,10 @@ impl EntityStore { ) .with_retain(); register_messages.push(service_register_message.clone()); - self.update(EntityRegistrationMessage::try_from(&service_register_message).unwrap())?; + self.update( + EntityRegistrationMessage::try_from(&service_register_message).unwrap(), + entity_external_id, + )?; } Ok(register_messages) @@ -289,7 +353,7 @@ pub struct EntityMetadata { pub topic_id: EntityTopicId, pub parent: Option, pub r#type: EntityType, - pub entity_id: String, + pub entity_id: EntityExternalId, pub other: serde_json::Value, } @@ -305,7 +369,7 @@ impl EntityMetadata { pub fn main_device(device_id: String) -> Self { Self { topic_id: EntityTopicId::default_main_device(), - entity_id: device_id, + entity_id: device_id.into(), r#type: EntityType::MainDevice, parent: None, other: serde_json::json!({}), @@ -316,7 +380,7 @@ impl EntityMetadata { pub fn child_device(child_device_id: String) -> Result { Ok(Self { topic_id: EntityTopicId::default_child_device(&child_device_id)?, - entity_id: child_device_id, + entity_id: child_device_id.into(), r#type: EntityType::ChildDevice, parent: Some(EntityTopicId::default_main_device()), other: serde_json::json!({}), @@ -335,16 +399,22 @@ pub enum Error { #[error("The main device was already registered at topic {0}")] MainDeviceAlreadyRegistered(Box), + + #[error("The specified entity {0} does not exist in the store")] + UnknownEntity(String), + + #[error("External ID not provided for {0}")] + ExternalIdNotGiven(EntityTopicId), } /// An object representing a valid entity registration message. #[derive(Debug, Clone, PartialEq, Eq)] pub struct EntityRegistrationMessage { - topic_id: EntityTopicId, - entity_id: Option, - r#type: EntityType, - parent: Option, - payload: serde_json::Value, + pub topic_id: EntityTopicId, + pub entity_id: Option, + pub r#type: EntityType, + pub parent: Option, + pub payload: serde_json::Value, } impl EntityRegistrationMessage { @@ -472,6 +542,7 @@ mod tests { json!({"@type": "child-device"}).to_string(), )) .unwrap(), + "child1".into(), ) .unwrap(); @@ -488,6 +559,7 @@ mod tests { json!({"@type": "child-device", "@parent": "device/main//"}).to_string(), )) .unwrap(), + "child2".into(), ) .unwrap(); assert_eq!(updated_entities, ["device/main//"]); @@ -509,13 +581,16 @@ mod tests { // Services are namespaced under devices, so `parent` is not necessary let updated_entities = store - .update(EntityRegistrationMessage { - r#type: EntityType::Service, - entity_id: None, - topic_id: EntityTopicId::default_main_service("service1").unwrap(), - parent: None, - payload: json!({}), - }) + .update( + EntityRegistrationMessage { + r#type: EntityType::Service, + entity_id: None, + topic_id: EntityTopicId::default_main_service("service1").unwrap(), + parent: None, + payload: json!({}), + }, + "service1".into(), + ) .unwrap(); assert_eq!(updated_entities, ["device/main//"]); @@ -525,13 +600,16 @@ mod tests { ); let updated_entities = store - .update(EntityRegistrationMessage { - r#type: EntityType::Service, - entity_id: None, - topic_id: EntityTopicId::default_main_service("service2").unwrap(), - parent: None, - payload: json!({}), - }) + .update( + EntityRegistrationMessage { + r#type: EntityType::Service, + entity_id: None, + topic_id: EntityTopicId::default_main_service("service2").unwrap(), + parent: None, + payload: json!({}), + }, + "service2".into(), + ) .unwrap(); assert_eq!(updated_entities, ["device/main//"]); @@ -560,13 +638,16 @@ mod tests { }) .unwrap(); - let res = store.update(EntityRegistrationMessage { - topic_id: EntityTopicId::default_child_device("another_main").unwrap(), - entity_id: Some("test-device".to_string()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({}), - }); + let res = store.update( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_child_device("another_main").unwrap(), + entity_id: Some("test-device".to_string()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({}), + }, + "another_main".into(), + ); assert_eq!( res, @@ -585,42 +666,257 @@ mod tests { }) .unwrap(); - let res = store.update(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - entity_id: None, - r#type: EntityType::ChildDevice, - parent: Some(EntityTopicId::default_child_device("myawesomeparent").unwrap()), - payload: json!({}), - }); + let res = store.update( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: None, + r#type: EntityType::ChildDevice, + parent: Some(EntityTopicId::default_child_device("myawesomeparent").unwrap()), + payload: json!({}), + }, + "myawesomedevice".into(), + ); assert!(matches!(res, Err(Error::NoParent(_)))); } #[test] - fn generates_entity_ids() { + fn list_ancestors() { let mut store = EntityStore::with_main_device(EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), entity_id: Some("test-device".to_string()), r#type: EntityType::MainDevice, parent: None, - payload: json!({}), + payload: json!({"@type": "device"}), }) .unwrap(); + // Assert no ancestors of main device + assert!(store + .ancestors(&EntityTopicId::default_main_device()) + .unwrap() + .is_empty()); + + // Register service on main store - .update(EntityRegistrationMessage { - topic_id: EntityTopicId::default_child_service("child001", "service001").unwrap(), - entity_id: None, - r#type: EntityType::ChildDevice, - parent: None, - payload: serde_json::json!({}), - }) + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/main/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(), + "collectd".into(), + ) + .unwrap(); + + // Assert ancestors of main device service + assert_eq!( + store + .ancestors(&EntityTopicId::default_main_service("collectd").unwrap()) + .unwrap(), + ["device/main//"] + ); + + // Register immediate child of main + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child1//").unwrap(), + json!({"@type": "child-device"}).to_string(), + )) + .unwrap(), + "child1".into(), + ) + .unwrap(); + + // Assert ancestors of child1 + assert_eq!( + store + .ancestors(&EntityTopicId::default_child_device("child1").unwrap()) + .unwrap(), + ["device/main//"] + ); + + // Register service on child1 + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child1/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(), + "child1_collectd".into(), + ) + .unwrap(); + + // Assert ancestors of child1 service + assert_eq!( + store + .ancestors(&EntityTopicId::default_child_service("child1", "collectd").unwrap()) + .unwrap(), + ["device/child1//", "device/main//"] + ); + + // Register child2 as child of child1 + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child2//").unwrap(), + json!({"@type": "child-device", "@parent": "device/child1//"}).to_string(), + )) + .unwrap(), + "child2".into(), + ) + .unwrap(); + + // Assert ancestors of child2 + assert_eq!( + store + .ancestors(&EntityTopicId::default_child_device("child2").unwrap()) + .unwrap(), + ["device/child1//", "device/main//"] + ); + + // Register service on child2 + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child2/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(), + "child2_collectd".into(), + ) + .unwrap(); + + // Assert ancestors of child2 service + assert_eq!( + store + .ancestors(&EntityTopicId::default_child_service("child2", "collectd").unwrap()) + .unwrap(), + ["device/child2//", "device/child1//", "device/main//"] + ); + } + + #[test] + fn list_ancestors_external_ids() { + let mut store = EntityStore::with_main_device(EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: Some("test-device".to_string()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({"@type": "device"}), + }) + .unwrap(); + + // Assert ancestor external ids of main device + assert!(store + .ancestors_external_ids(&EntityTopicId::default_main_device()) + .unwrap() + .is_empty()); + + // Register service on main + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/main/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(), + "collectd".into(), + ) .unwrap(); - let entity1 = store.get_by_id("test-device:device:child001:service:service001"); + // Assert ancestor external id of main device service assert_eq!( - entity1.unwrap().entity_id, - "test-device:device:child001:service:service001" + store + .ancestors_external_ids(&EntityTopicId::default_main_service("collectd").unwrap()) + .unwrap(), + ["test-device"] + ); + + // Register immediate child of main + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child1//").unwrap(), + json!({"@type": "child-device"}).to_string(), + )) + .unwrap(), + "child1".into(), + ) + .unwrap(); + + // Assert ancestor external ids of child1 + assert_eq!( + store + .ancestors_external_ids(&EntityTopicId::default_child_device("child1").unwrap()) + .unwrap(), + ["test-device"] + ); + + // Register service on child1 + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child1/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(), + "child1_collectd".into(), + ) + .unwrap(); + + // Assert ancestor external ids of child1 service + assert_eq!( + store + .ancestors_external_ids( + &EntityTopicId::default_child_service("child1", "collectd").unwrap() + ) + .unwrap(), + ["child1", "test-device"] + ); + + // Register child2 as child of child1 + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child2//").unwrap(), + json!({"@type": "child-device", "@parent": "device/child1//"}).to_string(), + )) + .unwrap(), + "child2".into(), + ) + .unwrap(); + + // Assert ancestor external ids of child2 + assert_eq!( + store + .ancestors_external_ids(&EntityTopicId::default_child_device("child2").unwrap()) + .unwrap(), + ["child1", "test-device"] + ); + + // Register service on child2 + store + .update( + EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child2/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(), + "child2_collectd".into(), + ) + .unwrap(); + + // Assert ancestor external ids of child2 service + assert_eq!( + store + .ancestors_external_ids( + &EntityTopicId::default_child_service("child2", "collectd").unwrap() + ) + .unwrap(), + ["child2", "child1", "test-device"] ); } } diff --git a/crates/core/tedge_api/src/event.rs b/crates/core/tedge_api/src/event.rs index 7fe1d007c3c..6370a0d5399 100644 --- a/crates/core/tedge_api/src/event.rs +++ b/crates/core/tedge_api/src/event.rs @@ -66,7 +66,7 @@ impl ThinEdgeEvent { Ok(Self { name: event_type.into(), data: event_data, - source: external_source, + source: external_source.map(|v| v.into()), }) } } diff --git a/crates/core/tedge_api/src/mqtt_topics.rs b/crates/core/tedge_api/src/mqtt_topics.rs index 81261a3feda..58a8eb13a6f 100644 --- a/crates/core/tedge_api/src/mqtt_topics.rs +++ b/crates/core/tedge_api/src/mqtt_topics.rs @@ -305,6 +305,10 @@ impl EntityTopicId { } .map(|parent_id| EntityTopicId(format!("device/{parent_id}//"))) } + + pub fn is_default_main_device(&self) -> bool { + self == &Self::default_main_device() + } } #[derive(Debug, thiserror::Error, PartialEq, Eq, Clone)] diff --git a/crates/extensions/c8y_mapper_ext/src/config.rs b/crates/extensions/c8y_mapper_ext/src/config.rs index 57b2d6785f3..277e37f4c48 100644 --- a/crates/extensions/c8y_mapper_ext/src/config.rs +++ b/crates/extensions/c8y_mapper_ext/src/config.rs @@ -153,8 +153,10 @@ impl C8yMapperConfig { } /// List of all possible external topics that Cumulocity mapper addresses. For testing purpose. + #[cfg(test)] pub fn default_external_topic_filter() -> TopicFilter { vec![ + "te/+/+/+/+", "te/+/+/+/+/m/+", "te/+/+/+/+/e/+", "te/+/+/+/+/a/+", diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 4e575f17acc..17fb5e09458 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -13,6 +13,8 @@ use c8y_api::json_c8y::C8yCreateEvent; use c8y_api::json_c8y::C8yUpdateSoftwareListResponse; use c8y_api::smartrest::error::OperationsError; use c8y_api::smartrest::error::SmartRestDeserializerError; +use c8y_api::smartrest::inventory::child_device_creation_message; +use c8y_api::smartrest::inventory::service_creation_message; use c8y_api::smartrest::message::collect_smartrest_messages; use c8y_api::smartrest::message::get_failure_reason_for_smartrest; use c8y_api::smartrest::message::get_smartrest_device_id; @@ -53,9 +55,11 @@ use std::path::PathBuf; use tedge_actors::LoggingSender; use tedge_actors::Sender; use tedge_api::entity_store; +use tedge_api::entity_store::EntityExternalId; use tedge_api::entity_store::EntityMetadata; use tedge_api::entity_store::EntityRegistrationMessage; use tedge_api::entity_store::EntityType; +use tedge_api::entity_store::Error; use tedge_api::event::error::ThinEdgeJsonDeserializerError; use tedge_api::event::ThinEdgeEvent; use tedge_api::messages::CommandStatus; @@ -231,6 +235,79 @@ impl CumulocityConverter { }) } + fn try_convert_entity_registration( + &mut self, + entity_topic_id: &EntityTopicId, + input: &EntityRegistrationMessage, + ) -> Result { + // Parse the optional fields + let display_name = input.payload.get("name").and_then(|v| v.as_str()); + let display_type = input.payload.get("type").and_then(|v| v.as_str()); + + let external_id = self + .entity_store + .get(entity_topic_id) + .map(|e| &e.entity_id) + .ok_or_else(|| Error::UnknownEntity(entity_topic_id.to_string()))?; + match input.r#type { + EntityType::MainDevice => Err(ConversionError::MainDeviceRegistrationNotSupported), + EntityType::ChildDevice => { + let ancestors_external_ids = + self.entity_store.ancestors_external_ids(entity_topic_id)?; + Ok(child_device_creation_message( + external_id.as_ref(), + display_name, + display_type, + &ancestors_external_ids, + )) + } + EntityType::Service => { + let ancestors_external_ids = + self.entity_store.ancestors_external_ids(entity_topic_id)?; + + Ok(service_creation_message( + external_id.as_ref(), + display_name.unwrap_or_else(|| { + entity_topic_id + .default_service_name() + .unwrap_or(external_id.as_ref()) + }), + display_type.unwrap_or("service"), + "up", + &ancestors_external_ids, + )) + } + } + } + + /// Generates external ID of the given entity. + /// + /// The external id is generated by transforming the EntityTopicId + /// by replacing the `/` characters with `:` and then adding the + /// main device id as a prefix, to namespace all the entities under that device. + /// + /// # Examples + /// - `device/main//` => `DEVICE_COMMON_NAME` + /// - `device/child001//` => `DEVICE_COMMON_NAME:device:child001` + /// - `device/child001/service/service001` => `DEVICE_COMMON_NAME:device:child001:service:service001` + /// - `factory01/hallA/packaging/belt001` => `DEVICE_COMMON_NAME:factory01:hallA:packaging:belt001` + fn map_to_c8y_external_id(&self, entity_topic_id: &EntityTopicId) -> EntityExternalId { + let external_id = if entity_topic_id.is_default_main_device() { + self.device_name.clone() + } else { + format!( + "{}:{}", + self.device_name.clone(), + entity_topic_id + .to_string() + .trim_end_matches('/') + .replace('/', ":") + ) + }; + + external_id.into() + } + fn try_convert_measurement( &mut self, source: &EntityTopicId, @@ -695,15 +772,39 @@ impl CumulocityConverter { match &channel { Channel::EntityMetadata => { if let Ok(register_message) = EntityRegistrationMessage::try_from(message) { - if let Err(e) = self.entity_store.update(register_message) { + // Generate the c8y external id, if an external id is not provided in the message + let external_id = register_message + .entity_id + .as_ref() + .map(|v| v.to_string().into()) + .unwrap_or_else(|| self.map_to_c8y_external_id(&source)); + + if let Err(e) = self + .entity_store + .update(register_message.clone(), external_id) + { error!("Could not update device registration: {e}"); } + let c8y_message = + self.try_convert_entity_registration(&source, ®ister_message)?; + registration_messages.push(c8y_message); } } _ => { // if device is unregistered register using auto-registration if self.entity_store.get(&source).is_none() { - registration_messages = match self.entity_store.auto_register_entity(&source) { + let entity_external_id = self.map_to_c8y_external_id(&source); + + // If the entity is a service, generate the external id of the parent device as well + let parent_external_id = source + .default_parent_identifier() + .map(|parent| self.map_to_c8y_external_id(&parent)); + + registration_messages = match self.entity_store.auto_register_entity( + &source, + entity_external_id, + parent_external_id, + ) { Ok(register_messages) => register_messages, Err(e) => { error!("Could not update device registration: {e}"); @@ -714,7 +815,7 @@ impl CumulocityConverter { if let Some(entity) = self.entity_store.get(&source) { if let Some(message) = external_device_registration_message(entity) { self.children - .insert(entity.entity_id.to_string(), Operations::default()); + .insert(entity.entity_id.clone().into(), Operations::default()); registration_messages.push(message); } } else { @@ -971,7 +1072,7 @@ fn add_external_device_registration_message( fn external_device_registration_message(entity: &EntityMetadata) -> Option { match entity.r#type { - EntityType::ChildDevice => Some(new_child_device_message(&entity.entity_id)), + EntityType::ChildDevice => Some(new_child_device_message(entity.entity_id.as_ref())), _ => None, } } @@ -1000,7 +1101,7 @@ impl CumulocityConverter { Some(device) => { let ops_dir = match device.r#type { EntityType::MainDevice => self.ops_dir.clone(), - EntityType::ChildDevice => self.ops_dir.clone().join(&device.entity_id), + EntityType::ChildDevice => self.ops_dir.clone().join(device.entity_id.as_ref()), EntityType::Service => { error!("Unsupported `restart` operation for a service: {target}"); return Ok(vec![]); @@ -1230,12 +1331,14 @@ mod tests { use rand::SeedableRng; use serde_json::json; use std::collections::HashMap; + use std::str::FromStr; use tedge_actors::Builder; use tedge_actors::LoggingSender; use tedge_actors::MessageReceiver; use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_actors::SimpleMessageBoxBuilder; + use tedge_api::mqtt_topics::EntityTopicId; use tedge_api::mqtt_topics::MqttSchema; use tedge_mqtt_ext::Message; use tedge_mqtt_ext::MqttMessage; @@ -1394,11 +1497,11 @@ mod tests { let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,child1,child1,thin-edge.io-child", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), - r#"{"externalSource":{"externalId":"child1","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, + r#"{"externalSource":{"externalId":"test-device:device:child1","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, ); // Test the first output messages contains SmartREST and C8Y JSON. @@ -1450,11 +1553,11 @@ mod tests { .collect(); let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,child1,child1,thin-edge.io-child", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), - r#"{"externalSource":{"externalId":"child1","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, + r#"{"externalSource":{"externalId":"test-device:device:child1","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, ); assert_eq!( out_second_messages, @@ -1479,11 +1582,11 @@ mod tests { .collect(); let expected_first_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,child1,child1,thin-edge.io-child", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", ); let expected_first_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), - r#"{"externalSource":{"externalId":"child1","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, + r#"{"externalSource":{"externalId":"test-device:device:child1","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, ); assert_eq!( out_first_messages, @@ -1504,11 +1607,11 @@ mod tests { .collect(); let expected_second_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,child2,child2,thin-edge.io-child", + "101,test-device:device:child2,test-device:device:child2,thin-edge.io-child", ); let expected_second_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), - r#"{"externalSource":{"externalId":"child2","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, + r#"{"externalSource":{"externalId":"test-device:device:child2","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"ThinEdgeMeasurement"}"#, ); assert_eq!( out_second_messages, @@ -1578,12 +1681,12 @@ mod tests { let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,child,child,thin-edge.io-child", + "101,test-device:device:child,test-device:device:child,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), - r#"{"externalSource":{"externalId":"child","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"test_type"}"#, + r#"{"externalSource":{"externalId":"test-device:device:child","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"test_type"}"#, ); // Test the output messages contains SmartREST and C8Y JSON. @@ -1612,12 +1715,12 @@ mod tests { let in_message = Message::new(&Topic::new_unchecked(in_topic), in_payload); let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,child2,child2,thin-edge.io-child", + "101,test-device:device:child2,test-device:device:child2,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), - r#"{"externalSource":{"externalId":"child2","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"type_in_payload"}"#, + r#"{"externalSource":{"externalId":"test-device:device:child2","type":"c8y_Serial"},"temp":{"temp":{"value":1.0}},"time":"2021-11-16T17:45:40.571760714+01:00","type":"type_in_payload"}"#, ); // Test the first output messages contains SmartREST and C8Y JSON. @@ -1871,9 +1974,11 @@ mod tests { let payload1 = &result[0].payload_str().unwrap(); let payload2 = &result[1].payload_str().unwrap(); - assert!(payload1.contains("101,child1,child1,thin-edge.io-child")); + assert!(payload1.contains( + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child" + )); assert!(payload2 .contains( - r#"{"externalSource":{"externalId":"child1","type":"c8y_Serial"},"temperature0":{"temperature0":{"value":0.0}},"# + r#"{"externalSource":{"externalId":"test-device:device:child1","type":"c8y_Serial"},"temperature0":{"temperature0":{"value":0.0}},"# )); assert!(payload2.contains(r#""type":"ThinEdgeMeasurement""#)); } @@ -2068,6 +2173,32 @@ mod tests { assert_eq!(supported_operations_counter, 4); } + #[test_case("device/main//", "test-device")] + #[test_case( + "device/main/service/tedge-agent", + "test-device:device:main:service:tedge-agent" + )] + #[test_case("device/child1//", "test-device:device:child1")] + #[test_case( + "device/child1/service/collectd", + "test-device:device:child1:service:collectd" + )] + #[test_case("custom_name///", "test-device:custom_name")] + #[tokio::test] + async fn entity_topic_id_to_c8y_external_id_mapping( + entity_topic_id: &str, + c8y_external_id: &str, + ) { + let tmp_dir = TempTedgeDir::new(); + let (converter, _http_proxy) = create_c8y_converter(&tmp_dir).await; + + let entity_topic_id = EntityTopicId::from_str(entity_topic_id).unwrap(); + assert_eq!( + converter.map_to_c8y_external_id(&entity_topic_id), + c8y_external_id.into() + ); + } + async fn create_c8y_converter( tmp_dir: &TempTedgeDir, ) -> ( diff --git a/crates/extensions/c8y_mapper_ext/src/error.rs b/crates/extensions/c8y_mapper_ext/src/error.rs index 75497dc9368..2275f3f5683 100644 --- a/crates/extensions/c8y_mapper_ext/src/error.rs +++ b/crates/extensions/c8y_mapper_ext/src/error.rs @@ -118,6 +118,12 @@ pub enum ConversionError { #[error(transparent)] FromC8yAlarmError(#[from] c8y_api::json_c8y::C8yAlarmError), + + #[error("Main device registration via MQTT is not supported. Use 'tedge connect c8y' command instead.")] + MainDeviceRegistrationNotSupported, + + #[error(transparent)] + FromEntityStoreError(#[from] tedge_api::entity_store::Error), } #[derive(thiserror::Error, Debug)] diff --git a/crates/extensions/c8y_mapper_ext/src/log_upload.rs b/crates/extensions/c8y_mapper_ext/src/log_upload.rs index 9c1bfe8baa9..e77caf8d23b 100644 --- a/crates/extensions/c8y_mapper_ext/src/log_upload.rs +++ b/crates/extensions/c8y_mapper_ext/src/log_upload.rs @@ -67,7 +67,10 @@ impl CumulocityConverter { let tedge_url = format!( "http://{}/tedge/file-transfer/{}/log_upload/{}-{}", - &self.config.tedge_http_host, target.entity_id, log_request.log_type, cmd_id + &self.config.tedge_http_host, + target.entity_id.as_ref(), + log_request.log_type, + cmd_id ); let request = LogUploadCmdPayload { @@ -106,7 +109,7 @@ impl CumulocityConverter { topic_id: topic_id.to_string(), } })?; - let external_id = device.entity_id.to_string(); + let external_id = &device.entity_id; let c8y_topic: C8yTopic = device.into(); let smartrest_topic = c8y_topic.to_topic()?; @@ -126,7 +129,7 @@ impl CumulocityConverter { let uploaded_file_path = self .config .file_transfer_dir - .join(&device.entity_id) + .join(device.entity_id.as_ref()) .join("log_upload") .join(format!("{}-{}", response.log_type, cmd_id)); let result = self @@ -134,7 +137,7 @@ impl CumulocityConverter { .upload_file( uploaded_file_path.as_std_path(), &response.log_type, - external_id, + external_id.as_ref().to_string(), ) .await; // We need to get rid of this await, otherwise it blocks @@ -202,7 +205,7 @@ impl CumulocityConverter { // Create a c8y_LogfileRequest operation file let dir_path = match device.r#type { EntityType::MainDevice => self.ops_dir.clone(), - EntityType::ChildDevice => self.ops_dir.join(&device.entity_id), + EntityType::ChildDevice => self.ops_dir.join(device.entity_id.as_ref()), EntityType::Service => { // No support for service log management return Ok(vec![]); diff --git a/crates/extensions/c8y_mapper_ext/src/serializer.rs b/crates/extensions/c8y_mapper_ext/src/serializer.rs index 0720e2e2f22..1d0c4196d13 100644 --- a/crates/extensions/c8y_mapper_ext/src/serializer.rs +++ b/crates/extensions/c8y_mapper_ext/src/serializer.rs @@ -67,7 +67,7 @@ impl C8yJsonSerializer { let _ = json.write_key("externalSource"); json.write_open_obj(); let _ = json.write_key("externalId"); - let _ = json.write_str(child_id); + let _ = json.write_str(child_id.as_ref()); let _ = json.write_key("type"); let _ = json.write_str("c8y_Serial"); json.write_close_obj(); diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index 669964676ff..b5dccac0088 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -75,6 +75,134 @@ async fn mapper_publishes_init_messages_on_startup() { .await; } +#[tokio::test] +async fn child_device_registration_mapping() { + let (mqtt, _http, _fs, mut timer) = spawn_c8y_mapper_actor(&TempTedgeDir::new(), true).await; + timer.send(Timeout::new(())).await.unwrap(); // Complete sync phase so that alarm mapping starts + let mut mqtt = mqtt.with_timeout(TEST_TIMEOUT_MS); + mqtt.skip(6).await; // Skip all init messages + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child1//"), + r#"{ "@type": "child-device", "type": "RaspberryPi", "name": "Child1" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us", + "101,test-device:device:child1,Child1,RaspberryPi", + )], + ) + .await; + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child2//"), + r#"{ "@type": "child-device", "@parent": "device/child1//" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us/test-device:device:child1", + "101,test-device:device:child2,test-device:device:child2,thin-edge.io-child", + )], + ) + .await; + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child3//"), + r#"{ "@type": "child-device", "@id": "child3", "@parent": "device/child2//" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us/test-device:device:child1/test-device:device:child2", + "101,child3,child3,thin-edge.io-child", + )], + ) + .await; +} + +#[tokio::test] +async fn service_registration_mapping() { + let (mqtt, _http, _fs, mut timer) = spawn_c8y_mapper_actor(&TempTedgeDir::new(), true).await; + timer.send(Timeout::new(())).await.unwrap(); // Complete sync phase so that alarm mapping starts + let mut mqtt = mqtt.with_timeout(TEST_TIMEOUT_MS); + mqtt.skip(6).await; // Skip all init messages + + // Create a direct child device: child1 and a nested child device: child2 + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child1//"), + r#"{ "@type": "child-device" }"#, + )) + .await + .unwrap(); + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child2//"), + r#"{ "@type": "child-device", "@parent": "device/child1//" }"#, + )) + .await + .unwrap(); + + mqtt.skip(2).await; // Skip mappings of above child device creation messages + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/main/service/collectd"), + r#"{ "@type": "service", "type": "systemd", "name": "Collectd" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us", + "102,test-device:device:main:service:collectd,systemd,Collectd,up", + )], + ) + .await; + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child1/service/collectd"), + r#"{ "@type": "service", "type": "systemd", "name": "Collectd" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us/test-device:device:child1", + "102,test-device:device:child1:service:collectd,systemd,Collectd,up", + )], + ) + .await; + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/device/child2/service/collectd"), + r#"{ "@type": "service", "type": "systemd", "name": "Collectd" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us/test-device:device:child1/test-device:device:child2", + "102,test-device:device:child2:service:collectd,systemd,Collectd,up", + )], + ) + .await; +} + #[tokio::test] async fn mapper_publishes_software_update_request() { // The test assures SM Mapper correctly receives software update request smartrest message on `c8y/s/ds` @@ -1419,6 +1547,8 @@ async fn mapper_converts_smartrest_logfile_req_to_log_upload_cmd_for_child_devic .await .expect("fail to register the child-device"); + mqtt.skip(6).await; // Skip the mapped child device registration message + // Simulate c8y_LogfileRequest SmartREST request mqtt.send(MqttMessage::new( &C8yTopic::downstream_topic(), @@ -1517,18 +1647,28 @@ async fn mapper_converts_log_upload_cmd_to_supported_op_and_types_for_child_devi [ ( "te/device/child1//", - r#"{ "@type":"child-device", "@id":"child1"}"#, + r#"{ "@type":"child-device", "@id":"test-device:device:child1"}"#, + ), + ( + "c8y/s/us", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", ), - ("c8y/s/us", "101,child1,child1,thin-edge.io-child"), ], ) .await; - assert_received_contains_str(&mut mqtt, [("c8y/s/us/child1", "118,typeA,typeB,typeC")]).await; + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us/test-device:device:child1", + "118,typeA,typeB,typeC", + )], + ) + .await; // Validate if the supported operation file is created assert!(ttd .path() - .join("operations/c8y/child1/c8y_LogfileRequest") + .join("operations/c8y/test-device:device:child1/c8y_LogfileRequest") .exists()); } @@ -1617,15 +1757,25 @@ async fn handle_log_upload_executing_and_failed_cmd_for_child_device() { [ ( "te/device/child1//", - r#"{ "@type":"child-device", "@id":"child1"}"#, + r#"{ "@type":"child-device", "@id":"test-device:device:child1"}"#, + ), + ( + "c8y/s/us", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", ), - ("c8y/s/us", "101,child1,child1,thin-edge.io-child"), ], ) .await; // Expect `501` smartrest message on `c8y/s/us/child1`. - assert_received_contains_str(&mut mqtt, [("c8y/s/us/child1", "501,c8y_LogfileRequest")]).await; + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us/test-device:device:child1", + "501,c8y_LogfileRequest", + )], + ) + .await; // Simulate log_upload command with "failed" state mqtt.send(MqttMessage::new( @@ -1649,7 +1799,7 @@ async fn handle_log_upload_executing_and_failed_cmd_for_child_device() { assert_received_contains_str( &mut mqtt, [( - "c8y/s/us/child1", + "c8y/s/us/test-device:device:child1", "502,c8y_LogfileRequest,\"Something went wrong\"", )], ) diff --git a/docs/src/contribute/design/mqtt-topic-design.md b/docs/src/contribute/design/mqtt-topic-design.md index c670d219282..975e4f12ff8 100644 --- a/docs/src/contribute/design/mqtt-topic-design.md +++ b/docs/src/contribute/design/mqtt-topic-design.md @@ -126,7 +126,7 @@ The equipment, which is a conveyor belt called "belt01", is located in factory " ```sh te2mqtt formats="v1" tedge mqtt pub -r 'te/factory01/hallA/packaging/belt001' '{ "@type": "child-device", - "displayName": "belt001", + "name": "belt001", "type": "ConveyorBelt", "factory": "factory01", "building": "hallA", diff --git a/docs/src/references/mappers/c8y-mapper.md b/docs/src/references/mappers/c8y-mapper.md index ab56d93ea08..f4485183aa8 100644 --- a/docs/src/references/mappers/c8y-mapper.md +++ b/docs/src/references/mappers/c8y-mapper.md @@ -37,7 +37,7 @@ te/device/child01 ```json5 title="Payload" { "@type": "child-device", - "displayName": "child01", + "name": "child01", "type": "SmartHomeHub" } ``` @@ -75,7 +75,7 @@ te/device/child01 { "@type": "child-device", "@id": "child01", - "displayName": "child01", + "name": "child01", "type": "SmartHomeHub" } ``` @@ -112,7 +112,7 @@ te/device/nested_child01 { "@type": "child-device", "@parent": "te/device/child01", - "displayName": "nested_child01", + "name": "nested_child01", "type": "BatterySensor" } ``` @@ -147,7 +147,7 @@ te/device/main/service/nodered ```json5 title="Payload" { "@type": "service", - "displayName": "Node-Red", + "name": "Node-Red", "type": "systemd" } ``` @@ -182,7 +182,7 @@ te/device/child01/service/nodered ```json5 title="Payload" { "@type": "service", - "displayName": "Node-Red", + "name": "Node-Red", "type": "systemd" } ``` diff --git a/docs/src/references/mqtt-api.md b/docs/src/references/mqtt-api.md index 7eef3b4da14..89bda187360 100644 --- a/docs/src/references/mqtt-api.md +++ b/docs/src/references/mqtt-api.md @@ -294,7 +294,7 @@ tedge mqtt pub -r 'te/device/main' '{ ```sh te2mqtt formats="v1" tedge mqtt pub -r 'te/device/main/service/nodered' '{ "@type": "service", - "displayName": "nodered", + "name": "nodered", "type": "systemd" }' ``` @@ -308,7 +308,7 @@ if the parent can not be derived from the topic directly: tedge mqtt pub -r 'te/component_namespace/service/nodered/instance-1' '{ "@type": "service", "@parent": "te/device/main", - "displayName": "nodered", + "name": "nodered", "type": "systemd" }' ``` @@ -318,7 +318,7 @@ tedge mqtt pub -r 'te/component_namespace/service/nodered/instance-1' '{ ```sh te2mqtt formats="v1" tedge mqtt pub -r 'te/device/child01' '{ "@type": "child-device", - "displayName": "child01", + "name": "child01", "type": "SmartHomeHub" }' ``` @@ -333,7 +333,7 @@ Nested child devices are registered in a similar fashion as an immediate child d tedge mqtt pub -r 'te/device/nested_child01' '{ "@type": "child-device", "@parent": "te/device/child01", - "displayName": "nested_child01" + "name": "nested_child01" }' ``` @@ -347,7 +347,7 @@ But, it is advised to declare it explicitly as follows: tedge mqtt pub -r 'te/device/child01/service/nodered' '{ "@type": "service", "@parent": "te/device/child01", - "displayName": "nodered", + "name": "nodered", "type": "systemd" }' ``` @@ -364,7 +364,7 @@ For example, a linux service runs on a device as it relies on physical hardware tedge mqtt pub -r 'te/device/nested_child01/service/nodered' '{ "@type": "service", "@parent": "te/device/nested_child01", - "displayName": "nodered", + "name": "nodered", "type": "systemd" }' ``` diff --git a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot index 86e5542ab4a..0d27c689c36 100644 --- a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot +++ b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot @@ -5,6 +5,7 @@ Library ThinEdgeIO Test Tags theme:c8y theme:registration Suite Setup Custom Setup +Test Setup Test Setup Test Teardown Get Logs ${DEVICE_SN} *** Test Cases *** @@ -17,7 +18,6 @@ Main device registration Child device registration - ThinEdgeIO.Set Device Context ${DEVICE_SN} Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} Restart Service tedge-mapper-c8y @@ -29,13 +29,80 @@ Child device registration # Check child device relationship Cumulocity.Set Device ${DEVICE_SN} - Cumulocity.Device Should Have A Child Devices ${CHILD_SN} + Cumulocity.Should Be A Child Device Of Device ${CHILD_SN} + +Register child device with defaults via MQTT + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device"}' + Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${CHILD_SN} child_name=${DEVICE_SN}:device:${CHILD_SN} child_type=thin-edge.io-child + +Register child device with custom name and type via MQTT + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device","name":"${CHILD_SN}","type":"linux-device-Aböut"}' + Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${CHILD_SN} child_name=${CHILD_SN} child_type=linux-device-Aböut + +Register child device with custom id via MQTT + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device","@id":"${CHILD_SN}","name":"custom-${CHILD_SN}"}' + Check Child Device parent_sn=${DEVICE_SN} child_sn=${CHILD_SN} child_name=custom-${CHILD_SN} child_type=thin-edge.io-child + +Register nested child device using default topic schema via MQTT + ${child_level1}= Get Random Name + ${child_level2}= Get Random Name + ${child_level3}= Get Random Name + + Execute Command tedge mqtt pub --retain 'te/device/${child_level1}//' '{"@type":"child-device","@parent":"device/main//"}' + Execute Command tedge mqtt pub --retain 'te/device/${child_level2}//' '{"@type":"child-device","@parent":"device/${child_level1}//","name":"${child_level2}"}' + Execute Command tedge mqtt pub --retain 'te/device/${child_level3}//' '{"@type":"child-device","@parent":"device/${child_level2}//","type":"child_level3"}' + Execute Command tedge mqtt pub --retain 'te/device/${child_level3}/service/custom-app' '{"@type":"service","@parent":"device/${child_level3}//","name":"custom-app","type":"service-level3"}' + + # Level 1 + Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${child_level1} child_name=${DEVICE_SN}:device:${child_level1} child_type=thin-edge.io-child + + # Level 2 + Check Child Device parent_sn=${DEVICE_SN}:device:${child_level1} child_sn=${DEVICE_SN}:device:${child_level2} child_name=${child_level2} child_type=thin-edge.io-child + + # Level 3 + Check Child Device parent_sn=${DEVICE_SN}:device:${child_level2} child_sn=${DEVICE_SN}:device:${child_level3} child_name=${DEVICE_SN}:device:${child_level3} child_type=child_level3 + Check Service child_sn=${DEVICE_SN}:device:${child_level3} service_sn=${DEVICE_SN}:device:${child_level3}:service:custom-app service_name=custom-app service_type=service-level3 service_status=up + + +Register service on a child device via MQTT + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device","name":"${CHILD_SN}","type":"linux-device-Aböut"}' + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}/service/custom-app' '{"@type":"service","@parent":"device/${CHILD_SN}//","name":"custom-app","type":"custom-type"}' + + # Check child registration + Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${CHILD_SN} child_name=${CHILD_SN} child_type=linux-device-Aböut + + # Check service registration + Check Service child_sn=${DEVICE_SN}:device:${CHILD_SN} service_sn=${DEVICE_SN}:device:${CHILD_SN}:service:custom-app service_name=custom-app service_type=custom-type service_status=up + *** Keywords *** +Check Child Device + [Arguments] ${parent_sn} ${child_sn} ${child_name} ${child_type} + ${child_mo}= Device Should Exist ${child_sn} + + ${child_mo}= Cumulocity.Device Should Have Fragment Values name\=${child_name} + Should Be Equal ${child_mo["owner"]} device_${DEVICE_SN} # The parent is the owner of the child + Should Be Equal ${child_mo["name"]} ${child_name} + Should Be Equal ${child_mo["type"]} ${child_type} + + # Check child device relationship + Cumulocity.Device Should Exist ${parent_sn} + Cumulocity.Should Be A Child Device Of Device ${child_sn} + +Check Service + [Arguments] ${child_sn} ${service_sn} ${service_name} ${service_type} ${service_status}=up + Cumulocity.Device Should Exist ${service_sn} show_info=${False} + Cumulocity.Device Should Exist ${child_sn} show_info=${False} + Should Have Services name=${service_name} service_type=${service_type} status=${service_status} + + +Test Setup + ${CHILD_SN}= Get Random Name + Set Test Variable $CHILD_SN + + ThinEdgeIO.Set Device Context ${DEVICE_SN} + Custom Setup ${DEVICE_SN}= Setup Set Suite Variable $DEVICE_SN - - ${CHILD_SN}= Setup bootstrap=${False} - Set Suite Variable $CHILD_SN diff --git a/tests/RobotFramework/tests/tedge/call_tedge_config_list.robot b/tests/RobotFramework/tests/tedge/call_tedge_config_list.robot index 3256b694b1d..2b7d32f98e0 100644 --- a/tests/RobotFramework/tests/tedge/call_tedge_config_list.robot +++ b/tests/RobotFramework/tests/tedge/call_tedge_config_list.robot @@ -105,7 +105,7 @@ set/unset c8y.topics ${unset} Execute Command tedge config list Should Contain ... ${unset} - ... c8y.topics=["te/+/+/+/+/m/+", "te/+/+/+/+/e/+", "te/+/+/+/+/a/+", "tedge/health/+", "tedge/health/+/+"] + ... c8y.topics=["te/+/+/+/+", "te/+/+/+/+/m/+", "te/+/+/+/+/e/+", "te/+/+/+/+/a/+", "tedge/health/+", "tedge/health/+/+"] set/unset az.root_cert_path Execute Command sudo tedge config set az.root_cert_path /etc/ssl/certs1 # Changing az.root_cert_path From 7b073d41dc36c65b9989a7f997ba584c4ca17ba4 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Thu, 21 Sep 2023 12:05:11 +0000 Subject: [PATCH 02/13] Fix external id of auto-registered devices in system tests --- .../cumulocity/log/log_operation_child.robot | 9 +++++---- .../telemetry/c8y_child_alarms_rpi.robot | 4 ++-- .../telemetry/child_device_telemetry.robot | 19 ++++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/RobotFramework/tests/cumulocity/log/log_operation_child.robot b/tests/RobotFramework/tests/cumulocity/log/log_operation_child.robot index 43322550416..e4ee73ac80e 100644 --- a/tests/RobotFramework/tests/cumulocity/log/log_operation_child.robot +++ b/tests/RobotFramework/tests/cumulocity/log/log_operation_child.robot @@ -23,13 +23,13 @@ Successful log operation *** Keywords *** Setup Child Device - ThinEdgeIO.Set Device Context ${CHILD_SN} + ThinEdgeIO.Set Device Context ${CHILD_ID} Execute Command sudo dpkg -i packages/tedge_*.deb Execute Command sudo tedge config set mqtt.client.host ${PARENT_IP} Execute Command sudo tedge config set mqtt.client.port 1883 Execute Command sudo tedge config set mqtt.topic_root te - Execute Command sudo tedge config set mqtt.device_topic_id "device/${CHILD_SN}//" + Execute Command sudo tedge config set mqtt.device_topic_id "device/${CHILD_ID}//" # Install plugin after the default settings have been updated to prevent it from starting up as the main plugin Execute Command sudo dpkg -i packages/tedge-log-plugin*.deb @@ -57,7 +57,8 @@ Custom Setup ThinEdgeIO.Service Health Status Should Be Up tedge-mapper-c8y # Child - ${child_sn}= Setup skip_bootstrap=${True} - Set Suite Variable $CHILD_SN ${child_sn} + ${CHILD_ID}= Setup skip_bootstrap=${True} + Set Suite Variable $CHILD_ID + Set Suite Variable $CHILD_SN ${PARENT_SN}:device:${CHILD_ID} Setup Child Device Cumulocity.Device Should Exist ${CHILD_SN} diff --git a/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot b/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot index 85e957e595e..5b4f93bccd6 100644 --- a/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot +++ b/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot @@ -18,8 +18,8 @@ ${CHILD_SN} *** Test Cases *** Define Child device 1 ID - ${name}= Get Random Name prefix=${EMPTY} - Set Suite Variable $CHILD_SN ${DEVICE_SN}-${name}-child01 + Set Suite Variable $CHILD_ID child01 + Set Suite Variable $CHILD_SN ${DEVICE_SN}:device:${CHILD_ID} Normal case when the child device does not exist on c8y cloud # Sending child alarm diff --git a/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot b/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot index 7e35d2c4d94..0e017b72467 100644 --- a/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot +++ b/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot @@ -9,50 +9,50 @@ Test Teardown Get Logs *** Test Cases *** Child devices support sending simple measurements - Execute Command tedge mqtt pub te/device/${CHILD_SN}///m/ '{ "temperature": 25 }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///m/ '{ "temperature": 25 }' ${measurements}= Device Should Have Measurements minimum=1 maximum=1 type=ThinEdgeMeasurement value=temperature series=temperature Log ${measurements} Child devices support sending simple measurements with custom type in topic - Execute Command tedge mqtt pub te/device/${CHILD_SN}///m/CustomType_topic '{ "temperature": 25 }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///m/CustomType_topic '{ "temperature": 25 }' ${measurements}= Device Should Have Measurements minimum=1 maximum=1 type=CustomType_topic value=temperature series=temperature Log ${measurements} Child devices support sending simple measurements with custom type in payload - Execute Command tedge mqtt pub te/device/${CHILD_SN}///m/CustomType_topic '{ "type":"CustomType_payload","temperature": 25 }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///m/CustomType_topic '{ "type":"CustomType_payload","temperature": 25 }' ${measurements}= Device Should Have Measurements minimum=1 maximum=1 type=CustomType_payload value=temperature series=temperature Log ${measurements} Child devices support sending custom measurements - Execute Command tedge mqtt pub te/device/${CHILD_SN}///m/ '{ "current": {"L1": 9.5, "L2": 1.3} }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///m/ '{ "current": {"L1": 9.5, "L2": 1.3} }' ${measurements}= Device Should Have Measurements minimum=1 maximum=1 type=ThinEdgeMeasurement value=current series=L1 Log ${measurements} Child devices support sending custom events - Execute Command tedge mqtt pub te/device/${CHILD_SN}///e/myCustomType1 '{ "text": "Some test event", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///e/myCustomType1 '{ "text": "Some test event", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' ${events}= Device Should Have Event/s expected_text=Some test event with_attachment=False minimum=1 maximum=1 type=myCustomType1 fragment=someOtherCustomFragment Log ${events} Child devices support sending custom events overriding the type - Execute Command tedge mqtt pub te/device/${CHILD_SN}///e/myCustomType '{"type": "otherType", "text": "Some test event", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///e/myCustomType '{"type": "otherType", "text": "Some test event", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' ${events}= Device Should Have Event/s expected_text=Some test event with_attachment=False minimum=1 maximum=1 type=otherType fragment=someOtherCustomFragment Log ${events} Child devices support sending custom events without type in topic - Execute Command tedge mqtt pub te/device/${CHILD_SN}///e/ '{"text": "Some test event", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///e/ '{"text": "Some test event", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' ${events}= Device Should Have Event/s expected_text=Some test event with_attachment=False minimum=1 maximum=1 type=ThinEdgeEvent fragment=someOtherCustomFragment Log ${events} Child devices support sending custom alarms #1699 [Tags] \#1699 - Execute Command tedge mqtt pub te/device/${CHILD_SN}///a/myCustomAlarmType '{ "severity": "critical", "text": "Some test alarm", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///a/myCustomAlarmType '{ "severity": "critical", "text": "Some test alarm", "someOtherCustomFragment": {"nested":{"value": "extra info"}} }' ${alarms}= Device Should Have Alarm/s expected_text=Some test alarm severity=CRITICAL minimum=1 maximum=1 type=myCustomAlarmType Should Be Equal ${alarms[0]["someOtherCustomFragment"]["nested"]["value"]} extra info Log ${alarms} @@ -84,7 +84,8 @@ Child device supports sending custom child device measurements directly to c8y Custom Setup ${DEVICE_SN}= Setup Set Suite Variable $DEVICE_SN - Set Suite Variable $CHILD_SN ${DEVICE_SN}_child1 + Set Suite Variable $CHILD_ID child1 + Set Suite Variable $CHILD_SN ${DEVICE_SN}:device:child1 Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} Restart Service tedge-mapper-c8y Device Should Exist ${DEVICE_SN} From 8ee34208f8b80c9bcfee9f618228a2056223b98d Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Fri, 22 Sep 2023 04:46:09 +0000 Subject: [PATCH 03/13] Fix alarm tests with updated external id logic --- crates/core/c8y_api/src/json_c8y.rs | 6 ++-- .../c8y_mapper_ext/src/converter.rs | 4 +-- crates/extensions/c8y_mapper_ext/src/tests.rs | 33 ++++++++++--------- .../telemetry/c8y_child_alarms_rpi.robot | 9 ++--- .../telemetry/child_device_telemetry.robot | 4 +-- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/crates/core/c8y_api/src/json_c8y.rs b/crates/core/c8y_api/src/json_c8y.rs index c69cbf97bbc..97e477464d1 100644 --- a/crates/core/c8y_api/src/json_c8y.rs +++ b/crates/core/c8y_api/src/json_c8y.rs @@ -415,7 +415,7 @@ mod tests { use serde_json::json; use tedge_api::alarm::ThinEdgeAlarm; use tedge_api::alarm::ThinEdgeAlarmData; - use tedge_api::entity_store::{EntityExternalId, EntityRegistrationMessage}; + use tedge_api::entity_store::EntityRegistrationMessage; use tedge_api::event::ThinEdgeEventData; use tedge_api::mqtt_topics::EntityTopicId; use test_case::test_case; @@ -781,7 +781,9 @@ mod tests { r#"{"@id": "external_source", "@type": "child-device"}"#, )) .unwrap(); - entity_store.update(child_registration, EntityExternalId::from("external_source")).unwrap(); + entity_store + .update(child_registration, "external_source".into()) + .unwrap(); let actual_c8y_alarm = C8yAlarm::try_from(&tedge_alarm, &entity_store).unwrap(); assert_eq!(actual_c8y_alarm, expected_c8y_alarm); diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 17fb5e09458..34ad1a01fc8 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -1429,12 +1429,12 @@ mod tests { let device_creation_msgs = converter.convert(&alarm_message).await; let first_msg = Message::new( &Topic::new_unchecked("te/device/external_sensor//"), - r#"{ "@type":"child-device", "@id":"external_sensor"}"#, + r#"{ "@type":"child-device", "@id":"test-device:device:external_sensor"}"#, ) .with_retain(); let second_msg = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,external_sensor,external_sensor,thin-edge.io-child", + "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", ); assert_eq!(device_creation_msgs[0], first_msg); assert_eq!(device_creation_msgs[1], second_msg); diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index b5dccac0088..5333eb7b70a 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -454,14 +454,14 @@ async fn c8y_mapper_child_alarm_mapping_to_smartrest() { [ ( "te/device/external_sensor//", - r#"{ "@type":"child-device", "@id":"external_sensor"}"#, + r#"{ "@type":"child-device", "@id":"test-device:device:external_sensor"}"#, ), ( "c8y/s/us", - "101,external_sensor,external_sensor,thin-edge.io-child", + "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", ), ( - "c8y/s/us/external_sensor", + "c8y/s/us/test-device:device:external_sensor", "303,temperature_high,\"Temperature high\"", ), ], @@ -548,11 +548,11 @@ async fn c8y_mapper_child_alarm_with_custom_fragment_mapping_to_c8y_json() { [ ( "te/device/external_sensor//", - r#"{ "@type":"child-device", "@id":"external_sensor"}"#, + r#"{ "@type":"child-device", "@id":"test-device:device:external_sensor"}"#, ), ( "c8y/s/us", - "101,external_sensor,external_sensor,thin-edge.io-child", + "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", ), ], ) @@ -574,7 +574,7 @@ async fn c8y_mapper_child_alarm_with_custom_fragment_mapping_to_c8y_json() { } }, "externalSource": { - "externalId":"external_sensor", + "externalId":"test-device:device:external_sensor", "type":"c8y_Serial" } }), @@ -658,7 +658,7 @@ async fn c8y_mapper_child_alarm_with_message_custom_fragment_mapping_to_c8y_json "text":"Pressure high", "message":"custom message", "externalSource":{ - "externalId":"external_sensor", + "externalId":"test-device:device:external_sensor", "type":"c8y_Serial" } }), @@ -702,7 +702,7 @@ async fn c8y_mapper_child_alarm_with_custom_message() { "text":"child_msg_to_text_pressure_alarm", "message":"Pressure high", "externalSource":{ - "externalId":"external_sensor", + "externalId":"test-device:device:external_sensor", "type":"c8y_Serial" } }), @@ -768,7 +768,10 @@ async fn c8y_mapper_child_alarm_empty_payload() { // Expect converted alarm SmartREST message assert_received_contains_str( &mut mqtt, - [("c8y/s/us/external_sensor", "306,empty_temperature_alarm")], + [( + "c8y/s/us/test-device:device:external_sensor", + "306,empty_temperature_alarm", + )], ) .await; } @@ -1048,7 +1051,7 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { let cfg_dir = TempTedgeDir::new(); create_thin_edge_child_operations( &cfg_dir, - "child1", + "test-device:device:child1", vec!["c8y_ChildTestOp1", "c8y_ChildTestOp2"], ); @@ -1062,7 +1065,7 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { cfg_dir .dir("operations") .dir("c8y") - .dir("child1") + .dir("test-device:device:child1") .file("c8y_ChildTestOp3") .to_path_buf(), )) @@ -1073,7 +1076,7 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { assert_received_contains_str( &mut mqtt, [( - "c8y/s/us/child1", + "c8y/s/us/test-device:device:child1", "114,c8y_ChildTestOp1,c8y_ChildTestOp2,c8y_ChildTestOp3", )], ) @@ -1095,20 +1098,20 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { assert_eq!(child_metadata.topic.name, "te/device/child1//"); assert_eq!( child_metadata.payload_str().unwrap(), - r#"{ "@type":"child-device", "@id":"child1"}"# + r#"{ "@type":"child-device", "@id":"test-device:device:child1"}"# ); let child_c8y_registration = mqtt.recv().await.unwrap(); assert_eq!(child_c8y_registration.topic.name, "c8y/s/us"); assert_eq!( child_c8y_registration.payload_str().unwrap(), - "101,child1,child1,thin-edge.io-child" + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child" ); // Expect an update list of capabilities with agent capabilities assert_received_contains_str( &mut mqtt, [( - "c8y/s/us/child1", + "c8y/s/us/test-device:device:child1", "114,c8y_ChildTestOp1,c8y_ChildTestOp2,c8y_ChildTestOp3,c8y_Restart", )], ) diff --git a/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot b/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot index 5b4f93bccd6..ab7bb2573ca 100644 --- a/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot +++ b/tests/RobotFramework/tests/cumulocity/telemetry/c8y_child_alarms_rpi.robot @@ -13,6 +13,7 @@ Suite Teardown Get Logs ${DEVICE_SN} ${CHILD_SN} +${CHILD_ID} *** Test Cases *** @@ -23,7 +24,7 @@ Define Child device 1 ID Normal case when the child device does not exist on c8y cloud # Sending child alarm - Execute Command sudo tedge mqtt pub 'te/device/${CHILD_SN}///a/temperature_high' '{ "severity": "critical", "text": "Temperature is very high", "time": "2021-01-01T05:30:45+00:00" }' -q 2 -r + Execute Command sudo tedge mqtt pub 'te/device/${CHILD_ID}///a/temperature_high' '{ "severity": "critical", "text": "Temperature is very high", "time": "2021-01-01T05:30:45+00:00" }' -q 2 -r # Check Child device creation Set Device ${DEVICE_SN} Should Be A Child Device Of Device ${CHILD_SN} @@ -35,7 +36,7 @@ Normal case when the child device does not exist on c8y cloud Normal case when the child device already exists #Sending child alarm again - Execute Command sudo tedge mqtt pub 'te/device/${CHILD_SN}///a/temperature_high' '{ "severity": "critical", "text": "Temperature is very high", "time": "2021-01-02T05:30:45+00:00" }' -q 2 -r + Execute Command sudo tedge mqtt pub 'te/device/${CHILD_ID}///a/temperature_high' '{ "severity": "critical", "text": "Temperature is very high", "time": "2021-01-02T05:30:45+00:00" }' -q 2 -r #Check created second alarm ${alarms}= Device Should Have Alarm/s minimum=1 maximum=1 updated_after=2021-01-02 @@ -43,7 +44,7 @@ Normal case when the child device already exists Reconciliation when the new alarm message arrives, restart the mapper Execute Command sudo systemctl stop tedge-mapper-c8y.service - Execute Command sudo tedge mqtt pub 'te/device/${CHILD_SN}///a/temperature_high' '{ "severity": "critical", "text": "Temperature is very high", "time": "2021-01-03T05:30:45+00:00" }' -q 2 -r + Execute Command sudo tedge mqtt pub 'te/device/${CHILD_ID}///a/temperature_high' '{ "severity": "critical", "text": "Temperature is very high", "time": "2021-01-03T05:30:45+00:00" }' -q 2 -r Execute Command sudo systemctl start tedge-mapper-c8y.service # Check created second alarm @@ -52,7 +53,7 @@ Reconciliation when the new alarm message arrives, restart the mapper Reconciliation when the alarm that is cleared Execute Command sudo systemctl stop tedge-mapper-c8y.service - Execute Command sudo tedge mqtt pub 'te/device/${CHILD_SN}///a/temperature_high' '' -q 2 -r + Execute Command sudo tedge mqtt pub 'te/device/${CHILD_ID}///a/temperature_high' '' -q 2 -r Execute Command sudo systemctl start tedge-mapper-c8y.service Device Should Not Have Alarm/s diff --git a/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot b/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot index 0e017b72467..1870511a69d 100644 --- a/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot +++ b/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot @@ -59,7 +59,7 @@ Child devices support sending custom alarms #1699 Child devices support sending alarms using text fragment - Execute Command tedge mqtt pub te/device/${CHILD_SN}///a/childAlarmType1 '{ "severity": "critical", "text": "Some test alarm" }' + Execute Command tedge mqtt pub te/device/${CHILD_ID}///a/childAlarmType1 '{ "severity": "critical", "text": "Some test alarm" }' ${alarms}= Device Should Have Alarm/s expected_text=Some test alarm severity=CRITICAL minimum=1 maximum=1 type=childAlarmType1 Log ${alarms} @@ -85,7 +85,7 @@ Custom Setup ${DEVICE_SN}= Setup Set Suite Variable $DEVICE_SN Set Suite Variable $CHILD_ID child1 - Set Suite Variable $CHILD_SN ${DEVICE_SN}:device:child1 + Set Suite Variable $CHILD_SN ${DEVICE_SN}:device:${CHILD_ID} Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} Restart Service tedge-mapper-c8y Device Should Exist ${DEVICE_SN} From 21d90050dcb91f99cb5dd7e826a380a36d616fe2 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Fri, 22 Sep 2023 16:38:26 +1000 Subject: [PATCH 04/13] add system test for registration devices on custom topics Signed-off-by: Reuben Miller --- .../registration/device_registration.robot | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot index 0d27c689c36..8936ef57c02 100644 --- a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot +++ b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot @@ -75,6 +75,40 @@ Register service on a child device via MQTT Check Service child_sn=${DEVICE_SN}:device:${CHILD_SN} service_sn=${DEVICE_SN}:device:${CHILD_SN}:service:custom-app service_name=custom-app service_type=custom-type service_status=up +Register devices using custom MQTT schema + [Documentation] Complex example showing how to use custom MQTT topics to register devices/services using + ... custom identity schemas + Execute Command tedge mqtt pub --retain 'te/base///' '{"@type":"main-device","name":"base","type":"te_gateway"}' + + Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/sensor1' '{"@type":"child-device","name":"sensor1","type":"SmartSensor"}' + Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/sensor2' '{"@type":"child-device","name":"sensor2","type":"SmartSensor"}' + + # Service of main device + Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/metrics' '{"@type":"service","name":"metrics","type":"PLCApplication"}' + + # Service of child device + Execute Command tedge mqtt pub --retain 'te/factory1/shop1/apps/sensor1' '{"@type":"service","@parent":"factory1/shop1/plc1/sensor1","name":"metrics","type":"PLCMonitorApplication"}' + Execute Command tedge mqtt pub --retain 'te/factory1/shop1/apps/sensor2' '{"@type":"service","@parent":"factory1/shop1/plc1/sensor2","name":"metrics","type":"PLCMonitorApplication"}' + + Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:factory1:shop1:plc1:sensor1 child_name=sensor1 child_type=SmartSensor + Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:factory1:shop1:plc1:sensor2 child_name=sensor2 child_type=SmartSensor + + # Check main device services + Cumulocity.Set Device ${DEVICE_SN} + Should Have Services name=metrics service_type=PLCApplication status=up + + # Check child services + Cumulocity.Set Device ${DEVICE_SN}:factory1:shop1:plc1:sensor1 + Should Have Services name=metrics service_type=PLCMonitorApplication status=up + + Cumulocity.Set Device ${DEVICE_SN}:factory1:shop1:plc1:sensor2 + Should Have Services name=metrics service_type=PLCMonitorApplication status=up + + # Publish to main device on custom topic + Execute Command cmd=tedge mqtt pub te/base////m/gateway_stats '{"runtime":1001}' + Cumulocity.Set Device ${DEVICE_SN} + Cumulocity.Device Should Have Measurements type=gateway_stats minimum=1 maximum=1 + *** Keywords *** Check Child Device From f6ccc3837fc7d8a05d9b66159371038987e8bd18 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Fri, 22 Sep 2023 14:41:02 +0000 Subject: [PATCH 05/13] Addressing review comments Signed-off-by: Didier Wenzek --- crates/core/c8y_api/src/json_c8y.rs | 22 +- .../core/c8y_api/src/smartrest/inventory.rs | 8 + crates/core/tedge_api/src/entity_store.rs | 364 ++++++++++-------- crates/core/tedge_api/src/mqtt_topics.rs | 34 ++ .../c8y_mapper_ext/src/converter.rs | 62 +-- .../c8y_mapper_ext/src/log_upload.rs | 5 +- crates/extensions/c8y_mapper_ext/src/tests.rs | 55 +++ .../registration/device_registration.robot | 15 +- 8 files changed, 340 insertions(+), 225 deletions(-) diff --git a/crates/core/c8y_api/src/json_c8y.rs b/crates/core/c8y_api/src/json_c8y.rs index 97e477464d1..e9610d80826 100644 --- a/crates/core/c8y_api/src/json_c8y.rs +++ b/crates/core/c8y_api/src/json_c8y.rs @@ -415,6 +415,7 @@ mod tests { use serde_json::json; use tedge_api::alarm::ThinEdgeAlarm; use tedge_api::alarm::ThinEdgeAlarmData; + use tedge_api::entity_store::EntityExternalId; use tedge_api::entity_store::EntityRegistrationMessage; use tedge_api::event::ThinEdgeEventData; use tedge_api::mqtt_topics::EntityTopicId; @@ -774,16 +775,15 @@ mod tests { )] fn check_alarm_translation(tedge_alarm: ThinEdgeAlarm, expected_c8y_alarm: C8yAlarm) { let main_device = EntityRegistrationMessage::main_device("test-main".into()); - let mut entity_store = EntityStore::with_main_device(main_device).unwrap(); + let mut entity_store = + EntityStore::with_main_device(main_device, dummy_external_id_mapper).unwrap(); let child_registration = EntityRegistrationMessage::new(&Message::new( &Topic::new_unchecked("te/device/external_source//"), r#"{"@id": "external_source", "@type": "child-device"}"#, )) .unwrap(); - entity_store - .update(child_registration, "external_source".into()) - .unwrap(); + entity_store.update(child_registration).unwrap(); let actual_c8y_alarm = C8yAlarm::try_from(&tedge_alarm, &entity_store).unwrap(); assert_eq!(actual_c8y_alarm, expected_c8y_alarm); @@ -803,7 +803,8 @@ mod tests { }; let main_device = EntityRegistrationMessage::main_device("test-main".into()); - let entity_store = EntityStore::with_main_device(main_device).unwrap(); + let entity_store = + EntityStore::with_main_device(main_device, dummy_external_id_mapper).unwrap(); match C8yAlarm::try_from(&tedge_alarm, &entity_store).unwrap() { C8yAlarm::Create(value) => { @@ -812,4 +813,15 @@ mod tests { C8yAlarm::Clear(_) => panic!("Must be C8yAlarm::Create"), }; } + + fn dummy_external_id_mapper( + entity_topic_id: &EntityTopicId, + _main_device_xid: &EntityExternalId, + ) -> EntityExternalId { + entity_topic_id + .to_string() + .trim_end_matches('/') + .replace('/', ":") + .into() + } } diff --git a/crates/core/c8y_api/src/smartrest/inventory.rs b/crates/core/c8y_api/src/smartrest/inventory.rs index fe89b224f05..5dea653703f 100644 --- a/crates/core/c8y_api/src/smartrest/inventory.rs +++ b/crates/core/c8y_api/src/smartrest/inventory.rs @@ -1,6 +1,11 @@ +//! This module provides some helper functions to create SmartREST messages +//! that can be used to create various managed objects in Cumulocity inventory. use crate::smartrest::topic::publish_topic_from_ancestors; use mqtt_channel::Message; +/// Create a SmartREST message for creating a child device under the given ancestors. +/// The provided ancestors list must contain all the parents of the given device +/// starting from its immediate parent device. pub fn child_device_creation_message( child_id: &str, device_name: Option<&str>, @@ -18,6 +23,9 @@ pub fn child_device_creation_message( ) } +/// Create a SmartREST message for creating a service on device. +/// The provided ancestors list must contain all the parents of the given service +/// starting from its immediate parent device. pub fn service_creation_message( service_id: &str, service_name: &str, diff --git a/crates/core/tedge_api/src/entity_store.rs b/crates/core/tedge_api/src/entity_store.rs index 6c72ab70521..541850cb3a6 100644 --- a/crates/core/tedge_api/src/entity_store.rs +++ b/crates/core/tedge_api/src/entity_store.rs @@ -56,6 +56,9 @@ impl From for String { } } +type ExternalIdMapperFn = + Box EntityExternalId + Send + Sync + 'static>; + /// A store for topic-based entity metadata lookup. /// /// This object is a hashmap from MQTT identifiers to entities (devices or @@ -82,24 +85,31 @@ impl From for String { /// ); /// let registration_message = EntityRegistrationMessage::try_from(&mqtt_message).unwrap(); /// -/// let mut entity_store = EntityStore::with_main_device(registration_message); +/// let mut entity_store = EntityStore::with_main_device(registration_message, |tid, xid| tid.to_string().into()); /// ``` -#[derive(Debug, Clone)] pub struct EntityStore { main_device: EntityTopicId, entities: HashMap, entity_id_index: HashMap, + external_id_mapper: ExternalIdMapperFn, } impl EntityStore { /// Creates a new entity store with a given main device. #[must_use] - pub fn with_main_device(main_device: EntityRegistrationMessage) -> Option { + pub fn with_main_device( + main_device: EntityRegistrationMessage, + external_id_mapper: F, + ) -> Option + where + F: Fn(&EntityTopicId, &EntityExternalId) -> EntityExternalId, + F: 'static + Send + Sync, + { if main_device.r#type != EntityType::MainDevice { return None; } - let entity_id: EntityExternalId = main_device.entity_id?.into(); + let entity_id: EntityExternalId = main_device.entity_id?; let metadata = EntityMetadata { topic_id: main_device.topic_id.clone(), entity_id: entity_id.clone(), @@ -112,6 +122,7 @@ impl EntityStore { main_device: main_device.topic_id.clone(), entities: HashMap::from([(main_device.topic_id.clone(), metadata)]), entity_id_index: HashMap::from([(entity_id, main_device.topic_id)]), + external_id_mapper: Box::new(external_id_mapper), }) } @@ -121,8 +132,8 @@ impl EntityStore { } /// Returns information for an entity under a given device/service id . - pub fn get_by_id(&self, entity_id: &str) -> Option<&EntityMetadata> { - let topic_id = self.entity_id_index.get(&entity_id.into())?; + pub fn get_by_external_id(&self, external_id: &EntityExternalId) -> Option<&EntityMetadata> { + let topic_id = self.entity_id_index.get(external_id)?; self.get(topic_id) } @@ -219,13 +230,13 @@ impl EntityStore { pub fn update( &mut self, message: EntityRegistrationMessage, - external_id: EntityExternalId, ) -> Result, Error> { if message.r#type == EntityType::MainDevice && message.topic_id != self.main_device { return Err(Error::MainDeviceAlreadyRegistered( self.main_device.to_string().into_boxed_str(), )); } + let topic_id = message.topic_id; let mut affected_entities = vec![]; @@ -234,7 +245,7 @@ impl EntityStore { EntityType::ChildDevice => message.parent.or_else(|| Some(self.main_device.clone())), EntityType::Service => message .parent - .or_else(|| message.topic_id.default_parent_identifier()) + .or_else(|| topic_id.default_parent_identifier()) .or_else(|| Some(self.main_device.clone())), }; @@ -247,8 +258,11 @@ impl EntityStore { affected_entities.push(parent.clone()); } + let external_id = message.entity_id.unwrap_or_else(|| { + (self.external_id_mapper)(&topic_id, &self.main_device_external_id()) + }); let entity_metadata = EntityMetadata { - topic_id: message.topic_id.clone(), + topic_id: topic_id.clone(), r#type: message.r#type, entity_id: external_id.clone(), parent, @@ -261,9 +275,9 @@ impl EntityStore { .insert(entity_metadata.topic_id.clone(), entity_metadata); if previous.is_some() { - affected_entities.push(message.topic_id); + affected_entities.push(topic_id); } else { - self.entity_id_index.insert(external_id, message.topic_id); + self.entity_id_index.insert(external_id, topic_id); } Ok(affected_entities) @@ -277,7 +291,7 @@ impl EntityStore { /// Performs auto-registration process for an entity under a given /// identifier. /// - /// If an entity is a service, its device is also auto-registered if it's + /// If an entity is a service, its parent device is also auto-registered if it's /// not already registered. /// /// It returns MQTT register messages for the given entities to be published @@ -286,65 +300,65 @@ impl EntityStore { pub fn auto_register_entity( &mut self, entity_topic_id: &EntityTopicId, - entity_external_id: EntityExternalId, - parent_external_id: Option, ) -> Result, entity_store::Error> { - let device_id = match entity_topic_id.default_device_name() { - Some(device_id) => device_id, - None => return Ok(vec![]), - }; + if entity_topic_id.matches_default_topic_scheme() { + if entity_topic_id.is_default_main_device() { + return Ok(vec![]); // Do nothing as the main device is always pre-registered + } - let mut register_messages = vec![]; + let mut register_messages = vec![]; + + let parent_device_id = entity_topic_id + .default_parent_identifier() + .expect("device id must be present as the topic id follows the default scheme"); + + if !parent_device_id.is_default_main_device() && self.get(&parent_device_id).is_none() { + let device_external_id = + (self.external_id_mapper)(&parent_device_id, &self.main_device_external_id()); + + let device_register_payload = format!( + "{{ \"@type\":\"child-device\", \"@id\":\"{}\"}}", + device_external_id.as_ref() + ); + + // FIXME: The root prefix should not be added this way. + // The simple fix is to change the signature of the method, + // returning (EntityTopicId, EntityMetadata) pairs instead of MQTT Messages. + let topic = Topic::new(&format!("{MQTT_ROOT}/{parent_device_id}")).unwrap(); + let device_register_message = + Message::new(&topic, device_register_payload).with_retain(); + register_messages.push(device_register_message.clone()); + self.update( + EntityRegistrationMessage::try_from(&device_register_message).unwrap(), + )?; + } - let parent_id = entity_topic_id.default_parent_identifier().unwrap(); - if self.get(&parent_id).is_none() { - let device_type = if device_id == "main" { - "device" - } else { - "child-device" - }; - let device_external_id = if device_id == "main" { - self.main_device_external_id() - } else { - parent_external_id - .clone() - .ok_or_else(|| Error::ExternalIdNotGiven(parent_id.clone()))? - }; - let device_register_payload = format!( - "{{ \"@type\":\"{device_type}\", \"@id\":\"{}\"}}", - device_external_id.as_ref() - ); - - // FIXME: The root prefix should not be added this way. - // The simple fix is to change the signature of the method, - // returning (EntityTopicId, EntityMetadata) pairs instead of MQTT Messages. - let topic = Topic::new(&format!("{MQTT_ROOT}/{parent_id}")).unwrap(); - let device_register_message = - Message::new(&topic, device_register_payload).with_retain(); - register_messages.push(device_register_message.clone()); - self.update( - EntityRegistrationMessage::try_from(&device_register_message).unwrap(), - parent_external_id.ok_or_else(|| Error::ExternalIdNotGiven(parent_id.clone()))?, - )?; - } + // if the entity is a service, register the service as well + if let Some(service_id) = entity_topic_id.default_service_name() { + let service_external_id = + (self.external_id_mapper)(entity_topic_id, &self.main_device_external_id()); - // register service itself - if let Some(service_id) = entity_topic_id.default_service_name() { - let service_topic = format!("{MQTT_ROOT}/device/{device_id}/service/{service_id}"); - let service_register_payload = r#"{"@type": "service", "type": "systemd"}"#.to_string(); - let service_register_message = Message::new( - &Topic::new(&service_topic).unwrap(), - service_register_payload, - ) - .with_retain(); - register_messages.push(service_register_message.clone()); - self.update( - EntityRegistrationMessage::try_from(&service_register_message).unwrap(), - entity_external_id, - )?; - } + let service_register_payload = format!( + "{{ \"@type\":\"service\", \"@id\":\"{}\", \"name\":\"{}\", \"type\": \"systemd\"}}", + service_external_id.as_ref(), + service_id + ); - Ok(register_messages) + let service_register_message = Message::new( + &Topic::new(&format!("{MQTT_ROOT}/{entity_topic_id}")).unwrap(), + service_register_payload, + ) + .with_retain(); + register_messages.push(service_register_message.clone()); + self.update( + EntityRegistrationMessage::try_from(&service_register_message).unwrap(), + )?; + } + + Ok(register_messages) + } else { + Err(Error::NonDefaultTopicScheme(entity_topic_id.clone())) + } } } @@ -403,15 +417,15 @@ pub enum Error { #[error("The specified entity {0} does not exist in the store")] UnknownEntity(String), - #[error("External ID not provided for {0}")] - ExternalIdNotGiven(EntityTopicId), + #[error("The specified topic id {0} does not match the default topic scheme: 'device//service/'")] + NonDefaultTopicScheme(EntityTopicId), } /// An object representing a valid entity registration message. #[derive(Debug, Clone, PartialEq, Eq)] pub struct EntityRegistrationMessage { pub topic_id: EntityTopicId, - pub entity_id: Option, + pub entity_id: Option, pub r#type: EntityType, pub parent: Option, pub payload: serde_json::Value, @@ -450,7 +464,7 @@ impl EntityRegistrationMessage { let entity_id = payload .get("@id") .and_then(|id| id.as_str()) - .map(|id| id.to_string()); + .map(|id| id.into()); let topic_id = message .topic @@ -468,10 +482,10 @@ impl EntityRegistrationMessage { } /// Creates a entity registration message for a main device. - pub fn main_device(entity_id: String) -> Self { + pub fn main_device(main_device_id: String) -> Self { Self { topic_id: EntityTopicId::default_main_device(), - entity_id: Some(entity_id), + entity_id: Some(main_device_id.into()), r#type: EntityType::MainDevice, parent: None, payload: serde_json::json!({}), @@ -507,15 +521,29 @@ mod tests { use super::*; + fn dummy_external_id_mapper( + entity_topic_id: &EntityTopicId, + _main_device_xid: &EntityExternalId, + ) -> EntityExternalId { + entity_topic_id + .to_string() + .trim_end_matches('/') + .replace('/', ":") + .into() + } + #[test] fn registers_main_device() { - let store = EntityStore::with_main_device(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".to_string()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({"@type": "device"}), - }) + let store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: Some("test-device".into()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({"@type": "device"}), + }, + dummy_external_id_mapper, + ) .unwrap(); assert_eq!(store.main_device(), &EntityTopicId::default_main_device()); @@ -524,13 +552,16 @@ mod tests { #[test] fn lists_child_devices() { - let mut store = EntityStore::with_main_device(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".to_string()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({"@type": "device"}), - }) + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: Some("test-device".into()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({"@type": "device"}), + }, + dummy_external_id_mapper, + ) .unwrap(); // If the @parent info is not provided, it is assumed to be an immediate @@ -542,7 +573,6 @@ mod tests { json!({"@type": "child-device"}).to_string(), )) .unwrap(), - "child1".into(), ) .unwrap(); @@ -559,7 +589,6 @@ mod tests { json!({"@type": "child-device", "@parent": "device/main//"}).to_string(), )) .unwrap(), - "child2".into(), ) .unwrap(); assert_eq!(updated_entities, ["device/main//"]); @@ -570,27 +599,27 @@ mod tests { #[test] fn lists_services() { - let mut store = EntityStore::with_main_device(EntityRegistrationMessage { - r#type: EntityType::MainDevice, - entity_id: Some("test-device".to_string()), - topic_id: EntityTopicId::default_main_device(), - parent: None, - payload: json!({}), - }) + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + r#type: EntityType::MainDevice, + entity_id: Some("test-device".into()), + topic_id: EntityTopicId::default_main_device(), + parent: None, + payload: json!({}), + }, + dummy_external_id_mapper, + ) .unwrap(); // Services are namespaced under devices, so `parent` is not necessary let updated_entities = store - .update( - EntityRegistrationMessage { - r#type: EntityType::Service, - entity_id: None, - topic_id: EntityTopicId::default_main_service("service1").unwrap(), - parent: None, - payload: json!({}), - }, - "service1".into(), - ) + .update(EntityRegistrationMessage { + r#type: EntityType::Service, + entity_id: None, + topic_id: EntityTopicId::default_main_service("service1").unwrap(), + parent: None, + payload: json!({}), + }) .unwrap(); assert_eq!(updated_entities, ["device/main//"]); @@ -600,16 +629,13 @@ mod tests { ); let updated_entities = store - .update( - EntityRegistrationMessage { - r#type: EntityType::Service, - entity_id: None, - topic_id: EntityTopicId::default_main_service("service2").unwrap(), - parent: None, - payload: json!({}), - }, - "service2".into(), - ) + .update(EntityRegistrationMessage { + r#type: EntityType::Service, + entity_id: None, + topic_id: EntityTopicId::default_main_service("service2").unwrap(), + parent: None, + payload: json!({}), + }) .unwrap(); assert_eq!(updated_entities, ["device/main//"]); @@ -629,25 +655,25 @@ mod tests { /// device on another topic is not allowed. #[test] fn forbids_multiple_main_devices() { - let mut store = EntityStore::with_main_device(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - r#type: EntityType::MainDevice, - entity_id: Some("test-device".to_string()), - parent: None, - payload: json!({}), - }) - .unwrap(); - - let res = store.update( + let mut store = EntityStore::with_main_device( EntityRegistrationMessage { - topic_id: EntityTopicId::default_child_device("another_main").unwrap(), - entity_id: Some("test-device".to_string()), + topic_id: EntityTopicId::default_main_device(), r#type: EntityType::MainDevice, + entity_id: Some("test-device".into()), parent: None, payload: json!({}), }, - "another_main".into(), - ); + dummy_external_id_mapper, + ) + .unwrap(); + + let res = store.update(EntityRegistrationMessage { + topic_id: EntityTopicId::default_child_device("another_main").unwrap(), + entity_id: Some("test-device".into()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({}), + }); assert_eq!( res, @@ -657,38 +683,41 @@ mod tests { #[test] fn forbids_nonexistent_parents() { - let mut store = EntityStore::with_main_device(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".to_string()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({}), - }) - .unwrap(); - - let res = store.update( + let mut store = EntityStore::with_main_device( EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: None, - r#type: EntityType::ChildDevice, - parent: Some(EntityTopicId::default_child_device("myawesomeparent").unwrap()), + entity_id: Some("test-device".into()), + r#type: EntityType::MainDevice, + parent: None, payload: json!({}), }, - "myawesomedevice".into(), - ); + dummy_external_id_mapper, + ) + .unwrap(); + + let res = store.update(EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: None, + r#type: EntityType::ChildDevice, + parent: Some(EntityTopicId::default_child_device("myawesomeparent").unwrap()), + payload: json!({}), + }); assert!(matches!(res, Err(Error::NoParent(_)))); } #[test] fn list_ancestors() { - let mut store = EntityStore::with_main_device(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".to_string()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({"@type": "device"}), - }) + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: Some("test-device".into()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({"@type": "device"}), + }, + dummy_external_id_mapper, + ) .unwrap(); // Assert no ancestors of main device @@ -705,7 +734,6 @@ mod tests { json!({"@type": "service"}).to_string(), )) .unwrap(), - "collectd".into(), ) .unwrap(); @@ -725,7 +753,6 @@ mod tests { json!({"@type": "child-device"}).to_string(), )) .unwrap(), - "child1".into(), ) .unwrap(); @@ -745,7 +772,6 @@ mod tests { json!({"@type": "service"}).to_string(), )) .unwrap(), - "child1_collectd".into(), ) .unwrap(); @@ -765,7 +791,6 @@ mod tests { json!({"@type": "child-device", "@parent": "device/child1//"}).to_string(), )) .unwrap(), - "child2".into(), ) .unwrap(); @@ -785,7 +810,6 @@ mod tests { json!({"@type": "service"}).to_string(), )) .unwrap(), - "child2_collectd".into(), ) .unwrap(); @@ -800,13 +824,16 @@ mod tests { #[test] fn list_ancestors_external_ids() { - let mut store = EntityStore::with_main_device(EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".to_string()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({"@type": "device"}), - }) + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + entity_id: Some("test-device".into()), + r#type: EntityType::MainDevice, + parent: None, + payload: json!({"@type": "device"}), + }, + dummy_external_id_mapper, + ) .unwrap(); // Assert ancestor external ids of main device @@ -823,7 +850,6 @@ mod tests { json!({"@type": "service"}).to_string(), )) .unwrap(), - "collectd".into(), ) .unwrap(); @@ -843,7 +869,6 @@ mod tests { json!({"@type": "child-device"}).to_string(), )) .unwrap(), - "child1".into(), ) .unwrap(); @@ -863,7 +888,6 @@ mod tests { json!({"@type": "service"}).to_string(), )) .unwrap(), - "child1_collectd".into(), ) .unwrap(); @@ -874,7 +898,7 @@ mod tests { &EntityTopicId::default_child_service("child1", "collectd").unwrap() ) .unwrap(), - ["child1", "test-device"] + ["device:child1", "test-device"] ); // Register child2 as child of child1 @@ -885,7 +909,6 @@ mod tests { json!({"@type": "child-device", "@parent": "device/child1//"}).to_string(), )) .unwrap(), - "child2".into(), ) .unwrap(); @@ -894,7 +917,7 @@ mod tests { store .ancestors_external_ids(&EntityTopicId::default_child_device("child2").unwrap()) .unwrap(), - ["child1", "test-device"] + ["device:child1", "test-device"] ); // Register service on child2 @@ -905,7 +928,6 @@ mod tests { json!({"@type": "service"}).to_string(), )) .unwrap(), - "child2_collectd".into(), ) .unwrap(); @@ -916,7 +938,7 @@ mod tests { &EntityTopicId::default_child_service("child2", "collectd").unwrap() ) .unwrap(), - ["child2", "child1", "test-device"] + ["device:child2", "device:child1", "test-device"] ); } } diff --git a/crates/core/tedge_api/src/mqtt_topics.rs b/crates/core/tedge_api/src/mqtt_topics.rs index 58a8eb13a6f..936a12c8009 100644 --- a/crates/core/tedge_api/src/mqtt_topics.rs +++ b/crates/core/tedge_api/src/mqtt_topics.rs @@ -271,6 +271,17 @@ impl EntityTopicId { format!("device/{child}/service/{service}").parse() } + /// Returns true if the current topic id matches the default topic scheme: + /// - device/// : for devices + /// - device//service/ : for services + /// + /// Returns false otherwise + pub fn matches_default_topic_scheme(&self) -> bool { + self.default_device_name() + .or(self.default_service_name()) + .is_some() + } + /// Returns the device name when the entity topic identifier is using the `device/+/service/+` pattern. /// /// Returns None otherwise. @@ -306,6 +317,7 @@ impl EntityTopicId { .map(|parent_id| EntityTopicId(format!("device/{parent_id}//"))) } + /// Returns true if the current topic identifier matches that of the main device pub fn is_default_main_device(&self) -> bool { self == &Self::default_main_device() } @@ -486,6 +498,8 @@ pub enum ChannelFilter { #[cfg(test)] mod tests { use super::*; + use test_case::test_case; + const MQTT_ROOT: &str = "test_te"; #[test] @@ -563,4 +577,24 @@ mod tests { assert!(entity_channel1.is_err()); assert!(entity_channel2.is_err()); } + + #[test_case("device/main//", true)] + #[test_case("device/child//", true)] + #[test_case("device/main/service/foo", true)] + #[test_case("device/child/service/foo", true)] + #[test_case("device/main//foo", false)] + #[test_case("custom///", false)] + #[test_case("custom/main//", false)] + #[test_case("custom/child//", false)] + #[test_case("custom/main/service/foo", false)] + #[test_case("custom/child/service/foo", false)] + #[test_case("device/main/custom_service/foo", false)] + fn default_topic_scheme_match(topic: &str, matches: bool) { + assert_eq!( + EntityTopicId::from_str(topic) + .unwrap() + .matches_default_topic_scheme(), + matches + ) + } } diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 34ad1a01fc8..92053aa336a 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -211,7 +211,8 @@ impl CumulocityConverter { }; let main_device = entity_store::EntityRegistrationMessage::main_device(device_id.clone()); - let entity_store = EntityStore::with_main_device(main_device).unwrap(); + let entity_store = + EntityStore::with_main_device(main_device, Self::map_to_c8y_external_id).unwrap(); Ok(CumulocityConverter { size_threshold, @@ -291,21 +292,23 @@ impl CumulocityConverter { /// - `device/child001//` => `DEVICE_COMMON_NAME:device:child001` /// - `device/child001/service/service001` => `DEVICE_COMMON_NAME:device:child001:service:service001` /// - `factory01/hallA/packaging/belt001` => `DEVICE_COMMON_NAME:factory01:hallA:packaging:belt001` - fn map_to_c8y_external_id(&self, entity_topic_id: &EntityTopicId) -> EntityExternalId { - let external_id = if entity_topic_id.is_default_main_device() { - self.device_name.clone() + fn map_to_c8y_external_id( + entity_topic_id: &EntityTopicId, + main_device_xid: &EntityExternalId, + ) -> EntityExternalId { + if entity_topic_id.is_default_main_device() { + main_device_xid.clone() } else { format!( "{}:{}", - self.device_name.clone(), + main_device_xid.as_ref(), entity_topic_id .to_string() .trim_end_matches('/') .replace('/', ":") ) - }; - - external_id.into() + .into() + } } fn try_convert_measurement( @@ -558,12 +561,13 @@ impl CumulocityConverter { smartrest: &str, ) -> Result, CumulocityMapperError> { let request = SmartRestRestartRequest::from_smartrest(smartrest)?; - let device_id = &request.device; - let target = self.entity_store.get_by_id(device_id).ok_or_else(|| { - CumulocityMapperError::UnknownDevice { - device_id: device_id.to_owned(), - } - })?; + let device_id = &request.device.into(); + let target = self + .entity_store + .get_by_external_id(device_id) + .ok_or_else(|| CumulocityMapperError::UnknownDevice { + device_id: device_id.as_ref().to_string(), + })?; let command = RestartCommand::new(target.topic_id.clone()); let message = command.command_message(&self.mqtt_schema); Ok(vec![message]) @@ -772,17 +776,7 @@ impl CumulocityConverter { match &channel { Channel::EntityMetadata => { if let Ok(register_message) = EntityRegistrationMessage::try_from(message) { - // Generate the c8y external id, if an external id is not provided in the message - let external_id = register_message - .entity_id - .as_ref() - .map(|v| v.to_string().into()) - .unwrap_or_else(|| self.map_to_c8y_external_id(&source)); - - if let Err(e) = self - .entity_store - .update(register_message.clone(), external_id) - { + if let Err(e) = self.entity_store.update(register_message.clone()) { error!("Could not update device registration: {e}"); } let c8y_message = @@ -793,18 +787,7 @@ impl CumulocityConverter { _ => { // if device is unregistered register using auto-registration if self.entity_store.get(&source).is_none() { - let entity_external_id = self.map_to_c8y_external_id(&source); - - // If the entity is a service, generate the external id of the parent device as well - let parent_external_id = source - .default_parent_identifier() - .map(|parent| self.map_to_c8y_external_id(&parent)); - - registration_messages = match self.entity_store.auto_register_entity( - &source, - entity_external_id, - parent_external_id, - ) { + registration_messages = match self.entity_store.auto_register_entity(&source) { Ok(register_messages) => register_messages, Err(e) => { error!("Could not update device registration: {e}"); @@ -2189,12 +2172,9 @@ mod tests { entity_topic_id: &str, c8y_external_id: &str, ) { - let tmp_dir = TempTedgeDir::new(); - let (converter, _http_proxy) = create_c8y_converter(&tmp_dir).await; - let entity_topic_id = EntityTopicId::from_str(entity_topic_id).unwrap(); assert_eq!( - converter.map_to_c8y_external_id(&entity_topic_id), + CumulocityConverter::map_to_c8y_external_id(&entity_topic_id, &"test-device".into()), c8y_external_id.into() ); } diff --git a/crates/extensions/c8y_mapper_ext/src/log_upload.rs b/crates/extensions/c8y_mapper_ext/src/log_upload.rs index e77caf8d23b..22c4ad1d762 100644 --- a/crates/extensions/c8y_mapper_ext/src/log_upload.rs +++ b/crates/extensions/c8y_mapper_ext/src/log_upload.rs @@ -51,11 +51,12 @@ impl CumulocityConverter { } let log_request = SmartRestLogRequest::from_smartrest(smartrest)?; + let device_external_id = log_request.device.into(); let target = self .entity_store - .get_by_id(&log_request.device) + .get_by_external_id(&device_external_id) .ok_or_else(|| UnknownDevice { - device_id: log_request.device.to_string(), + device_id: device_external_id.into(), })?; let cmd_id = nanoid!(); diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index 5333eb7b70a..6e356bc7113 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -131,6 +131,61 @@ async fn child_device_registration_mapping() { .await; } +#[tokio::test] +async fn custom_topic_scheme_registration_mapping() { + let (mqtt, _http, _fs, mut timer) = spawn_c8y_mapper_actor(&TempTedgeDir::new(), true).await; + timer.send(Timeout::new(())).await.unwrap(); // Complete sync phase so that alarm mapping starts + let mut mqtt = mqtt.with_timeout(TEST_TIMEOUT_MS); + mqtt.skip(6).await; // Skip all init messages + + // Child device with custom scheme + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/custom///"), + r#"{ "@type": "child-device", "type": "RaspberryPi", "name": "Child1" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [("c8y/s/us", "101,test-device:custom,Child1,RaspberryPi")], + ) + .await; + + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/custom/child1//"), + r#"{ "@type": "child-device", "type": "RaspberryPi", "name": "Child1" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us", + "101,test-device:custom:child1,Child1,RaspberryPi", + )], + ) + .await; + + // Service with custom scheme + mqtt.send(MqttMessage::new( + &Topic::new_unchecked("te/custom/service/collectd/"), + r#"{ "@type": "service", "type": "systemd", "name": "Collectd" }"#, + )) + .await + .unwrap(); + + assert_received_contains_str( + &mut mqtt, + [( + "c8y/s/us", + "102,test-device:custom:service:collectd,systemd,Collectd,up", + )], + ) + .await; +} + #[tokio::test] async fn service_registration_mapping() { let (mqtt, _http, _fs, mut timer) = spawn_c8y_mapper_actor(&TempTedgeDir::new(), true).await; diff --git a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot index 8936ef57c02..a4fc480f4f7 100644 --- a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot +++ b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot @@ -78,7 +78,9 @@ Register service on a child device via MQTT Register devices using custom MQTT schema [Documentation] Complex example showing how to use custom MQTT topics to register devices/services using ... custom identity schemas - Execute Command tedge mqtt pub --retain 'te/base///' '{"@type":"main-device","name":"base","type":"te_gateway"}' + + # Main device registration via MQTT is not supported at the moment + # Execute Command tedge mqtt pub --retain 'te/base///' '{"@type":"main-device","name":"base","type":"te_gateway"}' Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/sensor1' '{"@type":"child-device","name":"sensor1","type":"SmartSensor"}' Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/sensor2' '{"@type":"child-device","name":"sensor2","type":"SmartSensor"}' @@ -94,8 +96,8 @@ Register devices using custom MQTT schema Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:factory1:shop1:plc1:sensor2 child_name=sensor2 child_type=SmartSensor # Check main device services - Cumulocity.Set Device ${DEVICE_SN} - Should Have Services name=metrics service_type=PLCApplication status=up + # Cumulocity.Set Device ${DEVICE_SN} + # Should Have Services name=metrics service_type=PLCApplication status=up # Check child services Cumulocity.Set Device ${DEVICE_SN}:factory1:shop1:plc1:sensor1 @@ -104,10 +106,11 @@ Register devices using custom MQTT schema Cumulocity.Set Device ${DEVICE_SN}:factory1:shop1:plc1:sensor2 Should Have Services name=metrics service_type=PLCMonitorApplication status=up + # Skipping as main device registration via MQTT is not supported at the moment # Publish to main device on custom topic - Execute Command cmd=tedge mqtt pub te/base////m/gateway_stats '{"runtime":1001}' - Cumulocity.Set Device ${DEVICE_SN} - Cumulocity.Device Should Have Measurements type=gateway_stats minimum=1 maximum=1 + # Execute Command cmd=tedge mqtt pub te/base////m/gateway_stats '{"runtime":1001}' + # Cumulocity.Set Device ${DEVICE_SN} + # Cumulocity.Device Should Have Measurements type=gateway_stats minimum=1 maximum=1 *** Keywords *** From 2e22e2ea94427a9944736e43145ded9993750208 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Mon, 25 Sep 2023 07:28:01 +0000 Subject: [PATCH 06/13] Fix device IDs in device restart tests --- .../tests/cumulocity/restart/restart_device_child.robot | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/RobotFramework/tests/cumulocity/restart/restart_device_child.robot b/tests/RobotFramework/tests/cumulocity/restart/restart_device_child.robot index 9ce1e0d27dd..64dc6bf9f47 100644 --- a/tests/RobotFramework/tests/cumulocity/restart/restart_device_child.robot +++ b/tests/RobotFramework/tests/cumulocity/restart/restart_device_child.robot @@ -61,13 +61,13 @@ Default restart timeout supports the default 60 second delay of the linux shutdo *** Keywords *** Setup Child Device - ThinEdgeIO.Set Device Context ${CHILD_SN} + ThinEdgeIO.Set Device Context ${CHILD_ID} Execute Command sudo dpkg -i packages/tedge_*.deb Execute Command sudo tedge config set mqtt.client.host ${PARENT_IP} Execute Command sudo tedge config set mqtt.client.port 1883 Execute Command sudo tedge config set mqtt.topic_root te - Execute Command sudo tedge config set mqtt.device_topic_id "device/${CHILD_SN}//" + Execute Command sudo tedge config set mqtt.device_topic_id "device/${CHILD_ID}//" # Install plugin after the default settings have been updated to prevent it from starting up as the main plugin Execute Command sudo dpkg -i packages/tedge-agent*.deb @@ -100,8 +100,9 @@ Custom Setup ThinEdgeIO.Service Health Status Should Be Up tedge-mapper-c8y # Child - ${child_sn}= Setup skip_bootstrap=${True} - Set Suite Variable $CHILD_SN ${child_sn} + ${CHILD_ID}= Setup skip_bootstrap=${True} + Set Suite Variable $CHILD_ID + Set Suite Variable $CHILD_SN ${PARENT_SN}:device:${CHILD_ID} Setup Child Device Cumulocity.Device Should Exist ${CHILD_SN} From fb509bd931a35586a01cb1a07cfecade34793cb9 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Mon, 25 Sep 2023 13:04:22 +0000 Subject: [PATCH 07/13] Remove redundant auto-registration logic --- crates/core/c8y_api/src/json_c8y.rs | 4 +- crates/core/c8y_api/src/smartrest/topic.rs | 6 +- crates/core/tedge_api/src/entity_store.rs | 107 +++++++++++++----- crates/core/tedge_api/src/event.rs | 2 +- .../c8y_mapper_ext/src/converter.rs | 63 ++++++----- .../c8y_mapper_ext/src/log_upload.rs | 8 +- .../c8y_mapper_ext/src/serializer.rs | 2 +- crates/extensions/c8y_mapper_ext/src/tests.rs | 79 ++++++++----- 8 files changed, 172 insertions(+), 99 deletions(-) diff --git a/crates/core/c8y_api/src/json_c8y.rs b/crates/core/c8y_api/src/json_c8y.rs index e9610d80826..18e8e002073 100644 --- a/crates/core/c8y_api/src/json_c8y.rs +++ b/crates/core/c8y_api/src/json_c8y.rs @@ -319,8 +319,8 @@ impl C8yAlarm { fn convert_source(entity: &EntityMetadata) -> Option { match entity.r#type { EntityType::MainDevice => None, - EntityType::ChildDevice => Some(make_c8y_source_fragment(entity.entity_id.as_ref())), - EntityType::Service => Some(make_c8y_source_fragment(entity.entity_id.as_ref())), + EntityType::ChildDevice => Some(make_c8y_source_fragment(entity.external_id.as_ref())), + EntityType::Service => Some(make_c8y_source_fragment(entity.external_id.as_ref())), } } diff --git a/crates/core/c8y_api/src/smartrest/topic.rs b/crates/core/c8y_api/src/smartrest/topic.rs index 18c47af9813..0e1ad096258 100644 --- a/crates/core/c8y_api/src/smartrest/topic.rs +++ b/crates/core/c8y_api/src/smartrest/topic.rs @@ -24,7 +24,7 @@ impl C8yTopic { match entity.r#type { EntityType::MainDevice => Some(C8yTopic::upstream_topic()), EntityType::ChildDevice | EntityType::Service => { - Self::ChildSmartRestResponse(entity.entity_id.clone().into()) + Self::ChildSmartRestResponse(entity.external_id.clone().into()) .to_topic() .ok() } @@ -108,7 +108,9 @@ impl From<&EntityMetadata> for C8yTopic { fn from(value: &EntityMetadata) -> Self { match value.r#type { EntityType::MainDevice => Self::SmartRestResponse, - EntityType::ChildDevice => Self::ChildSmartRestResponse(value.entity_id.clone().into()), + EntityType::ChildDevice => { + Self::ChildSmartRestResponse(value.external_id.clone().into()) + } EntityType::Service => Self::SmartRestResponse, // TODO how services are handled by c8y? } } diff --git a/crates/core/tedge_api/src/entity_store.rs b/crates/core/tedge_api/src/entity_store.rs index 541850cb3a6..bb11d3b0f8d 100644 --- a/crates/core/tedge_api/src/entity_store.rs +++ b/crates/core/tedge_api/src/entity_store.rs @@ -14,6 +14,8 @@ use crate::mqtt_topics::EntityTopicId; use crate::mqtt_topics::TopicIdError; use mqtt_channel::Message; use mqtt_channel::Topic; +use serde_json::Map; +use serde_json::Value; /// Represents an "Entity topic identifier" portion of the MQTT topic /// @@ -109,10 +111,10 @@ impl EntityStore { return None; } - let entity_id: EntityExternalId = main_device.entity_id?; + let entity_id: EntityExternalId = main_device.external_id?; let metadata = EntityMetadata { topic_id: main_device.topic_id.clone(), - entity_id: entity_id.clone(), + external_id: entity_id.clone(), r#type: main_device.r#type, parent: None, other: main_device.payload, @@ -146,7 +148,7 @@ impl EntityStore { /// Returns the external id of the main device. pub fn main_device_external_id(&self) -> EntityExternalId { - self.get(&self.main_device).unwrap().entity_id.clone() + self.get(&self.main_device).unwrap().external_id.clone() } /// Returns an ordered list of ancestors of the given entity @@ -186,7 +188,7 @@ impl EntityStore { .map(|tid| { self.entities .get(tid) - .map(|e| e.entity_id.clone().into()) + .map(|e| e.external_id.clone().into()) .unwrap() }) .collect(); @@ -258,13 +260,13 @@ impl EntityStore { affected_entities.push(parent.clone()); } - let external_id = message.entity_id.unwrap_or_else(|| { + let external_id = message.external_id.unwrap_or_else(|| { (self.external_id_mapper)(&topic_id, &self.main_device_external_id()) }); let entity_metadata = EntityMetadata { topic_id: topic_id.clone(), r#type: message.r#type, - entity_id: external_id.clone(), + external_id: external_id.clone(), parent, other: message.payload, }; @@ -300,7 +302,7 @@ impl EntityStore { pub fn auto_register_entity( &mut self, entity_topic_id: &EntityTopicId, - ) -> Result, entity_store::Error> { + ) -> Result, entity_store::Error> { if entity_topic_id.matches_default_topic_scheme() { if entity_topic_id.is_default_main_device() { return Ok(vec![]); // Do nothing as the main device is always pre-registered @@ -327,10 +329,10 @@ impl EntityStore { let topic = Topic::new(&format!("{MQTT_ROOT}/{parent_device_id}")).unwrap(); let device_register_message = Message::new(&topic, device_register_payload).with_retain(); + let device_register_message = + EntityRegistrationMessage::try_from(&device_register_message).unwrap(); register_messages.push(device_register_message.clone()); - self.update( - EntityRegistrationMessage::try_from(&device_register_message).unwrap(), - )?; + self.update(device_register_message)?; } // if the entity is a service, register the service as well @@ -349,10 +351,10 @@ impl EntityStore { service_register_payload, ) .with_retain(); + let service_register_message = + EntityRegistrationMessage::try_from(&service_register_message).unwrap(); register_messages.push(service_register_message.clone()); - self.update( - EntityRegistrationMessage::try_from(&service_register_message).unwrap(), - )?; + self.update(service_register_message)?; } Ok(register_messages) @@ -367,7 +369,7 @@ pub struct EntityMetadata { pub topic_id: EntityTopicId, pub parent: Option, pub r#type: EntityType, - pub entity_id: EntityExternalId, + pub external_id: EntityExternalId, pub other: serde_json::Value, } @@ -383,7 +385,7 @@ impl EntityMetadata { pub fn main_device(device_id: String) -> Self { Self { topic_id: EntityTopicId::default_main_device(), - entity_id: device_id.into(), + external_id: device_id.into(), r#type: EntityType::MainDevice, parent: None, other: serde_json::json!({}), @@ -394,7 +396,7 @@ impl EntityMetadata { pub fn child_device(child_device_id: String) -> Result { Ok(Self { topic_id: EntityTopicId::default_child_device(&child_device_id)?, - entity_id: child_device_id.into(), + external_id: child_device_id.into(), r#type: EntityType::ChildDevice, parent: Some(EntityTopicId::default_main_device()), other: serde_json::json!({}), @@ -425,7 +427,7 @@ pub enum Error { #[derive(Debug, Clone, PartialEq, Eq)] pub struct EntityRegistrationMessage { pub topic_id: EntityTopicId, - pub entity_id: Option, + pub external_id: Option, pub r#type: EntityType, pub parent: Option, pub payload: serde_json::Value, @@ -474,7 +476,7 @@ impl EntityRegistrationMessage { Some(Self { topic_id: topic_id.parse().ok()?, - entity_id, + external_id: entity_id, r#type, parent, payload, @@ -485,7 +487,7 @@ impl EntityRegistrationMessage { pub fn main_device(main_device_id: String) -> Self { Self { topic_id: EntityTopicId::default_main_device(), - entity_id: Some(main_device_id.into()), + external_id: Some(main_device_id.into()), r#type: EntityType::MainDevice, parent: None, payload: serde_json::json!({}), @@ -501,6 +503,39 @@ impl TryFrom<&Message> for EntityRegistrationMessage { } } +impl From<&EntityRegistrationMessage> for Message { + fn from(value: &EntityRegistrationMessage) -> Self { + let entity_topic_id = value.topic_id.clone(); + + let mut register_payload: Map = Map::new(); + + let entity_type = match value.r#type { + EntityType::MainDevice => "device", + EntityType::ChildDevice => "child-device", + EntityType::Service => "service", + }; + register_payload.insert("@type".into(), Value::String(entity_type.to_string())); + + if let Some(external_id) = &value.external_id { + register_payload.insert("@id".into(), Value::String(external_id.as_ref().into())); + } + + if let Some(parent_id) = &value.parent { + register_payload.insert("@parent".into(), Value::String(parent_id.to_string())); + } + + if let Value::Object(other_keys) = value.payload.clone() { + register_payload.extend(other_keys) + } + + Message::new( + &Topic::new(&format!("{MQTT_ROOT}/{entity_topic_id}")).unwrap(), + serde_json::to_string(&Value::Object(register_payload)).unwrap(), + ) + .with_retain() + } +} + /// Parse a MQTT message payload as an entity registration payload. /// /// Returns `Some(register_payload)` if a payload is valid JSON and is a @@ -537,7 +572,7 @@ mod tests { let store = EntityStore::with_main_device( EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), r#type: EntityType::MainDevice, parent: None, payload: json!({"@type": "device"}), @@ -555,7 +590,7 @@ mod tests { let mut store = EntityStore::with_main_device( EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), r#type: EntityType::MainDevice, parent: None, payload: json!({"@type": "device"}), @@ -602,7 +637,7 @@ mod tests { let mut store = EntityStore::with_main_device( EntityRegistrationMessage { r#type: EntityType::MainDevice, - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), topic_id: EntityTopicId::default_main_device(), parent: None, payload: json!({}), @@ -615,7 +650,7 @@ mod tests { let updated_entities = store .update(EntityRegistrationMessage { r#type: EntityType::Service, - entity_id: None, + external_id: None, topic_id: EntityTopicId::default_main_service("service1").unwrap(), parent: None, payload: json!({}), @@ -631,7 +666,7 @@ mod tests { let updated_entities = store .update(EntityRegistrationMessage { r#type: EntityType::Service, - entity_id: None, + external_id: None, topic_id: EntityTopicId::default_main_service("service2").unwrap(), parent: None, payload: json!({}), @@ -659,7 +694,7 @@ mod tests { EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), r#type: EntityType::MainDevice, - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), parent: None, payload: json!({}), }, @@ -669,7 +704,7 @@ mod tests { let res = store.update(EntityRegistrationMessage { topic_id: EntityTopicId::default_child_device("another_main").unwrap(), - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), r#type: EntityType::MainDevice, parent: None, payload: json!({}), @@ -686,7 +721,7 @@ mod tests { let mut store = EntityStore::with_main_device( EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), r#type: EntityType::MainDevice, parent: None, payload: json!({}), @@ -697,7 +732,7 @@ mod tests { let res = store.update(EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: None, + external_id: None, r#type: EntityType::ChildDevice, parent: Some(EntityTopicId::default_child_device("myawesomeparent").unwrap()), payload: json!({}), @@ -711,7 +746,7 @@ mod tests { let mut store = EntityStore::with_main_device( EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), r#type: EntityType::MainDevice, parent: None, payload: json!({"@type": "device"}), @@ -827,7 +862,7 @@ mod tests { let mut store = EntityStore::with_main_device( EntityRegistrationMessage { topic_id: EntityTopicId::default_main_device(), - entity_id: Some("test-device".into()), + external_id: Some("test-device".into()), r#type: EntityType::MainDevice, parent: None, payload: json!({"@type": "device"}), @@ -941,4 +976,16 @@ mod tests { ["device:child2", "device:child1", "test-device"] ); } + + #[test] + fn entity_registration_message_into_mqtt_message() { + let entity_reg_message = EntityRegistrationMessage::new(&Message::new( + &Topic::new("te/device/child2/service/collectd").unwrap(), + json!({"@type": "service"}).to_string(), + )) + .unwrap(); + + let message: Message = (&entity_reg_message).into(); + println!("{}", message.payload_str().unwrap()); + } } diff --git a/crates/core/tedge_api/src/event.rs b/crates/core/tedge_api/src/event.rs index 6370a0d5399..ada6b130fde 100644 --- a/crates/core/tedge_api/src/event.rs +++ b/crates/core/tedge_api/src/event.rs @@ -61,7 +61,7 @@ impl ThinEdgeEvent { }; // Parent exists means the device is child device - let external_source = entity.parent.as_ref().map(|_| entity.entity_id.clone()); + let external_source = entity.parent.as_ref().map(|_| entity.external_id.clone()); Ok(Self { name: event_type.into(), diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 92053aa336a..028d333c39f 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -248,7 +248,7 @@ impl CumulocityConverter { let external_id = self .entity_store .get(entity_topic_id) - .map(|e| &e.entity_id) + .map(|e| &e.external_id) .ok_or_else(|| Error::UnknownEntity(entity_topic_id.to_string()))?; match input.r#type { EntityType::MainDevice => Err(ConversionError::MainDeviceRegistrationNotSupported), @@ -787,22 +787,25 @@ impl CumulocityConverter { _ => { // if device is unregistered register using auto-registration if self.entity_store.get(&source).is_none() { - registration_messages = match self.entity_store.auto_register_entity(&source) { - Ok(register_messages) => register_messages, - Err(e) => { - error!("Could not update device registration: {e}"); - vec![] + let auto_registration_messages = + self.entity_store.auto_register_entity(&source)?; + + for auto_registration_message in &auto_registration_messages { + if auto_registration_message.r#type == EntityType::ChildDevice { + self.children.insert( + auto_registration_message + .external_id + .clone() + .unwrap() + .into(), + Operations::default(), + ); } - }; - if let Some(entity) = self.entity_store.get(&source) { - if let Some(message) = external_device_registration_message(entity) { - self.children - .insert(entity.entity_id.clone().into(), Operations::default()); - registration_messages.push(message); - } - } else { - error!("Cannot auto-register entity with non-standard MQTT identifier: {source}"); + registration_messages.push(auto_registration_message.into()); + let c8y_message = self + .try_convert_entity_registration(&source, auto_registration_message)?; + registration_messages.push(c8y_message); } } } @@ -1053,13 +1056,6 @@ fn add_external_device_registration_message( false } -fn external_device_registration_message(entity: &EntityMetadata) -> Option { - match entity.r#type { - EntityType::ChildDevice => Some(new_child_device_message(entity.entity_id.as_ref())), - _ => None, - } -} - fn create_inventory_fragments_message( device_name: &str, cfg_dir: &Path, @@ -1084,7 +1080,9 @@ impl CumulocityConverter { Some(device) => { let ops_dir = match device.r#type { EntityType::MainDevice => self.ops_dir.clone(), - EntityType::ChildDevice => self.ops_dir.clone().join(device.entity_id.as_ref()), + EntityType::ChildDevice => { + self.ops_dir.clone().join(device.external_id.as_ref()) + } EntityType::Service => { error!("Unsupported `restart` operation for a service: {target}"); return Ok(vec![]); @@ -1303,6 +1301,7 @@ pub fn check_tedge_agent_status(message: &Message) -> Result( + device_creation_msgs[0].payload_str().unwrap() + ) + .unwrap(), + json!({ "@type":"child-device", "@id":"test-device:device:external_sensor" }) + ); + let second_msg = Message::new( &Topic::new_unchecked("c8y/s/us"), "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", ); - assert_eq!(device_creation_msgs[0], first_msg); assert_eq!(device_creation_msgs[1], second_msg); // During the sync phase, alarms are not converted immediately, but only cached to be synced later diff --git a/crates/extensions/c8y_mapper_ext/src/log_upload.rs b/crates/extensions/c8y_mapper_ext/src/log_upload.rs index 22c4ad1d762..682ff15117d 100644 --- a/crates/extensions/c8y_mapper_ext/src/log_upload.rs +++ b/crates/extensions/c8y_mapper_ext/src/log_upload.rs @@ -69,7 +69,7 @@ impl CumulocityConverter { let tedge_url = format!( "http://{}/tedge/file-transfer/{}/log_upload/{}-{}", &self.config.tedge_http_host, - target.entity_id.as_ref(), + target.external_id.as_ref(), log_request.log_type, cmd_id ); @@ -110,7 +110,7 @@ impl CumulocityConverter { topic_id: topic_id.to_string(), } })?; - let external_id = &device.entity_id; + let external_id = &device.external_id; let c8y_topic: C8yTopic = device.into(); let smartrest_topic = c8y_topic.to_topic()?; @@ -130,7 +130,7 @@ impl CumulocityConverter { let uploaded_file_path = self .config .file_transfer_dir - .join(device.entity_id.as_ref()) + .join(device.external_id.as_ref()) .join("log_upload") .join(format!("{}-{}", response.log_type, cmd_id)); let result = self @@ -206,7 +206,7 @@ impl CumulocityConverter { // Create a c8y_LogfileRequest operation file let dir_path = match device.r#type { EntityType::MainDevice => self.ops_dir.clone(), - EntityType::ChildDevice => self.ops_dir.join(device.entity_id.as_ref()), + EntityType::ChildDevice => self.ops_dir.join(device.external_id.as_ref()), EntityType::Service => { // No support for service log management return Ok(vec![]); diff --git a/crates/extensions/c8y_mapper_ext/src/serializer.rs b/crates/extensions/c8y_mapper_ext/src/serializer.rs index 1d0c4196d13..c974b7b8b68 100644 --- a/crates/extensions/c8y_mapper_ext/src/serializer.rs +++ b/crates/extensions/c8y_mapper_ext/src/serializer.rs @@ -59,7 +59,7 @@ impl C8yJsonSerializer { json.write_open_obj(); if entity.r#type == EntityType::ChildDevice { - let child_id = &entity.entity_id; + let child_id = &entity.external_id; // In case the measurement is addressed to a child-device use fragment // "externalSource" to tell c8Y identity API to use child-device // object referenced by "externalId", instead of root device object diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index 6e356bc7113..480b7244e74 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -503,14 +503,19 @@ async fn c8y_mapper_child_alarm_mapping_to_smartrest() { .await .unwrap(); + // Expect auto-registration message + assert_received_includes_json( + &mut mqtt, + [( + "te/device/external_sensor//", + json!({"@type":"child-device","@id":"test-device:device:external_sensor"}), + )], + ) + .await; // Expect child device creation and converted temperature alarm messages assert_received_contains_str( &mut mqtt, [ - ( - "te/device/external_sensor//", - r#"{ "@type":"child-device", "@id":"test-device:device:external_sensor"}"#, - ), ( "c8y/s/us", "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", @@ -597,14 +602,20 @@ async fn c8y_mapper_child_alarm_with_custom_fragment_mapping_to_c8y_json() { .await .unwrap(); + // Expect auto-registration message + assert_received_includes_json( + &mut mqtt, + [( + "te/device/external_sensor//", + json!({"@type":"child-device","@id":"test-device:device:external_sensor"}), + )], + ) + .await; + // Expect child device creation message assert_received_contains_str( &mut mqtt, [ - ( - "te/device/external_sensor//", - r#"{ "@type":"child-device", "@id":"test-device:device:external_sensor"}"#, - ), ( "c8y/s/us", "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", @@ -1152,8 +1163,8 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { let child_metadata = mqtt.recv().await.unwrap(); assert_eq!(child_metadata.topic.name, "te/device/child1//"); assert_eq!( - child_metadata.payload_str().unwrap(), - r#"{ "@type":"child-device", "@id":"test-device:device:child1"}"# + serde_json::from_str::(child_metadata.payload_str().unwrap()).unwrap(), + json!({"@type":"child-device","@id":"test-device:device:child1"}) ); let child_c8y_registration = mqtt.recv().await.unwrap(); assert_eq!(child_c8y_registration.topic.name, "c8y/s/us"); @@ -1700,18 +1711,22 @@ async fn mapper_converts_log_upload_cmd_to_supported_op_and_types_for_child_devi .await .expect("Send failed"); + // Expect auto-registration message + assert_received_includes_json( + &mut mqtt, + [( + "te/device/child1//", + json!({"@type":"child-device","@id":"test-device:device:child1"}), + )], + ) + .await; + assert_received_contains_str( &mut mqtt, - [ - ( - "te/device/child1//", - r#"{ "@type":"child-device", "@id":"test-device:device:child1"}"#, - ), - ( - "c8y/s/us", - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", - ), - ], + [( + "c8y/s/us", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + )], ) .await; assert_received_contains_str( @@ -1810,18 +1825,22 @@ async fn handle_log_upload_executing_and_failed_cmd_for_child_device() { .await .expect("Send failed"); + // Expect auto-registration message + assert_received_includes_json( + &mut mqtt, + [( + "te/device/child1//", + json!({"@type":"child-device","@id":"test-device:device:child1"}), + )], + ) + .await; + assert_received_contains_str( &mut mqtt, - [ - ( - "te/device/child1//", - r#"{ "@type":"child-device", "@id":"test-device:device:child1"}"#, - ), - ( - "c8y/s/us", - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", - ), - ], + [( + "c8y/s/us", + "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + )], ) .await; From 6efb329cb70122bc08448220eae217787cec0762 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Mon, 25 Sep 2023 14:41:33 +0000 Subject: [PATCH 08/13] Fix external id of children registered via file watcher --- crates/extensions/c8y_mapper_ext/src/actor.rs | 23 ++++-- .../c8y_mapper_ext/src/converter.rs | 80 +++++++++++-------- crates/extensions/c8y_mapper_ext/src/tests.rs | 9 ++- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/crates/extensions/c8y_mapper_ext/src/actor.rs b/crates/extensions/c8y_mapper_ext/src/actor.rs index 2b4bd366dc2..6bf325caaa2 100644 --- a/crates/extensions/c8y_mapper_ext/src/actor.rs +++ b/crates/extensions/c8y_mapper_ext/src/actor.rs @@ -2,11 +2,11 @@ use super::config::C8yMapperConfig; use super::converter::CumulocityConverter; use super::dynamic_discovery::process_inotify_events; use async_trait::async_trait; -use c8y_api::smartrest::topic::SMARTREST_PUBLISH_TOPIC; use c8y_auth_proxy::url::ProxyUrlGenerator; use c8y_http_proxy::handle::C8YHttpProxy; use c8y_http_proxy::messages::C8YRestRequest; use c8y_http_proxy::messages::C8YRestResult; +use serde_json::json; use std::path::PathBuf; use std::time::Duration; use tedge_actors::adapt; @@ -26,10 +26,11 @@ use tedge_actors::Sender; use tedge_actors::ServiceProvider; use tedge_actors::SimpleMessageBox; use tedge_actors::SimpleMessageBoxBuilder; +use tedge_api::entity_store::EntityRegistrationMessage; +use tedge_api::entity_store::EntityType; +use tedge_api::mqtt_topics::EntityTopicId; use tedge_file_system_ext::FsWatchEvent; -use tedge_mqtt_ext::Message; use tedge_mqtt_ext::MqttMessage; -use tedge_mqtt_ext::Topic; use tedge_mqtt_ext::TopicFilter; use tedge_timer_ext::SetTimeout; use tedge_timer_ext::Timeout; @@ -127,11 +128,17 @@ impl C8yMapperActor { FsWatchEvent::DirectoryCreated(path) => { if let Some(directory_name) = path.file_name() { let child_id = directory_name.to_string_lossy().to_string(); - let message = Message::new( - &Topic::new_unchecked(SMARTREST_PUBLISH_TOPIC), - format!("101,{child_id},{child_id},thin-edge.io-child"), - ); - self.mqtt_publisher.send(message).await?; + let child_topic_id = EntityTopicId::default_child_device(&child_id).unwrap(); + let child_device_reg_msg = EntityRegistrationMessage { + topic_id: child_topic_id, + external_id: None, + r#type: EntityType::ChildDevice, + parent: Some(EntityTopicId::default_main_device()), + payload: json!({}), + }; + self.converter + .try_convert_entity_registration(&child_device_reg_msg) + .unwrap(); } } FsWatchEvent::FileCreated(path) diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 028d333c39f..74fb1095ac5 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -56,7 +56,6 @@ use tedge_actors::LoggingSender; use tedge_actors::Sender; use tedge_api::entity_store; use tedge_api::entity_store::EntityExternalId; -use tedge_api::entity_store::EntityMetadata; use tedge_api::entity_store::EntityRegistrationMessage; use tedge_api::entity_store::EntityType; use tedge_api::entity_store::Error; @@ -236,15 +235,15 @@ impl CumulocityConverter { }) } - fn try_convert_entity_registration( + pub fn try_convert_entity_registration( &mut self, - entity_topic_id: &EntityTopicId, input: &EntityRegistrationMessage, ) -> Result { // Parse the optional fields let display_name = input.payload.get("name").and_then(|v| v.as_str()); let display_type = input.payload.get("type").and_then(|v| v.as_str()); + let entity_topic_id = &input.topic_id; let external_id = self .entity_store .get(entity_topic_id) @@ -292,7 +291,7 @@ impl CumulocityConverter { /// - `device/child001//` => `DEVICE_COMMON_NAME:device:child001` /// - `device/child001/service/service001` => `DEVICE_COMMON_NAME:device:child001:service:service001` /// - `factory01/hallA/packaging/belt001` => `DEVICE_COMMON_NAME:factory01:hallA:packaging:belt001` - fn map_to_c8y_external_id( + pub fn map_to_c8y_external_id( entity_topic_id: &EntityTopicId, main_device_xid: &EntityExternalId, ) -> EntityExternalId { @@ -717,7 +716,10 @@ impl CumulocityConverter { self.children.insert(child_id.clone(), ops.clone()); let ops_msg = ops.create_smartrest_ops_message()?; - let topic_str = format!("{SMARTREST_PUBLISH_TOPIC}/{}", child_id); + let child_topic_id = EntityTopicId::default_child_device(&child_id).unwrap(); + let child_external_id = + Self::map_to_c8y_external_id(&child_topic_id, &self.device_name.as_str().into()); + let topic_str = format!("{SMARTREST_PUBLISH_TOPIC}/{}", child_external_id.as_ref()); let topic = Topic::new_unchecked(&topic_str); messages_vec.push(Message::new(&topic, ops_msg)); } @@ -779,8 +781,7 @@ impl CumulocityConverter { if let Err(e) = self.entity_store.update(register_message.clone()) { error!("Could not update device registration: {e}"); } - let c8y_message = - self.try_convert_entity_registration(&source, ®ister_message)?; + let c8y_message = self.try_convert_entity_registration(®ister_message)?; registration_messages.push(c8y_message); } } @@ -803,8 +804,8 @@ impl CumulocityConverter { } registration_messages.push(auto_registration_message.into()); - let c8y_message = self - .try_convert_entity_registration(&source, auto_registration_message)?; + let c8y_message = + self.try_convert_entity_registration(auto_registration_message)?; registration_messages.push(c8y_message); } } @@ -907,9 +908,9 @@ impl CumulocityConverter { &self.cfg_dir, )); - let supported_operations_message = self.wrap_error(create_supported_operations( - &self.cfg_dir.join("operations").join("c8y"), - )); + let supported_operations_message = self.wrap_error( + self.create_supported_operations(&self.cfg_dir.join("operations").join("c8y")), + ); let cloud_child_devices_message = create_request_for_cloud_child_devices(); @@ -929,6 +930,28 @@ impl CumulocityConverter { ]) } + fn create_supported_operations(&self, path: &Path) -> Result { + if is_child_operation_path(path) { + // operations for child + let child_id = get_child_id(&path.to_path_buf())?; + let child_topic_id = EntityTopicId::default_child_device(&child_id).unwrap(); + let child_external_id = + Self::map_to_c8y_external_id(&child_topic_id, &self.device_name.as_str().into()); + let stopic = format!("{SMARTREST_PUBLISH_TOPIC}/{}", child_external_id.as_ref()); + + Ok(Message::new( + &Topic::new_unchecked(&stopic), + Operations::try_new(path)?.create_smartrest_ops_message()?, + )) + } else { + // operations for parent + Ok(Message::new( + &Topic::new_unchecked(SMARTREST_PUBLISH_TOPIC), + Operations::try_new(path)?.create_smartrest_ops_message()?, + )) + } + } + pub fn sync_messages(&mut self) -> Vec { let sync_messages: Vec = self.alarm_converter.sync(); self.alarm_converter = AlarmConverter::Synced; @@ -946,7 +969,7 @@ impl CumulocityConverter { { // Re populate the operations irrespective add/remove/modify event self.operations = get_operations(message.ops_dir.clone())?; - Ok(Some(create_supported_operations(&message.ops_dir)?)) + Ok(Some(self.create_supported_operations(&message.ops_dir)?)) // operation for child } else if message.ops_dir.eq(&self @@ -960,7 +983,7 @@ impl CumulocityConverter { get_operations(message.ops_dir.clone())?, ); - Ok(Some(create_supported_operations(&message.ops_dir)?)) + Ok(Some(self.create_supported_operations(&message.ops_dir)?)) } else { Ok(None) } @@ -1020,25 +1043,6 @@ fn is_child_operation_path(path: &Path) -> bool { } } -fn create_supported_operations(path: &Path) -> Result { - if is_child_operation_path(path) { - // operations for child - let child_id = get_child_id(&path.to_path_buf())?; - let stopic = format!("{SMARTREST_PUBLISH_TOPIC}/{}", child_id); - - Ok(Message::new( - &Topic::new_unchecked(&stopic), - Operations::try_new(path)?.create_smartrest_ops_message()?, - )) - } else { - // operations for parent - Ok(Message::new( - &Topic::new_unchecked(SMARTREST_PUBLISH_TOPIC), - Operations::try_new(path)?.create_smartrest_ops_message()?, - )) - } -} - fn create_request_for_cloud_child_devices() -> Message { Message::new(&Topic::new_unchecked("c8y/s/us"), "105") } @@ -1081,7 +1085,13 @@ impl CumulocityConverter { let ops_dir = match device.r#type { EntityType::MainDevice => self.ops_dir.clone(), EntityType::ChildDevice => { - self.ops_dir.clone().join(device.external_id.as_ref()) + let child_dir_name = + if let Some(child_local_id) = target.default_device_name() { + child_local_id + } else { + device.external_id.as_ref() + }; + self.ops_dir.clone().join(child_dir_name) } EntityType::Service => { error!("Unsupported `restart` operation for a service: {target}"); @@ -1091,7 +1101,7 @@ impl CumulocityConverter { let ops_file = ops_dir.join("c8y_Restart"); create_directory_with_defaults(&ops_dir)?; create_file_with_defaults(ops_file, None)?; - let device_operations = create_supported_operations(&ops_dir)?; + let device_operations = self.create_supported_operations(&ops_dir)?; Ok(vec![device_operations]) } } diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index 480b7244e74..ffb93619023 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -1041,7 +1041,10 @@ async fn mapper_publishes_supported_operations_for_child_device() { &mut mqtt, [ ("c8y/s/us", "101,child1,child1,thin-edge.io-child"), - ("c8y/s/us/child1", "114,c8y_ChildTestOp1,c8y_ChildTestOp2\n"), + ( + "c8y/s/us/test-device:device:child1", + "114,c8y_ChildTestOp1,c8y_ChildTestOp2\n", + ), ], ) .await; @@ -1117,7 +1120,7 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { let cfg_dir = TempTedgeDir::new(); create_thin_edge_child_operations( &cfg_dir, - "test-device:device:child1", + "child1", vec!["c8y_ChildTestOp1", "c8y_ChildTestOp2"], ); @@ -1131,7 +1134,7 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { cfg_dir .dir("operations") .dir("c8y") - .dir("test-device:device:child1") + .dir("child1") .file("c8y_ChildTestOp3") .to_path_buf(), )) From fe340d42071f369966b7d6577ea299ca26fe0608 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Mon, 25 Sep 2023 16:35:50 +0000 Subject: [PATCH 09/13] Auto-register child devices via file system --- crates/extensions/c8y_mapper_ext/src/actor.rs | 6 +- .../c8y_mapper_ext/src/converter.rs | 75 +++++++++++++------ crates/extensions/c8y_mapper_ext/src/tests.rs | 30 +++++++- 3 files changed, 85 insertions(+), 26 deletions(-) diff --git a/crates/extensions/c8y_mapper_ext/src/actor.rs b/crates/extensions/c8y_mapper_ext/src/actor.rs index 6bf325caaa2..fe923f0be32 100644 --- a/crates/extensions/c8y_mapper_ext/src/actor.rs +++ b/crates/extensions/c8y_mapper_ext/src/actor.rs @@ -133,11 +133,11 @@ impl C8yMapperActor { topic_id: child_topic_id, external_id: None, r#type: EntityType::ChildDevice, - parent: Some(EntityTopicId::default_main_device()), - payload: json!({}), + parent: None, + payload: json!({ "name": child_id }), }; self.converter - .try_convert_entity_registration(&child_device_reg_msg) + .register_and_convert_entity(&child_device_reg_msg) .unwrap(); } } diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 74fb1095ac5..4f00cfc58d0 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -45,6 +45,7 @@ use plugin_sm::operation_logs::OperationLogs; use plugin_sm::operation_logs::OperationLogsError; use serde::Deserialize; use serde::Serialize; +use serde_json::json; use service_monitor::convert_health_status_message; use std::collections::HashMap; use std::fs; @@ -707,18 +708,31 @@ impl CumulocityConverter { for child_id in difference { // here we register new child devices, sending the 101 code - messages_vec.push(new_child_device_message(child_id)); + let child_topic_id = EntityTopicId::default_child_device(child_id).unwrap(); + let child_device_reg_msg = EntityRegistrationMessage { + topic_id: child_topic_id, + external_id: None, + r#type: EntityType::ChildDevice, + parent: None, + payload: json!({ "name": child_id }), + }; + let mut reg_messages = self + .register_and_convert_entity(&child_device_reg_msg) + .unwrap(); + + messages_vec.append(&mut reg_messages); } // loop over all local child devices and update the operations for child_id in local_child_devices { // update the children cache with the operations supported let ops = Operations::try_new(path_to_child_devices.join(&child_id))?; - self.children.insert(child_id.clone(), ops.clone()); - - let ops_msg = ops.create_smartrest_ops_message()?; let child_topic_id = EntityTopicId::default_child_device(&child_id).unwrap(); let child_external_id = Self::map_to_c8y_external_id(&child_topic_id, &self.device_name.as_str().into()); + + self.children + .insert(child_external_id.as_ref().into(), ops.clone()); + let ops_msg = ops.create_smartrest_ops_message()?; let topic_str = format!("{SMARTREST_PUBLISH_TOPIC}/{}", child_external_id.as_ref()); let topic = Topic::new_unchecked(&topic_str); messages_vec.push(Message::new(&topic, ops_msg)); @@ -792,21 +806,9 @@ impl CumulocityConverter { self.entity_store.auto_register_entity(&source)?; for auto_registration_message in &auto_registration_messages { - if auto_registration_message.r#type == EntityType::ChildDevice { - self.children.insert( - auto_registration_message - .external_id - .clone() - .unwrap() - .into(), - Operations::default(), - ); - } - - registration_messages.push(auto_registration_message.into()); - let c8y_message = - self.try_convert_entity_registration(auto_registration_message)?; - registration_messages.push(c8y_message); + registration_messages.append( + &mut self.register_and_convert_entity(auto_registration_message)?, + ); } } } @@ -860,6 +862,32 @@ impl CumulocityConverter { Ok(registration_messages) } + pub fn register_and_convert_entity( + &mut self, + registration_message: &EntityRegistrationMessage, + ) -> Result, ConversionError> { + let entity_topic_id = ®istration_message.topic_id; + self.entity_store.update(registration_message.clone())?; + if registration_message.r#type == EntityType::ChildDevice { + self.children.insert( + self.entity_store + .get(entity_topic_id) + .expect("Should have been registered in the previous step") + .external_id + .as_ref() + .into(), + Operations::default(), + ); + } + + let mut registration_messages = vec![]; + registration_messages.push(registration_message.into()); + let c8y_message = self.try_convert_entity_registration(registration_message)?; + registration_messages.push(c8y_message); + + Ok(registration_messages) + } + async fn try_convert_tedge_topics( &mut self, message: &Message, @@ -1351,7 +1379,12 @@ mod tests { "c8y_Command", ]; - const EXPECTED_CHILD_DEVICES: &[&str] = &["child-0", "child-1", "child-2", "child-3"]; + const EXPECTED_CHILD_DEVICES: &[&str] = &[ + "test-device:device:child-0", + "test-device:device:child-1", + "test-device:device:child-2", + "test-device:device:child-3", + ]; #[tokio::test] async fn test_sync_alarms() { @@ -2127,7 +2160,7 @@ mod tests { /// then supported operations will be published to the cloud and the device will be cached. #[test_case("106", EXPECTED_CHILD_DEVICES; "cloud representation is empty")] #[test_case("106,child-one,child-two", EXPECTED_CHILD_DEVICES; "cloud representation is completely different")] - #[test_case("106,child-3,child-one,child-1", &["child-0", "child-2"]; "cloud representation has some similar child devices")] + #[test_case("106,child-3,child-one,child-1", &["test-device:device:child-0", "test-device:device:child-2"]; "cloud representation has some similar child devices")] #[test_case("106,child-0,child-1,child-2,child-3", &[]; "cloud representation has seen all child devices")] #[tokio::test] async fn test_child_device_cache_is_updated( diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index ffb93619023..5f4bff46a8e 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -1005,10 +1005,23 @@ async fn mapper_publishes_child_device_create_message() { .await .expect("Send failed"); + // Expect auto-registration message + assert_received_includes_json( + &mut mqtt, + [( + "te/device/child1//", + json!({"@type":"child-device", "name": "child1"}), + )], + ) + .await; + // Expect smartrest message on `c8y/s/us` with expected payload "101,child1,child1,thin-edge.io-child". assert_received_contains_str( &mut mqtt, - [("c8y/s/us", "101,child1,child1,thin-edge.io-child")], + [( + "c8y/s/us", + "101,test-device:device:child1,child1,thin-edge.io-child", + )], ) .await; } @@ -1036,11 +1049,24 @@ async fn mapper_publishes_supported_operations_for_child_device() { .await .expect("Send failed"); + // Expect auto-registration message + assert_received_includes_json( + &mut mqtt, + [( + "te/device/child1//", + json!({"@type":"child-device", "name": "child1"}), + )], + ) + .await; + // Expect smartrest message on `c8y/s/us/child1` with expected payload "114,c8y_ChildTestOp1,c8y_ChildTestOp2. assert_received_contains_str( &mut mqtt, [ - ("c8y/s/us", "101,child1,child1,thin-edge.io-child"), + ( + "c8y/s/us", + "101,test-device:device:child1,child1,thin-edge.io-child", + ), ( "c8y/s/us/test-device:device:child1", "114,c8y_ChildTestOp1,c8y_ChildTestOp2\n", From 2942a6ec4b1ad309159911a4415ed59dd1d86a96 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Mon, 25 Sep 2023 16:59:03 +0000 Subject: [PATCH 10/13] Simplify entity auto-registration message creation --- crates/core/tedge_api/src/entity_store.rs | 41 +++++++++-------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/crates/core/tedge_api/src/entity_store.rs b/crates/core/tedge_api/src/entity_store.rs index bb11d3b0f8d..6b82e87a7b0 100644 --- a/crates/core/tedge_api/src/entity_store.rs +++ b/crates/core/tedge_api/src/entity_store.rs @@ -14,6 +14,7 @@ use crate::mqtt_topics::EntityTopicId; use crate::mqtt_topics::TopicIdError; use mqtt_channel::Message; use mqtt_channel::Topic; +use serde_json::json; use serde_json::Map; use serde_json::Value; @@ -318,19 +319,13 @@ impl EntityStore { let device_external_id = (self.external_id_mapper)(&parent_device_id, &self.main_device_external_id()); - let device_register_payload = format!( - "{{ \"@type\":\"child-device\", \"@id\":\"{}\"}}", - device_external_id.as_ref() - ); - - // FIXME: The root prefix should not be added this way. - // The simple fix is to change the signature of the method, - // returning (EntityTopicId, EntityMetadata) pairs instead of MQTT Messages. - let topic = Topic::new(&format!("{MQTT_ROOT}/{parent_device_id}")).unwrap(); - let device_register_message = - Message::new(&topic, device_register_payload).with_retain(); - let device_register_message = - EntityRegistrationMessage::try_from(&device_register_message).unwrap(); + let device_register_message = EntityRegistrationMessage { + topic_id: parent_device_id.clone(), + external_id: Some(device_external_id), + r#type: EntityType::ChildDevice, + parent: None, + payload: json!({}), + }; register_messages.push(device_register_message.clone()); self.update(device_register_message)?; } @@ -340,19 +335,13 @@ impl EntityStore { let service_external_id = (self.external_id_mapper)(entity_topic_id, &self.main_device_external_id()); - let service_register_payload = format!( - "{{ \"@type\":\"service\", \"@id\":\"{}\", \"name\":\"{}\", \"type\": \"systemd\"}}", - service_external_id.as_ref(), - service_id - ); - - let service_register_message = Message::new( - &Topic::new(&format!("{MQTT_ROOT}/{entity_topic_id}")).unwrap(), - service_register_payload, - ) - .with_retain(); - let service_register_message = - EntityRegistrationMessage::try_from(&service_register_message).unwrap(); + let service_register_message = EntityRegistrationMessage { + topic_id: entity_topic_id.clone(), + external_id: Some(service_external_id), + r#type: EntityType::Service, + parent: Some(parent_device_id), + payload: json!({ "name": service_id, "type": "systemd" }), + }; register_messages.push(service_register_message.clone()); self.update(service_register_message)?; } From 941e85e790c8dc286fa312711e3f0ffb31058a47 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Mon, 25 Sep 2023 20:56:58 +0000 Subject: [PATCH 11/13] Custom topic scheme support for main device --- crates/core/tedge_api/src/entity_store.rs | 213 ++++++++++++++---- .../c8y_mapper_ext/src/converter.rs | 52 +++-- crates/extensions/c8y_mapper_ext/src/error.rs | 3 - crates/extensions/c8y_mapper_ext/src/tests.rs | 30 ++- .../registration/device_registration.robot | 45 ++-- 5 files changed, 241 insertions(+), 102 deletions(-) diff --git a/crates/core/tedge_api/src/entity_store.rs b/crates/core/tedge_api/src/entity_store.rs index 6b82e87a7b0..0c9964447dc 100644 --- a/crates/core/tedge_api/src/entity_store.rs +++ b/crates/core/tedge_api/src/entity_store.rs @@ -234,11 +234,6 @@ impl EntityStore { &mut self, message: EntityRegistrationMessage, ) -> Result, Error> { - if message.r#type == EntityType::MainDevice && message.topic_id != self.main_device { - return Err(Error::MainDeviceAlreadyRegistered( - self.main_device.to_string().into_boxed_str(), - )); - } let topic_id = message.topic_id; let mut affected_entities = vec![]; @@ -261,9 +256,12 @@ impl EntityStore { affected_entities.push(parent.clone()); } - let external_id = message.external_id.unwrap_or_else(|| { - (self.external_id_mapper)(&topic_id, &self.main_device_external_id()) - }); + let external_id = match message.r#type { + EntityType::MainDevice => self.main_device_external_id(), + _ => message.external_id.unwrap_or_else(|| { + (self.external_id_mapper)(&topic_id, &self.main_device_external_id()) + }), + }; let entity_metadata = EntityMetadata { topic_id: topic_id.clone(), r#type: message.r#type, @@ -316,6 +314,7 @@ impl EntityStore { .expect("device id must be present as the topic id follows the default scheme"); if !parent_device_id.is_default_main_device() && self.get(&parent_device_id).is_none() { + let device_local_id = entity_topic_id.default_device_name().unwrap(); let device_external_id = (self.external_id_mapper)(&parent_device_id, &self.main_device_external_id()); @@ -324,7 +323,7 @@ impl EntityStore { external_id: Some(device_external_id), r#type: EntityType::ChildDevice, parent: None, - payload: json!({}), + payload: json!({ "name": device_local_id }), }; register_messages.push(device_register_message.clone()); self.update(device_register_message)?; @@ -541,6 +540,9 @@ fn parse_entity_register_payload(payload: &[u8]) -> Option { #[cfg(test)] mod tests { + use std::str::FromStr; + + use assert_matches::assert_matches; use serde_json::json; use super::*; @@ -672,39 +674,6 @@ mod tests { .any(|&e| e == &EntityTopicId::default_main_service("service2").unwrap())); } - /// Forbids creating multiple main devices. - /// - /// Publishing new registration message on a topic where main device is - /// registered updates the main device and is allowed. Creating a new main - /// device on another topic is not allowed. - #[test] - fn forbids_multiple_main_devices() { - let mut store = EntityStore::with_main_device( - EntityRegistrationMessage { - topic_id: EntityTopicId::default_main_device(), - r#type: EntityType::MainDevice, - external_id: Some("test-device".into()), - parent: None, - payload: json!({}), - }, - dummy_external_id_mapper, - ) - .unwrap(); - - let res = store.update(EntityRegistrationMessage { - topic_id: EntityTopicId::default_child_device("another_main").unwrap(), - external_id: Some("test-device".into()), - r#type: EntityType::MainDevice, - parent: None, - payload: json!({}), - }); - - assert_eq!( - res, - Err(Error::MainDeviceAlreadyRegistered("device/main//".into())) - ); - } - #[test] fn forbids_nonexistent_parents() { let mut store = EntityStore::with_main_device( @@ -977,4 +946,164 @@ mod tests { let message: Message = (&entity_reg_message).into(); println!("{}", message.payload_str().unwrap()); } + + #[test] + fn auto_register_service() { + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + r#type: EntityType::MainDevice, + external_id: Some("test-device".into()), + parent: None, + payload: json!({}), + }, + dummy_external_id_mapper, + ) + .unwrap(); + + let service_topic_id = EntityTopicId::default_child_service("child1", "service1").unwrap(); + let res = store.auto_register_entity(&service_topic_id).unwrap(); + assert_eq!( + res, + [ + EntityRegistrationMessage { + topic_id: EntityTopicId::from_str("device/child1//").unwrap(), + r#type: EntityType::ChildDevice, + external_id: Some("device:child1".into()), + parent: None, + payload: json!({ "name": "child1" }), + }, + EntityRegistrationMessage { + topic_id: EntityTopicId::from_str("device/child1/service/service1").unwrap(), + r#type: EntityType::Service, + external_id: Some("device:child1:service:service1".into()), + parent: Some(EntityTopicId::from_str("device/child1//").unwrap()), + payload: json!({ "name": "service1", "type": "systemd" }), + } + ] + ); + } + + #[test] + fn auto_register_child_device() { + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + r#type: EntityType::MainDevice, + external_id: Some("test-device".into()), + parent: None, + payload: json!({}), + }, + dummy_external_id_mapper, + ) + .unwrap(); + + let child_topic_id = EntityTopicId::default_child_device("child2").unwrap(); + let res = store.auto_register_entity(&child_topic_id).unwrap(); + + assert_eq!( + res, + [EntityRegistrationMessage { + topic_id: EntityTopicId::from_str("device/child2//").unwrap(), + r#type: EntityType::ChildDevice, + external_id: Some("device:child2".into()), + parent: None, + payload: json!({ "name": "child2" }), + },] + ); + } + + #[test] + fn auto_register_custom_topic_scheme_not_supported() { + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + r#type: EntityType::MainDevice, + external_id: Some("test-device".into()), + parent: None, + payload: json!({}), + }, + dummy_external_id_mapper, + ) + .unwrap(); + assert_matches!( + store.auto_register_entity(&EntityTopicId::from_str("custom/child2//").unwrap()), + Err(Error::NonDefaultTopicScheme(_)) + ); + } + + #[test] + fn register_main_device_custom_scheme() { + let mut store = EntityStore::with_main_device( + EntityRegistrationMessage { + topic_id: EntityTopicId::default_main_device(), + r#type: EntityType::MainDevice, + external_id: Some("test-device".into()), + parent: None, + payload: json!({}), + }, + dummy_external_id_mapper, + ) + .unwrap(); + + // Register main device with custom topic scheme + let main_topic_id = EntityTopicId::from_str("custom/main//").unwrap(); + store + .update(EntityRegistrationMessage { + topic_id: main_topic_id.clone(), + r#type: EntityType::MainDevice, + external_id: None, + parent: None, + payload: json!({}), + }) + .unwrap(); + + let expected_entity_metadata = EntityMetadata { + topic_id: main_topic_id.clone(), + parent: None, + r#type: EntityType::MainDevice, + external_id: "test-device".into(), + other: json!({}), + }; + // Assert main device registered with custom topic scheme + assert_eq!( + store.get(&main_topic_id).unwrap(), + &expected_entity_metadata + ); + assert_eq!( + store.get_by_external_id(&"test-device".into()).unwrap(), + &expected_entity_metadata + ); + + // Register service on main device with custom scheme + let service_topic_id = EntityTopicId::from_str("custom/main/service/collectd").unwrap(); + store + .update(EntityRegistrationMessage { + topic_id: service_topic_id.clone(), + r#type: EntityType::Service, + external_id: None, + parent: Some(main_topic_id.clone()), + payload: json!({}), + }) + .unwrap(); + + let expected_entity_metadata = EntityMetadata { + topic_id: service_topic_id.clone(), + parent: Some(main_topic_id), + r#type: EntityType::Service, + external_id: "custom:main:service:collectd".into(), + other: json!({}), + }; + // Assert service registered under main device with custom topic scheme + assert_eq!( + store.get(&service_topic_id).unwrap(), + &expected_entity_metadata + ); + assert_eq!( + store + .get_by_external_id(&"custom:main:service:collectd".into()) + .unwrap(), + &expected_entity_metadata + ); + } } diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 4f00cfc58d0..74332235ea5 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -239,7 +239,7 @@ impl CumulocityConverter { pub fn try_convert_entity_registration( &mut self, input: &EntityRegistrationMessage, - ) -> Result { + ) -> Result, ConversionError> { // Parse the optional fields let display_name = input.payload.get("name").and_then(|v| v.as_str()); let display_type = input.payload.get("type").and_then(|v| v.as_str()); @@ -251,22 +251,26 @@ impl CumulocityConverter { .map(|e| &e.external_id) .ok_or_else(|| Error::UnknownEntity(entity_topic_id.to_string()))?; match input.r#type { - EntityType::MainDevice => Err(ConversionError::MainDeviceRegistrationNotSupported), + EntityType::MainDevice => { + self.entity_store.update(input.clone())?; + Ok(vec![]) + } EntityType::ChildDevice => { let ancestors_external_ids = self.entity_store.ancestors_external_ids(entity_topic_id)?; - Ok(child_device_creation_message( + let child_creation_message = child_device_creation_message( external_id.as_ref(), display_name, display_type, &ancestors_external_ids, - )) + ); + Ok(vec![child_creation_message]) } EntityType::Service => { let ancestors_external_ids = self.entity_store.ancestors_external_ids(entity_topic_id)?; - Ok(service_creation_message( + let service_creation_message = service_creation_message( external_id.as_ref(), display_name.unwrap_or_else(|| { entity_topic_id @@ -276,7 +280,8 @@ impl CumulocityConverter { display_type.unwrap_or("service"), "up", &ancestors_external_ids, - )) + ); + Ok(vec![service_creation_message]) } } } @@ -788,15 +793,16 @@ impl CumulocityConverter { channel: Channel, message: &Message, ) -> Result, ConversionError> { - let mut registration_messages = vec![]; + let mut registration_messages: Vec = vec![]; match &channel { Channel::EntityMetadata => { if let Ok(register_message) = EntityRegistrationMessage::try_from(message) { if let Err(e) = self.entity_store.update(register_message.clone()) { error!("Could not update device registration: {e}"); } - let c8y_message = self.try_convert_entity_registration(®ister_message)?; - registration_messages.push(c8y_message); + let mut c8y_message = + self.try_convert_entity_registration(®ister_message)?; + registration_messages.append(&mut c8y_message); } } _ => { @@ -882,8 +888,8 @@ impl CumulocityConverter { let mut registration_messages = vec![]; registration_messages.push(registration_message.into()); - let c8y_message = self.try_convert_entity_registration(registration_message)?; - registration_messages.push(c8y_message); + let mut c8y_message = self.try_convert_entity_registration(registration_message)?; + registration_messages.append(&mut c8y_message); Ok(registration_messages) } @@ -1461,12 +1467,16 @@ mod tests { device_creation_msgs[0].payload_str().unwrap() ) .unwrap(), - json!({ "@type":"child-device", "@id":"test-device:device:external_sensor" }) + json!({ + "@type":"child-device", + "@id":"test-device:device:external_sensor", + "name": "external_sensor" + }) ); let second_msg = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", + "101,test-device:device:external_sensor,external_sensor,thin-edge.io-child", ); assert_eq!(device_creation_msgs[1], second_msg); @@ -1528,7 +1538,7 @@ mod tests { let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + "101,test-device:device:child1,child1,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), @@ -1584,7 +1594,7 @@ mod tests { .collect(); let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + "101,test-device:device:child1,child1,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), @@ -1613,7 +1623,7 @@ mod tests { .collect(); let expected_first_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + "101,test-device:device:child1,child1,thin-edge.io-child", ); let expected_first_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), @@ -1638,7 +1648,7 @@ mod tests { .collect(); let expected_second_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:child2,test-device:device:child2,thin-edge.io-child", + "101,test-device:device:child2,child2,thin-edge.io-child", ); let expected_second_c8y_json_message = Message::new( &Topic::new_unchecked("c8y/measurement/measurements/create"), @@ -1712,7 +1722,7 @@ mod tests { let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:child,test-device:device:child,thin-edge.io-child", + "101,test-device:device:child,child,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( @@ -1746,7 +1756,7 @@ mod tests { let in_message = Message::new(&Topic::new_unchecked(in_topic), in_payload); let expected_smart_rest_message = Message::new( &Topic::new_unchecked("c8y/s/us"), - "101,test-device:device:child2,test-device:device:child2,thin-edge.io-child", + "101,test-device:device:child2,child2,thin-edge.io-child", ); let expected_c8y_json_message = Message::new( @@ -2005,9 +2015,7 @@ mod tests { let payload1 = &result[0].payload_str().unwrap(); let payload2 = &result[1].payload_str().unwrap(); - assert!(payload1.contains( - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child" - )); + assert!(payload1.contains("101,test-device:device:child1,child1,thin-edge.io-child")); assert!(payload2 .contains( r#"{"externalSource":{"externalId":"test-device:device:child1","type":"c8y_Serial"},"temperature0":{"temperature0":{"value":0.0}},"# )); diff --git a/crates/extensions/c8y_mapper_ext/src/error.rs b/crates/extensions/c8y_mapper_ext/src/error.rs index 2275f3f5683..9e4559fb704 100644 --- a/crates/extensions/c8y_mapper_ext/src/error.rs +++ b/crates/extensions/c8y_mapper_ext/src/error.rs @@ -119,9 +119,6 @@ pub enum ConversionError { #[error(transparent)] FromC8yAlarmError(#[from] c8y_api::json_c8y::C8yAlarmError), - #[error("Main device registration via MQTT is not supported. Use 'tedge connect c8y' command instead.")] - MainDeviceRegistrationNotSupported, - #[error(transparent)] FromEntityStoreError(#[from] tedge_api::entity_store::Error), } diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index 5f4bff46a8e..62a6f8bd5cd 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -508,7 +508,11 @@ async fn c8y_mapper_child_alarm_mapping_to_smartrest() { &mut mqtt, [( "te/device/external_sensor//", - json!({"@type":"child-device","@id":"test-device:device:external_sensor"}), + json!({ + "@type":"child-device", + "@id":"test-device:device:external_sensor", + "name": "external_sensor" + }), )], ) .await; @@ -518,7 +522,7 @@ async fn c8y_mapper_child_alarm_mapping_to_smartrest() { [ ( "c8y/s/us", - "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", + "101,test-device:device:external_sensor,external_sensor,thin-edge.io-child", ), ( "c8y/s/us/test-device:device:external_sensor", @@ -615,12 +619,10 @@ async fn c8y_mapper_child_alarm_with_custom_fragment_mapping_to_c8y_json() { // Expect child device creation message assert_received_contains_str( &mut mqtt, - [ - ( - "c8y/s/us", - "101,test-device:device:external_sensor,test-device:device:external_sensor,thin-edge.io-child", - ), - ], + [( + "c8y/s/us", + "101,test-device:device:external_sensor,external_sensor,thin-edge.io-child", + )], ) .await; @@ -1193,13 +1195,17 @@ async fn mapper_dynamically_updates_supported_operations_for_child_device() { assert_eq!(child_metadata.topic.name, "te/device/child1//"); assert_eq!( serde_json::from_str::(child_metadata.payload_str().unwrap()).unwrap(), - json!({"@type":"child-device","@id":"test-device:device:child1"}) + json!({ + "@type":"child-device", + "@id":"test-device:device:child1", + "name": "child1" + }) ); let child_c8y_registration = mqtt.recv().await.unwrap(); assert_eq!(child_c8y_registration.topic.name, "c8y/s/us"); assert_eq!( child_c8y_registration.payload_str().unwrap(), - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child" + "101,test-device:device:child1,child1,thin-edge.io-child" ); // Expect an update list of capabilities with agent capabilities @@ -1754,7 +1760,7 @@ async fn mapper_converts_log_upload_cmd_to_supported_op_and_types_for_child_devi &mut mqtt, [( "c8y/s/us", - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + "101,test-device:device:child1,child1,thin-edge.io-child", )], ) .await; @@ -1868,7 +1874,7 @@ async fn handle_log_upload_executing_and_failed_cmd_for_child_device() { &mut mqtt, [( "c8y/s/us", - "101,test-device:device:child1,test-device:device:child1,thin-edge.io-child", + "101,test-device:device:child1,child1,thin-edge.io-child", )], ) .await; diff --git a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot index a4fc480f4f7..34629507b11 100644 --- a/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot +++ b/tests/RobotFramework/tests/cumulocity/registration/device_registration.robot @@ -18,30 +18,30 @@ Main device registration Child device registration - Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} + Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_ID} Restart Service tedge-mapper-c8y # Check registration ${child_mo}= Device Should Exist ${CHILD_SN} - ${child_mo}= Cumulocity.Device Should Have Fragment Values name\=${CHILD_SN} + ${child_mo}= Cumulocity.Device Should Have Fragment Values name\=${CHILD_ID} Should Be Equal ${child_mo["owner"]} device_${DEVICE_SN} # The parent is the owner of the child - Should Be Equal ${child_mo["name"]} ${CHILD_SN} + Should Be Equal ${child_mo["name"]} ${CHILD_ID} # Check child device relationship Cumulocity.Set Device ${DEVICE_SN} Cumulocity.Should Be A Child Device Of Device ${CHILD_SN} Register child device with defaults via MQTT - Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device"}' - Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${CHILD_SN} child_name=${DEVICE_SN}:device:${CHILD_SN} child_type=thin-edge.io-child + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_ID}//' '{"@type":"child-device"}' + Check Child Device parent_sn=${DEVICE_SN} child_sn=${CHILD_SN} child_name=${CHILD_SN} child_type=thin-edge.io-child Register child device with custom name and type via MQTT - Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device","name":"${CHILD_SN}","type":"linux-device-Aböut"}' - Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${CHILD_SN} child_name=${CHILD_SN} child_type=linux-device-Aböut + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_ID}//' '{"@type":"child-device","name":"${CHILD_ID}","type":"linux-device-Aböut"}' + Check Child Device parent_sn=${DEVICE_SN} child_sn=${CHILD_SN} child_name=${CHILD_ID} child_type=linux-device-Aböut Register child device with custom id via MQTT - Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device","@id":"${CHILD_SN}","name":"custom-${CHILD_SN}"}' - Check Child Device parent_sn=${DEVICE_SN} child_sn=${CHILD_SN} child_name=custom-${CHILD_SN} child_type=thin-edge.io-child + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_ID}//' '{"@type":"child-device","@id":"custom-${CHILD_SN}","name":"custom-${CHILD_ID}"}' + Check Child Device parent_sn=${DEVICE_SN} child_sn=custom-${CHILD_SN} child_name=custom-${CHILD_ID} child_type=thin-edge.io-child Register nested child device using default topic schema via MQTT ${child_level1}= Get Random Name @@ -65,22 +65,21 @@ Register nested child device using default topic schema via MQTT Register service on a child device via MQTT - Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}//' '{"@type":"child-device","name":"${CHILD_SN}","type":"linux-device-Aböut"}' - Execute Command tedge mqtt pub --retain 'te/device/${CHILD_SN}/service/custom-app' '{"@type":"service","@parent":"device/${CHILD_SN}//","name":"custom-app","type":"custom-type"}' + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_ID}//' '{"@type":"child-device","name":"${CHILD_ID}","type":"linux-device-Aböut"}' + Execute Command tedge mqtt pub --retain 'te/device/${CHILD_ID}/service/custom-app' '{"@type":"service","@parent":"device/${CHILD_ID}//","name":"custom-app","type":"custom-type"}' # Check child registration - Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:device:${CHILD_SN} child_name=${CHILD_SN} child_type=linux-device-Aböut + Check Child Device parent_sn=${DEVICE_SN} child_sn=${CHILD_SN} child_name=${CHILD_ID} child_type=linux-device-Aböut # Check service registration - Check Service child_sn=${DEVICE_SN}:device:${CHILD_SN} service_sn=${DEVICE_SN}:device:${CHILD_SN}:service:custom-app service_name=custom-app service_type=custom-type service_status=up + Check Service child_sn=${CHILD_SN} service_sn=${CHILD_SN}:service:custom-app service_name=custom-app service_type=custom-type service_status=up Register devices using custom MQTT schema [Documentation] Complex example showing how to use custom MQTT topics to register devices/services using ... custom identity schemas - # Main device registration via MQTT is not supported at the moment - # Execute Command tedge mqtt pub --retain 'te/base///' '{"@type":"main-device","name":"base","type":"te_gateway"}' + Execute Command tedge mqtt pub --retain 'te/base///' '{"@type":"device","name":"base","type":"te_gateway"}' Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/sensor1' '{"@type":"child-device","name":"sensor1","type":"SmartSensor"}' Execute Command tedge mqtt pub --retain 'te/factory1/shop1/plc1/sensor2' '{"@type":"child-device","name":"sensor2","type":"SmartSensor"}' @@ -96,8 +95,8 @@ Register devices using custom MQTT schema Check Child Device parent_sn=${DEVICE_SN} child_sn=${DEVICE_SN}:factory1:shop1:plc1:sensor2 child_name=sensor2 child_type=SmartSensor # Check main device services - # Cumulocity.Set Device ${DEVICE_SN} - # Should Have Services name=metrics service_type=PLCApplication status=up + Cumulocity.Set Device ${DEVICE_SN} + Should Have Services name=metrics service_type=PLCApplication status=up # Check child services Cumulocity.Set Device ${DEVICE_SN}:factory1:shop1:plc1:sensor1 @@ -106,11 +105,10 @@ Register devices using custom MQTT schema Cumulocity.Set Device ${DEVICE_SN}:factory1:shop1:plc1:sensor2 Should Have Services name=metrics service_type=PLCMonitorApplication status=up - # Skipping as main device registration via MQTT is not supported at the moment # Publish to main device on custom topic - # Execute Command cmd=tedge mqtt pub te/base////m/gateway_stats '{"runtime":1001}' - # Cumulocity.Set Device ${DEVICE_SN} - # Cumulocity.Device Should Have Measurements type=gateway_stats minimum=1 maximum=1 + Execute Command cmd=tedge mqtt pub te/base////m/gateway_stats '{"runtime":1001}' + Cumulocity.Set Device ${DEVICE_SN} + Cumulocity.Device Should Have Measurements type=gateway_stats minimum=1 maximum=1 *** Keywords *** @@ -135,8 +133,9 @@ Check Service Test Setup - ${CHILD_SN}= Get Random Name - Set Test Variable $CHILD_SN + ${CHILD_ID}= Get Random Name + Set Test Variable $CHILD_ID + Set Test Variable $CHILD_SN ${DEVICE_SN}:device:${CHILD_ID} ThinEdgeIO.Set Device Context ${DEVICE_SN} From 525e47fa30198e032c0bf21545a3e67b632375be Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Tue, 26 Sep 2023 08:33:08 +0000 Subject: [PATCH 12/13] Fix child ids in system tests --- .../extensions/c8y_mapper_ext/src/converter.rs | 17 +++++++++++++++++ .../extensions/c8y_mapper_ext/src/log_upload.rs | 13 ++++++++++--- .../firmware_operation_child_device.robot | 13 +++++++------ .../firmware_operation_child_device_retry.robot | 5 +++-- .../telemetry/child_device_telemetry.robot | 4 ++-- .../tests/customizing/data_path_config.robot | 7 ++++--- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index 74332235ea5..c857495cc64 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -34,6 +34,7 @@ use c8y_api::smartrest::smartrest_serializer::SmartRestSerializer; use c8y_api::smartrest::smartrest_serializer::SmartRestSetOperationToExecuting; use c8y_api::smartrest::smartrest_serializer::SmartRestSetOperationToFailed; use c8y_api::smartrest::smartrest_serializer::SmartRestSetOperationToSuccessful; +use c8y_api::smartrest::topic::publish_topic_from_ancestors; use c8y_api::smartrest::topic::C8yTopic; use c8y_api::smartrest::topic::MapperSubscribeTopic; use c8y_api::smartrest::topic::SMARTREST_PUBLISH_TOPIC; @@ -286,6 +287,22 @@ impl CumulocityConverter { } } + pub fn publish_topic_for_entity( + &self, + entity_topic_id: &EntityTopicId, + ) -> Result { + let entity = self.entity_store.get(entity_topic_id).ok_or_else(|| { + CumulocityMapperError::UnregisteredDevice { + topic_id: entity_topic_id.to_string(), + } + })?; + + let mut ancestors_external_ids = + self.entity_store.ancestors_external_ids(entity_topic_id)?; + ancestors_external_ids.insert(0, entity.external_id.as_ref().into()); + Ok(publish_topic_from_ancestors(&ancestors_external_ids)) + } + /// Generates external ID of the given entity. /// /// The external id is generated by transforming the EntityTopicId diff --git a/crates/extensions/c8y_mapper_ext/src/log_upload.rs b/crates/extensions/c8y_mapper_ext/src/log_upload.rs index 682ff15117d..359095a0a5d 100644 --- a/crates/extensions/c8y_mapper_ext/src/log_upload.rs +++ b/crates/extensions/c8y_mapper_ext/src/log_upload.rs @@ -206,7 +206,14 @@ impl CumulocityConverter { // Create a c8y_LogfileRequest operation file let dir_path = match device.r#type { EntityType::MainDevice => self.ops_dir.clone(), - EntityType::ChildDevice => self.ops_dir.join(device.external_id.as_ref()), + EntityType::ChildDevice => { + let child_dir_name = if let Some(child_local_id) = topic_id.default_device_name() { + child_local_id + } else { + device.external_id.as_ref() + }; + self.ops_dir.clone().join(child_dir_name) + } EntityType::Service => { // No support for service log management return Ok(vec![]); @@ -221,7 +228,7 @@ impl CumulocityConverter { let supported_log_types = types.join(","); let payload = format!("118,{supported_log_types}"); - let c8y_topic: C8yTopic = device.into(); - Ok(vec![MqttMessage::new(&c8y_topic.to_topic()?, payload)]) + let c8y_topic = self.publish_topic_for_entity(topic_id)?; + Ok(vec![MqttMessage::new(&c8y_topic, payload)]) } } diff --git a/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device.robot b/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device.robot index e22e86414d3..76886875264 100644 --- a/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device.robot +++ b/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device.robot @@ -89,7 +89,7 @@ Restart Firmware plugin ThinEdgeIO.Restart Service c8y-firmware-plugin.service Child device delete firmware file - Set Device Context ${CHILD_SN} + Set Device Context ${CHILD_ID} Execute Command sudo rm -f firmware1 Create child device @@ -98,9 +98,9 @@ Create child device ... If the order is opposite, the child device name will start with "MQTT Device". Set Device Context ${PARENT_SN} Sleep 3s - Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} + Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_ID} Sleep 3s - ThinEdgeIO.Transfer To Device ${CURDIR}/c8y_Firmware /etc/tedge/operations/c8y/${CHILD_SN}/ + ThinEdgeIO.Transfer To Device ${CURDIR}/c8y_Firmware /etc/tedge/operations/c8y/${CHILD_ID}/ Cumulocity.Device Should Exist ${CHILD_SN} Validate child Name @@ -152,7 +152,7 @@ Validate firmware update request Set Suite Variable $cache_key ${cache_key[0]} Child device response on update request - Set Device Context ${CHILD_SN} + Set Device Context ${CHILD_ID} Set Test Variable $topic_res tedge/${CHILD_SN}/commands/res/firmware_update Execute Command mosquitto_pub -h ${PARENT_IP} -t ${topic_res} -m '{"id":"${op_id}", "status":"executing"}' @@ -172,5 +172,6 @@ Custom Setup Set Suite Variable $PARENT_IP ${parent_ip} # Child - ${child_sn}= Setup skip_bootstrap=True - Set Suite Variable $CHILD_SN ${child_sn} + ${child_id}= Setup skip_bootstrap=True + Set Suite Variable $CHILD_ID ${child_id} + Set Suite Variable $CHILD_SN ${PARENT_SN}:device:${CHILD_ID} diff --git a/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device_retry.robot b/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device_retry.robot index b9f1acf8fb4..c961dc158ec 100644 --- a/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device_retry.robot +++ b/tests/RobotFramework/tests/cumulocity/firmware/firmware_operation_child_device_retry.robot @@ -30,8 +30,9 @@ Firmware plugin supports restart via service manager #1932 Custom Setup ${DEVICE_SN}= Setup Set Suite Variable $DEVICE_SN - Set Suite Variable $CHILD_SN ${DEVICE_SN}_child1 - Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} + Set Suite Variable $CHILD_ID ${DEVICE_SN}_child1 + Set Suite Variable $CHILD_SN ${DEVICE_SN}:device:${CHILD_ID} + Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_ID} Restart Service tedge-mapper-c8y Device Should Exist ${DEVICE_SN} Device Should Exist ${CHILD_SN} diff --git a/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot b/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot index 1870511a69d..ebd0b693fce 100644 --- a/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot +++ b/tests/RobotFramework/tests/cumulocity/telemetry/child_device_telemetry.robot @@ -84,9 +84,9 @@ Child device supports sending custom child device measurements directly to c8y Custom Setup ${DEVICE_SN}= Setup Set Suite Variable $DEVICE_SN - Set Suite Variable $CHILD_ID child1 + Set Suite Variable $CHILD_ID ${DEVICE_SN}_child1 Set Suite Variable $CHILD_SN ${DEVICE_SN}:device:${CHILD_ID} - Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} + Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_ID} Restart Service tedge-mapper-c8y Device Should Exist ${DEVICE_SN} Device Should Exist ${CHILD_SN} diff --git a/tests/RobotFramework/tests/customizing/data_path_config.robot b/tests/RobotFramework/tests/customizing/data_path_config.robot index bd76894f58a..2cc80502fbc 100644 --- a/tests/RobotFramework/tests/customizing/data_path_config.robot +++ b/tests/RobotFramework/tests/customizing/data_path_config.robot @@ -38,7 +38,8 @@ Validate updated data path used by c8y-firmware-plugin Custom Setup ${PARENT_SN}= Setup Set Suite Variable $PARENT_SN - Set Suite Variable $CHILD_SN ${PARENT_SN}_child + Set Suite Variable $CHILD_ID ${PARENT_SN}_child + Set Suite Variable $CHILD_SN ${PARENT_SN}:device:${CHILD_ID} Execute Command sudo mkdir /var/test Execute Command sudo chown tedge:tedge /var/test Execute Command sudo tedge config set data.path /var/test @@ -49,9 +50,9 @@ Custom Teardown Get Logs Bootstrap child device with firmware operation support - Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_SN} + Execute Command mkdir -p /etc/tedge/operations/c8y/${CHILD_ID} Sleep 3s - Execute Command touch /etc/tedge/operations/c8y/${CHILD_SN}/c8y_Firmware + Execute Command touch /etc/tedge/operations/c8y/${CHILD_ID}/c8y_Firmware Sleep 3s Cumulocity.Device Should Exist ${CHILD_SN} From cdb1143334ac0d25c90211b3fb241e1e639cd563 Mon Sep 17 00:00:00 2001 From: Albin Suresh Date: Tue, 26 Sep 2023 10:58:29 +0000 Subject: [PATCH 13/13] Addressing review comments --- crates/core/c8y_api/src/smartrest/topic.rs | 13 ----- crates/core/tedge_api/src/entity_store.rs | 49 +------------------ .../c8y_mapper_ext/src/converter.rs | 39 ++++++++++++++- .../c8y_mapper_ext/src/log_upload.rs | 6 +-- crates/extensions/c8y_mapper_ext/src/tests.rs | 2 +- 5 files changed, 41 insertions(+), 68 deletions(-) diff --git a/crates/core/c8y_api/src/smartrest/topic.rs b/crates/core/c8y_api/src/smartrest/topic.rs index 0e1ad096258..2a469dc4df0 100644 --- a/crates/core/c8y_api/src/smartrest/topic.rs +++ b/crates/core/c8y_api/src/smartrest/topic.rs @@ -103,19 +103,6 @@ impl From for TopicFilter { } } -// FIXME this From conversion is error prone as this can only be used for responses. -impl From<&EntityMetadata> for C8yTopic { - fn from(value: &EntityMetadata) -> Self { - match value.r#type { - EntityType::MainDevice => Self::SmartRestResponse, - EntityType::ChildDevice => { - Self::ChildSmartRestResponse(value.external_id.clone().into()) - } - EntityType::Service => Self::SmartRestResponse, // TODO how services are handled by c8y? - } - } -} - impl From<&C8yAlarm> for C8yTopic { fn from(value: &C8yAlarm) -> Self { match value { diff --git a/crates/core/tedge_api/src/entity_store.rs b/crates/core/tedge_api/src/entity_store.rs index 0c9964447dc..e63dcaf81cf 100644 --- a/crates/core/tedge_api/src/entity_store.rs +++ b/crates/core/tedge_api/src/entity_store.rs @@ -13,10 +13,7 @@ use crate::entity_store; use crate::mqtt_topics::EntityTopicId; use crate::mqtt_topics::TopicIdError; use mqtt_channel::Message; -use mqtt_channel::Topic; use serde_json::json; -use serde_json::Map; -use serde_json::Value; /// Represents an "Entity topic identifier" portion of the MQTT topic /// @@ -491,39 +488,6 @@ impl TryFrom<&Message> for EntityRegistrationMessage { } } -impl From<&EntityRegistrationMessage> for Message { - fn from(value: &EntityRegistrationMessage) -> Self { - let entity_topic_id = value.topic_id.clone(); - - let mut register_payload: Map = Map::new(); - - let entity_type = match value.r#type { - EntityType::MainDevice => "device", - EntityType::ChildDevice => "child-device", - EntityType::Service => "service", - }; - register_payload.insert("@type".into(), Value::String(entity_type.to_string())); - - if let Some(external_id) = &value.external_id { - register_payload.insert("@id".into(), Value::String(external_id.as_ref().into())); - } - - if let Some(parent_id) = &value.parent { - register_payload.insert("@parent".into(), Value::String(parent_id.to_string())); - } - - if let Value::Object(other_keys) = value.payload.clone() { - register_payload.extend(other_keys) - } - - Message::new( - &Topic::new(&format!("{MQTT_ROOT}/{entity_topic_id}")).unwrap(), - serde_json::to_string(&Value::Object(register_payload)).unwrap(), - ) - .with_retain() - } -} - /// Parse a MQTT message payload as an entity registration payload. /// /// Returns `Some(register_payload)` if a payload is valid JSON and is a @@ -543,6 +507,7 @@ mod tests { use std::str::FromStr; use assert_matches::assert_matches; + use mqtt_channel::Topic; use serde_json::json; use super::*; @@ -935,18 +900,6 @@ mod tests { ); } - #[test] - fn entity_registration_message_into_mqtt_message() { - let entity_reg_message = EntityRegistrationMessage::new(&Message::new( - &Topic::new("te/device/child2/service/collectd").unwrap(), - json!({"@type": "service"}).to_string(), - )) - .unwrap(); - - let message: Message = (&entity_reg_message).into(); - println!("{}", message.payload_str().unwrap()); - } - #[test] fn auto_register_service() { let mut store = EntityStore::with_main_device( diff --git a/crates/extensions/c8y_mapper_ext/src/converter.rs b/crates/extensions/c8y_mapper_ext/src/converter.rs index c857495cc64..d530a549144 100644 --- a/crates/extensions/c8y_mapper_ext/src/converter.rs +++ b/crates/extensions/c8y_mapper_ext/src/converter.rs @@ -47,6 +47,8 @@ use plugin_sm::operation_logs::OperationLogsError; use serde::Deserialize; use serde::Serialize; use serde_json::json; +use serde_json::Map; +use serde_json::Value; use service_monitor::convert_health_status_message; use std::collections::HashMap; use std::fs; @@ -287,7 +289,9 @@ impl CumulocityConverter { } } - pub fn publish_topic_for_entity( + /// Return the SmartREST publish topic for the given entity + /// derived from its ancestors. + pub fn smartrest_publish_topic_for_entity( &self, entity_topic_id: &EntityTopicId, ) -> Result { @@ -904,13 +908,44 @@ impl CumulocityConverter { } let mut registration_messages = vec![]; - registration_messages.push(registration_message.into()); + registration_messages.push(self.convert_entity_registration_message(registration_message)); let mut c8y_message = self.try_convert_entity_registration(registration_message)?; registration_messages.append(&mut c8y_message); Ok(registration_messages) } + fn convert_entity_registration_message(&self, value: &EntityRegistrationMessage) -> Message { + let entity_topic_id = value.topic_id.clone(); + + let mut register_payload: Map = Map::new(); + + let entity_type = match value.r#type { + EntityType::MainDevice => "device", + EntityType::ChildDevice => "child-device", + EntityType::Service => "service", + }; + register_payload.insert("@type".into(), Value::String(entity_type.to_string())); + + if let Some(external_id) = &value.external_id { + register_payload.insert("@id".into(), Value::String(external_id.as_ref().into())); + } + + if let Some(parent_id) = &value.parent { + register_payload.insert("@parent".into(), Value::String(parent_id.to_string())); + } + + if let Value::Object(other_keys) = value.payload.clone() { + register_payload.extend(other_keys) + } + + Message::new( + &Topic::new(&format!("{}/{entity_topic_id}", self.mqtt_schema.root)).unwrap(), + serde_json::to_string(&Value::Object(register_payload)).unwrap(), + ) + .with_retain() + } + async fn try_convert_tedge_topics( &mut self, message: &Message, diff --git a/crates/extensions/c8y_mapper_ext/src/log_upload.rs b/crates/extensions/c8y_mapper_ext/src/log_upload.rs index 359095a0a5d..1a232fa708d 100644 --- a/crates/extensions/c8y_mapper_ext/src/log_upload.rs +++ b/crates/extensions/c8y_mapper_ext/src/log_upload.rs @@ -9,7 +9,6 @@ use c8y_api::smartrest::smartrest_serializer::SmartRestSerializer; use c8y_api::smartrest::smartrest_serializer::SmartRestSetOperationToExecuting; use c8y_api::smartrest::smartrest_serializer::SmartRestSetOperationToFailed; use c8y_api::smartrest::smartrest_serializer::SmartRestSetOperationToSuccessful; -use c8y_api::smartrest::topic::C8yTopic; use nanoid::nanoid; use tedge_api::entity_store::EntityType; use tedge_api::messages::CommandStatus; @@ -112,8 +111,7 @@ impl CumulocityConverter { })?; let external_id = &device.external_id; - let c8y_topic: C8yTopic = device.into(); - let smartrest_topic = c8y_topic.to_topic()?; + let smartrest_topic = self.smartrest_publish_topic_for_entity(topic_id)?; let payload = message.payload_str()?; let response = &LogUploadCmdPayload::from_json(payload)?; @@ -228,7 +226,7 @@ impl CumulocityConverter { let supported_log_types = types.join(","); let payload = format!("118,{supported_log_types}"); - let c8y_topic = self.publish_topic_for_entity(topic_id)?; + let c8y_topic = self.smartrest_publish_topic_for_entity(topic_id)?; Ok(vec![MqttMessage::new(&c8y_topic, payload)]) } } diff --git a/crates/extensions/c8y_mapper_ext/src/tests.rs b/crates/extensions/c8y_mapper_ext/src/tests.rs index 62a6f8bd5cd..0fd330b9363 100644 --- a/crates/extensions/c8y_mapper_ext/src/tests.rs +++ b/crates/extensions/c8y_mapper_ext/src/tests.rs @@ -1776,7 +1776,7 @@ async fn mapper_converts_log_upload_cmd_to_supported_op_and_types_for_child_devi // Validate if the supported operation file is created assert!(ttd .path() - .join("operations/c8y/test-device:device:child1/c8y_LogfileRequest") + .join("operations/c8y/child1/c8y_LogfileRequest") .exists()); }