Add tools to transform opcua data to NGSI-LD, SHACl, OWL and Data bin…

…dings

This PR provides a transfomrational tool which takes nodeset2 owl files and creates NGSI-LD, SHACL and OWL based machine
descriptions. Also, bindings are created to map opcua data to the external data model. The PR also contains:

* Updated Dataservice to work with bindings created by this extractType tool.
* Updated Datamodel to pull NGSI-LD subcomponents based on the OPCUA metadata
* Additional E2e tests for extractType tool

related EPIC: #514
related User Story: #555, #571

Signed-off-by: marcel <wagmarcel@web.de>

semantic-model/datamodel/tools/getSubcomponents.js

@@ -44,6 +44,11 @@ const argv = yargs
  description: 'Entity Files containing description of attributes',
  type: 'array'
  })
+ .option('shacl', {
+ alias: 's',
+ description: 'SHACL File for the object model',
+ type: 'string'
+ })
  .option('token', {
  alias: 't',
  description: 'Token for rest call',
@@ -60,16 +65,16 @@ function namespace (baseIRI) {
 const NGSILD = namespace('https://uri.etsi.org/ngsi-ld/');
 
 // Read in all the entity files
-const entitiesstore = new N3.Store();
+const ontstore = new N3.Store();
 
-const loadEntities = async (entities) => {
- if (entities && entities.length > 0) {
- for (const entity of entities) {
+const loadOntologies = async (ont) => {
+ if (ont && ont.length > 0) {
+ for (const entity of ont) {
  const parser = new N3.Parser();
  const ttlContent = fs.readFileSync(entity, 'utf8');
  parser.parse(ttlContent, (error, quad) => {
  if (quad) {
- entitiesstore.addQuad(quad);
+ ontstore.addQuad(quad);
  } else if (error) {
  console.error('Parsing error:', error);
  }
@@ -157,38 +162,47 @@ const analyseNgsildObject = async (id, brokerUrl, token) => {
  return;
  }
 
- const quadsFromEntitiesStore = entitiesstore.getQuads(null, null, null, null);
+ const quadsFromEntitiesStore = ontstore.getQuads(null, null, null, null);
  store.addQuads(quadsFromEntitiesStore);
 
  const bindingsStream = await myEngine.queryBindings(`
- PREFIX base: <https://industryfusion.github.io/contexts/ontology/v0/base/>
- PREFIX ngsild: <https://uri.etsi.org/ngsi-ld/>
- SELECT ?s ?id
+ PREFIX base: <https://industryfusion.github.io/contexts/ontology/v0/base/>
+ PREFIX ngsild: <https://uri.etsi.org/ngsi-ld/>
+ PREFIX sh: <http://www.w3.org/ns/shacl#>
+ SELECT ?id ?type ?attribute
  WHERE {
- ?b a ngsild:Relationship .
- ?s a base:SubComponentRelationship .
- ?id ?s ?b .
+ ?id a ?type .
+ ?id ?attribute ?blank .
+ ?blank a ngsild:Relationship .
+ ?shape a sh:NodeShape .
+ ?shape sh:property ?property .
+ ?property sh:path ?attribute .
+ ?property a base:SubComponentRelationship .
  }`,
  { sources: [store] }
  );
 
  const bindings = await bindingsStream.toArray();
  for (const binding of bindings) {
- const s = binding.get('s').value;
+ const s = binding.get('attribute').value;
  const triples = store.getQuads(null, s, null, null);
  for (const quad of triples) {
  const ngsildObjects = store.getQuads(quad.object, NGSILD('hasObject'), null, null);
  for (const ngsildObject of ngsildObjects) {
  const subId = ngsildObject.object.value;
- subids.push(subId);
- await analyseNgsildObject(subId, brokerUrl, token);
+ if (!subids.includes(subId)) {
+ subids.push(subId);
+ await analyseNgsildObject(subId, brokerUrl, token);
+ }
  }
  }
  }
 };
 
 (async () => {
- await loadEntities(argv.entities);
+ const ontologies = argv.entities;
+ ontologies.push(argv.shacl);
+ await loadOntologies(ontologies);
  await analyseNgsildObject(argv._[0], argv['broker-url'], argv.token);
 
  let cmdlineargs = '';

semantic-model/datamodel/tools/lib/owlUtils.js

@@ -21,7 +21,6 @@ const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#');
 const RDFS = $rdf.Namespace('http://www.w3.org/2000/01/rdf-schema#');
 const OWL = $rdf.Namespace('http://www.w3.org/2002/07/owl#');
 const NGSILD = $rdf.Namespace('https://uri.etsi.org/ngsi-ld/');
-const BASE = $rdf.Namespace('https://industryfusion.github.io/contexts/ontology/v0/base/');
 
 const globalAttributes = [];
 const globalEntities = [];
@@ -81,11 +80,6 @@ function dumpAttribute (attribute, entity, store) {
  store.add($rdf.sym(attribute.attributeName), RDFS('range'), NGSILD('Property'));
  } else {
  store.add($rdf.sym(attribute.attributeName), RDFS('range'), NGSILD('Relationship'));
- if (attribute.isSubcomponent) {
- store.add($rdf.sym(attribute.attributeName), RDF('type'), BASE('SubComponentRelationship'));
- } else {
- store.add($rdf.sym(attribute.attributeName), RDF('type'), BASE('PeerRelationship'));
- }
  }
  }
 }

semantic-model/datamodel/tools/lib/shaclUtils.js

@@ -25,6 +25,7 @@ const myParser = new ContextParser();
 const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#');
 const SHACL = $rdf.Namespace('http://www.w3.org/ns/shacl#');
 const IFFK = $rdf.Namespace('https://industry-fusion.org/knowledge/v0.1/');
+const BASE = $rdf.Namespace('https://industryfusion.github.io/contexts/ontology/v0/base/');
 
 let globalContext;
 let globalPrefixHash;
@@ -49,13 +50,14 @@ class NodeShape {
 }
 
 class PropertyShape {
- constructor (mincount, maxcount, nodeKind, path, isProperty) {
+ constructor (mincount, maxcount, nodeKind, path, isProperty, isSubComponent) {
  this.mincount = mincount;
  this.maxcount = maxcount;
  this.nodeKind = nodeKind;
  this.path = path;
  this.constraints = [];
  this.isProperty = isProperty;
+ this.isSubComponent = isSubComponent;
  }
 
  addConstraint (constraint) {
@@ -83,6 +85,13 @@ function dumpPropertyShape (propertyShape, store) {
  store.add(propNode, SHACL('minCount'), propertyShape.mincount);
  store.add(propNode, SHACL('maxCount'), propertyShape.maxcount);
  store.add(propNode, SHACL('nodeKind'), SHACL('BlankNode'));
+ if (propertyShape.isSubComponent && !propertyShape.isProperty) {
+ store.add(propNode, RDF('type'), BASE('SubComponentRelationship'));
+ } else if (!propertyShape.isSubComponent && !propertyShape.isProperty) {
+ store.add(propNode, RDF('type'), BASE('PeerRelationship'));
+ } else {
+ store.add(propNode, RDF('type'), BASE('Property'));
+ }
  store.add(propNode, SHACL('path'), propertyShape.path);
  const attributeNode = $rdf.blankNode();
  store.add(propNode, SHACL('property'), attributeNode);
@@ -143,12 +152,16 @@ function scanProperties (nodeShape, typeschema) {
  let nodeKind = SHACL('Literal');
  let klass = null;
  let isProperty = true;
+ let isSubComponent = false;
  if ('relationship' in typeschema.properties[property]) {
  nodeKind = SHACL('IRI');
  klass = typeschema.properties[property].relationship;
  klass = globalContext.expandTerm(klass, true);
  isProperty = false;
  }
+ if ('relationship_type' in typeschema.properties[property] && typeschema.properties[property].relationship_type === 'subcomponent') {
+ isSubComponent = true;
+ }
  let mincount = 0;
  const maxcount = 1;
  if (required.includes(property)) {
@@ -158,7 +171,7 @@ function scanProperties (nodeShape, typeschema) {
  if (!ContextUtil.isValidIri(path)) {
  path = globalContext.expandTerm(path, true);
  }
- const propertyShape = new PropertyShape(mincount, maxcount, nodeKind, $rdf.sym(path), isProperty);
+ const propertyShape = new PropertyShape(mincount, maxcount, nodeKind, $rdf.sym(path), isProperty, isSubComponent);
  nodeShape.addPropertyShape(propertyShape);
  if (klass !== null) {
  propertyShape.addConstraint(new Constraint(SHACL('class'), $rdf.sym(klass)));

semantic-model/datamodel/tools/tests/schema2owl/schema3_c0.json_result

@@ -1,11 +1,10 @@
 @prefix owl: <http://www.w3.org/2002/07/owl#>.
 @prefix iffb: <https://industry-fusion.org/base/v0.1/>.
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-@prefix base: <https://industryfusion.github.io/contexts/ontology/v0/base/>.
 @prefix ngsi-ld: <https://uri.etsi.org/ngsi-ld/>.
 
 iffb:hasFilter
- a owl:Property, base:PeerRelationship;
+ a owl:Property;
  rdfs:domain <https://industry-fusion.org/eclass%230173-1%2301-AKJ975%23017>;
  rdfs:range ngsi-ld:Relationship.
 iffb:machine_state

semantic-model/datamodel/tools/tests/schema2owl/schema4_c0.json_result

@@ -1,11 +1,10 @@
 @prefix owl: <http://www.w3.org/2002/07/owl#>.
 @prefix iffb: <https://industry-fusion.org/base/v0.1/>.
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-@prefix base: <https://industryfusion.github.io/contexts/ontology/v0/base/>.
 @prefix ngsi-ld: <https://uri.etsi.org/ngsi-ld/>.
 
 iffb:hasCartridge
- a owl:Property, base:SubComponentRelationship;
+ a owl:Property;
  rdfs:domain <https://industry-fusion.org/eclass%230173-1%2301-ACK991%23016>;
  rdfs:range ngsi-ld:Relationship.
 iffb:machine_state

semantic-model/datamodel/tools/tests/schema2shacl/c0

@@ -33,6 +33,10 @@
  "sh": {
  "@id": "http://www.w3.org/ns/shacl#",
  "@prefix": true
+ },
+ "base": {
+ "@id": "https://industryfusion.github.io/contexts/ontology/v0/base/",
+ "@prefix": true
  }
  }
 ]

semantic-model/datamodel/tools/tests/schema2shacl/schema3_c0.json_result

@@ -1,11 +1,13 @@
 @prefix iffb: <https://industry-fusion.org/base/v0.1/>.
 @prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix base: <https://industryfusion.github.io/contexts/ontology/v0/base/>.
 @prefix ngsi-ld: <https://uri.etsi.org/ngsi-ld/>.
 
 <https://industry-fusion.org/knowledge/v0.1/0173-1#01-AKJ975#017Shape>
  a sh:NodeShape;
  sh:property
  [
+ a base:PeerRelationship;
  sh:maxCount 1;
  sh:minCount 0;
  sh:nodeKind sh:BlankNode;
@@ -21,6 +23,7 @@
  ]
  ],
  [
+ a base:Property;
  sh:maxCount 1;
  sh:minCount 1;
  sh:nodeKind sh:BlankNode;

semantic-model/datamodel/tools/tests/schema2shacl/schema4_c0.json_result

@@ -1,37 +1,40 @@
 @prefix iffb: <https://industry-fusion.org/base/v0.1/>.
 @prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix base: <https://industryfusion.github.io/contexts/ontology/v0/base/>.
 @prefix ngsi-ld: <https://uri.etsi.org/ngsi-ld/>.
 
 <https://industry-fusion.org/knowledge/v0.1/0173-1#01-AKE795#017Shape>
  a sh:NodeShape;
  sh:property
  [
+ a base:Property;
  sh:maxCount 1;
- sh:minCount 0;
+ sh:minCount 1;
  sh:nodeKind sh:BlankNode;
- sh:path iffb:hasIdentification;
+ sh:path iffb:waste_class;
  sh:property
  [
- sh:class
- <https://industry-fusion.org/eclass#0173-1#01-ADN228#012>;
+ sh:in ( "WC0" "WC1" "WC2" "WC3" );
  sh:maxCount 1;
  sh:minCount 1;
- sh:nodeKind sh:IRI;
- sh:path ngsi-ld:hasObject
+ sh:nodeKind sh:Literal;
+ sh:path ngsi-ld:hasValue
  ]
  ],
  [
+ a base:SubComponentRelationship;
  sh:maxCount 1;
- sh:minCount 1;
+ sh:minCount 0;
  sh:nodeKind sh:BlankNode;
- sh:path iffb:waste_class;
+ sh:path iffb:hasIdentification;
  sh:property
  [
- sh:in ( "WC0" "WC1" "WC2" "WC3" );
+ sh:class
+ <https://industry-fusion.org/eclass#0173-1#01-ADN228#012>;
  sh:maxCount 1;
  sh:minCount 1;
- sh:nodeKind sh:Literal;
- sh:path ngsi-ld:hasValue
+ sh:nodeKind sh:IRI;
+ sh:path ngsi-ld:hasObject
  ]
  ];
  sh:targetClass <https://industry-fusion.org/eclass#0173-1#01-AKE795#017>.

semantic-model/datamodel/tools/tests/schema2shacl/schema5_c0.json_result

@@ -1,10 +1,12 @@
 @prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix base: <https://industryfusion.github.io/contexts/ontology/v0/base/>.
 @prefix ngsi-ld: <https://uri.etsi.org/ngsi-ld/>.
 
 <https://industry-fusion.org/knowledge/v0.1/0173-1#01-ADN228#012Shape>
  a sh:NodeShape;
  sh:property
  [
+ a base:Property;
  sh:maxCount 1;
  sh:minCount 0;
  sh:nodeKind sh:BlankNode;
@@ -19,6 +21,7 @@
  ]
  ],
  [
+ a base:Property;
  sh:maxCount 1;
  sh:minCount 1;
  sh:nodeKind sh:BlankNode;

semantic-model/datamodel/tools/tests/testOwlUtils.js

@@ -26,7 +26,6 @@ const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#');
 const RDFS = $rdf.Namespace('http://www.w3.org/2000/01/rdf-schema#');
 const OWL = $rdf.Namespace('http://www.w3.org/2002/07/owl#');
 const NGSILD = $rdf.Namespace('https://uri.etsi.org/ngsi-ld/');
-const BASE = $rdf.Namespace('https://industryfusion.github.io/contexts/ontology/v0/base/');
 
 describe('Test class Entity', function () {
  it('Should create Entity with IRI', function () {
@@ -76,8 +75,7 @@ describe('Test dumpAttribute', function () {
  const expected = [
  [$rdf.namedNode(attributeName), RDF('type'), OWL('Property')],
  [$rdf.namedNode(attributeName), RDFS('domain'), $rdf.namedNode(entityName)],
- [$rdf.namedNode(attributeName), RDFS('range'), NGSILD('Relationship')],
- [$rdf.namedNode(attributeName), RDF('type'), BASE('PeerRelationship')]
+ [$rdf.namedNode(attributeName), RDFS('range'), NGSILD('Relationship')]
  ];
  const store = {
  add: (s, p, o) => { added.push([s, p, o]); }