Fix manifest docs

buildingmotif/dataclasses/model.py

@@ -137,7 +137,9 @@ def add_graph(self, graph: rdflib.Graph) -> None:
         self.graph += graph
 
     def validate(
-        self, shape_collections: Optional[List[ShapeCollection]] = None
+        self,
+        shape_collections: Optional[List[ShapeCollection]] = None,
+        error_on_missing_imports: bool = True,
     ) -> "ValidationContext":
         """Validates this model against the given list of ShapeCollections.
         If no list is provided, the model will be validated against the model's "manifest".
@@ -150,6 +152,10 @@ def validate(
             graph should be validated. If an empty list or None is provided, the
             model will be validated against the model's manifest.
         :type shape_collections: List[ShapeCollection]
+        :param error_on_missing_imports: if True, raises an error if any of the dependency
+            ontologies are missing (i.e. they need to be loaded into BuildingMOTIF), defaults
+            to True
+        :type error_on_missing_imports: bool, optional
         :return: An object containing useful properties/methods to deal with
             the validation results
         :rtype: ValidationContext
@@ -162,8 +168,10 @@ def validate(
             shape_collections = [self.get_manifest()]
         # aggregate shape graphs
         for sc in shape_collections:
-            # inline sh:node for interpretability
-            shapeg += sc.graph
+            shapeg += sc.resolve_imports(
+                error_on_missing_imports=error_on_missing_imports
+            ).graph
+        # inline sh:node for interpretability
         shapeg = rewrite_shape_graph(shapeg)
         # TODO: do we want to preserve the materialized triples added to data_graph via reasoning?
         data_graph = copy_graph(self.graph)
@@ -296,3 +304,12 @@ def get_manifest(self) -> ShapeCollection:
         :rtype: ShapeCollection
         """
         return ShapeCollection.load(self._manifest_id)
+
+    def update_manifest(self, manifest: ShapeCollection):
+        """Updates the manifest for this model by adding in the contents
+        of the shape graph inside the provided SHapeCollection
+
+        :param manifest: the ShapeCollection containing additional shapes against which to validate this model
+        :type manifest: ShapeCollection
+        """
+        self.get_manifest().graph += manifest.graph

buildingmotif/dataclasses/shape_collection.py

@@ -65,6 +65,15 @@ def id(self) -> Optional[int]:
     def id(self, new_id):
         raise AttributeError("Cannot modify db id")
 
+    @property
+    def graph_name(self) -> Optional[URIRef]:
+        """
+        Returns the name of the graph (subject of "a owl:Ontology")
+        if one exists
+        """
+        # will be None if this is not found
+        return self.graph.value(predicate=RDF.type, object=OWL.Ontology)  # type: ignore
+
     def add_triples(self, *triples: Triple) -> None:
         """Add the given triples to the graph.
 
@@ -97,7 +106,9 @@ def _cbd(self, shape_name, self_contained=True):
             cbd = new_cbd
         return cbd
 
-    def resolve_imports(self, recursive_limit: int = -1) -> "ShapeCollection":
+    def resolve_imports(
+        self, recursive_limit: int = -1, error_on_missing_imports: bool = True
+    ) -> "ShapeCollection":
         """Resolves `owl:imports` to as many levels as requested.
 
         By default, all `owl:imports` are recursively resolved. This limit can
@@ -107,11 +118,20 @@ def resolve_imports(self, recursive_limit: int = -1) -> "ShapeCollection":
         :param recursive_limit: how many levels of `owl:imports` to resolve,
             defaults to -1 (all)
         :type recursive_limit: int, optional
+        :param error_on_missing_imports: if True, raises an error if any of the dependency
+            ontologies are missing (i.e. they need to be loaded into BuildingMOTIF), defaults
+            to True
+        :type error_on_missing_imports: bool, optional
         :return: a new ShapeCollection with the types resolved
         :rtype: ShapeCollection
         """
         resolved_namespaces: Set[rdflib.URIRef] = set()
-        resolved = _resolve_imports(self.graph, recursive_limit, resolved_namespaces)
+        resolved = _resolve_imports(
+            self.graph,
+            recursive_limit,
+            resolved_namespaces,
+            error_on_missing_imports=error_on_missing_imports,
+        )
         new_sc = ShapeCollection.create()
         new_sc.add_graph(resolved)
         return new_sc
@@ -223,10 +243,15 @@ def get_shapes_about_class(
 
 
 def _resolve_imports(
-    graph: rdflib.Graph, recursive_limit: int, seen: Set[rdflib.URIRef]
+    graph: rdflib.Graph,
+    recursive_limit: int,
+    seen: Set[rdflib.URIRef],
+    error_on_missing_imports: bool = True,
 ) -> rdflib.Graph:
     from buildingmotif.dataclasses.library import Library
 
+    bm = get_building_motif()
+
     logger = logging.getLogger(__name__)
 
     if recursive_limit == 0:
@@ -240,14 +265,37 @@ def _resolve_imports(
         # go find the graph definition from our libraries
         try:
             lib = Library.load(name=ontology)
+            sc_to_add = lib.get_shape_collection()
         except Exception as e:
-            logger.error(
-                "Could not resolve import of library/shape collection: %s", ontology
+            logger.warning(
+                "Could not resolve import of %s from Libraries (%s). Trying shape collections",
+                ontology,
+                e,
+            )
+            sc_to_add = None
+
+        # search through our shape collections for a graph with the provided name
+        if sc_to_add is None:
+            for shape_collection in bm.table_connection.get_all_db_shape_collections():
+                sc = ShapeCollection.load(shape_collection.id)
+                if sc.graph_name == ontology:
+                    sc_to_add = sc
+                    break
+            logger.warning(
+                "Could not resolve import of %s from Libraries. Trying shape collections",
+                ontology,
             )
-            raise e
+
+        if sc_to_add is None:
+            if error_on_missing_imports:
+                raise Exception("Could not resolve import of %s", ontology)
+            continue
+
         dependency = _resolve_imports(
-            lib.get_shape_collection().graph, recursive_limit - 1, seen
+            sc_to_add.graph,
+            recursive_limit - 1,
+            seen,
+            error_on_missing_imports=error_on_missing_imports,
         )
         new_g += dependency
-        print(f"graph size now {len(new_g)}")
     return new_g

docs/conf.py

@@ -14,7 +14,7 @@
 execution_allow_errors = False
 execution_excludepatterns = []
 execution_in_temp = False
-execution_timeout = 30
+execution_timeout = 300
 extensions = ['sphinx_togglebutton', 'sphinx_copybutton', 'myst_nb', 'jupyter_book', 'sphinx_thebe', 'sphinx_comments', 'sphinx_external_toc', 'sphinx.ext.intersphinx', 'sphinx_design', 'sphinx_book_theme', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.viewcode', 'sphinxcontrib.bibtex', 'sphinx_jupyterbook_latex']
 external_toc_exclude_missing = False
 external_toc_path = '_toc.yml'

docs/tutorials/model_validation.md

@@ -40,7 +40,8 @@ Validating a model is the process of ensuring that the model is both *correct* (
 
 ## Setup
 
-We create an in-memory BuildingMOTIF instance, load the model from the previous tutorial, and load some libraries to create the manifest with. The `constraints.ttl` library we load is a special library with some custom constraints defined that are helpful for writing manifests.
+We create an in-memory BuildingMOTIF instance, load the model from the previous tutorial, and load some libraries to create the manifest with.
+The `constraints.ttl` library we load is a special library with some custom constraints defined that are helpful for writing manifests.
 
 ```{margin}
 ```{warning}
@@ -75,7 +76,7 @@ g36 = Library.load(directory="../../libraries/ashrae/guideline36")
 
 ## Model Validation - Ontology
 
-BuildingMOTIF organizes Shapes into `Shape Collections`. The shape collection associated with a library (if there is one) can be retrieved with the `get_shape_collection` property. Below, we use Brick's shape collection to ensure that the model is using Brick correctly:
+BuildingMOTIF organizes Shapes into `Shape Collections`. The shape collection associated with a library (if there is one) can be retrieved with the `get_shape_collection` method. Below, we use Brick's shape collection to ensure that the model is using Brick correctly:
 
 ```{code-cell}
 # pass a list of shape collections to .validate()
@@ -94,7 +95,8 @@ Success! The model is valid according to the Brick ontology.
 A `manifest` is an RDF graph with a set of `Shapes` inside, which place constraints and requirements on what metadata must be contained within a metadata model. 
 ```
 
-For now, we will write a `manifest` file directly; in the future, BuildingMOTIF will contain features that make manifests easier to write. Here is the header of a manifest file. This should also suffice for most of your own manifests.
+For now, we will write a `manifest` file directly; in the future, BuildingMOTIF will contain features that make manifests easier to write.
+Here is the header of a manifest file. This should also suffice for most of your own manifests.
 
 ```ttl
 @prefix brick: <https://brickschema.org/schema/Brick#> .
@@ -103,7 +105,10 @@ For now, we will write a `manifest` file directly; in the future, BuildingMOTIF
 @prefix constraint: <https://nrel.gov/BuildingMOTIF/constraints#> .
 @prefix : <urn:my_site_constraints/> .
 
-: a owl:Ontology .
+: a owl:Ontology ;
+    owl:imports <https://brickschema.org/schema/1.3/Brick>,
+                <https://nrel.gov/BuildingMOTIF/constraints>,
+                <urn:ashrae/g36> .
 ```
 
 We will now add a constraint stating that the model should contain exactly 1 Brick AHU.
@@ -119,7 +124,7 @@ We will now add a constraint stating that the model should contain exactly 1 Bri
 This basic structure can be changed to require different numbers of different Brick classes. Just don't forget to change the name of the shape (`:ahu-count`, above) when you copy-paste!
 
 ```{attention}
-As an exercise, try writing shapes that require the model to have the following.
+As an exercise, try writing shapes that require the model to contain the following.
 - (1) Brick Supply_Fan
 - (1) Brick Damper
 - (1) Brick Cooling_Coil
@@ -166,6 +171,8 @@ As an exercise, try writing shapes that require the model to have the following.
 
 Put all of the above in a new file called `tutorial2_manifest.ttl`. We'll also add a shape called `sz-vav-ahu-control-sequences`, which is a use case shape to validate the model against in the next section.
 
+The following block of code puts all of the above in the `tutorial2_manifest.ttl` file for you:
+
 ```{code-cell}
 with open("tutorial2_manifest.ttl", "w") as f:
     f.write("""
@@ -175,7 +182,10 @@ with open("tutorial2_manifest.ttl", "w") as f:
 @prefix constraint: <https://nrel.gov/BuildingMOTIF/constraints#> .
 @prefix : <urn:my_site_constraints/> .
 
-: a owl:Ontology .
+: a owl:Ontology ;
+    owl:imports <https://brickschema.org/schema/1.3/Brick>,
+                <https://nrel.gov/BuildingMOTIF/constraints>,
+                <urn:ashrae/g36> .
 
 :ahu-count a sh:NodeShape ;
     sh:message "need 1 AHU" ;
@@ -214,14 +224,44 @@ with open("tutorial2_manifest.ttl", "w") as f:
 """)
 ```
 
-### Validating the Model
+### Adding the Manifest to the Model
+
+We associate the manifest with our model so that BuildingMOTIF knows that we want validate the model against these specific shapes.
+We can always update this manifest, or validate our model against other shapes; however, validating a model against its manifest is
+the most common use case, so this is treated specially in BuildingMOTIF.
 
-We can now ask BuildingMOTIF to validate the model against the manifest and ask BuildingMOTIF for some details if it fails. We also have to be sure to include the supporting shape collections containing the definitions used in the manifest.
 
 ```{code-cell}
 # load manifest into BuildingMOTIF as its own library!
 manifest = Library.load(ontology_graph="tutorial2_manifest.ttl")
+# set it as the manifest for the model
+model.update_manifest(manifest)
+```
+
+### Validating the Model
+
+We can now ask BuildingMOTIF to validate the model against the manifest and ask BuildingMOTIF for some details if it fails.
+By default, BuildingMOTIF will include all shape collections imported by the manifest (`owl:imports`). BuildingMOTIF will
+complain if the manifest requires ontologies that have not yet been loaded into BuildingMOTIF; this is why we are careful
+to load in the Brick and Guideline36 libraries at the top of this tutorial.
+
+
+```{code-cell}
+validation_result = model.validate()
+print(f"Model is valid? {validation_result.valid}")
 
+# print reasons
+for diff in validation_result.diffset:
+    print(f" - {diff.reason()}")
+```
+
+```{admonition} Tip on supplying extra shape collections
+:class: dropdown
+
+We can also provide a list of shape collections directly to `Model.validate`; BuildingMOTIF
+will use these shape collections to validate the model *instead of* the manifest.
+
+```python
 # gather shape collections into a list for ease of use
 shape_collections = [
     brick.get_shape_collection(),
@@ -231,7 +271,7 @@ shape_collections = [
 ]
 
 # pass a list of shape collections to .validate()
-validation_result = model.validate(shape_collections)
+validation_result = model.validate()
 print(f"Model is valid? {validation_result.valid}")
 
 # print reasons
@@ -266,8 +306,7 @@ print(model.graph.serialize())
 We can see that the heating coil was added to the model and connected to the AHU so let's check if the manifest validation failure was fixed.
 
 ```{code-cell}
-# pass a list of shape collections to .validate()
-validation_result = model.validate(shape_collections)
+validation_result = model.validate()
 print(f"Model is valid? {validation_result.valid}")
 
 # print reasons

libraries/ashrae/223p/ontology/223p.ttl

@@ -1363,8 +1363,6 @@ FILTER (?system != ?homeSystem) .
     sh:returnType xsd:boolean .
 
 <http://data.ashrae.org/standard223/1.0/extension/function/shacl> rdfs:comment "This graph contains some SPARQLFunction definitions, which are part of SHACL Advanced Features (SHACL AF). Thus, they may not be supported by all SHACL implementations." ;
-    owl:imports <http://data.ashrae.org/standard223/1.0/model/all>,
-        sh: ;
     owl:versionInfo "Created with TopBraid Composer" .
 
 g36:AirFlowStationAnnotation a sh:NodeShape ;
@@ -1427,36 +1425,6 @@ g36:VAV_4-1 a s223:Class,
                             sh:path [ sh:inversePath s223:contains ] ] ] ;
             sh:path s223:connectedTo ] .
 
-<http://data.ashrae.org/standard223/1.0/extensions/settings> rdfs:comment """This file can be used to control which extensions are to be loaded with the 223 ontology by importing the appropriate graph(s). Options include:
-For access to magic properties and other SPIN functions - http://data.ashraw.org/standard223/1.0/function/spin.  
-For validation of the schema, model, or data - http://data.ashrae.org/standard223/1.0/validation/schema (or model or data).  
-For inferred properties - http://data.ashrae.org/standard223/1.0/inference/model-rules (and/or schema-rules, data-rules)""" ;
-    owl:imports <http://data.ashrae.org/standard223/1.0/extension/equipments>,
-        <http://data.ashrae.org/standard223/1.0/extension/equipments-extensions-rules>,
-        <http://data.ashrae.org/standard223/1.0/extension/g36>,
-        <http://data.ashrae.org/standard223/1.0/inference/data-rules>,
-        <http://data.ashrae.org/standard223/1.0/inference/model-rules>,
-        <http://data.ashrae.org/standard223/1.0/inference/owl-subset>,
-        <http://data.ashrae.org/standard223/1.0/validation/data>,
-        <http://data.ashrae.org/standard223/1.0/validation/model>,
-        <http://data.ashrae.org/standard223/1.0/validation/schema> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/function/spin> owl:imports <http://spinrdf.org/spl> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/model/core> spin:imports <http://topbraid.org/spin/owlrl-all> ;
-    owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/model/device> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/model/fnblck> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> .
-
-<http://data.ashrae.org/standard223/1.0/vocab/enumeration> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
 spin:allProperties a spin:MagicProperty ;
     rdfs:label "Find all Properties" ;
     spin:body [ a sp:Select ;
@@ -1676,13 +1644,6 @@ s223:produces a rdf:Property ;
 
 s223:scp a rdf:Property .
 
-<http://data.ashrae.org/standard223/1.0/extension/equipments> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> .
-
-<http://data.ashrae.org/standard223/1.0/extension/equipments-extensions-rules> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> .
-
-<http://data.ashrae.org/standard223/1.0/extension/g36> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
 g36:AirFlowStation a s223:Class,
         sh:NodeShape ;
     rdfs:label "Air Flow Station" ;
@@ -1770,23 +1731,6 @@ g36:ZoneRule a sh:NodeShape ;
                                                     sh:path [ sh:inversePath s223:hasMeasurementLocation ] ] ] ;
                                     sh:path s223:connectedTo ] ] ] ] .
 
-<http://data.ashrae.org/standard223/1.0/inference/data-rules> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/inference/model-rules> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/inference/owl-subset> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/validation/data> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/validation/model> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
-
-<http://data.ashrae.org/standard223/1.0/validation/schema> owl:imports <http://data.ashrae.org/standard223/1.0/model/all> ;
-    owl:versionInfo "Created with TopBraid Composer" .
 
 rdfs:comment a rdf:Property ;
     rdfs:label "Description of constraints for s223" ;