Validation finds unexpected elements (#542)

CHANGELOG.md

@@ -21,6 +21,8 @@
   @oruebel (#551)
 - Add tutoral for new `AlignedDynamicTable` type. @oruebel (#571)
 - Equality check for `DynamicTable` now also checks that the name and description of the table are the same. @rly (#566)
+- Add `ExtraFieldWarning` validator warning, which is created when fields that are not part of the spec are
+  encountered. @dsleiter, @t-b, @rly (#542)
 
 ### Internal improvements
 - Update CI and copyright year. @rly (#523, #524)

src/hdmf/build/manager.py

@@ -605,7 +605,7 @@ def get_builder_ns(self, **kwargs):
         builder = getargs('builder', kwargs)
         if isinstance(builder, LinkBuilder):
             builder = builder.builder
-        ret = builder.attributes.get('namespace')
+        ret = builder.attributes.get(self.namespace_catalog.group_spec_cls.namespace_key())
         return ret
 
     @docval({'name': 'builder', 'type': Builder,
@@ -713,7 +713,7 @@ def register_container_type(self, **kwargs):
         self.__data_types.setdefault(container_cls, (namespace, data_type))
         if not isinstance(container_cls, TypeSource):
             setattr(container_cls, spec.type_key(), data_type)
-            setattr(container_cls, 'namespace', namespace)
+            setattr(container_cls, spec.namespace_key(), namespace)
 
     @docval({"name": "container_cls", "type": type,
              "doc": "the AbstractContainer class for which the given ObjectMapper class gets used for"},
@@ -751,7 +751,7 @@ def build(self, **kwargs):
 
         # add additional attributes (namespace, data_type, object_id) to builder
         namespace, data_type = self.get_container_ns_dt(container)
-        builder.set_attribute('namespace', namespace)
+        builder.set_attribute(obj_mapper.spec.namespace_key(), namespace)
         builder.set_attribute(self.__type_key(obj_mapper.spec), data_type)
         builder.set_attribute(obj_mapper.spec.id_key(), container.object_id)
         return builder

src/hdmf/spec/spec.py

@@ -314,6 +314,7 @@ class BaseStorageSpec(Spec):
 
     __inc_key = 'data_type_inc'
     __def_key = 'data_type_def'
+    __namespace_key = 'namespace'
     __type_key = 'data_type'
     __id_key = 'object_id'
 
@@ -437,13 +438,27 @@ def get_data_type_spec(cls, data_type_def):
 
     @classmethod
     def get_namespace_spec(cls):
-        return AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False)
+        return AttributeSpec(cls.namespace_key(), 'the namespace for the data type of this object',
+                             'text', required=False)
 
     @property
     def attributes(self):
         ''' The attributes for this specification '''
         return tuple(self.get('attributes', tuple()))
 
+    @classmethod
+    def reserved_attrs(cls):
+        ''' Get the names of the reserved attributes '''
+        return (cls.namespace_key(), cls.type_key(), cls.id_key())
+
+    @classmethod
+    def namespace_key(cls):
+        ''' Get the key used to store namespace on an instance
+
+        Override this method to use a different name for 'namespace'
+        '''
+        return cls.__namespace_key
+
     @property
     def linkable(self):
         ''' True if object can be a link, False otherwise '''

src/hdmf/testing/testcase.py

@@ -4,13 +4,15 @@
 import re
 import unittest
 from abc import ABCMeta, abstractmethod
+from warnings import warn
 
 from .utils import remove_test_file
 from ..backends.hdf5 import HDF5IO
 from ..build import Builder
 from ..common import validate as common_validate, get_manager
 from ..container import AbstractContainer, Container, Data
 from ..query import HDMFDataset
+from ..validate.errors import WARNING_SEVERITY
 
 
 class TestCase(unittest.TestCase):
@@ -225,14 +227,25 @@ def validate(self, experimental=False):
         """Validate the written and exported files, if they exist."""
         if os.path.exists(self.filename):
             with HDF5IO(self.filename, manager=get_manager(), mode='r') as io:
-                errors = common_validate(io, experimental=experimental)
+                result = common_validate(io, experimental=experimental)
+                errors, warnings = self._split_validation_result(result)
                 if errors:
-                    for err in errors:
-                        raise Exception(err)
+                    raise Exception("Validation Errors %s" % errors)
+                if warnings:
+                    warn("Validation Warnings: %s" % warnings)
 
         if os.path.exists(self.export_filename):
             with HDF5IO(self.filename, manager=get_manager(), mode='r') as io:
-                errors = common_validate(io, experimental=experimental)
+                result = common_validate(io, experimental=experimental)
+                errors, warnings = self._split_validation_result(result)
                 if errors:
-                    for err in errors:
-                        raise Exception(err)
+                    raise Exception("Validation Errors %s" % errors)
+                if warnings:
+                    warn("Validation Warnings: %s" % warnings)
+
+    @staticmethod
+    def _split_validation_result(result):
+        """Split validation result into errors and warnings"""
+        errors = [alert for alert in result if alert.severity > WARNING_SEVERITY]
+        warnings = [alert for alert in result if alert.severity == WARNING_SEVERITY]
+        return errors, warnings

src/hdmf/validate/errors.py

@@ -12,19 +12,25 @@
     "MissingDataType",
     "IllegalLinkError",
     "IncorrectDataType",
-    "IncorrectQuantityError"
+    "IncorrectQuantityError",
+    "ExtraFieldWarning"
 ]
 
+WARNING_SEVERITY = 0
+ERROR_SEVERITY = 10
+
 
 class Error:
 
     @docval({'name': 'name', 'type': str, 'doc': 'the name of the component that is erroneous'},
             {'name': 'reason', 'type': str, 'doc': 'the reason for the error'},
+            {'name': 'severity', 'type': int, 'doc': 'severity of the error', 'default': ERROR_SEVERITY},
             {'name': 'location', 'type': str, 'doc': 'the location of the error', 'default': None})
     def __init__(self, **kwargs):
         self.__name = getargs('name', kwargs)
         self.__reason = getargs('reason', kwargs)
         self.__location = getargs('location', kwargs)
+        self.__severity = getargs('severity', kwargs)
         if self.__location is not None:
             self.__str = "%s (%s): %s" % (self.__name, self.__location, self.__reason)
         else:
@@ -42,6 +48,10 @@ def reason(self):
     def location(self):
         return self.__location
 
+    @property
+    def severity(self):
+        return self.__severity
+
     @location.setter
     def location(self, loc):
         self.__location = loc
@@ -176,3 +186,13 @@ def __init__(self, **kwargs):
         reason = "incorrect data_type - expected '%s', got '%s'" % (expected, received)
         loc = getargs('location', kwargs)
         super().__init__(name, reason, location=loc)
+
+
+class ExtraFieldWarning(Error):
+    @docval({'name': 'name', 'type': str, 'doc': 'the name of the component with the extra field'},
+            {'name': 'location', 'type': str, 'doc': 'the location of the warning', 'default': None})
+    def __init__(self, **kwargs):
+        name = getargs('name', kwargs)
+        reason = "encountered extra field"
+        loc = getargs('location', kwargs)
+        super().__init__(name, reason, location=loc, severity=WARNING_SEVERITY)