Error when reading NWB Zarr file containing array values that were replaced using Zarr

[rly]

A user wanted to update the spiketimes of an existing NWB file. The new spiketimes are the same size, just re-referenced to a different time base, so they're updating the zarr_root["units"]["spike_times"] field using Zarr in append mode. This seems to work, but when they moved the zarr folder to a different location, opened the file with NWBZarrIO, and tried to read the file with io.read, they got the following error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[3], line 1
----> 1 nwbfile = io.read()

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/backends/io.py:60, in HDMFIO.read(self, **kwargs)
     57 if all(len(v) == 0 for v in f_builder.values()):
     58     # TODO also check that the keys are appropriate. print a better error message
     59     raise UnsupportedOperation('Cannot build data. There are no values.')
---> 60 container = self.__manager.construct(f_builder)
     61 container.read_io = self
     62 if self.herd_path is not None:

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:286, in BuildManager.construct(self, **kwargs)
    282     result = self.__type_map.construct(builder, self, parent)
    283 else:
    284     # we are at the top of the hierarchy,
    285     # so it must be time to resolve parents
--> 286     result = self.__type_map.construct(builder, self, None)
    287     self.__resolve_parents(result)
    288 self.prebuilt(result, builder)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:813, in TypeMap.construct(self, **kwargs)
    811     raise ValueError('No ObjectMapper found for builder of type %s' % dt)
    812 else:
--> 813     return obj_mapper.construct(builder, build_manager, parent)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1319, in ObjectMapper.construct(self, **kwargs)
   1317 cls = manager.get_cls(builder)
   1318 # gather all subspecs
-> 1319 subspecs = self.__get_subspec_values(builder, self.spec, manager)
   1320 # get the constructor argument that each specification corresponds to
   1321 const_args = dict()

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1248, in ObjectMapper.__get_subspec_values(self, builder, spec, manager)
   1246                 ret[subspec] = self.__flatten(sub_builder, subspec, manager)
   1247     # now process groups and datasets
-> 1248     self.__get_sub_builders(groups, spec.groups, manager, ret)
   1249     self.__get_sub_builders(datasets, spec.datasets, manager, ret)
   1250 elif isinstance(spec, DatasetSpec):

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1301, in ObjectMapper.__get_sub_builders(self, sub_builders, subspecs, manager, ret)
   1299     ret.update(self.__get_subspec_values(sub_builder, subspec, manager))
   1300 else:
-> 1301     ret[subspec] = manager.construct(sub_builder)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:282, in BuildManager.construct(self, **kwargs)
    280 if parent_builder is not None:
    281     parent = self._get_proxy_builder(parent_builder)
--> 282     result = self.__type_map.construct(builder, self, parent)
    283 else:
    284     # we are at the top of the hierarchy,
    285     # so it must be time to resolve parents
    286     result = self.__type_map.construct(builder, self, None)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:813, in TypeMap.construct(self, **kwargs)
    811     raise ValueError('No ObjectMapper found for builder of type %s' % dt)
    812 else:
--> 813     return obj_mapper.construct(builder, build_manager, parent)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1319, in ObjectMapper.construct(self, **kwargs)
   1317 cls = manager.get_cls(builder)
   1318 # gather all subspecs
-> 1319 subspecs = self.__get_subspec_values(builder, self.spec, manager)
   1320 # get the constructor argument that each specification corresponds to
   1321 const_args = dict()

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1249, in ObjectMapper.__get_subspec_values(self, builder, spec, manager)
   1247     # now process groups and datasets
   1248     self.__get_sub_builders(groups, spec.groups, manager, ret)
-> 1249     self.__get_sub_builders(datasets, spec.datasets, manager, ret)
   1250 elif isinstance(spec, DatasetSpec):
   1251     if not isinstance(builder, DatasetBuilder):  # pragma: no cover

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1301, in ObjectMapper.__get_sub_builders(self, sub_builders, subspecs, manager, ret)
   1299     ret.update(self.__get_subspec_values(sub_builder, subspec, manager))
   1300 else:
-> 1301     ret[subspec] = manager.construct(sub_builder)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:282, in BuildManager.construct(self, **kwargs)
    280 if parent_builder is not None:
    281     parent = self._get_proxy_builder(parent_builder)
--> 282     result = self.__type_map.construct(builder, self, parent)
    283 else:
    284     # we are at the top of the hierarchy,
    285     # so it must be time to resolve parents
    286     result = self.__type_map.construct(builder, self, None)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:813, in TypeMap.construct(self, **kwargs)
    811     raise ValueError('No ObjectMapper found for builder of type %s' % dt)
    812 else:
--> 813     return obj_mapper.construct(builder, build_manager, parent)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1319, in ObjectMapper.construct(self, **kwargs)
   1317 cls = manager.get_cls(builder)
   1318 # gather all subspecs
-> 1319 subspecs = self.__get_subspec_values(builder, self.spec, manager)
   1320 # get the constructor argument that each specification corresponds to
   1321 const_args = dict()

File /opt/conda/lib/python3.9/site-packages/hdmf/build/objectmapper.py:1216, in ObjectMapper.__get_subspec_values(self, builder, spec, manager)
   1214     continue
   1215 if isinstance(attr_val, (GroupBuilder, DatasetBuilder)):
-> 1216     ret[attr_spec] = manager.construct(attr_val)
   1217 elif isinstance(attr_val, ReferenceBuilder):
   1218     ret[attr_spec] = manager.construct(attr_val.builder)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:282, in BuildManager.construct(self, **kwargs)
    280 if parent_builder is not None:
    281     parent = self._get_proxy_builder(parent_builder)
--> 282     result = self.__type_map.construct(builder, self, parent)
    283 else:
    284     # we are at the top of the hierarchy,
    285     # so it must be time to resolve parents
    286     result = self.__type_map.construct(builder, self, None)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:808, in TypeMap.construct(self, **kwargs)
    806 if build_manager is None:
    807     build_manager = BuildManager(self)
--> 808 obj_mapper = self.get_map(builder)
    809 if obj_mapper is None:
    810     dt = builder.attributes[self.namespace_catalog.group_spec_cls.type_key()]

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:727, in TypeMap.get_map(self, **kwargs)
    725     data_type = self.get_builder_dt(obj)
    726     namespace = self.get_builder_ns(obj)
--> 727     container_cls = self.get_cls(obj)
    728 # now build the ObjectMapper class
    729 mapper = self.__mappers.get(container_cls)

File /opt/conda/lib/python3.9/site-packages/hdmf/utils.py:577, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    575 def func_call(*args, **kwargs):
    576     pargs = _check_args(args, kwargs)
--> 577     return func(args[0], **pargs)

File /opt/conda/lib/python3.9/site-packages/hdmf/build/manager.py:652, in TypeMap.get_cls(self, **kwargs)
    650 data_type = self.get_builder_dt(builder)
    651 if data_type is None:
--> 652     raise ValueError("No data_type found for builder %s" % builder.path)
    653 namespace = self.get_builder_ns(builder)
    654 if namespace is None:

ValueError: No data_type found for builder root/units/spike_times

Upon further investigation, the spiketimes array was replaced in Zarr using something like:

z = zarr.open(file_path, "a")
z["units"]["spike_times"] = new_array

Apparently, this also resets the attributes (.zattrs) of the spiketimes array to an empty dictionary. So in the new zarr NWB, the {nwb zarr root}/units/spike_times/.zattrs file is missing. But the .zattrs are important because they signal that the spike_times array has data type VectorData and other important information relevant to NWB.

The correct approach for replacing the values of the spiketimes array that preserves .zattrs is

z = zarr.open(file_path, "a")
z["units"]["spike_times"][:] = new_array

Setting with [:] means replace just the values. Setting without [:] means replace the whole array, and the attrs are a property of the array.

Note that replacing the values using PyNWB currently raises an error: ValueError: Starting with Zarr 2.11.0, stores must be subclasses of BaseStore, if your store exposes the MutableMapping interface wrap it in Zarr.storage.KVStore. so replacing the values of a zarr array can only be done in Zarr, and only with an array of the same size and dtype.

Based on troubleshooting with @alejoe91

[rly]

See the solution at the end of the question!