Changes for sequence with variable element size #13

config/dpkg/changelog

@@ -1,5 +1,5 @@
-dtfabric (20230512-1) unstable; urgency=low
+dtfabric (20230518-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Joachim Metz <joachim.metz@gmail.com>  Fri, 12 May 2023 05:57:04 +0200
+ -- Joachim Metz <joachim.metz@gmail.com>  Thu, 18 May 2023 05:39:28 +0200

dtfabric/__init__.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 """Data type fabric."""
 
-__version__ = '20230512'
+__version__ = '20230518'

dtfabric/reader.py

@@ -401,14 +401,6 @@ def _ReadElementSequenceDataTypeDefinition(
       raise errors.DefinitionReaderError(definition_name, (
           f'undefined element data type: {element_data_type:s}'))
 
-    element_byte_size = element_data_type_definition.GetByteSize()
-    element_type_indicator = element_data_type_definition.TYPE_INDICATOR
-    if not element_byte_size and element_type_indicator != (
-        definitions.TYPE_INDICATOR_STRING):
-      raise errors.DefinitionReaderError(definition_name, (
-          f'unsupported variable size element data type: '
-          f'{element_data_type:s}'))
-
     aliases = definition_values.get('aliases', None)
     description = definition_values.get('description', None)
     urls = definition_values.get('urls', None)

dtfabric/runtime/data_maps.py

@@ -63,6 +63,8 @@ def __init__(self, byte_size, is_complete=False):
 class DataTypeMap(object):
   """Data type map."""
 
+  _MAXIMUM_RECURSION_DEPTH = 10
+
   def __init__(self, data_type_definition):
     """Initializes a data type map.
 
@@ -238,8 +240,6 @@ def MapByteStream(self, byte_stream, **unused_kwargs):
 class PrimitiveDataTypeMap(StorageDataTypeMap):
   """Primitive data type map."""
 
-  # pylint: disable=arguments-differ
-
   def __init__(self, data_type_definition):
     """Initializes a primitive data type map.
 
@@ -529,8 +529,6 @@ def GetStructFormatString(self):
 class UUIDMap(StorageDataTypeMap):
   """UUID (or GUID) data type map."""
 
-  # pylint: disable=arguments-differ
-
   def __init__(self, data_type_definition):
     """Initializes an UUID (or GUID) data type map.
 
@@ -612,8 +610,6 @@ def MapByteStream(
 class ElementSequenceDataTypeMap(StorageDataTypeMap):
   """Element sequence data type map."""
 
-  # pylint: disable=arguments-differ
-
   def __init__(self, data_type_definition):
     """Initializes a sequence data type map.
 
@@ -945,13 +941,15 @@ def _CompositeFoldByteStream(
     # TODO: implement.
 
   def _CompositeMapByteStream(
-      self, byte_stream, byte_offset=0, context=None, **unused_kwargs):
+      self, byte_stream, byte_offset=0, context=None, recursion_depth=0,
+      **unused_kwargs):
     """Maps a sequence of composite data types on a byte stream.
 
     Args:
       byte_stream (bytes): byte stream.
       byte_offset (Optional[int]): offset into the byte stream where to start.
       context (Optional[DataTypeMapContext]): data type map context.
+      recursion_depth (Optional[int]): recursion depth.
 
     Returns:
       tuple[object, ...]: mapped values.
@@ -961,6 +959,9 @@ def _CompositeMapByteStream(
       MappingError: if the data type definition cannot be mapped on
           the byte stream.
     """
+    if recursion_depth > self._MAXIMUM_RECURSION_DEPTH:
+      raise errors.MappingError('At maximum recursion depth')
+
     elements_data_size = None
     elements_terminator = None
     number_of_elements = None
@@ -1008,7 +1009,8 @@ def _CompositeMapByteStream(
           break
 
         element_value = self._element_data_type_map.MapByteStream(
-            byte_stream, byte_offset=byte_offset, context=subcontext)
+            byte_stream, byte_offset=byte_offset, context=subcontext,
+            recursion_depth=recursion_depth + 1)
 
         byte_offset += subcontext.byte_size
         elements_data_offset += subcontext.byte_size
@@ -1173,11 +1175,12 @@ def GetStructFormatString(self):
 
     return f'{number_of_elements:d}{format_string:s}'
 
-  def MapByteStream(self, byte_stream, **kwargs):
+  def MapByteStream(self, byte_stream, recursion_depth=0, **kwargs):
     """Maps the data type on a byte stream.
 
     Args:
       byte_stream (bytes): byte stream.
+      recursion_depth (Optional[int]): recursion depth.
 
     Returns:
       tuple[object, ...]: mapped values.
@@ -1186,14 +1189,13 @@ def MapByteStream(self, byte_stream, **kwargs):
       MappingError: if the data type definition cannot be mapped on
           the byte stream.
     """
-    return self._map_byte_stream(byte_stream, **kwargs)
+    return self._map_byte_stream(
+        byte_stream, recursion_depth=recursion_depth, **kwargs)
 
 
 class StreamMap(ElementSequenceDataTypeMap):
   """Stream data type map."""
 
-  # pylint: disable=arguments-differ
-
   def __init__(self, data_type_definition):
     """Initializes a stream data type map.
 
@@ -1335,8 +1337,6 @@ def MapByteStream(
 class PaddingMap(DataTypeMap):
   """Padding data type map."""
 
-  # pylint: disable=arguments-differ
-
   def _CalculatePaddingSize(self, byte_offset):
     """Calculates the padding size.
 
@@ -1454,9 +1454,7 @@ def MapValue(self, value):
 class StringMap(StreamMap):
   """String data type map."""
 
-  # pylint: disable=arguments-differ
-
-  def FoldByteStream(self, mapped_value, **kwargs):
+  def FoldByteStream(self, mapped_value, **kwargs):  # pylint: disable=arguments-differ
     """Folds the data type into a byte stream.
 
     Args:
@@ -1479,7 +1477,7 @@ def FoldByteStream(self, mapped_value, **kwargs):
 
     return super(StringMap, self).FoldByteStream(byte_stream, **kwargs)
 
-  def MapByteStream(self, byte_stream, byte_offset=0, **kwargs):
+  def MapByteStream(self, byte_stream, byte_offset=0, **kwargs):  # pylint: disable=arguments-differ
     """Maps the data type on a byte stream.
 
     Args:
@@ -1526,8 +1524,6 @@ def MapByteStream(self, byte_stream, byte_offset=0, **kwargs):
 class StructureMap(StorageDataTypeMap):
   """Structure data type map."""
 
-  # pylint: disable=arguments-differ
-
   def __init__(self, data_type_definition):
     """Initializes a structure data type map.
 
@@ -1642,13 +1638,15 @@ def _CompositeFoldByteStream(
     return b''.join(data_attributes)
 
   def _CompositeMapByteStream(
-      self, byte_stream, byte_offset=0, context=None, **unused_kwargs):
+      self, byte_stream, byte_offset=0, context=None, recursion_depth=0,
+      **unused_kwargs):
     """Maps a sequence of composite data types on a byte stream.
 
     Args:
       byte_stream (bytes): byte stream.
       byte_offset (Optional[int]): offset into the byte stream where to start.
       context (Optional[DataTypeMapContext]): data type map context.
+      recursion_depth (Optional[int]): recursion depth.
 
     Returns:
       object: mapped value.
@@ -1658,6 +1656,9 @@ def _CompositeMapByteStream(
       MappingError: if the data type definition cannot be mapped on
           the byte stream.
     """
+    if recursion_depth > self._MAXIMUM_RECURSION_DEPTH:
+      raise errors.MappingError('At maximum recursion depth')
+
     context_state = getattr(context, 'state', {})
     context_values = getattr(context, 'values', {})
 
@@ -1702,7 +1703,8 @@ def _CompositeMapByteStream(
 
       try:
         value = data_type_map.MapByteStream(
-            byte_stream, byte_offset=byte_offset, context=subcontext)
+            byte_stream, byte_offset=byte_offset, context=subcontext,
+            recursion_depth=recursion_depth + 1)
         setattr(mapped_values, attribute_name, value)
 
       except errors.ByteStreamTooSmallError:
@@ -1965,11 +1967,12 @@ def GetStructFormatString(self):
 
     return self._format_string
 
-  def MapByteStream(self, byte_stream, **kwargs):
+  def MapByteStream(self, byte_stream, recursion_depth=0, **kwargs):
     """Maps the data type on a byte stream.
 
     Args:
       byte_stream (bytes): byte stream.
+      recursion_depth (Optional[int]): recursion depth.
 
     Returns:
       object: mapped value.
@@ -1978,7 +1981,8 @@ def MapByteStream(self, byte_stream, **kwargs):
       MappingError: if the data type definition cannot be mapped on
           the byte stream.
     """
-    return self._map_byte_stream(byte_stream, **kwargs)
+    return self._map_byte_stream(
+        byte_stream, recursion_depth=recursion_depth, **kwargs)
 
 
 class SemanticDataTypeMap(DataTypeMap):
@@ -2182,7 +2186,7 @@ def GetByteSize(self):  # pylint: disable=redundant-returns-doc
     """
     return None
 
-  def GetSizeHint(self, context=None, **kwargs):  # pylint: disable=arguments-differ
+  def GetSizeHint(self, context=None, **kwargs):
     """Retrieves a hint about the size.
 
     Args:
@@ -2202,7 +2206,7 @@ def GetSizeHint(self, context=None, **kwargs):  # pylint: disable=arguments-diff
 
     return member_data_type_map.GetSizeHint(context=context, **kwargs)
 
-  def MapByteStream(self, byte_stream, context=None, **kwargs):  # pylint: disable=arguments-differ
+  def MapByteStream(self, byte_stream, context=None, **kwargs):
     """Maps the data type on a byte stream.
 
     Args:

tests/reader.py

@@ -1089,9 +1089,13 @@ def testReadFileObjectSequence(self):
     definitions_file = self._GetTestFilePath(['sequence_with_structure.yaml'])
     self._SkipIfPathNotExists(definitions_file)
 
-    with self.assertRaises(errors.FormatError):
-      with open(definitions_file, 'rb') as file_object:
-        definitions_reader.ReadFileObject(definitions_registry, file_object)
+    with open(definitions_file, 'rb') as file_object:
+      definitions_reader.ReadFileObject(definitions_registry, file_object)
+
+    self.assertEqual(len(definitions_registry._definitions), 3)
+
+    data_type_definition = definitions_registry.GetDefinitionByName('vectors')
+    self.assertIsInstance(data_type_definition, data_types.SequenceDefinition)
 
   def testReadFileObjectStream(self):
     """Tests the ReadFileObject function of a stream data type."""

tests/runtime/data_maps.py

@@ -909,6 +909,9 @@ def testCompositeMapByteStream(self):
     with self.assertRaises(errors.MappingError):
       data_type_map._CompositeMapByteStream(None)
 
+    with self.assertRaises(errors.MappingError):
+      data_type_map._CompositeMapByteStream(byte_stream, recursion_depth=999)
+
     with self.assertRaises(errors.ByteStreamTooSmallError):
       data_type_map._CompositeMapByteStream(b'\x12\x34\x56')
 
@@ -1259,6 +1262,9 @@ def testCompositeMapByteStream(self):
     self.assertEqual(utf16_string.size, len(text_stream))
     self.assertEqual(utf16_string.text, 'dtFabric')
 
+    with self.assertRaises(errors.MappingError):
+      data_type_map._CompositeMapByteStream(byte_stream, recursion_depth=999)
+
     byte_stream = b''.join([bytes(bytearray([3, 0])), text_stream])
 
     with self.assertRaises(errors.MappingError):