Migrated systemd journal construct-based plugin to use dtfabric log2t…

…imeline#1893

plaso/parsers/dtfabric_parser.py

@@ -247,14 +247,19 @@ def _ReadStructureFromFileObject(
       ValueError: if file-like object or data type map is missing.
     """
     context = None
+    data = b''
     last_data_size = 0
 
     data_size = data_type_map.GetByteSize()
     if not data_size:
       data_size = data_type_map.GetSizeHint()
 
     while data_size != last_data_size:
-      data = self._ReadData(file_object, file_offset, data_size)
+      read_offset = file_offset + last_data_size
+      read_size = data_size - last_data_size
+      data_segment = self._ReadData(file_object, read_offset, read_size)
+
+      data = b''.join([data, data_segment])
 
       try:
         context = dtfabric_data_maps.DataTypeMapContext()

plaso/parsers/systemd_journal.py

@@ -81,7 +81,7 @@ def _ParseDataObject(self, file_object, file_offset):
           of the file-like object.
 
     Returns:
-      systemd_journal_data_object: data object.
+      bytes: data.
 
     Raises:
       ParseError: if the data object cannot be parsed.
@@ -105,7 +105,28 @@ def _ParseDataObject(self, file_object, file_offset):
       raise errors.ParseError('Unsupported object flags: 0x{0:02x}.'.format(
           data_object.object_flags))
 
-    return data_object
+    # The data is read seperately for performance reasons.
+    data_size = data_object.data_size - 64
+    data = file_object.read(data_size)
+
+    if data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_XZ:
+      data = lzma.decompress(data)
+
+    elif data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_LZ4:
+      uncompressed_size_map = self._GetDataTypeMap('uint32le')
+
+      try:
+        uncompressed_size = self._ReadStructureFromByteStream(
+            data, file_offset + 64, uncompressed_size_map)
+      except (ValueError, errors.ParseError) as exception:
+        raise errors.ParseError((
+            'Unable to parse LZ4 uncompressed size at offset: 0x{0:08x} with '
+            'error: {1!s}').format(file_offset + 64, exception))
+
+      data = lz4.block.decompress(
+          data[8:], uncompressed_size=uncompressed_size)
+
+    return data
 
   def _ParseEntryArrayObject(self, file_object, file_offset):
     """Parses an entry array object.
@@ -245,26 +266,7 @@ def _ParseJournalEntryItem(self, file_object, file_offset):
     Raises:
       ParseError: if the LZ4 uncompressed size cannot be parsed.
     """
-    data_object = self._ParseDataObject(file_object, file_offset)
-
-    event_data = data_object.data
-    if data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_XZ:
-      event_data = lzma.decompress(event_data)
-
-    elif data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_LZ4:
-      uncompressed_size_map = self._GetDataTypeMap('uint32le')
-
-      try:
-        uncompressed_size = self._ReadStructureFromByteStream(
-            event_data, file_offset + 64, uncompressed_size_map)
-      except (ValueError, errors.ParseError) as exception:
-        raise errors.ParseError((
-            'Unable to parse LZ4 uncompressed size at offset: 0x{0:08x} with '
-            'error: {1!s}').format(file_offset + 64, exception))
-
-      event_data = lz4.block.decompress(
-          event_data[8:], uncompressed_size=uncompressed_size)
-
+    event_data = self._ParseDataObject(file_object, file_offset)
     event_string = event_data.decode('utf-8')
     return event_string.split('=', 1)
 

plaso/parsers/systemd_journal.yaml

@@ -105,22 +105,6 @@ members:
 - name: tail_entry_monotonic
   data_type: uint64
 ---
-name: systemd_journal_object_header
-type: structure
-attributes:
-  byte_order: little-endian
-members:
-- name: object_type
-  data_type: uint8
-- name: object_flags
-  data_type: uint8
-- name: reserved1
-  type: stream
-  element_data_type: byte
-  elements_data_size: 6
-- name: data_size
-  data_type: uint64
----
 name: systemd_journal_data_object
 type: structure
 attributes:
@@ -146,12 +130,6 @@ members:
   data_type: uint64
 - name: entry_array_offset
   data_type: uint64
-- name: number_of_entries
-  data_type: uint64
-- name: data
-  type: stream
-  element_data_type: byte
-  elements_data_size: systemd_journal_data_object.data_size - 64
 ---
 name: systemd_journal_entry_item
 type: structure

tests/parsers/systemd_journal.py

@@ -11,13 +11,16 @@
 except ImportError:
   systemd_journal = None
 
+from tests import test_lib as shared_test_lib
 from tests.parsers import test_lib
 
 
 @unittest.skipIf(systemd_journal is None, 'requires LZMA compression support')
 class SystemdJournalParserTest(test_lib.ParserTestCase):
   """Tests for the Systemd Journal parser."""
 
+  @shared_test_lib.skipUnlessHasTestFile([
+      'systemd', 'journal', 'system.journal'])
   def testParse(self):
     """Tests the Parse function."""
     parser = systemd_journal.SystemdJournalParser()
@@ -47,6 +50,8 @@ def testParse(self):
     expected_short_message = '{0:s}...'.format(expected_message[:77])
     self._TestGetMessageStrings(event, expected_message, expected_short_message)
 
+  @shared_test_lib.skipUnlessHasTestFile([
+      'systemd', 'journal', 'system.journal.lz4'])
   def testParseLZ4(self):
     """Tests the Parse function on a journal with LZ4 compressed events."""
     parser = systemd_journal.SystemdJournalParser()
@@ -81,6 +86,9 @@ def testParseLZ4(self):
     expected_short_message = '{0:s}...'.format(expected_message[:77])
     self._TestGetMessageStrings(event, expected_message, expected_short_message)
 
+  @shared_test_lib.skipUnlessHasTestFile([
+      'systemd', 'journal',
+      'system@00053f9c9a4c1e0e-2e18a70e8b327fed.journalTILDE'])
   def testParseDirty(self):
     """Tests the Parse function on a 'dirty' journal file."""
     storage_writer = self._CreateStorageWriter()