Clean up of dtFabric-based parsers log2timeline#2142

.travis.yml

@@ -39,15 +39,6 @@ jobs:
     python: 3.8
     services:
     - docker
-  - name: "Ubuntu Focal (20.04) (Docker) with Python 3.5 (tox)"
-    env:
-    - TOXENV="py35"
-    - UBUNTU_VERSION="20.04"
-    group: edge
-    language: python
-    python: 3.5
-    services:
-    - docker
   - name: "Ubuntu Focal (20.04) (Docker) with Python 3.6 (tox)"
     env:
     - TOXENV="py36"

config/dpkg/control

@@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline)
 
 Package: python3-plaso
 Architecture: all
-Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200501), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends}
+Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200501), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20200621), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of plaso (log2timeline)
  Plaso (log2timeline) is a framework to create super timelines. Its
  purpose is to extract timestamps from various files found on typical

dependencies.ini

@@ -73,7 +73,7 @@ version_property: __version__
 
 [dtfabric]
 dpkg_name: python3-dtfabric
-minimum_version: 20181128
+minimum_version: 20200621
 rpm_name: python3-dtfabric
 version_property: __version__
 

plaso/dependencies.py

@@ -30,7 +30,7 @@
     'dfdatetime': ('__version__', '20200501', None, True),
     'dfvfs': ('__version__', '20200604', None, True),
     'dfwinreg': ('__version__', '20180712', None, True),
-    'dtfabric': ('__version__', '20181128', None, True),
+    'dtfabric': ('__version__', '20200621', None, True),
     'elasticsearch': ('__versionstr__', '6.0', None, False),
     'future': ('__version__', '0.16.0', None, True),
     'idna': ('__version__', '2.5', None, True),

plaso/parsers/asl.py

@@ -63,8 +63,6 @@ class ASLParser(dtfabric_parser.DtFabricBaseParser):
 
   _DEFINITION_FILE = 'asl.yaml'
 
-  _FILE_SIGNATURE = b'ASL DB\x00\x00\x00\x00\x00\x00'
-
   # Most significant bit of a 64-bit string offset.
   _STRING_OFFSET_MSB = 1 << 63
 
@@ -262,7 +260,8 @@ def GetFormatSpecification(cls):
       FormatSpecification: format specification.
     """
     format_specification = specification.FormatSpecification(cls.NAME)
-    format_specification.AddNewSignature(cls._FILE_SIGNATURE, offset=0)
+    format_specification.AddNewSignature(
+        b'ASL DB\x00\x00\x00\x00\x00\x00', offset=0)
     return format_specification
 
   def ParseFileObject(self, parser_mediator, file_object):
@@ -286,9 +285,6 @@ def ParseFileObject(self, parser_mediator, file_object):
           'Unable to parse file header with error: {0!s}'.format(
               exception))
 
-    if file_header.signature != self._FILE_SIGNATURE:
-      raise errors.UnableToParseFile('Invalid file signature.')
-
     # TODO: generate event for creation time.
 
     file_size = file_object.get_size()

plaso/parsers/asl.yaml

@@ -47,6 +47,7 @@ members:
   type: stream
   element_data_type: byte
   elements_data_size: 12
+  value: "ASL DB\x00\x00\x00\x00\x00\x00"
 - name: format_version
   data_type: uint32
 - name: first_log_entry_offset

plaso/parsers/bsm.py

@@ -175,8 +175,6 @@ class BSMParser(dtfabric_parser.DtFabricBaseParser):
       0x82: 'bsm_token_data_sockunix',
   }
 
-  _TRAILER_TOKEN_SIGNATURE = 0xb105
-
   _TOKEN_DATA_FORMAT_FUNCTIONS = {
       0x11: '_FormatOtherFileToken',
       0x21: '_FormatDataToken',
@@ -684,9 +682,6 @@ def _ParseRecord(self, parser_mediator, file_object):
                 token_values['error'], token_values['token_status'],
                 token_values['call_status'])
 
-    if token_data.signature != self._TRAILER_TOKEN_SIGNATURE:
-      raise errors.ParseError('Unsupported signature in trailer token.')
-
     if token_data.record_size != header_record_size:
       raise errors.ParseError(
           'Mismatch of event record size between header and trailer token.')

plaso/parsers/bsm.yaml

@@ -611,6 +611,7 @@ attributes:
 members:
 - name: signature
   data_type: uint16
+  value: 0xb105
 - name: record_size
   data_type: uint32
 ---

plaso/parsers/chrome_cache.py

@@ -113,8 +113,6 @@ class ChromeCacheIndexFileParser(dtfabric_parser.DtFabricBaseParser):
 
   _DEFINITION_FILE = 'chrome_cache.yaml'
 
-  _FILE_SIGNATURE = 0xc103cac3
-
   def __init__(self):
     """Initializes an index file."""
     super(ChromeCacheIndexFileParser, self).__init__()
@@ -140,9 +138,6 @@ def _ParseFileHeader(self, file_object):
           'Unable to parse index file header with error: {0!s}'.format(
               exception))
 
-    if file_header.signature != self._FILE_SIGNATURE:
-      raise errors.ParseError('Unsupported index file signature')
-
     format_version = '{0:d}.{1:d}'.format(
         file_header.major_version, file_header.minor_version)
     if format_version not in ('2.0', '2.1'):
@@ -208,8 +203,6 @@ class ChromeCacheDataBlockFileParser(dtfabric_parser.DtFabricBaseParser):
 
   _DEFINITION_FILE = 'chrome_cache.yaml'
 
-  _FILE_SIGNATURE = 0xc104cac3
-
   def _ParseFileHeader(self, file_object):
     """Parses the file header.
 
@@ -230,9 +223,6 @@ def _ParseFileHeader(self, file_object):
           'Unable to parse data block file header with error: {0!s}'.format(
               exception))
 
-    if file_header.signature != self._FILE_SIGNATURE:
-      raise errors.ParseError('Unsupported data block file signature')
-
     format_version = '{0:d}.{1:d}'.format(
         file_header.major_version, file_header.minor_version)
     if format_version not in ('2.0', '2.1'):

plaso/parsers/chrome_cache.yaml

@@ -46,6 +46,7 @@ attributes:
 members:
 - name: signature
   data_type: uint32
+  value: 0xc104cac3
 - name: minor_version
   data_type: uint16
 - name: major_version
@@ -130,6 +131,7 @@ attributes:
 members:
 - name: signature
   data_type: uint32
+  value: 0xc103cac3
 - name: minor_version
   data_type: uint16
 - name: major_version

plaso/parsers/custom_destinations.py

@@ -34,8 +34,6 @@ class CustomDestinationsParser(dtfabric_parser.DtFabricBaseParser):
   _LNK_GUID = (
       b'\x01\x14\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x46')
 
-  _FILE_FOOTER_SIGNATURE = 0xbabffbab
-
   def _ParseLNKFile(
       self, parser_mediator, file_entry, file_offset, remaining_file_size):
     """Parses a LNK file stored within the .customDestinations-ms file.
@@ -184,11 +182,6 @@ def ParseFileObject(self, parser_mediator, file_object):
           file_footer, _ = self._ReadStructureFromFileObject(
               file_object, file_offset, file_footer_map)
 
-          if file_footer.signature != self._FILE_FOOTER_SIGNATURE:
-            parser_mediator.ProduceExtractionWarning(
-                'invalid entry header signature at offset: 0x{0:08x}'.format(
-                    file_offset))
-
         except (ValueError, errors.ParseError) as exception:
           parser_mediator.ProduceExtractionWarning((
               'unable to parse footer at offset: 0x{0:08x} with error: '
@@ -212,10 +205,6 @@ def ParseFileObject(self, parser_mediator, file_object):
       file_footer, _ = self._ReadStructureFromFileObject(
           file_object, file_offset, file_footer_map)
 
-      if file_footer.signature != self._FILE_FOOTER_SIGNATURE:
-        parser_mediator.ProduceExtractionWarning(
-            'invalid footer signature at offset: 0x{0:08x}'.format(file_offset))
-
     except (ValueError, errors.ParseError) as exception:
       parser_mediator.ProduceExtractionWarning((
           'unable to parse footer at offset: 0x{0:08x} with error: '

plaso/parsers/custom_destinations.yaml

@@ -73,6 +73,7 @@ attributes:
 members:
 - name: signature
   data_type: uint32
+  value: 0xbabffbab
 ---
 name: custom_entry_header
 type: structure

plaso/parsers/fseventsd.py

@@ -56,8 +56,6 @@ class FseventsdParser(dtfabric_parser.DtFabricBaseParser):
   # The version 2 format was introduced in MacOS High Sierra (10.13).
   _DLS_V2_SIGNATURE = b'2SLD'
 
-  _DLS_SIGNATURES = [_DLS_V1_SIGNATURE, _DLS_V2_SIGNATURE]
-
   _DEFINITION_FILE = 'fseventsd.yaml'
 
   @classmethod
@@ -99,11 +97,6 @@ def _ParseDLSPageHeader(self, file_object, page_offset):
           'Unable to parse page header at offset: 0x{0:08x} '
           'with error: {1!s}'.format(page_offset, exception))
 
-    if page_header.signature not in self._DLS_SIGNATURES:
-      raise errors.ParseError(
-          'Unsupported page header signature at offset: 0x{0:08x}'.format(
-              page_offset))
-
     return page_header, page_size
 
   def _BuildEventData(self, record):
@@ -170,9 +163,6 @@ def ParseFileObject(self, parser_mediator, file_object):
           'Unable to parse page header with error: {0!s}'.format(
               exception))
 
-    if page_header.signature not in self._DLS_SIGNATURES:
-      raise errors.UnableToParseFile('Invalid file signature')
-
     current_page_end = page_header.page_size
 
     file_entry = parser_mediator.GetFileEntry()

plaso/parsers/fseventsd.yaml

@@ -32,6 +32,7 @@ members:
   type: stream
   element_data_type: byte
   number_of_elements: 4
+  values: ["1SLD", "2SLD"]
 - name: padding
   type: stream
   element_data_type: byte

plaso/parsers/mac_keychain.py

@@ -120,8 +120,6 @@ class KeychainParser(dtfabric_parser.DtFabricBaseParser):
 
   _DEFINITION_FILE = 'mac_keychain.yaml'
 
-  _FILE_SIGNATURE = b'kych'
-
   _MAJOR_VERSION = 1
   _MINOR_VERSION = 0
 
@@ -324,9 +322,6 @@ def _ReadFileHeader(self, file_object):
     file_header, _ = self._ReadStructureFromFileObject(
         file_object, 0, data_type_map)
 
-    if file_header.signature != self._FILE_SIGNATURE:
-      raise errors.ParseError('Unsupported file signature.')
-
     if (file_header.major_format_version != self._MAJOR_VERSION or
         file_header.minor_format_version != self._MINOR_VERSION):
       raise errors.ParseError('Unsupported format version: {0:s}.{1:s}'.format(
@@ -855,7 +850,7 @@ def GetFormatSpecification(cls):
       FormatSpecification: format specification.
     """
     format_specification = specification.FormatSpecification(cls.NAME)
-    format_specification.AddNewSignature(cls._FILE_SIGNATURE, offset=0)
+    format_specification.AddNewSignature(b'kych', offset=0)
     return format_specification
 
   def ParseFileObject(self, parser_mediator, file_object):

plaso/parsers/mac_keychain.yaml

@@ -84,6 +84,7 @@ members:
   type: stream
   element_data_type: byte
   elements_data_size: 4
+  value: "kych"
 - name: major_format_version
   data_type: uint16
 - name: minor_format_version

plaso/parsers/recycler.py

@@ -202,13 +202,10 @@ def _ParseInfo2Record(
             'Unable to map record data at offset: 0x{0:08x} with error: '
             '{1!s}').format(record_offset, exception))
 
-      unicode_filename = unicode_filename.rstrip('\x00')
-
     if record.deletion_time == 0:
       date_time = dfdatetime_semantic_time.NotSet()
     else:
-      date_time = dfdatetime_filetime.Filetime(
-          timestamp=record.deletion_time)
+      date_time = dfdatetime_filetime.Filetime(timestamp=record.deletion_time)
 
     event_data = WinRecycleBinEventData()
     event_data.drive_number = record.drive_number

plaso/parsers/recycler.yaml

@@ -54,6 +54,7 @@ description: Windows Recycler INFO2 file entry Unicode original filename string
 type: string
 encoding: utf-16-le
 element_data_type: wchar16
+elements_data_size: 520
 elements_terminator: "\x00\x00"
 ---
 name: recycler_info2_file_entry

plaso/parsers/safari_cookies.py

@@ -51,8 +51,6 @@ class BinaryCookieParser(dtfabric_parser.DtFabricBaseParser):
 
   _DEFINITION_FILE = 'safari_cookies.yaml'
 
-  _SIGNATURE = b'cook'
-
   def __init__(self):
     """Initializes a parser object."""
     super(BinaryCookieParser, self).__init__()
@@ -220,9 +218,6 @@ def ParseFileObject(self, parser_mediator, file_object):
       raise errors.UnableToParseFile(
           'Unable to read file header with error: {0!s}.'.format(exception))
 
-    if file_header.signature != self._SIGNATURE:
-      raise errors.UnableToParseFile('Unsupported file signature.')
-
     file_offset = file_header_data_size
 
     # TODO: move page sizes array into file header, this will require dtFabric

plaso/parsers/safari_cookies.yaml

@@ -60,6 +60,7 @@ members:
   type: stream
   element_data_type: byte
   number_of_elements: 4
+  value: "cook"
 - name: number_of_pages
   data_type: uint32
 ---

plaso/parsers/systemd_journal.py

@@ -48,8 +48,6 @@ class SystemdJournalParser(dtfabric_parser.DtFabricBaseParser):
 
   _DEFINITION_FILE = 'systemd_journal.yaml'
 
-  _FILE_SIGNATURE = b'LPKSHHRH'
-
   _OBJECT_COMPRESSED_FLAG_XZ = 1
   _OBJECT_COMPRESSED_FLAG_LZ4 = 2
 
@@ -271,7 +269,7 @@ def GetFormatSpecification(cls):
       FormatSpecification: format specification.
     """
     format_specification = specification.FormatSpecification(cls.NAME)
-    format_specification.AddNewSignature(cls._FILE_SIGNATURE, offset=0)
+    format_specification.AddNewSignature(b'LPKSHHRH', offset=0)
     return format_specification
 
   def ParseFileObject(self, parser_mediator, file_object):
@@ -294,9 +292,6 @@ def ParseFileObject(self, parser_mediator, file_object):
           'Unable to parse file header with error: {0!s}'.format(
               exception))
 
-    if file_header.signature != self._FILE_SIGNATURE:
-      raise errors.UnableToParseFile('Invalid file signature.')
-
     if file_header.header_size not in self._SUPPORTED_FILE_HEADER_SIZES:
       raise errors.UnableToParseFile(
           'Unsupported file header size: {0:d}.'.format(

plaso/parsers/systemd_journal.yaml

@@ -48,6 +48,7 @@ members:
   type: stream
   element_data_type: byte
   elements_data_size: 8
+  value: "LPKSHHRH"
 - name: compatible_flags
   data_type: uint32
 - name: incompatible_flags

requirements.txt

@@ -9,10 +9,10 @@ cffi >= 1.9.1
 chardet >= 2.0.1
 cryptography >= 2.0.2
 defusedxml >= 0.5.0
-dfdatetime >= 20180704
+dfdatetime >= 20200501
 dfvfs >= 20200604
 dfwinreg >= 20180712
-dtfabric >= 20181128
+dtfabric >= 20200621
 elasticsearch >= 6.0
 future >= 0.16.0
 idna >= 2.5

setup.cfg

@@ -51,10 +51,10 @@ requires = libbde-python3 >= 20140531
            python3-cryptography >= 2.0.2
            python3-dateutil >= 1.5
            python3-defusedxml >= 0.5.0
-           python3-dfdatetime >= 20180704
+           python3-dfdatetime >= 20200501
            python3-dfvfs >= 20200604
            python3-dfwinreg >= 20180712
-           python3-dtfabric >= 20181128
+           python3-dtfabric >= 20200621
            python3-elasticsearch >= 6.0
            python3-future >= 0.16.0
            python3-idna >= 2.5