From 7a0e8e2efe1f82157cac5045864f0159b32e19b1 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Thu, 13 Jan 2022 08:20:44 +0100 Subject: [PATCH] Changes to expose data stream extents #597 --- appveyor.yml | 6 +++ config/dpkg/control | 2 +- dependencies.ini | 2 +- dfvfs.ini | 1 + dfvfs/vfs/hfs_file_entry.py | 25 +++++++++++ dfvfs/vfs/tsk_file_entry.py | 15 ++++--- requirements.txt | 2 +- setup.cfg | 2 +- tests/vfs/hfs_file_entry.py | 86 +++++++++++++++++++++++-------------- tests/vfs/tsk_file_entry.py | 24 ++++++++++- 10 files changed, 122 insertions(+), 43 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3c31a3e3..6ce1c8d0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,6 @@ environment: + pypi_token: + secure: /FwQrmudDyj+Mu3DaxLEowyvwBaY7x1GRt6gYJrVerEAo4PujrTDfMs9/K6PJSN7KkCL/6LQK2VfTD91bbnUwGMiTjfeItu2+aernJtwLLtoDJ22sHgMiajGMqficrHlOc7uNhFMjQsGa7WiiGGo12c/b7z55dNmU2N0EIc086/Z2G6O+n2+oBeT5SbFu5j5XXkwrd98vnW8hryuZPjLauV1mxc6MMNiv3dOgVL8gtWDjW5xZVJvfOTcYA+7MMLPUbMbqMcXkTSRshqUrX/6mw== matrix: - DESCRIPTION: "Windows with 32-bit Python 3.10" MACHINE_TYPE: "x86" @@ -35,3 +37,7 @@ test_script: artifacts: - path: dist\*.whl + +deploy_script: +- ps: If ($env:APPVEYOR_REPO_TAG -eq "true" -And $isWindows -And $env:MACHINE_TYPE -eq "x86") { + Invoke-Expression "${env:PYTHON}\\python.exe -m twine upload dist/*.whl --username __token__ --password ${env:PYPI_TOKEN} --skip-existing" } diff --git a/config/dpkg/control b/config/dpkg/control index 4db2ee6c..d10f55a5 100644 --- a/config/dpkg/control +++ b/config/dpkg/control @@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs Package: python3-dfvfs Architecture: all -Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20220112), libfshfs-python3 (>= 20210722), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220110), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211113), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends} +Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20220112), libfshfs-python3 (>= 20220113), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220110), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211113), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends} Description: Python 3 module of dfVFS dfVFS, or Digital Forensics Virtual File System, provides read-only access to file-system objects from various storage media types and file formats. The goal diff --git a/dependencies.ini b/dependencies.ini index 82f2d69f..538a07ac 100644 --- a/dependencies.ini +++ b/dependencies.ini @@ -66,7 +66,7 @@ version_property: get_version() [pyfshfs] dpkg_name: libfshfs-python3 l2tbinaries_name: libfshfs -minimum_version: 20210722 +minimum_version: 20220113 pypi_name: libfshfs-python rpm_name: libfshfs-python3 version_property: get_version() diff --git a/dfvfs.ini b/dfvfs.ini index 2c84f240..ce574bd7 100644 --- a/dfvfs.ini +++ b/dfvfs.ini @@ -11,3 +11,4 @@ description_long: dfVFS, or Digital Forensics Virtual File System, provides read of dfVFS is to provide a generic interface for accessing file-system objects, for which it uses several back-ends that provide the actual implementation of the various storage media types, volume systems and file systems. +pypi_token: /FwQrmudDyj+Mu3DaxLEowyvwBaY7x1GRt6gYJrVerEAo4PujrTDfMs9/K6PJSN7KkCL/6LQK2VfTD91bbnUwGMiTjfeItu2+aernJtwLLtoDJ22sHgMiajGMqficrHlOc7uNhFMjQsGa7WiiGGo12c/b7z55dNmU2N0EIc086/Z2G6O+n2+oBeT5SbFu5j5XXkwrd98vnW8hryuZPjLauV1mxc6MMNiv3dOgVL8gtWDjW5xZVJvfOTcYA+7MMLPUbMbqMcXkTSRshqUrX/6mw== diff --git a/dfvfs/vfs/hfs_file_entry.py b/dfvfs/vfs/hfs_file_entry.py index 237a1e9f..f47eec83 100644 --- a/dfvfs/vfs/hfs_file_entry.py +++ b/dfvfs/vfs/hfs_file_entry.py @@ -8,6 +8,7 @@ from dfvfs.lib import errors from dfvfs.path import hfs_path_spec from dfvfs.vfs import attribute +from dfvfs.vfs import extent from dfvfs.vfs import file_entry from dfvfs.vfs import hfs_attribute from dfvfs.vfs import hfs_directory @@ -221,6 +222,30 @@ def size(self): """int: size of the file entry in bytes or None if not available.""" return self._fshfs_file_entry.size + def GetExtents(self, data_stream_name=''): + """Retrieves extents of a specific data stream. + + Returns: + list[Extent]: extents of the data stream. + """ + extents = [] + if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and + not data_stream_name): + for extent_index in range(self._fshfs_file_entry.number_of_extents): + extent_offset, extent_size, extent_flags = ( + self._fshfs_file_entry.get_extent(extent_index)) + + if extent_flags & 0x1: + extent_type = definitions.EXTENT_TYPE_SPARSE + else: + extent_type = definitions.EXTENT_TYPE_DATA + + data_stream_extent = extent.Extent( + extent_type=extent_type, offset=extent_offset, size=extent_size) + extents.append(data_stream_extent) + + return extents + def GetHFSFileEntry(self): """Retrieves the HFS file entry. diff --git a/dfvfs/vfs/tsk_file_entry.py b/dfvfs/vfs/tsk_file_entry.py index efc34ef3..69571fda 100644 --- a/dfvfs/vfs/tsk_file_entry.py +++ b/dfvfs/vfs/tsk_file_entry.py @@ -709,8 +709,6 @@ def GetExtents(self, data_stream_name=''): for pytsk_attribute in self._tsk_file: if getattr(pytsk_attribute, 'info', None): attribute_type = getattr(pytsk_attribute.info, 'type', None) - if attribute_type in self._TSK_INTERNAL_ATTRIBUTE_TYPES: - continue name = getattr(pytsk_attribute.info, 'name', None) if name: @@ -721,10 +719,7 @@ def GetExtents(self, data_stream_name=''): raise errors.BackEndError( 'pytsk3 returned a non UTF-8 formatted name.') - # The data stream is returned as a name-less attribute of type - # pytsk3.TSK_FS_ATTR_TYPE_DEFAULT. - if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and - attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and + if attribute_type == pytsk3.TSK_FS_ATTR_TYPE_HFS_DATA and ( not name and not data_stream_name): data_pytsk_attribute = pytsk_attribute break @@ -734,6 +729,14 @@ def GetExtents(self, data_stream_name=''): data_pytsk_attribute = pytsk_attribute break + # The data stream is returned as a name-less attribute of type + # pytsk3.TSK_FS_ATTR_TYPE_DEFAULT. + if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and + attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and + not name and not data_stream_name): + data_pytsk_attribute = pytsk_attribute + break + extents = [] if data_pytsk_attribute: tsk_file_system = self._file_system.GetFsInfo() diff --git a/requirements.txt b/requirements.txt index ab94e740..ab834025 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ libbde-python >= 20140531 libewf-python >= 20131210 libfsapfs-python >= 20201107 libfsext-python >= 20220112 -libfshfs-python >= 20210722 +libfshfs-python >= 20220113 libfsntfs-python >= 20211229 libfsxfs-python >= 20210726 libfvde-python >= 20160719 diff --git a/setup.cfg b/setup.cfg index b1442e73..07404370 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,7 @@ requires = libbde-python3 >= 20140531 libewf-python3 >= 20131210 libfsapfs-python3 >= 20201107 libfsext-python3 >= 20220112 - libfshfs-python3 >= 20210722 + libfshfs-python3 >= 20220113 libfsntfs-python3 >= 20211229 libfsxfs-python3 >= 20210726 libfvde-python3 >= 20160719 diff --git a/tests/vfs/hfs_file_entry.py b/tests/vfs/hfs_file_entry.py index b59aa31a..91b28486 100644 --- a/tests/vfs/hfs_file_entry.py +++ b/tests/vfs/hfs_file_entry.py @@ -55,10 +55,10 @@ def testInitialize(self): def testAccessTime(self): """Test the access_time property.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) @@ -67,10 +67,10 @@ def testAccessTime(self): def testAddedTime(self): """Test the added_time property.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) @@ -79,10 +79,10 @@ def testAddedTime(self): def testChangeTime(self): """Test the change_time property.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) @@ -91,10 +91,10 @@ def testChangeTime(self): def testCreationTime(self): """Test the creation_time property.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) @@ -103,10 +103,10 @@ def testCreationTime(self): def testModificationTime(self): """Test the modification_time property.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) @@ -115,10 +115,10 @@ def testModificationTime(self): def testSize(self): """Test the size property.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) @@ -127,10 +127,9 @@ def testSize(self): def testGetAttributes(self): """Tests the _GetAttributes function.""" - test_location = '/a_directory/a_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_FILE, - location=test_location, parent=self._raw_path_spec) + location='/a_directory/a_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -149,10 +148,10 @@ def testGetAttributes(self): def testGetStat(self): """Tests the _GetStat function.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -181,10 +180,10 @@ def testGetStat(self): def testGetStatAttribute(self): """Tests the _GetStatAttribute function.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -201,6 +200,32 @@ def testGetStatAttribute(self): self.assertEqual(stat_attribute.size, 22) self.assertEqual(stat_attribute.type, stat_attribute.TYPE_FILE) + def testGetExtents(self): + """Tests the GetExtents function.""" + path_spec = path_spec_factory.Factory.NewPathSpec( + definitions.TYPE_INDICATOR_HFS, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', + parent=self._raw_path_spec) + file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) + self.assertIsNotNone(file_entry) + + extents = file_entry.GetExtents() + self.assertEqual(len(extents), 1) + + self.assertEqual(extents[0].extent_type, definitions.EXTENT_TYPE_DATA) + self.assertEqual(extents[0].offset, 1134592) + self.assertEqual(extents[0].size, 4096) + + path_spec = path_spec_factory.Factory.NewPathSpec( + definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_DIRECTORY, + location='/a_directory', parent=self._raw_path_spec) + file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) + self.assertIsNotNone(file_entry) + + extents = file_entry.GetExtents() + self.assertEqual(len(extents), 0) + def testGetFileEntryByPathSpec(self): """Tests the GetFileEntryByPathSpec function.""" path_spec = path_spec_factory.Factory.NewPathSpec( @@ -212,10 +237,9 @@ def testGetFileEntryByPathSpec(self): def testGetLinkedFileEntry(self): """Tests the GetLinkedFileEntry function.""" - test_location = '/a_link' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_LINK, - location=test_location, parent=self._raw_path_spec) + location='/a_link', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -227,10 +251,10 @@ def testGetLinkedFileEntry(self): def testGetParentFileEntry(self): """Tests the GetParentFileEntry function.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -243,10 +267,10 @@ def testGetParentFileEntry(self): def testIsFunctions(self): """Tests the Is? functions.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -262,10 +286,9 @@ def testIsFunctions(self): self.assertFalse(file_entry.IsPipe()) self.assertFalse(file_entry.IsSocket()) - test_location = '/a_directory' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_DIRECTORY, - location=test_location, parent=self._raw_path_spec) + location='/a_directory', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -335,10 +358,10 @@ def testSubFileEntries(self): def testDataStreams(self): """Tests the data streams functionality.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -351,10 +374,9 @@ def testDataStreams(self): self.assertEqual(data_stream_names, ['']) - test_location = '/a_directory' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_DIRECTORY, - location=test_location, parent=self._raw_path_spec) + location='/a_directory', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) @@ -368,10 +390,10 @@ def testDataStreams(self): def testGetDataStream(self): """Tests the GetDataStream function.""" - test_location = '/a_directory/another_file' path_spec = path_spec_factory.Factory.NewPathSpec( definitions.TYPE_INDICATOR_HFS, - identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location, + identifier=self._IDENTIFIER_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) self.assertIsNotNone(file_entry) diff --git a/tests/vfs/tsk_file_entry.py b/tests/vfs/tsk_file_entry.py index 6fb432cf..ed5ff1fa 100644 --- a/tests/vfs/tsk_file_entry.py +++ b/tests/vfs/tsk_file_entry.py @@ -1093,7 +1093,29 @@ def testSize(self): self.assertIsNotNone(file_entry) self.assertEqual(file_entry.size, 22) - # TODO: add tests for GetExtents + def testGetExtents(self): + """Tests the GetExtents function.""" + path_spec = path_spec_factory.Factory.NewPathSpec( + definitions.TYPE_INDICATOR_TSK, inode=self._INODE_ANOTHER_FILE, + location='/a_directory/another_file', parent=self._raw_path_spec) + file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) + self.assertIsNotNone(file_entry) + + extents = file_entry.GetExtents() + self.assertEqual(len(extents), 1) + + self.assertEqual(extents[0].extent_type, definitions.EXTENT_TYPE_DATA) + self.assertEqual(extents[0].offset, 1134592) + self.assertEqual(extents[0].size, 4096) + + path_spec = path_spec_factory.Factory.NewPathSpec( + definitions.TYPE_INDICATOR_TSK, inode=self._INODE_A_DIRECTORY, + location='/a_directory', parent=self._raw_path_spec) + file_entry = self._file_system.GetFileEntryByPathSpec(path_spec) + self.assertIsNotNone(file_entry) + + extents = file_entry.GetExtents() + self.assertEqual(len(extents), 0) def testGetFileObject(self): """Tests the GetFileObject function."""