Merge branch 'DelayMapUpdateWhenVisible' of https://github.com/sepinf…

…-inc/IPED.git into DelayMapUpdateWhenVisible
sepinf-inc · Oct 15, 2023 · 46c4fba · 46c4fba
2 parents efb2aa2 + 1006bc1
commit 46c4fba
Show file tree

Hide file tree

Showing 103 changed files with 1,695 additions and 1,194 deletions.
diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt
@@ -1,3 +1,29 @@
+16/08/2023: IPED-4.1.4:
+News:
+#1294: Support parsing iCloud backup LZFSE compressed files (@lfcnassif)
+#1525: Support parsing MacOS XXXXX.partial.emlx emails attachments (@FelipeFcosta, @lfcnassif)
+#1798: Support iLBC (Internet Low Bitrate Codec) audios (@tc-wleite)
+#1786: Improve the detection of Matroska files: MKV, MKA and WEBM (@tc-wleite)
+#1815: Improve the detection of WhatsApp iOS account plist file (@lfcnassif)
+#1793: Improve the detection of Apple iWork 13 documents (@lfcnassif)
+#1809: Make VideoThumbTask work on videos in mounted paths longer than 256 chars on Windows (@lfcnassif, @tc-wleite)
+Fixes: 
+#1769: Map renders locations but a blank background [regression 4.1.0] (@lfcnassif)
+#1774: Old WhatsApp databases parsing affected by forwarded message feature [regression 4.1.3] (@hauck-jvsh, @lfcnassif)
+#1791: WhatsApp parser may lose recent messages (@tc-wleite)
+#1765: Aborting IOException from AudioTranscriptTask (@lfcnassif)
+#1801: Never add video: prefix to transcription properties (@lfcnassif)
+#1782: Error opening items inside an E01 from an unmounted READ ONLY Windows network share (@tc-wleite)
+#1762: --remove evidence option fails if graph feature wasn't enabled (@lfcnassif)
+#1757: Infinite loop in IOUtil.ignoreInputStream() causing thread leak and wasting CPU (@hauck-jvsh, @lfcnassif)
+#1814: Corrupted ISO caused an "infinite recursion loop" in SevenZipParser (@tc-wleite, @lfcnassif)
+#1752: TorTcParser timestamp in UTC although not informed (@patrickdalla, @lfcnassif)
+#1795: TorTcParser giving different values on Linux and Windows (@lfcnassif)
+#1750: Absence of JavaFX aborting processing if user clicks on "Preview Case" (@lfcnassif)
+#1806: Mimetype of encrypted Office documents without extension ending with a hyphen (@lfcnassif)
+#1807: Avoid minor NPE while reading AD1 contained file after parsing timeout (@lfcnassif)
+
+
 30/06/2023: IPED-4.1.3:
 News:
 #1287: Flag WhatsApp Forwarded messages (@tc-wleite, @gfd2020)

diff --git a/iped-app/pom.xml b/iped-app/pom.xml
@@ -301,7 +301,7 @@
                                 <artifactItem>
                                     <groupId>org.imagemagick</groupId>
                                     <artifactId>imagemagick-zip</artifactId>
-                                    <version>7.1.0-q8-x64</version>
+                                    <version>7.1.1-19-q8-x64</version>
                                     <type>zip</type>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${tools.dir}</outputDirectory>
@@ -320,7 +320,7 @@
                                 <artifactItem>
                                     <groupId>tesseract</groupId>
                                     <artifactId>tesseract-zip</artifactId>
-                                    <version>5.0.0-alpha.extra.langs</version>
+                                    <version>5.3.2-24-g3922</version>
                                     <type>zip</type>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${tools.dir}</outputDirectory>

diff --git a/iped-app/resources/config/IPEDConfig.txt b/iped-app/resources/config/IPEDConfig.txt
@@ -92,10 +92,14 @@ enableIndexToElasticSearch = false
 # You must configure at least host/port options in conf/MinIOConfig.txt
 enableMinIO = false
 
+# Enables OCR on images and scanned PDF files. It can increase processing time a lot.
+# OCR language dictionary and other advanced configurations can be found in conf/OCRConfig.txt.
+enableOCR = false
+
 # Enable audio transcription.
-# Default implementation uses VOSK transcription [EXPERIMENTAL] on local CPU (slow and medium quality).
-# We include Vosk portable models for 'en' and 'pt-BR', the lang model is taken from 'locale' in LocalConfig.txt
-# You can change it from Vosk to Microsoft Azure or Google Cloud services (faster and better quality).
+# Default implementation uses VOSK transcription on local CPU (slow and not good accuracy).
+# You can change it to a local Facebook Wav2Vec2 implementation (slower on CPU, faster on GPU and good accuracy)
+# or remote Microsoft Azure or Google Cloud services (faster and good accuracy).
 # Configure it in conf/AudioTranscriptConfig.txt
 enableAudioTranscription = false
 

diff --git a/iped-app/resources/config/conf/AudioTranscriptConfig.txt b/iped-app/resources/config/conf/AudioTranscriptConfig.txt
@@ -45,7 +45,7 @@ language = auto
 convertCommand = mplayer -benchmark -vo null -vc null -srate 16000 -af format=s16le,channels=1 -ao pcm:fast:file=$OUTPUT $INPUT
 
 # Mime types or supertypes to process. If you want to add videos use ; as separator and update 'convertCommand'.
-mimesToProcess = audio/3gpp; audio/3gpp2; audio/vnd.3gpp.iufp; audio/x-aac; audio/x-aiff; audio/amr; audio/amr-wb; audio/amr-wb+; audio/mp4; audio/ogg; audio/vorbis; audio/x-oggflac; audio/x-oggpcm; audio/opus; audio/speex; audio/qcelp; audio/vnd.wave; audio/x-caf; audio/x-ms-wma; audio/x-opus+ogg
+mimesToProcess = audio/3gpp; audio/3gpp2; audio/vnd.3gpp.iufp; audio/x-aac; audio/x-aiff; audio/amr; audio/amr-wb; audio/amr-wb+; audio/mp4; audio/ogg; audio/vorbis; audio/x-oggflac; audio/x-oggpcm; audio/opus; audio/speex; audio/qcelp; audio/vnd.wave; audio/x-caf; audio/x-ms-wma; audio/x-opus+ogg; audio/ilbc
 
 # Skip known audios found in the hash lookup database.
 skipKnownFiles = true

diff --git a/iped-app/resources/config/conf/CarverConfig.xml b/iped-app/resources/config/conf/CarverConfig.xml
@@ -237,6 +237,17 @@
             <mediaType>application/x-ares-galaxy</mediaType>
         </carverType>
 
+        <carverType>
+            <name>SD</name>
+            <signatures>
+                <headerSignature>SDL?\00\00\00\FF</headerSignature>
+            </signatures>
+            <minLength>128</minLength>
+            <maxLength>65536</maxLength>
+            <mediaType>application/x-shareaza-download</mediaType>
+            <stopOnNextHeader>true</stopOnNextHeader>
+        </carverType>
+
         <!--carverType>
             <name>PLIST</name>
             <signatures>

diff --git a/iped-app/resources/config/conf/CategoriesConfig.json b/iped-app/resources/config/conf/CategoriesConfig.json
@@ -5,13 +5,13 @@
       {"name": "RTF Documents", "mimes": ["application/rtf"]},
       {"name": "PDF Documents", "mimes": ["application/pdf"]},
       {"name": "HTML Documents", "mimes": ["text/html", "application/xhtml+xml", "application/vnd.wap.xhtml+xml", "application/x-asp", "text/asp", "text/aspdotnet"]},
-      {"name": "Text Documents", "mimes": ["application/vnd.ms-wordml", "application/x-tika-ooxml-protected-docx", "application/vnd.apple.pages", "application/msword", "application/msword2", "application/msword5", "application/vnd.ms-word.document.macroenabled.12", "application/vnd.ms-word.template.macroenabled.12", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.wordprocessingml.template", "application/vnd.oasis.opendocument.text", "application/vnd.oasis.opendocument.text-master", "application/vnd.oasis.opendocument.text-template", "application/vnd.oasis.opendocument.text-web", "application/vnd.sun.xml.writer", "application/vnd.sun.xml.writer.global", "application/vnd.sun.xml.writer.template", "application/vnd.stardivision.writer", "application/vnd.oasis.opendocument.flat.text"]},
-      {"name": "Other Documents", "mimes": ["application/vnd.ms-xpsdocument", "application/vnd.visio", "application/vnd.ms-visio.drawing", "application/vnd.ms-project", "application/x-project", "application/vnd.ms-works", "application/x-quattro-pro", "application/sldworks", "application/x-tika-ooxml-protected", "application/onenote", "application/x-msmoney", "application/x-mspublisher", "application/x-ms-owner", "model/vnd.dwfx+xps", "application/vnd.ms-graph", "application/postscript"]}
+      {"name": "Text Documents", "mimes": ["application/vnd.ms-wordml", "application/x-tika-ooxml-protected-docx", "application/vnd.apple.pages", "application/vnd.apple.pages.13", "application/vnd.apple.pages.18", "application/msword", "application/msword2", "application/msword5", "application/vnd.ms-word.document.macroenabled.12", "application/vnd.ms-word.template.macroenabled.12", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.wordprocessingml.template", "application/vnd.oasis.opendocument.text", "application/vnd.oasis.opendocument.text-master", "application/vnd.oasis.opendocument.text-template", "application/vnd.oasis.opendocument.text-web", "application/vnd.sun.xml.writer", "application/vnd.sun.xml.writer.global", "application/vnd.sun.xml.writer.template", "application/vnd.stardivision.writer", "application/vnd.oasis.opendocument.flat.text"]},
+      {"name": "Other Documents", "mimes": ["application/vnd.ms-xpsdocument", "application/vnd.visio", "application/vnd.ms-visio.drawing", "application/vnd.ms-project", "application/x-project", "application/vnd.ms-works", "application/x-quattro-pro", "application/sldworks", "application/x-tika-ooxml-protected", "application/onenote", "application/x-msmoney", "application/x-mspublisher", "application/x-ms-owner", "model/vnd.dwfx+xps", "application/vnd.ms-graph", "application/postscript", "application/vnd.apple.iwork"]}
     ]},
     {"name": "XML Files", "mimes": ["application/xml", "application/rdf+xml", "application/rss+xml"]},
     {"name": "Georeferenced Files", "mimes": ["application/vnd.google-earth.kml+xml", "application/gpx"]},
-    {"name": "Spreadsheets", "mimes": ["application/vnd.ms-spreadsheetml", "application/x-tika-ooxml-protected-xlsx", "text/csv", "application/vnd.apple.numbers", "application/vnd.ms-excel", "application/vnd.ms-excel.addin.macroenabled.12", "application/vnd.ms-excel.sheet.macroenabled.12", "application/vnd.ms-excel.sheet.binary.macroenabled.12", "application/vnd.ms-excel.template.macroenabled.12", "application/x-tika-msworks-spreadsheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.template", "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.oasis.opendocument.spreadsheet-template", "application/vnd.sun.xml.calc", "application/vnd.sun.xml.calc.template", "application/vnd.stardivision.calc", "application/x-tika-old-excel", "application/vnd.oasis.opendocument.flat.spreadsheet"]},
-    {"name": "Presentations", "mimes": ["application/x-tika-ooxml-protected-pptx", "application/vnd.apple.keynote", "application/x-corelpresentations", "application/vnd.ms-powerpoint", "application/vnd.ms-powerpoint.addin.macroenabled.12", "application/vnd.ms-powerpoint.presentation.macroenabled.12", "application/vnd.ms-powerpoint.slide.macroenabled.12", "application/vnd.ms-powerpoint.slideshow.macroenabled.12", "application/vnd.ms-powerpoint.template.macroenabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.openxmlformats-officedocument.presentationml.slide", "application/vnd.openxmlformats-officedocument.presentationml.template", "application/vnd.openxmlformats-officedocument.presentationml.slideshow", "application/vnd.oasis.opendocument.presentation", "application/vnd.oasis.opendocument.presentation-template", "application/vnd.sun.xml.impress", "application/vnd.sun.xml.impress.template", "application/vnd.stardivision.impress", "application/vnd.oasis.opendocument.flat.presentation"]},
+    {"name": "Spreadsheets", "mimes": ["application/vnd.ms-spreadsheetml", "application/x-tika-ooxml-protected-xlsx", "text/csv", "application/vnd.apple.numbers", "application/vnd.apple.numbers.13", "application/vnd.apple.numbers.18", "application/vnd.ms-excel", "application/vnd.ms-excel.addin.macroenabled.12", "application/vnd.ms-excel.sheet.macroenabled.12", "application/vnd.ms-excel.sheet.binary.macroenabled.12", "application/vnd.ms-excel.template.macroenabled.12", "application/x-tika-msworks-spreadsheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.template", "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.oasis.opendocument.spreadsheet-template", "application/vnd.sun.xml.calc", "application/vnd.sun.xml.calc.template", "application/vnd.stardivision.calc", "application/x-tika-old-excel", "application/vnd.oasis.opendocument.flat.spreadsheet"]},
+    {"name": "Presentations", "mimes": ["application/x-tika-ooxml-protected-pptx", "application/vnd.apple.keynote", "application/vnd.apple.keynote.13", "application/vnd.apple.keynote.18", "application/x-corelpresentations", "application/vnd.ms-powerpoint", "application/vnd.ms-powerpoint.addin.macroenabled.12", "application/vnd.ms-powerpoint.presentation.macroenabled.12", "application/vnd.ms-powerpoint.slide.macroenabled.12", "application/vnd.ms-powerpoint.slideshow.macroenabled.12", "application/vnd.ms-powerpoint.template.macroenabled.12", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.openxmlformats-officedocument.presentationml.slide", "application/vnd.openxmlformats-officedocument.presentationml.template", "application/vnd.openxmlformats-officedocument.presentationml.slideshow", "application/vnd.oasis.opendocument.presentation", "application/vnd.oasis.opendocument.presentation-template", "application/vnd.sun.xml.impress", "application/vnd.sun.xml.impress.template", "application/vnd.stardivision.impress", "application/vnd.oasis.opendocument.flat.presentation"]},
     {"name": "OLE files", "mimes": ["application/x-tika-msoffice"]},
     {"name": "Links", "categories":[
       {"name": "Link files", "mimes": ["application/x-lnk"]},
@@ -74,7 +74,7 @@
       {"name": "GDrive File Entries", "mimes": ["application/x-gdrive-cloud-graph-registry", "application/x-gdrive-snapshot-registry"]}
     ]},
     {"name": "Databases", "mimes": ["application/x-edb", "application/x-edb-table", "application/irpf", "application/x-msaccess", "application/x-dbf", "application/vnd.oasis.opendocument.database", "application/x-sqlite3", "application/x-mysql-db", "application/x-berkeley-db", "application/x-mssql-data", "application/x-database-table"]},
-    {"name": "Compressed Archives", "mimes": ["application/x-tika-ooxml", "application/zlib", "application/applefile", "application/vnd.ms-tnef", "application/zip", "application/x-rar-compressed", "application/x-tar", "application/gzip", "application/x-gzip", "application/x-xz", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed", "application/x-arj", "application/x-gtar", "application/x-archive", "application/x-cpio", "application/x-tika-unix-dump", "application/x-snappy-framed", "application/x-snappy", "application/x-snappy-raw", "application/x-compress", "application/x-java-pack200", "application/x-lzma", "application/x-lz4", "application/x-lz4-block", "application/x-brotli", "application/zstd", "application/deflate64", "image/x-emf-compressed"]},
+    {"name": "Compressed Archives", "mimes": ["application/x-tika-ooxml", "application/zlib", "application/applefile", "application/vnd.ms-tnef", "application/zip", "application/x-rar-compressed", "application/x-tar", "application/gzip", "application/x-gzip", "application/x-xz", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed", "application/x-arj", "application/x-gtar", "application/x-archive", "application/x-cpio", "application/x-tika-unix-dump", "application/x-snappy-framed", "application/x-snappy", "application/x-snappy-raw", "application/x-compress", "application/x-java-pack200", "application/x-lzma", "application/x-lz4", "application/x-lz4-block", "application/x-brotli", "application/zstd", "application/deflate64", "image/x-emf-compressed", "application/x-lzfse"]},
     {"name": "Contacts", "mimes": ["text/x-vcard", "application/x-vcard-html", "application/windows-adress-book", "application/outlook-contact", "application/x-livecontacts", "application/x-livecontacts-table", "contact/x-skype-contact", "application/x-whatsapp-wadb", "application/x-whatsapp-contactsv2", "contact/x-whatsapp-contact", "application/x-ufed-html-contacts", "application/x-ufed-contact", "contact/x-telegram-contact", "application/x-ios-addressbook-db", "application/x-win10-mail-contact"]},
     {"name": "Chats", "categories":[
         {"name": "WhatsApp", "mimes":["application/x-whatsapp-db", "application/x-whatsapp-chatstorage", "application/x-whatsapp-chat","application/x-ufed-chat-whatsapp","application/x-ufed-chat-preview-whatsapp"]},
@@ -101,7 +101,7 @@
     {"name": "Peer-to-peer", "categories":[
       {"name": "Ares Galaxy", "mimes": ["application/x-ares-galaxy","application/x-ares-galaxy-entry"]},
       {"name": "E-Mule", "mimes": ["application/x-emule", "application/x-emule-part-met", "application/x-emule-searches", "application/x-emule-preferences-ini", "application/x-emule-preferences-dat", "application/x-emule-known-met-entry", "application/x-emule-part-met-entry"]},
-      {"name": "Shareaza", "mimes": ["application/x-shareaza-searches-dat", "application/x-shareaza-library-dat", "application/x-shareaza-library-dat-entry"]},
+      {"name": "Shareaza", "mimes": ["application/x-shareaza-searches-dat", "application/x-shareaza-library-dat", "application/x-shareaza-library-dat-entry", "application/x-shareaza-download"]},
       {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat", "application/x-bittorrent"]},
       {"name": "Other Peer-to-peer", "mimes": ["application/x-p2p"]}
     ]},

diff --git a/iped-app/resources/config/conf/CustomSignatures.xml b/iped-app/resources/config/conf/CustomSignatures.xml
@@ -413,7 +413,16 @@
     <mime-type type="application/x-shareaza-library-dat-entry">
         <sub-class-of type="application/x-metadata-entry"/>
     </mime-type>
-
+
+    <mime-type type="application/x-shareaza-download">
+        <_comment>Shareaza download control file (.sd)</_comment>
+        <sub-class-of type="application/x-p2p"/>
+    	<magic priority="60">
+            <match value="0x53444C00000000FF" type="string" offset="0" mask="0xFFFFFF00FFFFFFFF"/>
+     	</magic>
+        <glob pattern="*.sd"/>
+    </mime-type>
+
     <mime-type type="application/x-bittorrent-resume-dat">
         <_comment>BitTorrent Client Resume.dat file</_comment>
         <sub-class-of type="application/x-p2p"/>
@@ -1494,5 +1503,49 @@
     	<sub-class-of type="message/x-chat-message"/>
     </mime-type>
 
+    <mime-type type="video/x-matroska">
+        <magic priority="60">
+            <match value="0x1A45DFA3" type="string" offset="0">
+                <match value="matroska" type="string" offset="4:64">
+                </match>
+            </match>
+        </magic>
+        <glob pattern="*.mkv" />
+    </mime-type>
+
+    <mime-type type="audio/x-matroska">
+        <sub-class-of type="video/x-matroska" />
+        <glob pattern="*.mka" />
+    </mime-type>
+
+    <mime-type type="video/webm">
+        <magic priority="60">
+            <match value="0x1A45DFA3" type="string" offset="0">
+                <match value="webm" type="string" offset="4:64">
+                </match>
+            </match>
+        </magic>
+        <glob pattern="*.webm" />
+    </mime-type>
+
+    <mime-type type="audio/ilbc">
+        <magic priority="60">
+            <match value="0x2321694C424333300A" type="string" offset="0"/>
+            <match value="0x2321694C424332300A" type="string" offset="0"/>
+        </magic>
+        <glob pattern="*.lbc"/>
+    </mime-type>
+
+    <mime-type type="application/vnd.apple.unknown.13">
+        <sub-class-of type="application/vnd.apple.iwork"/>
+    </mime-type>
+
+    <mime-type type="application/x-lzfse">
+        <magic priority="50">
+            <match value="bvx" type="string" offset="0"/>
+        </magic>
+        <glob pattern="*.lzfse"/>
+    </mime-type>
+
 </mime-info>
 
diff --git a/iped-app/resources/config/conf/OCRConfig.txt b/iped-app/resources/config/conf/OCRConfig.txt
@@ -4,10 +4,6 @@
 
 # You can use the parameter -ocr "bookmark_name" to restrict the OCR to a specific bookmark (just works when creating reports from cmd line).
 
-# Enables OCR in scanned images and PDFs. It can strongly increase processing time.
-# Results depend on the quality and resolution of the images, and on the size and type of the fonts used.
-enableOCR = false
-
 # Dictionary language to be used for OCR. You can specify multiple languages e.g. por+eng but you will pay an important processing cost.
 # Languages supported: eng, por, ita, deu, spa
 OCRLanguage = por