Timestomp (#942)

* lib/analyzers: Create Timestomp Analyzer

Create naive Analyzer to detect timestomping on NTFS.

* lib/analyzers: Add implementation to timestomp.py

Add naive implementation and a basic test.

* lib/analyzers: Fix formatting for timestomp_test

* lib/analyzers: Fix formatting in timestomp.py

* lib/analyzers/timestomp: Fix vanishing timestamps

File_info s were created multiple times and
inserted into the dict. As a result
timestamps went missing.

* lib/analyzers/timestomp: Refactor $ add test-fame

* lib/analyzers/timestomp: Fix linter in test.

* lib/analyzers/timestomp: Add doc-strings.

* lib/analyzers: Add tests for timestomp analyzer.

* Refactor to match requested changes.

* Fix line length.

* lib/analyzers/ntfs_timestomp: Fix broken indentation.

data/timesketch.conf

@@ -179,6 +179,10 @@ DOMAIN_ANALYZER_WATCHED_DOMAINS_SCORE_THRESHOLD = 0.75
 # in the "phishy" domain comparison, mostly CDNs and similar.
 DOMAIN_ANALYZER_WHITELISTED_DOMAINS = ['ytimg.com', 'gstatic.com', 'yimg.com', 'akamaized.net', 'akamaihd.net', 's-microsoft.com', 'images-amazon.com', 'ssl-images-amazon.com', 'wikimedia.org', 'redditmedia.com', 'googleusercontent.com', 'googleapis.com', 'wikipedia.org', 'github.io', 'github.com']
 
+# The threshold in minutes which the difference in timestamps has to cross in order to be
+# detected as 'timestomping'.
+NTFS_TIMESTOMP_ANALYZER_THRESHOLD = 10
+
 #-------------------------------------------------------------------------------
 # Enable experimental UI features.
 

timesketch/lib/analyzers/__init__.py

@@ -27,4 +27,5 @@
 from timesketch.lib.analyzers import similarity_scorer
 from timesketch.lib.analyzers import ssh_sessionizer
 from timesketch.lib.analyzers import gcp_servicekey
+from timesketch.lib.analyzers import ntfs_timestomp
 from timesketch.lib.analyzers import yetiindicators

timesketch/lib/analyzers/ntfs_timestomp.py

@@ -0,0 +1,138 @@
+"""Sketch analyzer plugin for ntfs timestomping detection."""
+from __future__ import unicode_literals
+
+from flask import current_app
+
+from timesketch.lib.analyzers import interface
+from timesketch.lib.analyzers import manager
+
+class FileInfo(object):
+    """Datastructure to track all timestamps for a file and timestamp type."""
+    def __init__(self, file_reference=None, timestamp_desc=None,
+                 std_info_event=None, std_info_timestamp=None, file_names=None):
+        self.file_reference = file_reference
+        self.timestamp_desc = timestamp_desc
+        self.std_info_event = std_info_event
+        self.std_info_timestamp = std_info_timestamp
+        self.file_names = file_names or []
+
+class NtfsTimestompSketchPlugin(interface.BaseSketchAnalyzer):
+    """Sketch analyzer for Timestomp."""
+
+    NAME = 'ntfs_timestomp'
+    STD_INFO = 16
+    FILE_NAME = 48
+
+    def __init__(self, index_name, sketch_id):
+        """Initialize The Sketch Analyzer.
+
+        Args:
+            index_name: Elasticsearch index name
+            sketch_id: Sketch ID
+        """
+        self.index_name = index_name
+        self.threshold = current_app.config.get(
+            'NTFS_TIMESTOMP_ANALYZER_THRESHOLD', 10) * 60000000
+        super(NtfsTimestompSketchPlugin, self).__init__(index_name, sketch_id)
+
+    def is_suspicious(self, file_info):
+        """Compares timestamps and adds diffs to events if timestomping was
+        detected.
+
+        Args:
+            file_info: FileInfo object for event to look at.
+
+        Returns:
+            Boolean, true if timestomping was detected.
+
+        """
+        if not file_info.std_info_event or not file_info.file_names:
+            return False
+
+        suspicious = True
+        diffs = []
+
+        for fn in file_info.file_names:
+            diff = fn[1] - file_info.std_info_timestamp
+            diffs.append(diff)
+
+            if abs(diff) > self.threshold:
+                fn[0].add_attributes({'time_delta': diff})
+            else:
+                suspicious = False
+                break
+
+        if suspicious:
+            for fn in file_info.file_names:
+                fn[0].commit()
+
+            file_info.std_info_event.add_attributes({'time_deltas': diffs})
+            file_info.std_info_event.commit()
+
+        return suspicious
+
+    def run(self):
+        """Entry point for the analyzer.
+
+        Returns:
+            String with summary of the analyzer result.
+        """
+
+        query = 'attribute_type:48 OR attribute_type:16'
+
+        return_fields = ['attribute_type', 'timestamp_desc',
+                         'file_reference', 'timestamp']
+
+
+        # Generator of events based on your query.
+        events = self.event_stream(
+            query_string=query, return_fields=return_fields)
+
+        # Dict timstamp_type + "&" + file_ref -> FileInfo
+        file_infos = dict()
+
+        for event in events:
+            attribute_type = event.source.get('attribute_type')
+            file_ref = event.source.get('file_reference')
+            timestamp_type = event.source.get('timestamp_desc')
+            timestamp = event.source.get('timestamp')
+
+            if not attribute_type or not timestamp_type:
+                continue
+
+            if not attribute_type in [self.FILE_NAME, self.STD_INFO]:
+                continue
+
+            key = '{0:s}&{1:s}'.format(timestamp_type, str(file_ref))
+
+            if key not in file_infos:
+                file_infos[key] = FileInfo()
+
+            file_info = file_infos[key]
+            file_info.file_reference = file_ref
+            file_info.timestamp_desc = timestamp_type
+
+            if attribute_type == self.STD_INFO:
+                file_info.std_info_timestamp = timestamp
+                file_info.std_info_event = event
+
+            if attribute_type == self.FILE_NAME:
+                file_info.file_names.append((event, timestamp))
+
+        timestomps = 0
+        for file_info in file_infos.values():
+            if self.handle_timestomp(file_info):
+                timestomps = timestomps + 1
+
+
+        if timestomps > 0:
+            self.sketch.add_view(
+                view_name='NtfsTimestomp', analyzer_name=self.NAME,
+                query_string='_exists_:time_delta or _exists:time_deltas')
+
+
+        return ('NtfsTimestomp Analyzer done, found {0:d} timestomped events'
+                .format(timestomps))
+
+
+manager.AnalysisManager.register_analyzer(NtfsTimestompSketchPlugin)

timesketch/lib/analyzers/ntfs_timestomp_test.py

@@ -0,0 +1,118 @@
+"""Tests for NtfsTimestompPlugin."""
+from __future__ import unicode_literals
+
+import mock
+
+from timesketch.lib.analyzers import ntfs_timestomp
+from timesketch.lib.testlib import BaseTest
+from timesketch.lib.testlib import MockDataStore
+
+class MockEvent(object):
+    def __init__(self, source=None):
+        if source:
+            self.source = source
+        else:
+            self.source = {}
+        self.label = ""
+
+    def add_attributes(self, attributes):
+        self.source = dict(self.source, **attributes)
+
+    def add_label(self, label):
+        self.label = label
+
+    def commit(self):
+        pass
+
+class FileInfoTestCase(object):
+    def __init__(self, name, std_info_timestamp, fn_timestamps,
+                 expected_si_diffs, expected_fn_diffs, is_timestomp):
+        self.name = name
+        ref = 7357
+        ts_desc = "TEST"
+        std_event = MockEvent()
+        file_names = [(MockEvent(), ts) for ts in fn_timestamps]
+
+        self.file_info = ntfs_timestomp.FileInfo(ref, ts_desc, std_event,
+                                                 std_info_timestamp, file_names)
+        self.expected_fn_diffs = expected_fn_diffs
+        self.expected_si_diffs = expected_si_diffs
+
+        self.is_timestomp = is_timestomp
+
+class TestNtfsTimestompPlugin(BaseTest):
+    """Tests the functionality of the analyzer."""
+
+    #def __init__(self, *args, **kwargs):
+    #    super(TestNtfsTimestompPlugin, self).__init__(*args, **kwargs)
+
+    @mock.patch(
+        u'timesketch.lib.analyzers.interface.ElasticsearchDataStore',
+        MockDataStore)
+    def test_is_suspicious(self):
+        """Test is_suspicious method."""
+        analyzer = ntfs_timestomp.NtfsTimestompSketchPlugin('is_suspicious', 1)
+
+        test_cases = [
+            FileInfoTestCase(
+                "no timestomp",
+                1000000000000,
+                [1000000000000],
+                None,
+                [None],
+                False
+            ),
+            FileInfoTestCase(
+                "multiple file_names and all of them are timestomped",
+                0,
+                [6000000000, 7000000000, 8000000000],
+                [6000000000, 7000000000, 8000000000],
+                [6000000000, 7000000000, 8000000000],
+                True
+            ),
+            FileInfoTestCase(
+                "one of the file_names matches exactly",
+                0,
+                [0, 7000000000, 8000000000],
+                None,
+                [None, None, None],
+                False
+            ),
+            FileInfoTestCase(
+                "file_name is within threshold",
+                0,
+                [analyzer.threshold, 7000000000, 8000000000],
+                None,
+                [None, None, None],
+                False
+            ),
+            FileInfoTestCase(
+                "file_name is within threshold",
+                0,
+                [600000000, 7000000000, 8000000000],
+                None,
+                [None, None, None],
+                False
+            )
+        ]
+
+        for tc in test_cases:
+            ret = analyzer.is_suspicious(tc.file_info)
+
+            std_diffs = tc.file_info.std_info_event.source.get('time_deltas')
+            fn_diffs = [event.source.get('time_delta') for event, _ in
+                        tc.file_info.file_names]
+
+            self.assertEqual(ret, tc.is_timestomp)
+            self.assertEqual(std_diffs, tc.expected_si_diffs)
+            self.assertEqual(fn_diffs, tc.expected_fn_diffs)
+
+
+    # Mock the Elasticsearch datastore.
+    @mock.patch(
+        u'timesketch.lib.analyzers.interface.ElasticsearchDataStore',
+        MockDataStore)
+    def test_analyzer(self):
+        """Test analyzer."""
+        # TODO: Write actual tests here.
+        self.assertEqual(True, True)