Bux fix #29 and deprecate 'resume' parameter #28.

#28 #29.  + Bug fix to clean_known_content

ConfigExamples/filter.yaml

@@ -3,12 +3,12 @@ collect:
     Audit.General: True
     Audit.AzureActiveDirectory: True
     Audit.SharePoint: True
-# Collect logs concerning spoofing prevention in Audit.General, deleted files from Audit.SharePoint
-# and login failures from Audit.AzureActiveDirectory
-filter:
-  Audit.General:
-    Policy: Spoof
-  Audit.AzureActiveDirectory:
-    Operation: UserLoginFailed
-  Audit.SharePoint:
-    Operation: FileDeleted
+  # Collect logs concerning spoofing prevention in Audit.General, deleted files from Audit.SharePoint
+  # and login failures from Audit.AzureActiveDirectory
+  filter:
+    Audit.General:
+      Policy: Spoof
+    Audit.AzureActiveDirectory:
+      Operation: UserLoginFailed
+    Audit.SharePoint:
+      Operation: FileDeleted

ConfigExamples/fullConfig.yaml

@@ -9,20 +9,20 @@ collect:  # Settings determining which audit logs to collect and how to do it
     Audit.SharePoint: True
     DLP.All: True
   rustEngine: True  # Use False to revert to the old Python engine. If running from python instead of executable, make sure to install the python wheel in the RustEngineWheels folder
-  schedule: 0 1 0  # How often to run in days/hours/minutes. Delete this line to just run once and exit.
+# schedule: 0 1 0  # How often to run in days/hours/minutes. Program will never exit and run on the schedule. Uncomment to use.
   maxThreads: 50  # Maximum number of simultaneous threads retrieving logs
   retries: 3  # Times to retry retrieving a content blob if it fails
   retryCooldown: 3  # Seconds to wait before retrying retrieving a content blob
   autoSubscribe: True  # Automatically subscribe to collected content types. Never unsubscribes from anything.
   skipKnownLogs: True  # Remember retrieved log ID's, don't collect them twice
-  resume: False  # Remember last run time, resume collecting from there next run
+  resume: False  # DEPRECATED, recommended to keep 'false'. Remember last run time, resume collecting from there next run
   hoursToCollect: 24  # Look back this many hours for audit logs (can be overwritten by resume)
-filter:  # Only logs that match ALL filters for a content type are collected. Leave empty to collect all
-  Audit.General:
-  Audit.AzureActiveDirectory:
-  Audit.Exchange:
-  Audit.SharePoint:
-  DLP.All:
+  filter:  # Only logs that match ALL filters for a content type are collected. Leave empty to collect all
+    Audit.General:
+    Audit.AzureActiveDirectory:
+    Audit.Exchange:
+    Audit.SharePoint:
+    DLP.All:
 output:
   file:  # CSV output
     enabled: False

Source/AuditLogCollector.py

@@ -56,6 +56,13 @@ def force_stop(self, *args):
         sys.exit(0)
 
     def run(self):
+
+        if self.config['collect', 'resume']:
+            logging.warning(
+                "WARNING: The 'resume' parameter is deprecated; it will be removed in a future version. It is known to "
+                "cause issues, since logs sometimes experience delay in being published to the APIs. It is recommended "
+                "to set 'resume' to false in your config. If you used 'resume' to prevent duplicate logs, set "
+                "'skipKnownLogs' to true instead.")
         if not self.config['collect', 'schedule']:
             self.run_once()
         else:
@@ -514,7 +521,7 @@ def _clean_known_content(self):
                     del known_content[content_id]
         if not known_content:
             return
-        with open('known_logs', 'w') as ofile:
+        with open('known_content', 'w') as ofile:
             for content_id, expire_date in known_content.items():
                 ofile.write("{},{}\n".format(content_id, expire_date))
 
@@ -547,7 +554,7 @@ def known_content(self):
                     if not line.strip():
                         continue
                     try:
-                        self._known_content[line.split(',')[0].strip()] = line.split(',')[1]
+                        self._known_content[line.split(',')[0].strip()] = line.split(',')[1].strip()
                     except:
                         continue
         return self._known_content