ENH: Add ability to copy more fields from the report

CHANGELOG.md

@@ -32,6 +32,8 @@
 - `intelmq.bots.parsers.shadowserver._config`:
   - Fetch schema before first run (PR#2482 by elsif2, fixes #2480).
 - `intelmq.bots.parsers.dataplane.parser`: Use `  |  ` as field delimiter, fix parsing of AS names including `|` (PR#2488 by DigitalTrustCenter).
+- all parsers: add `copy_collector_provided_fields` parameter allowing copying additional fields from the report, e.g. `extra.file_name`.
+  (PR#2513 by Kamil Mankowski).
 
 #### Experts
 - `intelmq.bots.experts.sieve.expert`:

docs/user/bots.md

@@ -1331,6 +1331,17 @@ tweet text is sent separately and if allowed, links to pastebin are followed and
 
 ## Parser Bots
 
+If not set differently during parsing, all parser bots copy the following fields from the report to an event:
+
+  - `feed.accuracy`
+  - `feed.code`
+  - `feed.documentation`
+  - `feed.name`
+  - `feed.provider`
+  - `feed.url`
+  - `rtir_id`
+  - `time.observation`
+
 ### Common parameters
 
 #### `default_fields`
@@ -1346,6 +1357,18 @@ defaults_fields:
   protocol.transport: tcp
 ```
 
+#### `copy_collector_provided_fields`
+
+(optional, list) List of additional fields to be copy from the report (only applied if parsing the
+event doesn't set the value).
+
+Example usage:
+
+```yaml
+copy_collector_provided_fields:
+  - extra.file_name
+```
+
 ---
 
 ### Abuse.ch Feodo Tracker <div id="intelmq.bots.parsers.abusech.parser_feodotracker" />

intelmq/lib/bot.py

@@ -1082,6 +1082,7 @@ class ParserBot(Bot):
     _default_message_type = 'Report'
 
     default_fields: Optional[dict] = {}
+    copy_collector_provided_fields: Optional[list] = []
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1127,6 +1128,11 @@ def _get_io_and_save_line_ending(self, raw: str) -> io.StringIO:
             self._line_ending = '\r\n'
         return data_io
 
+    def new_event(self, *args, **kwargs):
+        if self.copy_collector_provided_fields:
+            kwargs['copy_collector_provided_fields'] = self.copy_collector_provided_fields
+        return super().new_event(*args, **kwargs)
+
     def parse_csv(self, report: libmessage.Report):
         """
         A basic CSV parser.

intelmq/lib/message.py

@@ -98,7 +98,7 @@ class Message(dict):
     _default_value_set = False
 
     def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
-                 harmonization: dict = None) -> None:
+                 harmonization: dict = None, **_) -> None:
         try:
             classname = message['__type'].lower()
             del message['__type']
@@ -522,9 +522,13 @@ def __contains__(self, item: str) -> bool:
 
 
 class Event(Message):
-
-    def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
-                 harmonization: Optional[dict] = None) -> None:
+    def __init__(
+        self,
+        message: Union[dict, tuple] = (),
+        auto: bool = False,
+        harmonization: Optional[dict] = None,
+        copy_collector_provided_fields: Optional[dict] = None,
+    ) -> None:
         """
         Parameters:
             message: Give a report and feed.name, feed.url and
@@ -551,6 +555,12 @@ def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
                 template['rtir_id'] = message['rtir_id']
             if 'time.observation' in message:
                 template['time.observation'] = message['time.observation']
+
+            if copy_collector_provided_fields:
+                for key in copy_collector_provided_fields:
+                    if key not in message:
+                        continue
+                    template[key] = message.get(key)
         else:
             template = message
         super().__init__(template, auto, harmonization)
@@ -559,7 +569,7 @@ def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
 class Report(Message):
 
     def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
-                 harmonization: Optional[dict] = None) -> None:
+                 harmonization: Optional[dict] = None, **_) -> None:
         """
         Parameters:
             message: Passed along to Message's and dict's init.

intelmq/tests/lib/test_parser_bot.py

@@ -167,6 +167,19 @@ def test_bad_default_fields_parameter_2(self):
         self.assertAnyLoglineEqual(message="Invalid value of key 'source.port' in default_fields parameter.",
                                    levelname="ERROR")
 
+    def test_copy_collector_provided_fields_from_report(self):
+        """Allow copying custom fields from the report message to support more context from reports"""
+        report = {**EXAMPLE_SHORT, "extra.file_name": "file.txt", "extra.field2": "value2"}
+        self.input_message = report
+
+        self.run_bot(parameters={"copy_collector_provided_fields":
+                                 ["extra.file_name", "extra.not_exists"]})
+
+        output_message = EXAMPLE_EVENT.copy()
+        output_message["extra.file_name"] = "file.txt"
+        self.assertMessageEqual(0, output_message)
+
+
     def test_missing_raw(self):
         """ Test DummyParserBot with missing raw. """
         self.input_message = EXAMPLE_EMPTY_REPORT