new: Add MISPFeed oputput bot.

intelmq/bots/BOTS

@@ -1030,6 +1030,16 @@
                 "field": "source.ip"
             }
         },
+        "MISP Feed": {
+            "description": "Generate an output in MISP Feed format.",
+            "module": "intelmq.bots.outputs.mispfeed.output",
+            "parameters": {
+                "misp_org_name": "<org name creating the events>",
+                "misp_org_uuid": "<org uuid creating the events>",
+                "output_dir": "/opt/intelmq/var/lib/bots/mispfeed-output",
+                "interval_event": "{\"hours\": 1}"
+            }
+        },
         "MongoDB": {
             "description": "MongoDB is the bot responsible to send events to a MongoDB database.",
             "module": "intelmq.bots.outputs.mongodb.output",

intelmq/bots/outputs/mispfeed/REQUIREMENTS.txt

@@ -0,0 +1 @@
+pymisp>=2.4.117.3

intelmq/bots/outputs/mispfeed/output.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+import datetime
+import json
+from pathlib import Path
+from uuid import uuid4
+import re
+
+from intelmq.lib.bot import OutputBot
+
+from pymisp import MISPEvent, MISPOrganisation, NewAttributeError
+from pymisp.tools import feed_meta_generator
+
+
+# NOTE: This module is compatible with Python 3.6+
+
+
+class MISPFeedOutputBot(OutputBot):
+    is_multithreadable = False
+
+    def init(self):
+        self.current_event = None
+
+        self.misp_org = MISPOrganisation()
+        self.misp_org.name = self.parameters.misp_org_name
+        self.misp_org.uuid = self.parameters.misp_org_uuid
+
+        self.output_dir = Path(self.parameters.output_dir)
+
+        if not hasattr(self.parameters, 'interval_event'):
+            self.timedelta = datetime.timedelta(hours=1)
+        else:
+            self.timedelta = datetime.timedelta(**json.loads(self.parameters.interval_event))
+
+        if (self.output_dir / '.current').exists():
+            with (self.output_dir / '.current').open() as f:
+                self.current_file = Path(f.read())
+            self.current_event = MISPEvent()
+            self.current_event.load_file(self.current_file)
+
+            last_min_time, last_max_time = re.findall('IntelMQ event (.*) - (.*)', self.current_event.info)[0]
+            last_min_time = datetime.datetime.strptime(last_min_time, '%Y-%m-%dT%H:%M:%S.%f')
+            last_max_time = datetime.datetime.strptime(last_max_time, '%Y-%m-%dT%H:%M:%S.%f')
+            if last_max_time < datetime.datetime.now():
+                self.min_time_current = datetime.datetime.now()
+                self.max_time_current = self.min_time_current + self.timedelta
+                self.current_event = None
+            else:
+                self.min_time_current = last_min_time
+                self.max_time_current = last_max_time
+        else:
+            self.min_time_current = datetime.datetime.now()
+            self.max_time_current = self.min_time_current + self.timedelta
+
+    def process(self):
+
+        if not self.current_event or datetime.datetime.now() > self.max_time_current:
+            self.min_time_current = datetime.datetime.now()
+            self.max_time_current = self.min_time_current + self.timedelta
+            self.current_event = MISPEvent()
+            self.current_event.info = f'IntelMQ event {self.min_time_current.isoformat()} - {self.max_time_current.isoformat()}'
+            self.current_event.set_date(datetime.date.today())
+            self.current_event.Orgc = self.misp_org
+            self.current_event.uuid = str(uuid4())
+            self.current_file = self.output_dir / f'{self.current_event.uuid}.json'
+            with (self.output_dir / '.current').open('w') as f:
+                f.write(str(self.current_file))
+
+        event = self.receive_message().to_dict(jsondict_as_string=True)
+
+        obj = self.current_event.add_object(name='intelmq_event')
+        for object_relation, value in event.items():
+            try:
+                obj.add_attribute(object_relation, value=value)
+            except NewAttributeError:
+                # This entry isn't listed in teh harmonization file, ignoring.
+                pass
+
+        feed_output = self.current_event.to_feed(with_meta=False)
+
+        with self.current_file.open('w') as f:
+            json.dump(feed_output, f)
+
+        feed_meta_generator(self.output_dir)
+        self.acknowledge_message()
+
+    @staticmethod
+    def check(parameters):
+        if 'output_dir' not in parameters:
+            return [["error", "Parameter 'output_dir' not given."]]
+        output_dir = Path(parameters.output_dir)
+        try:
+            output_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
+        except IOError:
+            return [["error", "Directory (%r) of parameter 'file' does not exist and could not be created." % output_dir]]
+        else:
+            return [["info", "Directory (%r) of parameter 'file' did not exist, but has now been created." % output_dir]]
+
+
+BOT = MISPFeedOutputBot

intelmq/tests/bots/outputs/mispfeed/test_output.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import unittest
+
+import intelmq.lib.test as test
+from intelmq.bots.outputs.mispfeed.output import MISPFeedOutputBot
+
+EXAMPLE_EVENT = {"classification.type": "malware",
+                 "destination.port": 9796,
+                 "feed.accuracy": 100.0,
+                 "destination.ip": "52.18.196.169",
+                 "malware.name": "salityp2p",
+                 "event_description.text": "Sinkhole attempted connection",
+                 "time.source": "2016-04-19T23:16:08+00:00",
+                 "source.ip": "152.166.119.2",
+                 "feed.url": "http://alerts.bitsighttech.com:8080/stream?",
+                 "source.geolocation.country": "Dominican Republic",
+                 "time.observation": "2016-04-19T23:16:08+00:00",
+                 "source.port": 65118,
+                 "__type": "Event",
+                 "feed.name": "BitSight",
+                 "extra.non_ascii": "ççãããã\x80\ua000 \164 \x80\x80 abcd \165\166",
+                 "raw": "eyJ0cm9qYW5mYW1pbHkiOiJTYWxpdHlwMnAiLCJlbnYiOnsic"
+                 "mVtb3RlX2FkZHIiOiIxNTIuMTY2LjExOS4yIiwicmVtb3RlX3"
+                 "BvcnQiOiI2NTExOCIsInNlcnZlcl9hZGRyIjoiNTIuMTguMTk"
+                 "2LjE2OSIsInNlcnZlcl9wb3J0IjoiOTc5NiJ9LCJfdHMiOjE0"
+                 "NjExMDc3NjgsIl9nZW9fZW52X3JlbW90ZV9hZGRyIjp7ImNvd"
+                 "W50cnlfbmFtZSI6IkRvbWluaWNhbiBSZXB1YmxpYyJ9fQ==",
+                 "__type": "Event",
+                 }
+
+
+class TestMISPFeedOutputBot(test.BotTestCase, unittest.TestCase):
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = MISPFeedOutputBot
+        cls.default_input_message = EXAMPLE_EVENT
+        cls.sysconfig = {"misp_org_name": 'IntelMQTestOrg',
+                         "misp_org_uuid": "b89da4c2-0f74-11ea-96a1-6fa873a0eb4d",
+                         "output_dir": "/opt/intelmq/var/lib/bots/mispfeed-output/",
+                         "interval_event": '{"hours": 1}'}
+
+    def test_event(self):
+        self.run_bot()
+
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()