From bf8114b1c3ca5c65437323ddee82d4efc6071d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 25 Nov 2019 16:59:17 +0100 Subject: [PATCH 1/2] new: Add MISPFeed oputput bot. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Vinot --- intelmq/bots/BOTS | 10 ++ .../bots/outputs/mispfeed/REQUIREMENTS.txt | 1 + intelmq/bots/outputs/mispfeed/__init__.py | 0 intelmq/bots/outputs/mispfeed/output.py | 98 +++++++++++++++++++ .../tests/bots/outputs/mispfeed/__init__.py | 0 .../bots/outputs/mispfeed/test_output.py | 48 +++++++++ 6 files changed, 157 insertions(+) create mode 100644 intelmq/bots/outputs/mispfeed/REQUIREMENTS.txt create mode 100644 intelmq/bots/outputs/mispfeed/__init__.py create mode 100644 intelmq/bots/outputs/mispfeed/output.py create mode 100644 intelmq/tests/bots/outputs/mispfeed/__init__.py create mode 100644 intelmq/tests/bots/outputs/mispfeed/test_output.py diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index 5e8e782e8..a94db18a0 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -1030,6 +1030,16 @@ "field": "source.ip" } }, + "MISP Feed": { + "description": "Generate an output in MISP Feed format.", + "module": "intelmq.bots.outputs.mispfeed.output", + "parameters": { + "misp_org_name": "", + "misp_org_uuid": "", + "output_dir": "/opt/intelmq/var/lib/bots/mispfeed-output", + "interval_event": "{\"hours\": 1}" + } + }, "MongoDB": { "description": "MongoDB is the bot responsible to send events to a MongoDB database.", "module": "intelmq.bots.outputs.mongodb.output", diff --git a/intelmq/bots/outputs/mispfeed/REQUIREMENTS.txt b/intelmq/bots/outputs/mispfeed/REQUIREMENTS.txt new file mode 100644 index 000000000..36e2d50d8 --- /dev/null +++ b/intelmq/bots/outputs/mispfeed/REQUIREMENTS.txt @@ -0,0 +1 @@ +pymisp>=2.4.117.3 diff --git a/intelmq/bots/outputs/mispfeed/__init__.py b/intelmq/bots/outputs/mispfeed/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/outputs/mispfeed/output.py b/intelmq/bots/outputs/mispfeed/output.py new file mode 100644 index 000000000..6f0407a98 --- /dev/null +++ b/intelmq/bots/outputs/mispfeed/output.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +import datetime +import json +from pathlib import Path +from uuid import uuid4 +import re + +from intelmq.lib.bot import OutputBot + +from pymisp import MISPEvent, MISPOrganisation, NewAttributeError +from pymisp.tools import feed_meta_generator + + +# NOTE: This module is compatible with Python 3.6+ + + +class MISPFeedOutputBot(OutputBot): + is_multithreadable = False + + def init(self): + self.current_event = None + + self.misp_org = MISPOrganisation() + self.misp_org.name = self.parameters.misp_org_name + self.misp_org.uuid = self.parameters.misp_org_uuid + + self.output_dir = Path(self.parameters.output_dir) + + if not hasattr(self.parameters, 'interval_event'): + self.timedelta = datetime.timedelta(hours=1) + else: + self.timedelta = datetime.timedelta(**json.loads(self.parameters.interval_event)) + + if (self.output_dir / '.current').exists(): + with (self.output_dir / '.current').open() as f: + self.current_file = Path(f.read()) + self.current_event = MISPEvent() + self.current_event.load_file(self.current_file) + + last_min_time, last_max_time = re.findall('IntelMQ event (.*) - (.*)', self.current_event.info)[0] + last_min_time = datetime.datetime.strptime(last_min_time, '%Y-%m-%dT%H:%M:%S.%f') + last_max_time = datetime.datetime.strptime(last_max_time, '%Y-%m-%dT%H:%M:%S.%f') + if last_max_time < datetime.datetime.now(): + self.min_time_current = datetime.datetime.now() + self.max_time_current = self.min_time_current + self.timedelta + else: + self.min_time_current = last_min_time + self.max_time_current = last_max_time + else: + self.min_time_current = datetime.datetime.now() + self.max_time_current = self.min_time_current + self.timedelta + + def process(self): + + if not self.current_event or datetime.datetime.now() > self.max_time_current: + self.min_time_current = datetime.datetime.now() + self.max_time_current = self.min_time_current + self.timedelta + self.current_event = MISPEvent() + self.current_event.info = f'IntelMQ event {self.min_time_current.isoformat()} - {self.max_time_current.isoformat()}' + self.current_event.set_date(datetime.date.today()) + self.current_event.Orgc = self.misp_org + self.current_event.uuid = str(uuid4()) + self.current_file = self.output_dir / f'{self.current_event.uuid}.json' + with (self.output_dir / '.current').open('w') as f: + f.write(str(self.current_file)) + + event = self.receive_message().to_dict(jsondict_as_string=True) + + obj = self.current_event.add_object(name='intelmq_event') + for object_relation, value in event.items(): + try: + obj.add_attribute(object_relation, value=value) + except NewAttributeError: + # This entry isn't listed in teh harmonization file, ignoring. + pass + + feed_output = self.current_event.to_feed(with_meta=False) + + with self.current_file.open('w') as f: + json.dump(feed_output, f) + + feed_meta_generator(self.output_dir) + self.acknowledge_message() + + @staticmethod + def check(parameters): + if 'output_dir' not in parameters: + return [["error", "Parameter 'output_dir' not given."]] + output_dir = Path(parameters.output_dir) + try: + output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + except IOError: + return [["error", "Directory (%r) of parameter 'file' does not exist and could not be created." % output_dir]] + else: + return [["info", "Directory (%r) of parameter 'file' did not exist, but has now been created." % output_dir]] + + +BOT = MISPFeedOutputBot diff --git a/intelmq/tests/bots/outputs/mispfeed/__init__.py b/intelmq/tests/bots/outputs/mispfeed/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/tests/bots/outputs/mispfeed/test_output.py b/intelmq/tests/bots/outputs/mispfeed/test_output.py new file mode 100644 index 000000000..cc6469019 --- /dev/null +++ b/intelmq/tests/bots/outputs/mispfeed/test_output.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +import unittest + +import intelmq.lib.test as test +from intelmq.bots.outputs.mispfeed.output import MISPFeedOutputBot + +EXAMPLE_EVENT = {"classification.type": "malware", + "destination.port": 9796, + "feed.accuracy": 100.0, + "destination.ip": "52.18.196.169", + "malware.name": "salityp2p", + "event_description.text": "Sinkhole attempted connection", + "time.source": "2016-04-19T23:16:08+00:00", + "source.ip": "152.166.119.2", + "feed.url": "http://alerts.bitsighttech.com:8080/stream?", + "source.geolocation.country": "Dominican Republic", + "time.observation": "2016-04-19T23:16:08+00:00", + "source.port": 65118, + "__type": "Event", + "feed.name": "BitSight", + "extra.non_ascii": "ççãããã\x80\ua000 \164 \x80\x80 abcd \165\166", + "raw": "eyJ0cm9qYW5mYW1pbHkiOiJTYWxpdHlwMnAiLCJlbnYiOnsic" + "mVtb3RlX2FkZHIiOiIxNTIuMTY2LjExOS4yIiwicmVtb3RlX3" + "BvcnQiOiI2NTExOCIsInNlcnZlcl9hZGRyIjoiNTIuMTguMTk" + "2LjE2OSIsInNlcnZlcl9wb3J0IjoiOTc5NiJ9LCJfdHMiOjE0" + "NjExMDc3NjgsIl9nZW9fZW52X3JlbW90ZV9hZGRyIjp7ImNvd" + "W50cnlfbmFtZSI6IkRvbWluaWNhbiBSZXB1YmxpYyJ9fQ==", + "__type": "Event", + } + + +class TestMISPFeedOutputBot(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = MISPFeedOutputBot + cls.default_input_message = EXAMPLE_EVENT + cls.sysconfig = {"misp_org_name": 'IntelMQTestOrg', + "misp_org_uuid": "b89da4c2-0f74-11ea-96a1-6fa873a0eb4d", + "output_dir": "/opt/intelmq/var/lib/bots/mispfeed-output/", + "interval_event": '{"hours": 1}'} + + def test_event(self): + self.run_bot() + + +if __name__ == '__main__': # pragma: no cover + unittest.main() From 6ebffd7ac30efae38b922bd2246aed138e8178f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 25 Nov 2019 17:21:20 +0100 Subject: [PATCH 2/2] fix: Force create new event if courrent is too old MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Vinot --- intelmq/bots/outputs/mispfeed/output.py | 1 + 1 file changed, 1 insertion(+) diff --git a/intelmq/bots/outputs/mispfeed/output.py b/intelmq/bots/outputs/mispfeed/output.py index 6f0407a98..318734ae1 100644 --- a/intelmq/bots/outputs/mispfeed/output.py +++ b/intelmq/bots/outputs/mispfeed/output.py @@ -43,6 +43,7 @@ def init(self): if last_max_time < datetime.datetime.now(): self.min_time_current = datetime.datetime.now() self.max_time_current = self.min_time_current + self.timedelta + self.current_event = None else: self.min_time_current = last_min_time self.max_time_current = last_max_time