From 849c8e7861696be0622240831f2d45bbe4a07d1a Mon Sep 17 00:00:00 2001 From: cmandich Date: Fri, 13 Dec 2024 08:14:48 -0800 Subject: [PATCH 1/4] Fix vuln import. --- .../src/shadowserver/stix_transform.py | 57 ++++--- .../tests/fixtures/report_type_blocklist.json | 152 +++++++++--------- .../shadowserver/tests/test_stix_transform.py | 6 + 3 files changed, 115 insertions(+), 100 deletions(-) diff --git a/external-import/shadowserver/src/shadowserver/stix_transform.py b/external-import/shadowserver/src/shadowserver/stix_transform.py index 111723abf9..dd307a50a3 100644 --- a/external-import/shadowserver/src/shadowserver/stix_transform.py +++ b/external-import/shadowserver/src/shadowserver/stix_transform.py @@ -10,6 +10,7 @@ from pycti import ObservedData as pycti_observed_data from pycti import OpenCTIConnectorHelper from pycti import Report as pycti_report +from pycti import Vulnerability as pycti_vulnerability from stix2 import ( Artifact, AutonomousSystem, @@ -531,12 +532,14 @@ def create_vulnerability(self, name: str): "object_marking_refs": self.marking_refs, } - opencti_obj = Vulnerability(id=Vulnerability.generate_id(name), **kwargs) + opencti_obj = Vulnerability(id=pycti_vulnerability.generate_id(name), **kwargs) if opencti_obj.get("id"): self.object_refs.append(opencti_obj.get("id")) self.stix_objects.append(opencti_obj) + return opencti_obj.get('id') + def create_asn(self, value: int, labels: list = []): """Creates an autonomous system STIX object.""" self.helper.connector_logger.debug(f"Creating ASN STIX object: {value}") @@ -652,9 +655,10 @@ def create_network_traffic( dst_ref: str = "", labels: list = [], ): + stix_object_id = str() description = [] if not protocol: - return None + return stix_object_id """Creates a network traffic STIX object.""" if labels is None: @@ -706,32 +710,32 @@ def create_network_traffic( self.helper.connector_logger.error( f"STIX object with ID {kwargs['id']} already exists. Aborting creation." ) - return None - - # Add description to custom properties - description_str = f"Shadowserver Network Traffic: {', '.join(description)}" - self.extend_stix_object(kwargs, labels) - if "custom_properties" not in kwargs: - kwargs["custom_properties"] = {} - kwargs["custom_properties"].update({"x_opencti_description": description_str}) + else: + # Add description to custom properties + description_str = f"Shadowserver Network Traffic: {', '.join(description)}" + self.extend_stix_object(kwargs, labels) + if "custom_properties" not in kwargs: + kwargs["custom_properties"] = {} + kwargs["custom_properties"].update({"x_opencti_description": description_str}) - stix_object = NetworkTraffic(**kwargs) + stix_object = NetworkTraffic(**kwargs) - if stix_object: - self.helper.connector_logger.debug( - f"Created network traffic STIX object: {stix_object.id}" - ) - self.object_refs.append(stix_object.id) - self.stix_objects.append(stix_object) - return stix_object.id - else: - self.helper.connector_logger.error( - f"Failed to create network traffic STIX object with ID {stix_object.id}" - ) - return None + if stix_object: + self.helper.connector_logger.debug( + f"Created network traffic STIX object: {stix_object.id}" + ) + stix_object_id = stix_object.get('id', str()) + self.object_refs.append(stix_object.id) + self.stix_objects.append(stix_object) + else: + self.helper.connector_logger.error( + f"Failed to create network traffic STIX object with ID {stix_object.id}" + ) + return stix_object_id def create_x509_certificate(self, data: dict, labels: list = []): """Creates an X509 certificate STIX object.""" + stix_object_id = str() self.helper.connector_logger.debug( f"Creating X509 certificate STIX object: {data}" ) @@ -770,7 +774,12 @@ def create_x509_certificate(self, data: dict, labels: list = []): ) self.object_refs.append(stix_object.get("id")) self.stix_objects.append(stix_object) - return stix_object.get("id") + stix_object_id = stix_object.get("id", str()) + else: + self.helper.connector_logger.error( + f"Failed to create X509 certificate STIX object: {data}" + ) + return stix_object_id def create_observed_data( self, diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json index c2d90b45d3..1b9135faa3 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json @@ -1,77 +1,77 @@ [ - { - "asn": 1234, - "city": "NORTHAMPTON", - "geo": "UK", - "hostname": "", - "ip": "192.168.0.1", - "naics": 517312, - "reason": "not-specified", - "region": "WEST NORTHAMPTONSHIRE", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "source": "random-source-1", - "tag": "", - "timestamp": "2024-01-04 08:04:37Z" - }, - { - "asn": 5678, - "city": "ROCKFORD", - "geo": "US", - "hostname": "", - "ip": "10.0.0.1", - "naics": 517311, - "reason": "not-specified", - "region": "ILLINOIS", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "source": "random-source-2", - "tag": "", - "timestamp": "2024-01-04 08:04:37Z" - }, - { - "asn": 9012, - "city": "WINNIPEG", - "geo": "CA", - "hostname": "", - "ip": "172.16.0.1", - "naics": 517311, - "reason": "not-specified", - "region": "MANITOBA", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "source": "random-source-3", - "tag": "", - "timestamp": "2024-01-04 08:04:37Z" - }, - { - "asn": 3456, - "city": "PENDER ISLAND", - "geo": "CA", - "hostname": "", - "ip": "192.168.1.1", - "naics": 517311, - "reason": "not-specified", - "region": "BRITISH COLUMBIA", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "source": "random-source-4", - "tag": "", - "timestamp": "2024-01-04 08:04:37Z" - }, - { - "asn": 7890, - "city": "MONTICELLO", - "geo": "US", - "hostname": "", - "ip": "192.168.2.1", - "naics": 517311, - "reason": "not-specified", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "source": "random-source-5", - "tag": "", - "timestamp": "2024-01-04 08:04:37Z" - } -] \ No newline at end of file + { + "asn": 1234, + "city": "NORTHAMPTON", + "geo": "UK", + "hostname": "", + "ip": "192.168.0.1", + "naics": 517312, + "reason": "not-specified", + "region": "WEST NORTHAMPTONSHIRE", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "source": "random-source-1", + "tag": "", + "timestamp": "2024-01-04 08:04:37Z" + }, + { + "asn": 5678, + "city": "ROCKFORD", + "geo": "US", + "hostname": "", + "ip": "10.0.0.1", + "naics": 517311, + "reason": "not-specified", + "region": "ILLINOIS", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "source": "random-source-2", + "tag": "", + "timestamp": "2024-01-04 08:04:37Z" + }, + { + "asn": 9012, + "city": "WINNIPEG", + "geo": "CA", + "hostname": "", + "ip": "172.16.0.1", + "naics": 517311, + "reason": "not-specified", + "region": "MANITOBA", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "source": "random-source-3", + "tag": "", + "timestamp": "2024-01-04 08:04:37Z" + }, + { + "asn": 3456, + "city": "PENDER ISLAND", + "geo": "CA", + "hostname": "", + "ip": "192.168.1.1", + "naics": 517311, + "reason": "not-specified", + "region": "BRITISH COLUMBIA", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "source": "random-source-4", + "tag": "", + "timestamp": "2024-01-04 08:04:37Z" + }, + { + "asn": 7890, + "city": "MONTICELLO", + "geo": "US", + "hostname": "", + "ip": "192.168.2.1", + "naics": 517311, + "reason": "not-specified", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "source": "random-source-5", + "tag": "cve-2024-38063", + "timestamp": "2024-01-04 08:04:37Z" + } +] diff --git a/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py b/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py index aa2fe4a541..b7e7a48add 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py @@ -151,6 +151,12 @@ def test_create_x509_certificate(self): self.assertTrue(isinstance(certificate, str)) self.assertTrue(certificate.startswith("x509-certificate--")) + def test_create_vulnerability(self): + name = "CVE-2022-1234" + vulnerability = self.transformation.create_vulnerability(name) + self.assertTrue(isinstance(vulnerability, str)) + self.assertTrue(vulnerability.startswith("vulnerability--")) + def test_create_opencti_case(self): self.transformation.create_opencti_case() self.assertTrue( From 7196b06b15ecd11a035f47b1dc26b500833202e3 Mon Sep 17 00:00:00 2001 From: cmandich Date: Fri, 13 Dec 2024 08:51:33 -0800 Subject: [PATCH 2/4] Add additional tests to validate stix transform. --- .../src/shadowserver/stix_transform.py | 10 +- .../tests/fixtures/report_type_blocklist.json | 2 +- .../tests/fixtures/report_type_device_id.json | 802 +++++++++--------- .../tests/fixtures/report_type_scan_snmp.json | 64 +- .../tests/fixtures/report_type_scan_ssl.json | 158 ++-- .../fixtures/report_type_scan_ssl_freak.json | 170 ++-- .../fixtures/report_type_scan_ssl_poodle.json | 170 ++-- .../src/shadowserver/tests/test_api.py | 34 +- .../shadowserver/tests/test_stix_transform.py | 5 +- .../src/shadowserver/tests/test_utils.py | 3 +- 10 files changed, 727 insertions(+), 691 deletions(-) diff --git a/external-import/shadowserver/src/shadowserver/stix_transform.py b/external-import/shadowserver/src/shadowserver/stix_transform.py index dd307a50a3..301a1726a3 100644 --- a/external-import/shadowserver/src/shadowserver/stix_transform.py +++ b/external-import/shadowserver/src/shadowserver/stix_transform.py @@ -498,7 +498,7 @@ def get_custom_labels(self, element): A list of custom labels extracted from the element. """ labels_list = [] - if element.get("tag") and ";" in element.get("tag"): + if element.get("tag"): custom_labels = element.get("tag").split(";") for label in custom_labels: if label.upper().startswith("CVE"): @@ -538,7 +538,7 @@ def create_vulnerability(self, name: str): self.object_refs.append(opencti_obj.get("id")) self.stix_objects.append(opencti_obj) - return opencti_obj.get('id') + return opencti_obj.get("id") def create_asn(self, value: int, labels: list = []): """Creates an autonomous system STIX object.""" @@ -716,7 +716,9 @@ def create_network_traffic( self.extend_stix_object(kwargs, labels) if "custom_properties" not in kwargs: kwargs["custom_properties"] = {} - kwargs["custom_properties"].update({"x_opencti_description": description_str}) + kwargs["custom_properties"].update( + {"x_opencti_description": description_str} + ) stix_object = NetworkTraffic(**kwargs) @@ -724,7 +726,7 @@ def create_network_traffic( self.helper.connector_logger.debug( f"Created network traffic STIX object: {stix_object.id}" ) - stix_object_id = stix_object.get('id', str()) + stix_object_id = stix_object.get("id", str()) self.object_refs.append(stix_object.id) self.stix_objects.append(stix_object) else: diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json index 1b9135faa3..38e74ac2c8 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_blocklist.json @@ -71,7 +71,7 @@ "sector": "Communications, Service Provider, and Hosting Service", "severity": "low", "source": "random-source-5", - "tag": "cve-2024-38063", + "tag": "CVE-2024-38063", "timestamp": "2024-01-04 08:04:37Z" } ] diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_device_id.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_device_id.json index 3172d42d10..fdd38abcf7 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_device_id.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_device_id.json @@ -1,402 +1,402 @@ [ - { - "asn": 12345, - "city": "ALBANY", - "device_model": "", - "device_type": "video-system", - "device_vendor": "Hikvision", - "device_version": "", - "geo": "US", - "hostname": "example1.com", - "hostname_source": "ptr", - "ip": "192.168.0.1", - "naics": 517311, - "port": 8080, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "http;iot", - "timestamp": "2024-01-04 00:43:10Z" - }, - { - "asn": 54321, - "city": "SARATOGA SPRINGS", - "device_model": "", - "device_type": "", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example2.com", - "hostname_source": "certificate", - "ip": "192.168.0.2", - "naics": 517311, - "port": 10443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 02:26:33Z" - }, - { - "asn": 98765, - "city": "SARATOGA SPRINGS", - "device_model": "FortiOS SSL-VPN", - "device_type": "vpn", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example3.com", - "hostname_source": "ptr", - "ip": "192.168.0.3", - "naics": 517311, - "port": 10443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "fortinet", - "timestamp": "2024-01-04 02:27:16Z" - }, - { - "asn": 24680, - "city": "PITTSFORD", - "device_model": "", - "device_type": "firewall", - "device_vendor": "SonicWall", - "device_version": "", - "geo": "US", - "hostname": "example4.com", - "hostname_source": "ptr", - "ip": "192.168.0.4", - "naics": 517311, - "port": 4433, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 02:33:19Z" - }, - { - "asn": 13579, - "city": "PITTSFORD", - "device_model": "", - "device_type": "firewall", - "device_vendor": "SonicWall", - "device_version": "", - "geo": "US", - "hostname": "example5.com", - "hostname_source": "ptr", - "ip": "192.168.0.5", - "naics": 517311, - "port": 80, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "http;vpn", - "timestamp": "2024-01-04 02:47:19Z" - }, - { - "asn": 98765, - "city": "NORMAL", - "device_model": "", - "device_type": "vpn", - "device_vendor": "Cisco", - "device_version": "", - "geo": "US", - "hostname": "example6.com", - "hostname_source": "certificate", - "ip": "192.168.0.6", - "naics": 336111, - "port": 443, - "protocol": "tcp", - "region": "ILLINOIS", - "sector": "", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 04:18:04Z" - }, - { - "asn": 24680, - "city": "PITTSFORD", - "device_model": "", - "device_type": "firewall", - "device_vendor": "SonicWall", - "device_version": "", - "geo": "US", - "hostname": "example7.com", - "hostname_source": "ptr", - "ip": "192.168.0.7", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 04:18:57Z" - }, - { - "asn": 86420, - "city": "HEALDSBURG", - "device_model": "", - "device_type": "vpn", - "device_vendor": "Cisco", - "device_version": "", - "geo": "US", - "hostname": "example8.com", - "hostname_source": "certificate", - "ip": "192.168.0.8", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "CALIFORNIA", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 07:06:27Z" - }, - { - "asn": 24680, - "city": "PITTSFORD", - "device_model": "", - "device_type": "firewall", - "device_vendor": "SonicWall", - "device_version": "", - "geo": "US", - "hostname": "example9.com", - "hostname_source": "ptr", - "ip": "192.168.0.9", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 09:53:33Z" - }, - { - "asn": 98765, - "city": "SCHENECTADY", - "device_model": "ZyWALL USG 50", - "device_type": "firewall", - "device_vendor": "Zyxel", - "device_version": "", - "geo": "US", - "hostname": "example10.com", - "hostname_source": "ptr", - "ip": "192.168.0.10", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "tag": "ssl;ssl-freak;vpn", - "timestamp": "2024-01-04 10:06:01Z" - }, - { - "asn": 13579, - "city": "GILROY", - "device_model": "FortiGate", - "device_type": "firewall", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example11.com", - "hostname_source": "", - "ip": "192.168.0.11", - "naics": 518210, - "port": 8443, - "protocol": "tcp", - "region": "CALIFORNIA", - "sector": "", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 12:04:05Z" - }, - { - "asn": 13579, - "city": "GILROY", - "device_model": "FortiGate", - "device_type": "firewall", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example12.com", - "hostname_source": "", - "ip": "192.168.0.12", - "naics": 518210, - "port": 8443, - "protocol": "tcp", - "region": "CALIFORNIA", - "sector": "", - "severity": "info", - "tag": "http-redirect;vpn", - "timestamp": "2024-01-04 12:06:21Z" - }, - { - "asn": 24680, - "city": "AUSTIN", - "device_model": "", - "device_type": "vpn", - "device_vendor": "Cisco", - "device_version": "", - "geo": "US", - "hostname": "example13.com", - "hostname_source": "certificate", - "ip": "192.168.0.13", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "TEXAS", - "sector": "", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 12:45:22Z" - }, - { - "asn": 13579, - "city": "GILROY", - "device_model": "FortiGate", - "device_type": "firewall", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example14.com", - "hostname_source": "", - "ip": "192.168.0.14", - "naics": 518210, - "port": 443, - "protocol": "tcp", - "region": "CALIFORNIA", - "sector": "", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 12:45:46Z" - }, - { - "asn": 98765, - "city": "SARATOGA SPRINGS", - "device_model": "FortiGate", - "device_type": "firewall", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example15.com", - "hostname_source": "ptr", - "ip": "192.168.0.15", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 12:45:56Z" - }, - { - "asn": 98765, - "city": "SARATOGA SPRINGS", - "device_model": "ProSafe", - "device_type": "firewall", - "device_vendor": "NETGEAR", - "device_version": "", - "geo": "US", - "hostname": "example16.com", - "hostname_source": "ptr", - "ip": "192.168.0.16", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "low", - "tag": "iot;ssl;ssl-poodle;vpn", - "timestamp": "2024-01-04 12:46:09Z" - }, - { - "asn": 24680, - "city": "NORMAL", - "device_model": "", - "device_type": "vpn", - "device_vendor": "Cisco", - "device_version": "", - "geo": "US", - "hostname": "example17.com", - "hostname_source": "certificate", - "ip": "192.168.0.17", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "ILLINOIS", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 12:46:14Z" - }, - { - "asn": 98765, - "city": "SARATOGA SPRINGS", - "device_model": "FortiOS SSL-VPN", - "device_type": "vpn", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example18.com", - "hostname_source": "ptr", - "ip": "192.168.0.18", - "naics": 517311, - "port": 443, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "fortinet", - "timestamp": "2024-01-04 12:55:44Z" - }, - { - "asn": 13579, - "city": "GILROY", - "device_model": "FortiOS SSL-VPN", - "device_type": "vpn", - "device_vendor": "Fortinet", - "device_version": "", - "geo": "US", - "hostname": "example19.com", - "hostname_source": "", - "ip": "192.168.0.19", - "naics": 518210, - "port": 443, - "protocol": "tcp", - "region": "CALIFORNIA", - "sector": "", - "severity": "info", - "tag": "fortinet", - "timestamp": "2024-01-04 12:56:17Z" - }, - { - "asn": 24680, - "city": "PITTSFORD", - "device_model": "", - "device_type": "firewall", - "device_vendor": "SonicWall", - "device_version": "", - "geo": "US", - "hostname": "example20.com", - "hostname_source": "ptr", - "ip": "192.168.0.20", - "naics": 517311, - "port": 4433, - "protocol": "tcp", - "region": "NEW YORK", - "sector": "Communications, Service Provider, and Hosting Service", - "severity": "info", - "tag": "ssl;vpn", - "timestamp": "2024-01-04 14:02:23Z" - } -] \ No newline at end of file + { + "asn": 12345, + "city": "ALBANY", + "device_model": "", + "device_type": "video-system", + "device_vendor": "Hikvision", + "device_version": "", + "geo": "US", + "hostname": "example1.com", + "hostname_source": "ptr", + "ip": "192.168.0.1", + "naics": 517311, + "port": 8080, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "http;iot;CVE-2024-38063", + "timestamp": "2024-01-04 00:43:10Z" + }, + { + "asn": 54321, + "city": "SARATOGA SPRINGS", + "device_model": "", + "device_type": "", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example2.com", + "hostname_source": "certificate", + "ip": "192.168.0.2", + "naics": 517311, + "port": 10443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 02:26:33Z" + }, + { + "asn": 98765, + "city": "SARATOGA SPRINGS", + "device_model": "FortiOS SSL-VPN", + "device_type": "vpn", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example3.com", + "hostname_source": "ptr", + "ip": "192.168.0.3", + "naics": 517311, + "port": 10443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "fortinet", + "timestamp": "2024-01-04 02:27:16Z" + }, + { + "asn": 24680, + "city": "PITTSFORD", + "device_model": "", + "device_type": "firewall", + "device_vendor": "SonicWall", + "device_version": "", + "geo": "US", + "hostname": "example4.com", + "hostname_source": "ptr", + "ip": "192.168.0.4", + "naics": 517311, + "port": 4433, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 02:33:19Z" + }, + { + "asn": 13579, + "city": "PITTSFORD", + "device_model": "", + "device_type": "firewall", + "device_vendor": "SonicWall", + "device_version": "", + "geo": "US", + "hostname": "example5.com", + "hostname_source": "ptr", + "ip": "192.168.0.5", + "naics": 517311, + "port": 80, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "http;vpn", + "timestamp": "2024-01-04 02:47:19Z" + }, + { + "asn": 98765, + "city": "NORMAL", + "device_model": "", + "device_type": "vpn", + "device_vendor": "Cisco", + "device_version": "", + "geo": "US", + "hostname": "example6.com", + "hostname_source": "certificate", + "ip": "192.168.0.6", + "naics": 336111, + "port": 443, + "protocol": "tcp", + "region": "ILLINOIS", + "sector": "", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 04:18:04Z" + }, + { + "asn": 24680, + "city": "PITTSFORD", + "device_model": "", + "device_type": "firewall", + "device_vendor": "SonicWall", + "device_version": "", + "geo": "US", + "hostname": "example7.com", + "hostname_source": "ptr", + "ip": "192.168.0.7", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 04:18:57Z" + }, + { + "asn": 86420, + "city": "HEALDSBURG", + "device_model": "", + "device_type": "vpn", + "device_vendor": "Cisco", + "device_version": "", + "geo": "US", + "hostname": "example8.com", + "hostname_source": "certificate", + "ip": "192.168.0.8", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "CALIFORNIA", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 07:06:27Z" + }, + { + "asn": 24680, + "city": "PITTSFORD", + "device_model": "", + "device_type": "firewall", + "device_vendor": "SonicWall", + "device_version": "", + "geo": "US", + "hostname": "example9.com", + "hostname_source": "ptr", + "ip": "192.168.0.9", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 09:53:33Z" + }, + { + "asn": 98765, + "city": "SCHENECTADY", + "device_model": "ZyWALL USG 50", + "device_type": "firewall", + "device_vendor": "Zyxel", + "device_version": "", + "geo": "US", + "hostname": "example10.com", + "hostname_source": "ptr", + "ip": "192.168.0.10", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "tag": "ssl;ssl-freak;vpn", + "timestamp": "2024-01-04 10:06:01Z" + }, + { + "asn": 13579, + "city": "GILROY", + "device_model": "FortiGate", + "device_type": "firewall", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example11.com", + "hostname_source": "", + "ip": "192.168.0.11", + "naics": 518210, + "port": 8443, + "protocol": "tcp", + "region": "CALIFORNIA", + "sector": "", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 12:04:05Z" + }, + { + "asn": 13579, + "city": "GILROY", + "device_model": "FortiGate", + "device_type": "firewall", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example12.com", + "hostname_source": "", + "ip": "192.168.0.12", + "naics": 518210, + "port": 8443, + "protocol": "tcp", + "region": "CALIFORNIA", + "sector": "", + "severity": "info", + "tag": "http-redirect;vpn", + "timestamp": "2024-01-04 12:06:21Z" + }, + { + "asn": 24680, + "city": "AUSTIN", + "device_model": "", + "device_type": "vpn", + "device_vendor": "Cisco", + "device_version": "", + "geo": "US", + "hostname": "example13.com", + "hostname_source": "certificate", + "ip": "192.168.0.13", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "TEXAS", + "sector": "", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 12:45:22Z" + }, + { + "asn": 13579, + "city": "GILROY", + "device_model": "FortiGate", + "device_type": "firewall", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example14.com", + "hostname_source": "", + "ip": "192.168.0.14", + "naics": 518210, + "port": 443, + "protocol": "tcp", + "region": "CALIFORNIA", + "sector": "", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 12:45:46Z" + }, + { + "asn": 98765, + "city": "SARATOGA SPRINGS", + "device_model": "FortiGate", + "device_type": "firewall", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example15.com", + "hostname_source": "ptr", + "ip": "192.168.0.15", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 12:45:56Z" + }, + { + "asn": 98765, + "city": "SARATOGA SPRINGS", + "device_model": "ProSafe", + "device_type": "firewall", + "device_vendor": "NETGEAR", + "device_version": "", + "geo": "US", + "hostname": "example16.com", + "hostname_source": "ptr", + "ip": "192.168.0.16", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "low", + "tag": "iot;ssl;ssl-poodle;vpn", + "timestamp": "2024-01-04 12:46:09Z" + }, + { + "asn": 24680, + "city": "NORMAL", + "device_model": "", + "device_type": "vpn", + "device_vendor": "Cisco", + "device_version": "", + "geo": "US", + "hostname": "example17.com", + "hostname_source": "certificate", + "ip": "192.168.0.17", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "ILLINOIS", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 12:46:14Z" + }, + { + "asn": 98765, + "city": "SARATOGA SPRINGS", + "device_model": "FortiOS SSL-VPN", + "device_type": "vpn", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example18.com", + "hostname_source": "ptr", + "ip": "192.168.0.18", + "naics": 517311, + "port": 443, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "fortinet", + "timestamp": "2024-01-04 12:55:44Z" + }, + { + "asn": 13579, + "city": "GILROY", + "device_model": "FortiOS SSL-VPN", + "device_type": "vpn", + "device_vendor": "Fortinet", + "device_version": "", + "geo": "US", + "hostname": "example19.com", + "hostname_source": "", + "ip": "192.168.0.19", + "naics": 518210, + "port": 443, + "protocol": "tcp", + "region": "CALIFORNIA", + "sector": "", + "severity": "info", + "tag": "fortinet", + "timestamp": "2024-01-04 12:56:17Z" + }, + { + "asn": 24680, + "city": "PITTSFORD", + "device_model": "", + "device_type": "firewall", + "device_vendor": "SonicWall", + "device_version": "", + "geo": "US", + "hostname": "example20.com", + "hostname_source": "ptr", + "ip": "192.168.0.20", + "naics": 517311, + "port": 4433, + "protocol": "tcp", + "region": "NEW YORK", + "sector": "Communications, Service Provider, and Hosting Service", + "severity": "info", + "tag": "ssl;vpn", + "timestamp": "2024-01-04 14:02:23Z" + } +] diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_snmp.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_snmp.json index 9cee9af1d0..cb4d2d9760 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_snmp.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_snmp.json @@ -1,33 +1,33 @@ [ - { - "amplification": "1.23", - "asn": 12345, - "city": "Sample City", - "community": "private", - "device_model": "Sample Model", - "device_sector": "Sample Sector", - "device_type": "Sample Type", - "device_vendor": "Sample Vendor", - "device_version": "1.0", - "geo": "US", - "hostname": "sample-host", - "hostname_source": "DNS", - "ip": "192.168.1.1", - "mac_address": "00:11:22:33:44:55", - "naics": 54321, - "port": 1234, - "protocol": "tcp", - "region": "Sample Region", - "response_size": 123, - "sector": "Sample Sector", - "severity": "low", - "sysdesc": "Sample Description", - "sysname": "Sample Name", - "tag": "sample", - "timestamp": "2022-01-01 12:34:56Z", - "uptime": 123456789, - "vendor": "Sample Vendor", - "vendor_id": 123, - "version": 3 - } -] \ No newline at end of file + { + "amplification": "1.23", + "asn": 12345, + "city": "Sample City", + "community": "private", + "device_model": "Sample Model", + "device_sector": "Sample Sector", + "device_type": "Sample Type", + "device_vendor": "Sample Vendor", + "device_version": "1.0", + "geo": "US", + "hostname": "sample-host", + "hostname_source": "DNS", + "ip": "192.168.1.1", + "mac_address": "00:11:22:33:44:55", + "naics": 54321, + "port": 1234, + "protocol": "tcp", + "region": "Sample Region", + "response_size": 123, + "sector": "Sample Sector", + "severity": "low", + "sysdesc": "Sample Description", + "sysname": "Sample Name", + "tag": "sample", + "timestamp": "2022-01-01 12:34:56Z", + "uptime": 123456789, + "vendor": "Sample Vendor", + "vendor_id": 123, + "version": 3 + } +] diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl.json index 2b28a88415..bc5b43aa1f 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl.json @@ -1,80 +1,80 @@ [ - { - "asn": 12345, - "browser_error": "Error message", - "browser_trusted": "Y", - "cert_expiration_date": "2022-12-31 23:59:59", - "cert_expired": "N", - "cert_issue_date": "2022-01-01 00:00:00", - "cert_length": 4096, - "cert_serial_number": "ABCDEF1234567890", - "cert_valid": "Y", - "cipher_suite": "TLS_AES_128_GCM_SHA256", - "city": "Sample City", - "content_length": 256, - "content_type": "application/json", - "freak_cipher_suite": "TLS_RSA_WITH_AES_256_CBC_SHA", - "freak_vulnerable": "Y", - "geo": "US", - "handshake": "TLSv1.2", - "hostname": "example.com", - "hostname_source": "DNS", - "http_code": 404, - "http_connection": "keep-alive", - "http_date": "2022-01-01 12:34:56", - "http_reason": "Not Found", - "http_response_type": "HTTP/2", - "ip": "192.168.0.1", - "issuer_business_category": "Category", - "issuer_common_name": "Issuer Common Name", - "issuer_country": "US", - "issuer_email_address": "issuer@example.com", - "issuer_given_name": "Issuer Given Name", - "issuer_locality_name": "Issuer Locality", - "issuer_organization_name": "Issuer Organization", - "issuer_organization_unit_name": "Issuer Unit", - "issuer_postal_code": "12345", - "issuer_serial_number": "1234567890", - "issuer_state_or_province_name": "Issuer State", - "issuer_street_address": "Issuer Street", - "issuer_surname": "Issuer Surname", - "jarm": "JARM value", - "key_algorithm": "ecdsaEncryption", - "md5_fingerprint": "MD5 fingerprint", - "naics": 98765, - "port": 443, - "protocol": "https", - "region": "Sample Region", - "sector": "Sample Sector", - "self_signed": "N", - "server_type": "Sample Server Type", - "set_cookie": "cookie=value", - "severity": "high", - "sha1_fingerprint": "SHA-1 fingerprint", - "sha256_fingerprint": "SHA-256 fingerprint", - "sha512_fingerprint": "SHA-512 fingerprint", - "signature_algorithm": "sha256WithECDSA", - "ssl_poodle": "N", - "ssl_version": 3, - "subject_business_category": "Category", - "subject_common_name": "Subject Common Name", - "subject_country": "US", - "subject_email_address": "subject@example.com", - "subject_given_name": "Subject Given Name", - "subject_locality_name": "Subject Locality", - "subject_organization_name": "Subject Organization", - "subject_organization_unit_name": "Subject Unit", - "subject_postal_code": "54321", - "subject_serial_number": "0987654321", - "subject_state_or_province_name": "Subject State", - "subject_street_address": "Subject Street", - "subject_surname": "Subject Surname", - "tag": "ssl;sample", - "timestamp": "2022-01-01 12:34:55Z", - "tlsv13_cipher": "TLS_AES_128_GCM_SHA256", - "tlsv13_support": "Y", - "transfer_encoding": "gzip", - "validation_level": "EV", - "www_authenticate": "Basic realm=\"Sample Realm\"" - } -] \ No newline at end of file + { + "asn": 12345, + "browser_error": "Error message", + "browser_trusted": "Y", + "cert_expiration_date": "2022-12-31 23:59:59", + "cert_expired": "N", + "cert_issue_date": "2022-01-01 00:00:00", + "cert_length": 4096, + "cert_serial_number": "ABCDEF1234567890", + "cert_valid": "Y", + "cipher_suite": "TLS_AES_128_GCM_SHA256", + "city": "Sample City", + "content_length": 256, + "content_type": "application/json", + "freak_cipher_suite": "TLS_RSA_WITH_AES_256_CBC_SHA", + "freak_vulnerable": "Y", + "geo": "US", + "handshake": "TLSv1.2", + "hostname": "example.com", + "hostname_source": "DNS", + "http_code": 404, + "http_connection": "keep-alive", + "http_date": "2022-01-01 12:34:56", + "http_reason": "Not Found", + "http_response_type": "HTTP/2", + "ip": "192.168.0.1", + "issuer_business_category": "Category", + "issuer_common_name": "Issuer Common Name", + "issuer_country": "US", + "issuer_email_address": "issuer@example.com", + "issuer_given_name": "Issuer Given Name", + "issuer_locality_name": "Issuer Locality", + "issuer_organization_name": "Issuer Organization", + "issuer_organization_unit_name": "Issuer Unit", + "issuer_postal_code": "12345", + "issuer_serial_number": "1234567890", + "issuer_state_or_province_name": "Issuer State", + "issuer_street_address": "Issuer Street", + "issuer_surname": "Issuer Surname", + "jarm": "JARM value", + "key_algorithm": "ecdsaEncryption", + "md5_fingerprint": "912ec803b2ce49e4a541068d495ab570", + "naics": 98765, + "port": 443, + "protocol": "https", + "region": "Sample Region", + "sector": "Sample Sector", + "self_signed": "N", + "server_type": "Sample Server Type", + "set_cookie": "cookie=value", + "severity": "high", + "sha1_fingerprint": "3f1e2d2c1b0a9e8d7c6b5a4f3e2d1c0b9a8e7d6c", + "sha256_fingerprint": "4e3d2c1b0a9e8d7c6b5a4f3e2d1c0b9a8e7d6c5b4a3d2c1b0a9e8d7c6b5a4f3e", + "sha512_fingerprint": "401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1", + "signature_algorithm": "sha256WithECDSA", + "ssl_poodle": "N", + "ssl_version": 3, + "subject_business_category": "Category", + "subject_common_name": "Subject Common Name", + "subject_country": "US", + "subject_email_address": "subject@example.com", + "subject_given_name": "Subject Given Name", + "subject_locality_name": "Subject Locality", + "subject_organization_name": "Subject Organization", + "subject_organization_unit_name": "Subject Unit", + "subject_postal_code": "54321", + "subject_serial_number": "0987654321", + "subject_state_or_province_name": "Subject State", + "subject_street_address": "Subject Street", + "subject_surname": "Subject Surname", + "tag": "ssl;sample", + "timestamp": "2022-01-01 12:34:55Z", + "tlsv13_cipher": "TLS_AES_128_GCM_SHA256", + "tlsv13_support": "Y", + "transfer_encoding": "gzip", + "validation_level": "EV", + "www_authenticate": "Basic realm=\"Sample Realm\"" + } +] diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_freak.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_freak.json index 524c210a3c..561ed2f88f 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_freak.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_freak.json @@ -1,86 +1,86 @@ [ - { - "asn": 12345, - "browser_error": "Error message", - "browser_trusted": "Y", - "cert_expiration_date": "2023-12-31 23:59:59", - "cert_expired": "N", - "cert_issue_date": "2020-01-01 00:00:00", - "cert_length": 2048, - "cert_serial_number": 987654321, - "cert_valid": "Y", - "cipher_suite": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "city": "Sample City", - "content_length": "1024", - "content_type": "application/json", - "device_model": "Sample Model", - "device_sector": "Sample Sector", - "device_type": "Sample Type", - "device_vendor": "Sample Vendor", - "device_version": "1.0", - "freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "freak_vulnerable": "N", - "geo": "US", - "handshake": "TLSv1.2", - "hostname": "sample.domain.com", - "hostname_source": "dns", - "http_code": 200, - "http_connection": "keep-alive", - "http_date": "2022-01-01 12:00:00", - "http_reason": "OK", - "http_response_type": "HTTP/2", - "ip": "192.168.0.1", - "issuer_business_category": "Sample Business Category", - "issuer_common_name": "Sample Issuer", - "issuer_country": "US", - "issuer_email_address": "sample@example.com", - "issuer_given_name": "John", - "issuer_locality_name": "Sample Locality", - "issuer_organization_name": "Sample Organization", - "issuer_organization_unit_name": "Sample Unit", - "issuer_postal_code": "12345", - "issuer_serial_number": "ABC123", - "issuer_state_or_province_name": "Sample State", - "issuer_street_address": "Sample Street", - "issuer_surname": "Doe", - "jarm": "0123456789abcdef", - "key_algorithm": "rsaEncryption", - "md5_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", - "naics": 54321, - "page_sha256fp": "abcdef0123456789", - "port": 8443, - "protocol": "tcp", - "raw_cert": "-----BEGIN CERTIFICATE-----\nSample Certificate\n-----END CERTIFICATE-----", - "raw_cert_chain": "-----BEGIN CERTIFICATE-----\nSample Certificate Chain\n-----END CERTIFICATE-----", - "region": "Sample Region", - "sector": "Sample Sector", - "self_signed": "N", - "server_type": "Sample Server Type", - "set_cookie": "sample_cookie=1234567890", - "severity": "medium", - "sha1_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67", - "sha256_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", - "sha512_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", - "signature_algorithm": "sha256WithRSAEncryption", - "subject_business_category": "Sample Business Category", - "subject_common_name": "Sample Subject", - "subject_country": "US", - "subject_email_address": "sample@example.com", - "subject_given_name": "Jane", - "subject_locality_name": "Sample Locality", - "subject_organization_name": "Sample Organization", - "subject_organization_unit_name": "Sample Unit", - "subject_postal_code": "54321", - "subject_serial_number": "XYZ789", - "subject_state_or_province_name": "Sample State", - "subject_street_address": "Sample Street", - "subject_surname": "Doe", - "tag": "sample;data;json", - "timestamp": "2022-01-01 12:00:00Z", - "tlsv13_cipher": "TLS_AES_128_GCM_SHA256", - "tlsv13_support": "Y", - "transfer_encoding": "gzip", - "validation_level": "high", - "www_authenticate": "Bearer token" - } -] \ No newline at end of file + { + "asn": 12345, + "browser_error": "Error message", + "browser_trusted": "Y", + "cert_expiration_date": "2023-12-31 23:59:59", + "cert_expired": "N", + "cert_issue_date": "2020-01-01 00:00:00", + "cert_length": 2048, + "cert_serial_number": 987654321, + "cert_valid": "Y", + "cipher_suite": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "city": "Sample City", + "content_length": "1024", + "content_type": "application/json", + "device_model": "Sample Model", + "device_sector": "Sample Sector", + "device_type": "Sample Type", + "device_vendor": "Sample Vendor", + "device_version": "1.0", + "freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", + "freak_vulnerable": "N", + "geo": "US", + "handshake": "TLSv1.2", + "hostname": "sample.domain.com", + "hostname_source": "dns", + "http_code": 200, + "http_connection": "keep-alive", + "http_date": "2022-01-01 12:00:00", + "http_reason": "OK", + "http_response_type": "HTTP/2", + "ip": "192.168.0.1", + "issuer_business_category": "Sample Business Category", + "issuer_common_name": "Sample Issuer", + "issuer_country": "US", + "issuer_email_address": "sample@example.com", + "issuer_given_name": "John", + "issuer_locality_name": "Sample Locality", + "issuer_organization_name": "Sample Organization", + "issuer_organization_unit_name": "Sample Unit", + "issuer_postal_code": "12345", + "issuer_serial_number": "ABC123", + "issuer_state_or_province_name": "Sample State", + "issuer_street_address": "Sample Street", + "issuer_surname": "Doe", + "jarm": "0123456789abcdef", + "key_algorithm": "rsaEncryption", + "md5_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", + "naics": 54321, + "page_sha256fp": "abcdef0123456789", + "port": 8443, + "protocol": "tcp", + "raw_cert": "-----BEGIN CERTIFICATE-----\nSample Certificate\n-----END CERTIFICATE-----", + "raw_cert_chain": "-----BEGIN CERTIFICATE-----\nSample Certificate Chain\n-----END CERTIFICATE-----", + "region": "Sample Region", + "sector": "Sample Sector", + "self_signed": "N", + "server_type": "Sample Server Type", + "set_cookie": "sample_cookie=1234567890", + "severity": "medium", + "sha1_fingerprint": "3f1e2d2c1b0a9e8d7c6b5a4f3e2d1c0b9a8e7d6c", + "sha256_fingerprint": "4e3d2c1b0a9e8d7c6b5a4f3e2d1c0b9a8e7d6c5b4a3d2c1b0a9e8d7c6b5a4f3e", + "sha512_fingerprint": "401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1", + "signature_algorithm": "sha256WithRSAEncryption", + "subject_business_category": "Sample Business Category", + "subject_common_name": "Sample Subject", + "subject_country": "US", + "subject_email_address": "sample@example.com", + "subject_given_name": "Jane", + "subject_locality_name": "Sample Locality", + "subject_organization_name": "Sample Organization", + "subject_organization_unit_name": "Sample Unit", + "subject_postal_code": "54321", + "subject_serial_number": "XYZ789", + "subject_state_or_province_name": "Sample State", + "subject_street_address": "Sample Street", + "subject_surname": "Doe", + "tag": "sample;data;json", + "timestamp": "2022-01-01 12:00:00Z", + "tlsv13_cipher": "TLS_AES_128_GCM_SHA256", + "tlsv13_support": "Y", + "transfer_encoding": "gzip", + "validation_level": "high", + "www_authenticate": "Bearer token" + } +] diff --git a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_poodle.json b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_poodle.json index 805a5132e7..39317dca76 100644 --- a/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_poodle.json +++ b/external-import/shadowserver/src/shadowserver/tests/fixtures/report_type_scan_ssl_poodle.json @@ -1,86 +1,86 @@ [ - { - "asn": 12345, - "browser_error": "Error message", - "browser_trusted": "Y", - "cert_expiration_date": "2024-12-31 23:59:59", - "cert_expired": "N", - "cert_issue_date": "2020-01-01 00:00:00", - "cert_length": 2048, - "cert_serial_number": "ABC123", - "cert_valid": "Y", - "cipher_suite": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "city": "Sample City", - "content_length": 256, - "content_type": "application/json", - "device_model": "Sample Model", - "device_sector": "Sample Sector", - "device_type": "Sample Type", - "device_vendor": "Sample Vendor", - "device_version": "1.0", - "geo": "US", - "handshake": "TLSv1.2", - "hostname": "sample.domain.com", - "hostname_source": "dns", - "http_code": 200, - "http_connection": "keep-alive", - "http_date": "2022-01-01 12:00:00", - "http_reason": "OK", - "http_response_type": "HTTP/1.1", - "ip": "192.168.0.1", - "issuer_business_category": "Sample Business Category", - "issuer_common_name": "Sample Issuer", - "issuer_country": "US", - "issuer_email_address": "sample@example.com", - "issuer_given_name": "John", - "issuer_locality_name": "Sample Locality", - "issuer_organization_name": "Sample Organization", - "issuer_organization_unit_name": "Sample Unit", - "issuer_postal_code": "12345", - "issuer_serial_number": "XYZ789", - "issuer_state_or_province_name": "Sample State", - "issuer_street_address": "Sample Street", - "issuer_surname": "Doe", - "jarm": "0123456789abcdef", - "key_algorithm": "rsaEncryption", - "md5_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", - "naics": 54321, - "page_sha256fp": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "port": 8443, - "protocol": "tcp", - "raw_cert": "-----BEGIN CERTIFICATE-----\nSample Certificate\n-----END CERTIFICATE-----", - "raw_cert_chain": "-----BEGIN CERTIFICATE-----\nSample Certificate Chain\n-----END CERTIFICATE-----", - "region": "Sample Region", - "sector": "Sample Sector", - "self_signed": "N", - "server_type": "Sample Server Type", - "set_cookie": "sample_cookie=1234567890", - "severity": "medium", - "sha1_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67", - "sha256_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", - "sha512_fingerprint": "01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef:01:23:45:67:89:ab:cd:ef", - "signature_algorithm": "sha256WithRSAEncryption", - "ssl_poodle": "N", - "ssl_version": 3, - "subject_business_category": "Sample Business Category", - "subject_common_name": "Sample Subject", - "subject_country": "US", - "subject_email_address": "sample@example.com", - "subject_given_name": "Jane", - "subject_locality_name": "Sample Locality", - "subject_organization_name": "Sample Organization", - "subject_organization_unit_name": "Sample Unit", - "subject_postal_code": "54321", - "subject_serial_number": "XYZ789", - "subject_state_or_province_name": "Sample State", - "subject_street_address": "Sample Street", - "subject_surname": "Doe", - "tag": "sample;data;json", - "timestamp": "2022-01-01 12:00:00Z", - "tlsv13_cipher": "TLS_AES_128_GCM_SHA256", - "tlsv13_support": "Y", - "transfer_encoding": "chunked", - "validation_level": "high", - "www_authenticate": "Bearer token" - } -] \ No newline at end of file + { + "asn": 12345, + "browser_error": "Error message", + "browser_trusted": "Y", + "cert_expiration_date": "2024-12-31 23:59:59", + "cert_expired": "N", + "cert_issue_date": "2020-01-01 00:00:00", + "cert_length": 2048, + "cert_serial_number": "ABC123", + "cert_valid": "Y", + "cipher_suite": "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "city": "Sample City", + "content_length": 256, + "content_type": "application/json", + "device_model": "Sample Model", + "device_sector": "Sample Sector", + "device_type": "Sample Type", + "device_vendor": "Sample Vendor", + "device_version": "1.0", + "geo": "US", + "handshake": "TLSv1.2", + "hostname": "sample.domain.com", + "hostname_source": "dns", + "http_code": 200, + "http_connection": "keep-alive", + "http_date": "2022-01-01 12:00:00", + "http_reason": "OK", + "http_response_type": "HTTP/1.1", + "ip": "192.168.0.1", + "issuer_business_category": "Sample Business Category", + "issuer_common_name": "Sample Issuer", + "issuer_country": "US", + "issuer_email_address": "sample@example.com", + "issuer_given_name": "John", + "issuer_locality_name": "Sample Locality", + "issuer_organization_name": "Sample Organization", + "issuer_organization_unit_name": "Sample Unit", + "issuer_postal_code": "12345", + "issuer_serial_number": "XYZ789", + "issuer_state_or_province_name": "Sample State", + "issuer_street_address": "Sample Street", + "issuer_surname": "Doe", + "jarm": "0123456789abcdef", + "key_algorithm": "rsaEncryption", + "md5_fingerprint": "05c12a287334386c94131ab8aa00d08a", + "naics": 54321, + "page_sha256fp": "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b", + "port": 8443, + "protocol": "tcp", + "raw_cert": "-----BEGIN CERTIFICATE-----\nSample Certificate\n-----END CERTIFICATE-----", + "raw_cert_chain": "-----BEGIN CERTIFICATE-----\nSample Certificate Chain\n-----END CERTIFICATE-----", + "region": "Sample Region", + "sector": "Sample Sector", + "self_signed": "N", + "server_type": "Sample Server Type", + "set_cookie": "sample_cookie=1234567890", + "severity": "medium", + "sha1_fingerprint": "74dc916419a178d22cb0fc8a04f62d345784ad7d", + "sha256_fingerprint": "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b", + "sha512_fingerprint": "401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1", + "signature_algorithm": "sha256WithRSAEncryption", + "ssl_poodle": "N", + "ssl_version": 3, + "subject_business_category": "Sample Business Category", + "subject_common_name": "Sample Subject", + "subject_country": "US", + "subject_email_address": "sample@example.com", + "subject_given_name": "Jane", + "subject_locality_name": "Sample Locality", + "subject_organization_name": "Sample Organization", + "subject_organization_unit_name": "Sample Unit", + "subject_postal_code": "54321", + "subject_serial_number": "XYZ789", + "subject_state_or_province_name": "Sample State", + "subject_street_address": "Sample Street", + "subject_surname": "Doe", + "tag": "sample;data;json", + "timestamp": "2022-01-01 12:00:00Z", + "tlsv13_cipher": "TLS_AES_128_GCM_SHA256", + "tlsv13_support": "Y", + "transfer_encoding": "chunked", + "validation_level": "high", + "www_authenticate": "Bearer token" + } +] diff --git a/external-import/shadowserver/src/shadowserver/tests/test_api.py b/external-import/shadowserver/src/shadowserver/tests/test_api.py index 1e55ec6443..4de37604e2 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_api.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_api.py @@ -1,10 +1,15 @@ import json import os +from unittest.mock import MagicMock import pytest import requests +from pycti import OpenCTIConnectorHelper + from shadowserver.api import ShadowserverAPI +FIXTURES_DIR = "fixtures" + class TestShadowserverAPI: api_key = "test_api_key" @@ -30,7 +35,7 @@ def load_fixture(self, filename): FileNotFoundError: If the fixture file does not exist. ValueError: If the fixture file is empty. """ - filepath = os.path.join(os.path.dirname(__file__), "fixtures", filename) + filepath = os.path.join(os.path.dirname(__file__), FIXTURES_DIR, filename) if not os.path.exists(filepath): raise FileNotFoundError(f"Fixture {filename} not found.") @@ -120,3 +125,30 @@ def test_get_report_type_invalid(self, shadow_server_api, mocker): shadow_server_api.get_report( report_id="test_report_id", report="invalid_report" ) + + def test_all_fixture_types(self, shadow_server_api, mocker): + + self.api_helper = MagicMock(spec=OpenCTIConnectorHelper) + self.api_helper.connector_logger = MagicMock() + self.api_helper.connector_logger.debug = MagicMock() + self.api_helper.connector_logger.info = MagicMock() + self.api_helper.connector_logger.error = MagicMock() + + filepath = os.path.join(os.path.dirname(__file__), FIXTURES_DIR) + report_files = [f for f in os.listdir(filepath) if f.startswith("report_type")] + for report_file in report_files: + report_type = report_file.replace("report_type_", "").replace(".json", "") + + self.shadowserver_fixture( + report_file, + fixture_mocker=mocker, + shadow_server_api=shadow_server_api, + ) + + report = {"id": "test_report_id", "report": report_type} + + stix_reports = shadow_server_api.get_stix_report( + report=report, api_helper=self.api_helper + ) + + assert len(stix_reports) > 0 diff --git a/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py b/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py index b7e7a48add..6f228af909 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py @@ -3,8 +3,6 @@ from unittest.mock import MagicMock, patch from pycti import CustomObjectCaseIncident, OpenCTIConnectorHelper -from shadowserver.stix_transform import ShadowserverStixTransformation -from shadowserver.utils import datetime_to_string from stix2 import ( Artifact, DomainName, @@ -15,6 +13,9 @@ MarkingDefinition, ) +from shadowserver.stix_transform import ShadowserverStixTransformation +from shadowserver.utils import datetime_to_string + class TestShadowserverStixTransformation(unittest.TestCase): def setUp(self): diff --git a/external-import/shadowserver/src/shadowserver/tests/test_utils.py b/external-import/shadowserver/src/shadowserver/tests/test_utils.py index 6272a3b17d..3a546b7939 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_utils.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_utils.py @@ -1,6 +1,8 @@ import unittest from datetime import datetime +from stix2 import DomainName, IPv4Address + from shadowserver.utils import ( check_ip_address, clean_dict, @@ -15,7 +17,6 @@ validate_date_format, validate_marking_refs, ) -from stix2 import DomainName, IPv4Address class TestValidateDateFormat(unittest.TestCase): From 5b65e36c6d4a3d321529379076a1e621e78c0b58 Mon Sep 17 00:00:00 2001 From: cmandich Date: Fri, 13 Dec 2024 08:59:43 -0800 Subject: [PATCH 3/4] Fix isort. --- external-import/shadowserver/src/main.py | 1 + .../shadowserver/src/shadowserver/tests/test_api.py | 1 - .../src/shadowserver/tests/test_stix_transform.py | 5 ++--- .../shadowserver/src/shadowserver/tests/test_utils.py | 3 +-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/external-import/shadowserver/src/main.py b/external-import/shadowserver/src/main.py index 6a3c695da8..def7af9904 100644 --- a/external-import/shadowserver/src/main.py +++ b/external-import/shadowserver/src/main.py @@ -5,6 +5,7 @@ from datetime import UTC, datetime, timedelta from lib.external_import import ExternalImportConnector + from shadowserver import ShadowserverAPI, get_tlp_keys, remove_duplicates # Lookback in days diff --git a/external-import/shadowserver/src/shadowserver/tests/test_api.py b/external-import/shadowserver/src/shadowserver/tests/test_api.py index 4de37604e2..19b67d95de 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_api.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_api.py @@ -5,7 +5,6 @@ import pytest import requests from pycti import OpenCTIConnectorHelper - from shadowserver.api import ShadowserverAPI FIXTURES_DIR = "fixtures" diff --git a/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py b/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py index 6f228af909..b7e7a48add 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_stix_transform.py @@ -3,6 +3,8 @@ from unittest.mock import MagicMock, patch from pycti import CustomObjectCaseIncident, OpenCTIConnectorHelper +from shadowserver.stix_transform import ShadowserverStixTransformation +from shadowserver.utils import datetime_to_string from stix2 import ( Artifact, DomainName, @@ -13,9 +15,6 @@ MarkingDefinition, ) -from shadowserver.stix_transform import ShadowserverStixTransformation -from shadowserver.utils import datetime_to_string - class TestShadowserverStixTransformation(unittest.TestCase): def setUp(self): diff --git a/external-import/shadowserver/src/shadowserver/tests/test_utils.py b/external-import/shadowserver/src/shadowserver/tests/test_utils.py index 3a546b7939..6272a3b17d 100644 --- a/external-import/shadowserver/src/shadowserver/tests/test_utils.py +++ b/external-import/shadowserver/src/shadowserver/tests/test_utils.py @@ -1,8 +1,6 @@ import unittest from datetime import datetime -from stix2 import DomainName, IPv4Address - from shadowserver.utils import ( check_ip_address, clean_dict, @@ -17,6 +15,7 @@ validate_date_format, validate_marking_refs, ) +from stix2 import DomainName, IPv4Address class TestValidateDateFormat(unittest.TestCase): From 16e72b32bb093aa086727e90fac936b6970feb1e Mon Sep 17 00:00:00 2001 From: cmandich Date: Fri, 13 Dec 2024 09:03:46 -0800 Subject: [PATCH 4/4] Try isort fix again from base dir. --- external-import/shadowserver/src/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/external-import/shadowserver/src/main.py b/external-import/shadowserver/src/main.py index def7af9904..6a3c695da8 100644 --- a/external-import/shadowserver/src/main.py +++ b/external-import/shadowserver/src/main.py @@ -5,7 +5,6 @@ from datetime import UTC, datetime, timedelta from lib.external_import import ExternalImportConnector - from shadowserver import ShadowserverAPI, get_tlp_keys, remove_duplicates # Lookback in days