[greynoisefeed] remove dedup function, add batching for bundle create and submit

external-import/greynoise-feed/docker-compose.yml

@@ -18,5 +18,5 @@ services:
       - "GREYNOISE_DESCRIPTION=GreyNoise collects and analyzes untargeted, widespread, and opportunistic scan and attack activity that reaches every server directly connected to the Internet."
       - GREYNOISE_LIMIT=10000
       - GREYNOISE_IMPORT_METADATA=false
-      - GREYNOISE_INTERVAL=12 # In hours (the connector will always pull last 2 days each run)
+      - GREYNOISE_INTERVAL=24 # In hours
     restart: always

external-import/greynoise-feed/src/config.yml.sample

@@ -14,8 +14,8 @@ greynoisefeed:
   feed_type: 'malicious' # set to benign, malicious, benign+malicious, or all
   indicator_score_malicious: 75
   indicator_score_benign: 20
-  name: 'GreyNoise'
+  name: 'GreyNoise Feed'
   description: 'GreyNoise collects and analyzes untargeted, widespread, and opportunistic scan and attack activity that reaches every server directly connected to the Internet.'
   limit: 10000
   import_metadata: false
-  interval: 12 # In hours (the connector will always pull last 2 days each run)
+  interval: 24 # In hours (the connector will always pull last 2 days each run)

external-import/greynoise-feed/src/main.py

@@ -81,7 +81,7 @@ def __init__(self):
             ["greynoisefeed", "interval"],
             config,
             isNumber=True,
-            default=12,
+            default=24,
         )
         self.identity = self.helper.api.identity.create(
             type="Organization",
@@ -564,12 +564,21 @@ def _process_data(self, work_id, session, ips_list):
         if len(bundle_entities) > 0:
             shuffle(bundle_relationships)
             bundle_objects = bundle_entities + bundle_relationships
-            bundle_objects = self.helper.stix2_deduplicate_objects(bundle_objects)
-            bundle = self.helper.stix2_create_bundle(bundle_objects)
-            self.helper.send_stix2_bundle(
-                bundle,
-                work_id=work_id,
-            )
+            batch_count = 0
+            bundle_objects_len = len(bundle_objects)
+            batch_size = 50000
+            for i in range(batch_count, bundle_objects_len, batch_size):
+                self.helper.log_info("Batch: " + str(int(i) / int(batch_size)))
+                x = i
+                self.helper.log_info("Creating Bundles")
+                bundle = self.helper.stix2_create_bundle(
+                    bundle_objects[x : x + batch_size]
+                )
+                self.helper.log_info("Submitting Bundles")
+                self.helper.send_stix2_bundle(
+                    bundle,
+                    work_id=work_id,
+                )
 
     def run(self):
         self.helper.log_info("GreyNoise feed - Initialization...")
@@ -581,6 +590,24 @@ def run(self):
                 current_state = self.helper.get_state()
                 if current_state is not None and "last_run_timestamp" in current_state:
                     last_run_timestamp = parse(current_state["last_run_timestamp"])
+
+                    pause_run = True
+
+                    while pause_run:
+                        self.helper.log_info(
+                            "GreyNoise feed - Checking Last Run Interval"
+                        )
+                        now = datetime.now(pytz.UTC)
+                        diff = now - last_run_timestamp
+                        days, seconds = diff.days, diff.seconds
+                        last_run_hours = days * 24 + seconds // 3600
+                        if last_run_hours < self.greynoise_interval:
+                            self.helper.log_info(
+                                "GreyNoise feed - Last Run Less than Interval - Waiting 60 minutes"
+                            )
+                            time.sleep(3600)
+                        else:
+                            pause_run = False
                 else:
                     last_run_timestamp = now - timedelta(days=1)
 
@@ -591,7 +618,7 @@ def run(self):
                 try:
                     ips_list = []
                     session = GreyNoise(
-                        api_key=self.api_key, integration_name="opencti-feed-v2.3"
+                        api_key=self.api_key, integration_name="opencti-feed-v2.4"
                     )
 
                     query = self.get_feed_query(self.feed_type)
@@ -653,7 +680,7 @@ def run(self):
                     self.helper.log_error(str(e))
 
                 # Wait
-                time.sleep(3600 * self.greynoise_interval)
+                # time.sleep(3600 * self.greynoise_interval)
             except (KeyboardInterrupt, SystemExit):
                 self.helper.log_info("Connector stop")
                 exit(0)

external-import/greynoise-feed/src/requirements.txt

@@ -1,4 +1,4 @@
+urllib3==2.2.3
 pycti==6.4.5
-urllib3==2.2.2
 certifi==2024.12.14
 greynoise==2.3.0