Fix taxi feed observables extraction (demisto#31120)

* fixed an isssue where taxi feed will fail to retreive all indicators due to wrong extraction from XML blocks. * added rn * update docker * update test_data * add secret to secrets ignore
riskive · Nov 28, 2023 · 4aff1a6 · 4aff1a6
1 parent 1578504
commit 4aff1a6
Show file tree

Hide file tree

Showing 10 changed files with 131 additions and 80 deletions.
diff --git a/Packs/FeedTAXII/.secrets-ignore b/Packs/FeedTAXII/.secrets-ignore
@@ -26,3 +26,4 @@ TOUMarking:Terms_Of_Use
 TOUMarking:TermsOfUseMarkingStructureType
 simpleMarking:SimpleMarkingStructureType
 tlpMarking:TLPMarkingStructureType
+http://test.org/?n
diff --git a/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.py b/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.py
diff --git a/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml b/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml
@@ -183,7 +183,7 @@ script:
     - contextPath: TAXII.Indicator.Rawjson
       description: The indicator rawJSON value.
       type: Unknown
-  dockerimage: demisto/taxii:1.0.0.76522
+  dockerimage: demisto/taxii:1.0.0.80809
   feed: true
   runonce: false
   script: ''

diff --git a/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII_test.py b/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII_test.py
@@ -24,11 +24,11 @@ def test_decode(self):
         xml_files_names = get_files_in_dir(self.FILE_PATH, 'xml')
         for xml_f_name in xml_files_names:
             file_path = f'{self.FILE_PATH}/{xml_f_name}'
-            with open(file_path, 'r') as xml_f:
+            with open(file_path) as xml_f:
                 stix_str = xml_f.read()
                 res = StixDecode.decode(stix_str)
                 res_path = f'{file_path.rstrip(".xml")}-result.json'
-                with open(res_path, 'r') as res_f:
+                with open(res_path) as res_f:
                     expected_res = json.load(res_f)
                     assert expected_res == list(res[1])
 
@@ -89,19 +89,19 @@ def test_interval_in_sec_6(self):
 class TestCommands:
     def test_fetch_indicators(self, mocker):
         client = TAXIIClient(collection='a collection')
-        with open('FeedTAXII_test/TestCommands/raw_indicators.json', 'r') as f:
+        with open('test_data/raw_indicators.json') as f:
             raw_indicators = json.load(f)
             mocker.patch.object(client, 'build_iterator', return_value=raw_indicators)
             res = fetch_indicators_command(client)
-            with open('FeedTAXII_test/TestCommands/indicators_results.json') as exp_f:
+            with open('test_data/indicators_results.json') as exp_f:
                 expected = json.load(exp_f)
                 assert res == expected
 
 
 def test_poll_collection(mocker):
     """
     Given:
-        - A collection of indicators in STIX format.
+        - A collection of indicators in STIX format where there are more than one indicator for some blocks.
 
     When:
         - fetch_indicators_command is running.
@@ -113,14 +113,14 @@ def test_poll_collection(mocker):
     from FeedTAXII import fetch_indicators_command
     client = TAXIIClient(collection='a collection', poll_service='http://example/taxii-data')
 
-    with open('FeedTAXII_test/TestCommands/collection_example.xml', 'rb') as xml_f:
+    with open('test_data/collection_example.xml', 'rb') as xml_f:
         stix_content = xml_f.read()
 
     with requests_mock.Mocker() as m:
         m.post('http://example/taxii-data', content=stix_content)
         res = fetch_indicators_command(client)
 
-    with open('FeedTAXII_test/TestCommands/indicators_example.json') as json_f:
+    with open('test_data/indicators_example.json') as json_f:
         expected_result = json.load(json_f)
 
     assert res == expected_result
@@ -137,7 +137,7 @@ def test_tags_parameter(mocker, tags):
     - Validate the tags supplied exists in the indicators
     """
     client = TAXIIClient(collection='a collection', feedTags=json.dumps(tags))
-    with open('FeedTAXII_test/TestCommands/raw_indicators.json', 'r') as f:
+    with open('test_data/raw_indicators.json') as f:
         raw_indicators = json.load(f)
         mocker.patch.object(client, 'build_iterator', return_value=raw_indicators)
         res = fetch_indicators_command(client)

diff --git a/..._test/TestCommands/collection_example.xml → ...eedTAXII/test_data/collection_example.xml b/..._test/TestCommands/collection_example.xml → ...eedTAXII/test_data/collection_example.xml
@@ -78,6 +78,37 @@
             </stix:STIX_Package>
         </taxii_11:Content>
         <taxii_11:Timestamp_Label>2021-10-20T06:43:08.843260+00:00</taxii_11:Timestamp_Label>
+        <taxii_11:Content_Binding binding_id="id"/>
+        <taxii_11:Content>
+            <stix:STIX_Package xmlns:cyboxCommon="http://example.org/common-2" xmlns:cybox="http://example.org/cybox-2" xmlns:cyboxVocabs="http://example.org/default_vocabularies-2" xmlns:URIObj="http://example.org/objects#URIObject-2" xmlns:marking="http://example.org/Marking-1" xmlns:simpleMarking="http://example.org/extensions/MarkingStructure#Simple-1" xmlns:tlpMarking="http://example.org/extensions/MarkingStructure#TLP-1" xmlns:TOUMarking="http://example.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:opensource="http://hailataxii.com" xmlns:edge="http://example.org/" xmlns:stixCommon="http://example.org/common-1" xmlns:stixVocabs="http://example.org/default_vocabularies-1" xmlns:stix="http://example.org/stix-1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="edge:Package-65bdcda9-54f2-47aa-a473-5176d4c2c1d8" version="1.1.1" timestamp="2021-10-20T06:43:08.841901+00:00">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure xsi:type="tlpMarking:TLPMarkingStructureType" color="WHITE"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>TBD</TOUMarking:Terms_Of_Use>
+                            </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                            </marking:Marking_Structure>
+                        </marking:Marking>
+                    </stix:Handling>
+                </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-335290e0-3496-4644-bdaa-25b323814b45" sighting_count="1">
+                        <cybox:Title>URL: http://test.org/?n...</cybox:Title>
+                        <cybox:Description>URL: http://test.org/?n| isOnline:yes| dateVerified:2021-10-19T04:13:43+00:00</cybox:Description>
+                        <cybox:Object id="opensource:URI-d63ca0ba-3e0d-490e-914d-3271a66dc805">
+                            <cybox:Properties xsi:type="URIObj:URIObjectType" type="URL">
+                                <URIObj:Value condition="Equals">http://test.org/?n</URIObj:Value>
+                            </cybox:Properties>
+                        </cybox:Object>
+                    </cybox:Observable>
+                </stix:Observables>
+            </stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2021-10-20T06:43:08.843260+00:00</taxii_11:Timestamp_Label>
     </taxii_11:Content_Block>
     <taxii_11:Content_Block xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:taxii_11="http://taxii.mitre.org/messages/taxii_xml_binding-1.1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1">
         <taxii_11:Content_Binding binding_id="id"/>

diff --git a/...test/TestCommands/indicators_example.json → ...edTAXII/test_data/indicators_example.json b/...test/TestCommands/indicators_example.json → ...edTAXII/test_data/indicators_example.json
@@ -48,6 +48,25 @@
             "value": "http://example.org/?n"
         }
     },
+    {
+        "value": "http://test.org/?n",
+        "type": "URL",
+        "title": "URL: http://test.org/?n...",
+        "description": "URL: http://test.org/?n| isOnline:yes| dateVerified:2021-10-19T04:13:43+00:00",
+        "stixindicatorname": null, "stixindicatordescription": null,
+        "confidence": null,
+        "fields": {},
+        "rawJSON": {
+            "indicator": "http://test.org/?n",
+            "type": "URL",
+            "indicator_ref": "opensource:Observable-335290e0-3496-4644-bdaa-25b323814b45",
+            "stix_title": "URL: http://test.org/?n...",
+            "stix_description": "URL: http://test.org/?n| isOnline:yes| dateVerified:2021-10-19T04:13:43+00:00",
+            "share_level": "white",
+            "TLP": "WHITE",
+            "value": "http://test.org/?n"
+        }
+    },
     {
         "value": "https://pancakeswap.finance.exchange-goswap.com/",
         "type": "URL",

diff --git a/...test/TestCommands/indicators_results.json → ...edTAXII/test_data/indicators_results.json b/...test/TestCommands/indicators_results.json → ...edTAXII/test_data/indicators_results.json
diff --git a/...XII_test/TestCommands/raw_indicators.json → ...s/FeedTAXII/test_data/raw_indicators.json b/...XII_test/TestCommands/raw_indicators.json → ...s/FeedTAXII/test_data/raw_indicators.json
diff --git a/Packs/FeedTAXII/ReleaseNotes/1_2_2.md b/Packs/FeedTAXII/ReleaseNotes/1_2_2.md
@@ -0,0 +1,7 @@
+
+#### Integrations
+
+##### TAXII Feed
+
+- Fixed an issue where TaxiiFeed failed to retrieve all indicators due to wrong extraction from XML blocks.
+- Updated the Docker image to: *demisto/taxii:1.0.0.80809*.
diff --git a/Packs/FeedTAXII/pack_metadata.json b/Packs/FeedTAXII/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "TAXII Feed",
     "description": "Ingest indicator feeds from TAXII 1 and TAXII 2 servers.",
     "support": "xsoar",
-    "currentVersion": "1.2.1",
+    "currentVersion": "1.2.2",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",