From 2a8cf79c38c967d78caebdb2f46f4de83aeaa3ae Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Fri, 25 Feb 2022 15:35:40 -0400 Subject: [PATCH 1/4] update stix validator to support multiple versions of stix2-validator --- bundle_validator/bundle_validator.py | 10 +++++----- bundle_validator/validate.sh | 25 ++++++++++++++++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/bundle_validator/bundle_validator.py b/bundle_validator/bundle_validator.py index 3608c81cb..51b96b644 100644 --- a/bundle_validator/bundle_validator.py +++ b/bundle_validator/bundle_validator.py @@ -10,15 +10,15 @@ def __main__(): with open(bundle_file) as f: bundle = json.load(f) results = validate_instance(bundle) - if results.is_valid is not True: + if results.is_valid: print_results(results) - raise Exception() + print("\n *** STIX bundle is valid but may contain warnings. Warnings marked as yellow [!] can be ignored but recommended to fix ***\n") + else: + print_results(results) + print("\n *** Invalid STIX Objects found in the bundle. Please fix the error marked as Red[X]. Warnings marked as yellow [!] can be ignored but recommended to fix ***\n") - print('*** STIX Bundle validated!!\n') except ValueError as ex: print("*** Malformed JSON in the STIX Bundle: " + str(ex)) - except Exception as ex: - print("\n *** Invalid STIX Objects found in the bundle. Please fix the error marked as Red[X]. Warnings marked as yellow [!] can be ingnored but recommended to fix ***\n") if __name__ == "__main__": __main__() \ No newline at end of file diff --git a/bundle_validator/validate.sh b/bundle_validator/validate.sh index 03f86c894..10835bbde 100755 --- a/bundle_validator/validate.sh +++ b/bundle_validator/validate.sh @@ -5,14 +5,29 @@ if [ -z "$1" ]; then exit 1 fi +if [ -z "$2" ]; then + echo "Specify the STIX version to validate against. Usage: ./validate.sh <2.0 or 2.1>" + exit 1 +fi + FILE=$1 +SPEC=$2 -if [ -f "$FILE" ]; then - echo "Validating STIX Bundle file: $FILE " - echo "" -else +if ! [ -f "$FILE" ]; then echo "$FILE does not exists. Place your bundle file inside bundle_validator/ folder" exit 1 +elif [ $SPEC != '2.0' ] && [ $SPEC != '2.1' ]; then + echo "$SPEC does not match 2.0 or 2.1." + exit 1 +else + echo "Validating bundle file $FILE against STIX $SPEC " + echo "" +fi + +if [ $SPEC == '2.0' ]; then + VALIDATOR_VERSION=1.1.2 +else + VALIDATOR_VERSION=3.0.2 fi { @@ -21,7 +36,7 @@ fi virtualenv -p python3 virtualenv - venv-run pip install stix2-validator==1.1.2 + venv-run pip install stix2-validator==$VALIDATOR_VERSION } &> /dev/null venv-run bundle_validator.py $FILE \ No newline at end of file From 6d011dc940d5eb5a0bc657bc259f29c4a0b98e93 Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Thu, 3 Mar 2022 13:45:27 -0400 Subject: [PATCH 2/4] Run CLI STIX validation on entire bundle --- .../src/json_to_stix/json_to_stix_translator.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index 2248eb30c..80a0515b8 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -33,6 +33,10 @@ def convert_to_stix(data_source, map_data, data, transformers, options, callback for _, value in ds2stix.unique_cybox_objects.items(): ds2stix.bundle["objects"].append(value) + if options.get('stix_validator'): + validated_result = validate_instance(ds2stix.bundle) + print_results(validated_result) + return ds2stix.bundle @@ -58,7 +62,6 @@ def __init__(self, data_source, ds_to_stix_map, transformers, options, callback= self.callback = callback # parse through options - self.stix_validator = options.get('stix_validator') self.cybox_default = options.get('cybox_default', True) self.properties = observable.properties @@ -439,10 +442,6 @@ def transform(self, obj): observation["object_refs"] = object_refs observation["spec_version"] = "2.1" - # Validate each STIX object - if self.stix_validator: - validated_result = validate_instance(observation) - print_results(validated_result) except Exception as e: raise Exception("Error in json_to_stix_translator.transform: %s" % e) From a0c32d4a76752ab0cf1d50fa3da626764155d743 Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Thu, 3 Mar 2022 13:50:47 -0400 Subject: [PATCH 3/4] update docs for STIX validator changes --- OVERVIEW.md | 30 +++++++++++++++---- adapter-guide/develop-translation-module.md | 2 +- bundle_validator/README.md | 2 ++ stix_shifter_modules/cybereason/README.md | 2 -- .../stix_translation/README.md | 2 +- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index 5f652dd38..6c0ad03b3 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -225,7 +225,7 @@ _pattern.txt_ `python main.py translate qradar query '{}' '' < /path/to/file/pattern.txt` -### 2. Translate a JSON data source query result to a STIX bundle of observable objects +### 2. Translate a JSON data source query result to a STIX 2.0 bundle of observable objects #### INPUT: JSON data source query result @@ -242,13 +242,14 @@ _pattern.txt_ ] ``` -#### OUTPUT: STIX bundle of observable objects +#### OUTPUT: STIX 2.0 bundle of observable objects ``` # STIX Observables { "type": "bundle", "id": "bundle--2042a6e9-7f34-4a03-a745-502e358594c3", + "spec_version": "2.0", "objects": [ { "type": "identity", @@ -297,15 +298,34 @@ Alternatively, you can run the CLI commands from the source. Open a terminal and The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. The STIX Identity object represents the data source and is passed in to allow stix-shifter to create a reference between the data source and the generated STIX observed objects. -Using the Qradar connector as an example: +Using the QRadar connector as an example: + +``` +python main.py translate qradar results \ +'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ +'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' +``` + +### Translating results into STIX 2.1 + +By default, JSON results are translated into STIX 2.0. To return STIX 2.1 results include `'{"stix_2.1": true}'` in the CLI command ``` python main.py translate qradar results \ '{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ -'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' --stix-validator +'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true}' ``` -The `--stix-validator` flag at the end will run validation on the returned STIX objects to ensure they conform to the STIX 2 standard. Alternatively, `'{ "stix_validator": true }'` can be passed in at the end as an options dictionary. + +### Validating translated STIX 2.1 bundle from the CLI + +You can validate translated STIX results from the CLI provided they conform to the 2.1 standard. The `--stix-validator` flag at the end will run validation on the returned STIX objects to ensure they conform to the STIX 2.1 standard. Alternatively, `'{ "stix_validator": true }'` can be passed in at the end as an options dictionary. + +``` +python main.py translate qradar results \ +'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ +'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true, "stix_validator: true}' +``` ### Results translation using an input file diff --git a/adapter-guide/develop-translation-module.md b/adapter-guide/develop-translation-module.md index 0a0184c80..138890436 100644 --- a/adapter-guide/develop-translation-module.md +++ b/adapter-guide/develop-translation-module.md @@ -439,7 +439,7 @@ python main.py translate abc_security_monitor results '{"type": "identity","id": 6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' '{ "stix_validator": true }' ``` -Adding the `stix_validator` option at the end will ensure the observed-data objects conform to the STIX 2 standard. +Adding the `stix_validator` option at the end will test if the translated STIX bundle conforms to the STIX 2.1 standard. 2. Visually verify that all expected data is in the returned STIX bundle. If a data source field in your sample results is mapped in `to_stix_map.json`, the value must be in the STIX bundle under the mapped STIX property. diff --git a/bundle_validator/README.md b/bundle_validator/README.md index cc4cb21a6..e371574f3 100644 --- a/bundle_validator/README.md +++ b/bundle_validator/README.md @@ -1,5 +1,7 @@ # Validate STIX bundle + + You can easily validate your stix bundle file by following the below steps: ## Prerequisites: diff --git a/stix_shifter_modules/cybereason/README.md b/stix_shifter_modules/cybereason/README.md index a43951b55..e2703b068 100644 --- a/stix_shifter_modules/cybereason/README.md +++ b/stix_shifter_modules/cybereason/README.md @@ -199,7 +199,6 @@ results translate cybereason results "{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"cybereason\",\"identity_class\":\"events\"}" "[ { \"Connection\": { \"hasSuspicions\": \"false\", \"isProcessLegit\": \"true\", \"aggregatedReceivedBytesCount\": \"1225449\", \"remotePort\": \"443\", \"state\": \"CONNECTION_OPEN\", \"portType\": \"SERVICE_HTTP\", \"transportProtocol\": \"TCP\", \"elementDisplayName\": \"1.1.0.0:50799 > 52.226.139.180:443\", \"aggregatedTransmittedBytesCount\": \"724812\", \"isWellKnownPort\": \"true\", \"isExternalConnection\": \"true\", \"localPort\": \"50799\", \"remoteAddressInternalExternalLocal\": \"EXTERNAL\", \"endTime\": \"1635034091852\", \"serverAddress\": \"52.226.139.180\", \"portDescription\": \"Hypertext Transfer Protocol over TLS/SSL (HTTPS)\", \"serverPort\": \"443\", \"isIncoming\": \"false\", \"calculatedCreationTime\": \"1633093769591\", \"hasMalops\": \"false\", \"direction\": \"OUTGOING\", \"isProcessMalware\": \"false\", \"localAddress\": \"1.1.0.0\", \"domainName\": \"client.wns.windows.com\", \"remoteAddress\": \"52.226.139.180\", \"ownerMachine\": \"d3cyber-win10-1\", \"ownerProcess\": \"svchost.exe\", \"dnsQuery\": \"client.wns.windows.com > 52.226.139.180\", \"urlDomains\": [ \"wns2-bl2p.wns.windows.com\", \"client.wns.windows.com\" ] } }, { \"Connection\": { \"hasSuspicions\": \"false\", \"isProcessLegit\": \"true\", \"aggregatedReceivedBytesCount\": \"183141\", \"remotePort\": \"443\", \"state\": \"CONNECTION_OPEN\", \"portType\": \"SERVICE_HTTP\", \"transportProtocol\": \"TCP\", \"elementDisplayName\": \"1.1.0.0:57412 > 40.83.240.146:443\", \"aggregatedTransmittedBytesCount\": \"106995\", \"isWellKnownPort\": \"true\", \"isExternalConnection\": \"true\", \"localPort\": \"57412\", \"remoteAddressInternalExternalLocal\": \"EXTERNAL\", \"endTime\": \"1634996831727\", \"serverAddress\": \"40.83.240.146\", \"portDescription\": \"Hypertext Transfer Protocol over TLS/SSL (HTTPS)\", \"serverPort\": \"443\", \"isIncoming\": \"false\", \"calculatedCreationTime\": \"1633417907599\", \"hasMalops\": \"false\", \"direction\": \"OUTGOING\", \"isProcessMalware\": \"false\", \"localAddress\": \"1.1.0.0\", \"domainName\": \"client.wns.windows.com\", \"remoteAddress\": \"40.83.240.146\", \"ownerMachine\": \"d3cyber-win10-1\", \"ownerProcess\": \"svchost.exe\", \"dnsQuery\": \"client.wns.windows.com > 40.83.240.146\", \"urlDomains\": \"client.wns.windows.com\" } } ]" -"{\"stix_validator\": true}" ``` #### STIX Translate results - output @@ -984,7 +983,6 @@ results translate cybereason results "{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"cybereason\",\"identity_class\":\"events\"}" "[ { \"Process\": { \"imageFile.maliciousClassificationType\": \"indifferent\", \"isImageFileVerified\": \"true\", \"imageFile.companyName\": \"Adobe Systems Incorporated\", \"productType\": \"ADOBE\", \"imageFileExtensionType\": \"EXECUTABLE_WINDOWS\", \"isAggregate\": \"false\", \"hasAutorun\": \"false\", \"hasClassification\": \"true\", \"imageFile.productName\": \"Adobe Acrobat Reader DC\", \"imageFile.signerInternalOrExternal\": \"Adobe Inc.\", \"imageFile.signedInternalOrExternal\": \"true\", \"hasInjectedChildren\": \"false\", \"creationTime\": \"1625048289019\", \"imageFile.sha256String\": \"cf40670e0eb0629a0d51f65325c692788d0a5503dea3f13db643b916701ab1da\", \"imageFile.md5String\": \"3c9b885b579ebadaae15e391ac8313af\", \"iconBase64\": \"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAKGSURBVFhHxZc/aBNRHMd/TZMmFy0pIhVdLCIOXZKaxQoSB1FRahwcXDTqELemBKqDYLrUTRBUsKJExMFR6CKo4B9wUbSIi4soDi4OdnFxeH6/T96RhF9ySe6O/uDT67v3u/f93u9+d+SJMWZDUU/W6/VEsVhMghQYC0mqVCqNNhqNEU2rbZDP50eve3LgcVaer24SEyVY89ktT/ZXKpU2I/4/uVwuddOTRe3iKLmflXmYSLQZ4J3X03JcuyAOrmXkYK1Ws5X470JkopmVt1pyHDzMymtoZqw2Gm4Eg91aYpxAc9IaqFarCQxmtKRevDt90vz5/s18vX1DnQ8CmlN8M5yBopbUC8avNy/tUZsPApq7Qht4umPCHrX5ICIx8PnSgj1q80GENrD+ac38Xf+9cQa+LC9ZcYY2H0RoA3wLGKyCNh9EaAM/Hj2wBhivZgtqTi9CGXDdz2AF2A88p+V2I5QB9/z5IeLd08TP1SdqbjeGNsA7dd1PUZpx487gec5r6wxtoLX7GawCjfCTTEGO1y6eMy+mp+yYaOsMbICL8lm7oGhn81GUzdlaERpqzXH0bYAl5zefizoD3RYdhL4NUJQGXLMxhnntOunbAEVdFRh8FFreoLQZOJaSOS2JsLkYNBKVOOm7AnHRaSC/4skHLTEO7nryHpo7rQH7B4OjSbmsJcfBXEpq0NyCJ+v/Kt4MSvNpuaddECULaVmBVgH7kDHfAHcrOLkNHDmUlKt3PPmoXRwGrnk4KVegsQ+MU9c3QLA5YS9MgllwCpwFFyLgPDgDToACGG82m/72zDdAuFtBQgZsB9NgLyiGZAbsAVtd2VtpG7SCHW2Cu9ooKJfL6s7YGCP/ABNgFQCEfBgYAAAAAElFTkSuQmCC\", \"hasSuspicions\": \"false\", \"hasUnresolvedDnsQueriesFromDomain\": \"false\", \"hasMalops\": \"false\", \"imageFile.sha1String\": \"d75dea803685620b22514689e32c2287206dcc63\", \"architecture\": \"wow64\", \"markedForPrevention\": \"false\", \"applicablePid\": \"10148\", \"endTime\": \"1625048301349\", \"integrity\": \"MEDIUM\", \"isExectuedByWmi\": \"false\", \"commandLine\": \"\\"C:\\Program Files (x86)\\Adobe\\Acrobat Reader DC\\Reader\\AcroRd32.exe\\" /l /slMode\", \"isWhiteListClassification\": \"false\", \"executionPrevented\": \"false\", \"isNotShellRunner\": \"true\", \"elementDisplayName\": \"acrord32.exe\", \"isIdentifiedProduct\": \"true\", \"hasChildren\": \"false\", \"calculatedUser\": \"desktop-trs61af\\kj\", \"ownerMachine\": \"desktop-trs61af\", \"logonSession\": \"desktop-trs61af > desktop-trs61af\", \"loadedModules\": [ \"cryptbase.dll\", \"ole32.dll\", \"rpcrt4.dll\", \"propsys.dll\", \"bcrypt.dll\", \"combase.dll\", \"oleaut32.dll\", \"samcli.dll\", \"libeay32.dll\", \"comctl32.dll\", \"dwmapi.dll\", \"shlwapi.dll\", \"urlmon.dll\", \"mpr.dll\", \"msvcrt.dll\", \"wow64win.dll\", \"sspicli.dll\", \"gdi32.dll\", \"winmmbase.dll\", \"firewallapi.dll\", \"windows.storage.dll\", \"user32.dll\", \"acrord32.exe\", \"wow64.dll\", \"uxtheme.dll\", \"sechost.dll\", \"msctf.dll\", \"fwbase.dll\", \"iphlpapi.dll\", \"version.dll\", \"bcryptprimitives.dll\", \"agm.dll\", \"profext.dll\", \"ntmarta.dll\", \"winmm.dll\", \"kernelbase.dll\", \"powrprof.dll\", \"user32.dll\", \"msvcp_win.dll\", \"kernel.appcore.dll\", \"ntdll.dll\", \"ntdll.dll\", \"ucrtbase.dll\", \"imm32.dll\", \"cfgmgr32.dll\", \"userenv.dll\", \"wow64cpu.dll\", \"win32u.dll\", \"kernel32.dll\", \"{FLOATING}\", \"profapi.dll\", \"acrord32.dll\", \"kernel32.dll\", \"ws2_32.dll\", \"ninput.dll\", \"iertutil.dll\", \"apphelp.dll\", \"winspool.drv\", \"shell32.dll\", \"fltlib.dll\", \"advapi32.dll\", \"shcore.dll\", \"gdi32full.dll\" ], \"modulesNotInLoaderDbList\": [ \"samcli.dll\", \"comctl32.dll\", \"dwmapi.dll\", \"winmmbase.dll\", \"firewallapi.dll\", \"user32.dll\", \"uxtheme.dll\", \"fwbase.dll\", \"winmm.dll\", \"apphelp.dll\" ], \"unsignedWithSignedVersionModules\": \"libeay32.dll\", \"imageFile\": \"acrord32.exe\" } } ] " - "{\"stix_validator\": true}" ``` #### STIX Translate results - output diff --git a/stix_shifter_modules/security_advisor/stix_translation/README.md b/stix_shifter_modules/security_advisor/stix_translation/README.md index 54677f416..857385747 100644 --- a/stix_shifter_modules/security_advisor/stix_translation/README.md +++ b/stix_shifter_modules/security_advisor/stix_translation/README.md @@ -17,7 +17,7 @@ Security Advisor data to Stix mapping is defined in `to_stix_map.json` which is This example Security Advisor data : -`python3 main.py translate security_advisor results '{"type": "identity", "id":"identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "SecurityAdvisor","identity_class": "events"}' '[{"author_accountId": "4263e551d4a9460e8cdaccc06414198b", "author_id": "iam-ServiceId-7b4398db-219e-4174-b4f5-c6e31970c7dc", "author_email": null, "name": "67035ffbd96ead38e7e5bd6bf40f364a/providers/config-advisor/occurrences/datacos-not_in_private_network-crn%3Av1%3Abluemix%3Apublic%3Acloud-object-storage%3Aglobal%3Aa%2F67035ffbd96ead38e7e5bd6bf40f364a%3A895aa3a5-905f-4e78-90c9-d7e86d3032e1%3A%3A", "id": "datacos-not_in_private_network-crn%3Av1%3Abluemix%3Apublic%3Acloud-object-storage%3Aglobal%3Aa%2F67035ffbd96ead38e7e5bd6bf40f364a%3A895aa3a5-905f-4e78-90c9-d7e86d3032e1%3A%3A", "noteName": "4263e551d4a9460e8cdaccc06414198b/providers/config-advisor/notes/datacos-not_in_private_network", "updateTime": "2020-01-08T18:03:59.825854Z", "createTime": "2020-01-08T18:03:59.825827Z", "shortDescription": "COS bucket is not in a private network", "providerId": "config-advisor", "providerName": "67035ffbd96ead38e7e5bd6bf40f364a/providers/config-advisor", "longDescription": "Bucket is not in a private network", "context_accountId": "67035ffbd96ead38e7e5bd6bf40f364a", "context_region": "us-south", "context_resourceType": "COS bucket", "context_resourceName": "Any", "context_resourceId": null, "context_resourceCrn": "Any", "context_serviceName": "COS service", "context_serviceCrn": "crn:v1:bluemix:public:cloud-object-storage:global:a/67035ffbd96ead38e7e5bd6bf40f364a:895aa3a5-905f-4e78-90c9-d7e86d3032e1::", "reportedBy_id": "appprotection", "reportedBy_title": "Config Advisor", "reportedBy_url": null, "finding_severity": "HIGH", "finding_certainty": "HIGH", "finding_networkConnection": null, "finding_nextSteps_0_title": "Cloud Object Storage Docs", "finding_nextSteps_0_url": "https://cloud.ibm.com/docs/services/cloud-object-storage?topic=cloud-object-storage-setting-a-firewall", "finding_nextSteps_1_title": "Bucket 'sa.67035ffbd96ead38e7e5bd6bf40f364a.telemetric.us-south' of COS-instance 'securityadvisor.67035ffbd96ead38e7e5bd6bf40f364a.instance' is not using private network. Use REST API to set the private network mask for the bucket", "finding_nextSteps_1_url": null, "finding_dataTransferred": null, "occurence_count": 1}]' '{"stix_validator": true}'` +`python3 main.py translate security_advisor results '{"type": "identity", "id":"identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "SecurityAdvisor","identity_class": "events"}' '[{"author_accountId": "4263e551d4a9460e8cdaccc06414198b", "author_id": "iam-ServiceId-7b4398db-219e-4174-b4f5-c6e31970c7dc", "author_email": null, "name": "67035ffbd96ead38e7e5bd6bf40f364a/providers/config-advisor/occurrences/datacos-not_in_private_network-crn%3Av1%3Abluemix%3Apublic%3Acloud-object-storage%3Aglobal%3Aa%2F67035ffbd96ead38e7e5bd6bf40f364a%3A895aa3a5-905f-4e78-90c9-d7e86d3032e1%3A%3A", "id": "datacos-not_in_private_network-crn%3Av1%3Abluemix%3Apublic%3Acloud-object-storage%3Aglobal%3Aa%2F67035ffbd96ead38e7e5bd6bf40f364a%3A895aa3a5-905f-4e78-90c9-d7e86d3032e1%3A%3A", "noteName": "4263e551d4a9460e8cdaccc06414198b/providers/config-advisor/notes/datacos-not_in_private_network", "updateTime": "2020-01-08T18:03:59.825854Z", "createTime": "2020-01-08T18:03:59.825827Z", "shortDescription": "COS bucket is not in a private network", "providerId": "config-advisor", "providerName": "67035ffbd96ead38e7e5bd6bf40f364a/providers/config-advisor", "longDescription": "Bucket is not in a private network", "context_accountId": "67035ffbd96ead38e7e5bd6bf40f364a", "context_region": "us-south", "context_resourceType": "COS bucket", "context_resourceName": "Any", "context_resourceId": null, "context_resourceCrn": "Any", "context_serviceName": "COS service", "context_serviceCrn": "crn:v1:bluemix:public:cloud-object-storage:global:a/67035ffbd96ead38e7e5bd6bf40f364a:895aa3a5-905f-4e78-90c9-d7e86d3032e1::", "reportedBy_id": "appprotection", "reportedBy_title": "Config Advisor", "reportedBy_url": null, "finding_severity": "HIGH", "finding_certainty": "HIGH", "finding_networkConnection": null, "finding_nextSteps_0_title": "Cloud Object Storage Docs", "finding_nextSteps_0_url": "https://cloud.ibm.com/docs/services/cloud-object-storage?topic=cloud-object-storage-setting-a-firewall", "finding_nextSteps_1_title": "Bucket 'sa.67035ffbd96ead38e7e5bd6bf40f364a.telemetric.us-south' of COS-instance 'securityadvisor.67035ffbd96ead38e7e5bd6bf40f364a.instance' is not using private network. Use REST API to set the private network mask for the bucket", "finding_nextSteps_1_url": null, "finding_dataTransferred": null, "occurence_count": 1}]'` Will return the following valid STIX Cyber Observable Object: ```json From 55cb309c7f7ebf50d536b29c16065429ce64f5d9 Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Thu, 3 Mar 2022 16:35:46 -0400 Subject: [PATCH 4/4] Bundle validator readme update --- OVERVIEW.md | 4 +++ adapter-guide/develop-translation-module.md | 11 ++++++-- bundle_validator/README.md | 29 ++++++++++++--------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index 6c0ad03b3..2f3d39800 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -327,6 +327,10 @@ python main.py translate qradar results \ '[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true, "stix_validator: true}' ``` +### Validating STIX 2.0 and 2.1 bundles with the validator script + +Refer to the [STIX validator](bundle_validator/README.md) + ### Results translation using an input file Create a JSON file with the results you wish to translate. The file can be used in the results translation call using standard input. diff --git a/adapter-guide/develop-translation-module.md b/adapter-guide/develop-translation-module.md index 138890436..2990b6136 100644 --- a/adapter-guide/develop-translation-module.md +++ b/adapter-guide/develop-translation-module.md @@ -436,10 +436,17 @@ If the translation module uses multiple from-STIX mapping files, you can append ``` python main.py translate abc_security_monitor results '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c- -6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' '{ "stix_validator": true }' +6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' ``` -Adding the `stix_validator` option at the end will test if the translated STIX bundle conforms to the STIX 2.1 standard. +By default, STIX 2.0 results will be returned. Adding the `{"stix_2.1": true}` option to the end of the CLI command will return STIX 2.1 objects. STIX 2.1 results can be validated against the 2.1 standard by including the `{"stix_validator": true }` option at the end. Only STIX 2.1 can be validated this way. For example: + +``` +python main.py translate abc_security_monitor results '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c- +6a4751cae5ff", "name": "abc_security_monitor", "identity_class": "events"}' '[{"Url": "www.example.com", "SourcePort": 3000, "DestinationPort": 1000, "SourceIpV4": "192.0.2.0", "DestinationIpV4": "198.51.100.0", "NetworkProtocol": "TCP"}]' '{"stix_2.1": true, "stix_validator": true }' +``` + +An alternative way to validate both STIX 2.0 and 2.1 results is to use the [Bundle validator](../bundle_validator/README.md). 2. Visually verify that all expected data is in the returned STIX bundle. If a data source field in your sample results is mapped in `to_stix_map.json`, the value must be in the STIX bundle under the mapped STIX property. diff --git a/bundle_validator/README.md b/bundle_validator/README.md index e371574f3..c8d744bb3 100644 --- a/bundle_validator/README.md +++ b/bundle_validator/README.md @@ -1,16 +1,13 @@ # Validate STIX bundle - - You can easily validate your stix bundle file by following the below steps: ## Prerequisites: The following needs to be installed on your local machine: -1. Python 3 -2. GIT - +* Python 3 +* GIT If you have not already cloned the [stix-shifter github project](https://github.com/opencybersecurityalliance/stix-shifter): ``` @@ -21,19 +18,25 @@ git clone https://github.com/opencybersecurityalliance/stix-shifter.git 1. Open a terminal 2. cd into /stix-shifter/bundle_validator -3. Copy your STIX bundle JSON file into the bundle_validator directory -4. Run the validate.sh script. You need to specify the bundle json file name: +3. Copy your STIX 2.0 or 2.1 bundle JSON file into the `bundle_validator` directory +4. Run the validate.sh script. You need to specify the bundle JSON file name and the STIX specification version (2.0 or 2.1) + + `./validate.sh <2.0 or 2.1>` + + Example: ``` - ./validate.sh + ./validate.sh my_stix_2_1_bundle.json 2.1 ``` -5. After successfull validation, you should see a messegae in your terminal: `STIX Bundle validated!!` -6. For unsuccessfull validation, you should see mainly two types of error- - 1. If JSON format is invalid: `Malformed JSON in the STIX Bundle: ` - 2. If the file contains invalid STIX Objects, you should see errors/warnings with heading `[X] STIX JSON: Invalid`. It is mandatory to fix the errors marked red as `[X]`. Warnings which are marked yellow as `[!]`, can be ingnored but recommended to fix. For example- +5. After successful validation, you should see a message in your terminal: `STIX Bundle validated!!` +6. For unsuccessful validation, you should see two types of errors: + * If the JSON format in the bundle is invalid: `Malformed JSON in the STIX Bundle: ` + * If the file contains invalid STIX Objects, you should see errors and/or warnings with the heading `[X] STIX JSON: Invalid`. It is mandatory to fix the errors marked red as `[X]`. Warnings which are marked yellow as `[!]`, can be ignored but are recommended to fix. For example: + + ``` [X] STIX JSON: Invalid [!] Warning: identity--33fa3e56-6511-40de-bc69-c5ffeb3838f9: {213} identity_class contains a value not in the identity-class-ov vocabulary. [X] observed-data--ed82dd61-cc41-485b-b608-d278469e6259: 'number_observed' is a required property ``` - To debug the above error[X], find `observed-data--ed82dd61-cc41-485b-b608-d278469e6259` "id" in the bundle file and you will see `number_observed` property is missing in the stix object. + To debug the above error `[X]`, find `observed-data--ed82dd61-cc41-485b-b608-d278469e6259` "id" in the bundle file and you will see the `number_observed` property is missing in the STIX object.