From ce729df73238cad61eaa2cba7e011cb19433f87a Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Tue, 21 Mar 2023 10:32:44 -0300 Subject: [PATCH 1/2] 5.1.1 changelog (#1395) --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16f680a11..cce9e3490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,35 @@ We have started this changelogs from version 4.0.0. So, changes on previously re -------------------------------------- +## 5.1.1 (2023-03-21) + +### Breaking changes: + +### Deprecations: + +### Changes: + +* Added process:x_unique_id property to Splunk [#1389](https://github.com/opencybersecurityalliance/stix-shifter/pull/1389) +* get configs [#1392](https://github.com/opencybersecurityalliance/stix-shifter/pull/1392) +* GitHub action update [#1385](https://github.com/opencybersecurityalliance/stix-shifter/pull/1385) + +### Fixes: + +* Added metadata changes for GCP Chronicle [#1393](https://github.com/opencybersecurityalliance/stix-shifter/pull/1393) +* Splunk: Fix MAC address to display in proper STIX format [#1386](https://github.com/opencybersecurityalliance/stix-shifter/pull/1386) +* Updated custom properties mapping in Okta with 'x_' prefix [#1387](https://github.com/opencybersecurityalliance/stix-shifter/pull/1387) +* Await async fixes [#1391](https://github.com/opencybersecurityalliance/stix-shifter/pull/1391) +* fix json loads of data arg in stix-shifter CLI [#1394](https://github.com/opencybersecurityalliance/stix-shifter/pull/1394) + +### Dependency update: + +* Bump aiohttp-retry from 2.4.0 to 2.8.3 in /stix_shifter [#1374](https://github.com/opencybersecurityalliance/stix-shifter/pull/1374) +* Consolidate `network-traffic`, `user-account`, `file` objects in the elastic_ecs connector mapping [#1378](https://github.com/opencybersecurityalliance/stix-shifter/pull/1378) +* Fix #1375, optimize get_pagesize() function call, and add testcases [#1384](https://github.com/opencybersecurityalliance/stix-shifter/pull/1384) +* Async changes for Okta UDI connector [#1383](https://github.com/opencybersecurityalliance/stix-shifter/pull/1383) + +-------------------------------------- + ## 5.1.0 (2023-03-08) ### Breaking changes: From 0b5dff947f41099e7b49ad098273ca2f12401bfb Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 27 Mar 2023 11:55:03 -0300 Subject: [PATCH 2/2] Elastic ecs module readme (#1400) --- stix_shifter_modules/elastic_ecs/README.md | 269 +++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 stix_shifter_modules/elastic_ecs/README.md diff --git a/stix_shifter_modules/elastic_ecs/README.md b/stix_shifter_modules/elastic_ecs/README.md new file mode 100644 index 000000000..176a25aad --- /dev/null +++ b/stix_shifter_modules/elastic_ecs/README.md @@ -0,0 +1,269 @@ +# Elastic ECS Connector + +Elastic ECS Connector can be used to search events such as logs and metrics stored Elasticsearch. The connector can search Elastic Common Schema (ECS) formatted events. + +## Supported version + +The connector supports Elasticsearch version 8.6 + +## API and Query Language + +Connector uses Elasticsearch [search REST API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to fetch events stored in Elasticsearch indices. API Endpoint: `GET //_search` + +For search, the connector uses Elasticsearch [Query DSL](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) (Domain Specific Language). The query syntax is based on JSON. + +## Dialects + +The connector supports two dialects. + +### Elastic Common Schema (ECS) + +The default dialects contains [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/8.6/ecs-reference.html). All the available ECS fields can be searched using the default dialect. The default dialect mapping is specified in [from_stix_map.json](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json) + +### Beats + +Beats dialect can be used if any [beats](https://www.elastic.co/guide/en/beats/libbeat/current/beats-reference.html) agents is installed in Elasticsearch datasource. The mapping of this dialect can be found in [beats_from_stix_map.json](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json) + + +### Format for making STIX translation calls via the CLI + +`python main.py ` + +Note the identity object is only used when converting from Elasticsearch JSON response to STIX, but due to positional arguments, an empty hash will need to be passed in when converting from STIX patterns to datasource query. + + +## Converting from STIX patterns to DSL queries + +This example input stix pattern for default dialect: + +`python main.py translate elastic_ecs query {} "[ipv4-addr:value = '192.168.1.2' OR network-traffic:src_port = 443] START t'2022-04-06T00:00:00.000Z' STOP t'2022-04-06T00:05:00.000Z'"` + +Returns the following DSL query string: + +`((source.port : \"443\" OR client.port : \"443\") OR (source.ip : \"192.168.1.2\" OR destination.ip : \"192.168.1.2\" OR client.ip : \"192.168.1.2\" OR server.ip : \"192.168.1.2\" OR host.ip : \"192.168.1.2\" OR dns.resolved_ip : \"192.168.1.2\")) AND (@timestamp:[\"2022-04-06T00:00:00.000Z\" TO \"2022-04-06T00:05:00.000Z\"])` + +This example input stix pattern for beats dialect: + +`python main.py translate elastic_ecs:beats query {} "[ipv4-addr:value = '192.168.1.2' OR network-traffic:src_port = 443] START t'2022-04-06T00:00:00.000Z' STOP t'2022-04-06T00:05:00.000Z'"` + +Returns the following DSL query string: + +`((source.port : \"443\" OR client.port : \"443\") OR (source.ip.keyword : \"192.168.1.2\" OR destination.ip.keyword : \"192.168.1.2\" OR client.ip : \"192.168.1.2\" OR server.ip : \"192.168.1.2\" OR host.ip.keyword : \"192.168.1.2\" OR dns.resolved_ip : \"192.168.1.2\")) AND (@timestamp:[\"2022-04-06T00:00:00.000Z\" TO \"2022-04-06T00:05:00.000Z\"])` + +## Sending Query to Search API + +This is a synchronous connector. Therefore, the connector can only uses results transmission call to send query to the API. Example results call: + +``` +python main.py transmit elastic_ecs '{"host":"", "port": , "selfSignedCert": "", "indices": "index1, index2"}' '{"auth":{"username":"","password":""}}' results '((source.port : "443" OR client.port : "443") OR (source.ip : "192.168.1.2" OR destination.ip : "192.168.1.2" OR client.ip : "192.168.1.2" OR server.ip : "192.168.1.2" OR host.ip : "192.168.1.2" OR dns.resolved_ip : "192.168.1.2")) AND (@timestamp:["2022-04-06T00:00:00.000Z" TO "2022-04-06T00:05:00.000Z"])' 0 1 +``` + +**Note** + 1. "indices" parameter is optional. If it is not specified all the indices will be queried. Multiple indices can be set for the search. + 2. If the connector supports key and token based authentication then specify them inside "auth" object. + +### Transmit Results Output + +``` +{ + "success": true, + "data": [ + { + "process": { + "args": "-D -e -f /etc/ssh/sshd_config -R", + "parent": { + "args": "-D -e -f /etc/ssh/sshd_config", + "name": "sshd", + "start": "2021-10-16T01:54:25.664830275Z", + "pid": 820693, + "command_line": "/path/sbin/sshd -D", + "executable": "/path/sbin/sshd" + }, + "name": "sshd", + "start": "2021-10-17T14:33:05.446908698Z", + "pid": 899531, + "thread": { + "id": 899531 + }, + "command_line": "/path/sbin/sshd -D", + "executable": "/path/sbin/sshd" + }, + "@timestamp": "2021-10-17T14:33:05.452045076Z", + "destination": { + "address": "0.0.0.0", + "port": 655, + "bytes": 0, + "ip": "0.0.0.0", + "packets": 0 + }, + "source": { + "address": "127.0.0.1", + "port": 422, + "bytes": 0, + "ip": "127.0.0.1", + "packets": 0 + }, + "event": { + "duration": 5354, + "kind": "event", + "start": "2021-10-17T14:33:05.452045076Z", + "action": "network-connection-traffic", + "end": "2021-10-17T14:33:05.45205043Z", + "sf_type": "NF", + "category": "network", + "type": "connection" + }, + "user": { + "name": "root", + "id": 0, + "group": { + "name": "root", + "id": 0 + } + }, + "network": { + "community_id": "1233242", + "protocol": "udp", + "bytes": 0, + "iana_number": "17" + } + } + ], + "metadata": [ + 1634481185452 + ] +} +``` + +***Note** "metadata" parameter is returned for pagination. The value of the metadata is used to fetch next batch of results. + +## Elasticsearch response results to STIX objects + +### Translate command +``` +python main.py translate elastic_ecs results '{"type":"identity","id":"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name":"elastic_ecs","identity_class":"events", "created": "2022-04-07T20:35:41.042Z", "modified": "2022-04-07T20:35:41.042Z"}' '[]' +``` +### STIX 2.0 Output + +``` +{ + "type": "bundle", + "id": "bundle--d0c6c6ce-69a2-45bb-82f3-a5e96a9af95e", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "elastic", + "identity_class": "events" + }, + { + "id": "observed-data--a0b6e2fa-1f8b-47de-9777-383646ad304e", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-03-23T18:47:27.536Z", + "modified": "2023-03-23T18:47:27.536Z", + "objects": { + "0": { + "type": "process", + "name": "sshd", + "created": "2021-10-16T01:54:25.664830275Z", + "pid": 820693, + "command_line": "/usr/sbin/sshd -D", + "binary_ref": "3" + }, + "1": { + "type": "process", + "parent_ref": "0", + "name": "sshd", + "created": "2021-10-17T14:33:05.446908698Z", + "pid": 899531, + "x_thread_id": 899531, + "command_line": "/path/sbin/sshd -D -e", + "binary_ref": "5", + "creator_user_ref": "12", + "opened_connection_refs": [ + "8" + ] + }, + "2": { + "type": "x-oca-event", + "parent_process_ref": "0", + "process_ref": "1", + "duration": 5354, + "kind": "event", + "start": "2021-10-17T14:33:05.452045076Z", + "action": "network-connection-traffic", + "end": "2021-10-17T14:33:05.45205043Z", + "category": "network", + "event_type": "connection", + "user_ref": "12", + "network_ref": "8" + }, + "3": { + "type": "file", + "name": "sshd", + "parent_directory_ref": "4" + }, + "4": { + "type": "directory", + "path": "/path/sbin" + }, + "5": { + "type": "file", + "name": "sshd", + "parent_directory_ref": "6", + "x_owner_ref": "12" + }, + "6": { + "type": "directory", + "path": "/path/sbin" + }, + "7": { + "type": "x-ecs-destination", + "address": "0.0.0.0" + }, + "8": { + "type": "network-traffic", + "dst_port": 655, + "dst_byte_count": 0, + "dst_ref": "9", + "dst_packets": 0, + "src_port": 42001, + "src_byte_count": 0, + "src_ref": "11", + "src_packets": 0, + "x_community_id": "123456", + "protocols": [ + "udp" + ] + }, + "9": { + "type": "ipv4-addr", + "value": "0.0.0.0" + }, + "10": { + "type": "x-ecs-source", + "address": "127.0.0.1" + }, + "11": { + "type": "ipv4-addr", + "value": "127.0.0.1" + }, + "12": { + "type": "user-account", + "user_id": "root", + "account_login": "root", + "x_group": { + "name": "root", + "id": 0 + } + } + }, + "first_observed": "2021-10-17T14:33:05.452045076Z", + "last_observed": "2021-10-17T14:33:05.452045076Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +```