From f1bdd638cd04405f78b26bbe4eef17a4f129cb46 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 8 Feb 2023 17:15:15 -0500 Subject: [PATCH 01/32] WIP: eliminate x-ecs-process from ECS connector mapping --- .../json/beats_from_stix_map.json | 59 +++++++------------ 1 file changed, 21 insertions(+), 38 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 7d7ba75c7..3b12bddba 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -108,55 +108,38 @@ "group_name": ["user.group.name"] } }, + "x-oca-thread" :{ + "process_ref": ["process.pid"], + "id": ["process.thread.id"], + "name": ["process.thread.name.keyword"] + }, "process" : { "fields": { "command_line": ["process.command_line.keyword", "powershell.command.value"], "created": ["process.start"], + "cwd": ["process.working_directory.keyword"], "pid": ["process.pid", "process.ppid", "process.parent.pid", "process.parent.ppid"], "name": ["process.name.keyword", "process.parent.name.keyword"], "creator_user_ref.user_id": ["user.name.keyword"], "parent_ref.pid": ["process.ppid", "process.parent.ppid"], "parent_ref.name": ["process.parent.name.keyword"], + "parent_ref.exit_code": ["process.parent.exit_code"], + "parent_ref.pgid": ["process.parent.pgid"], + "parent_ref.title": ["process.parent.title.keyword"], + "parent_ref.thread.id": ["process.parent.thread.id"], + "parent_ref.thread.name": ["process.parent.thread.name.keyword"], + "parent_ref.uptime": ["process.parent.uptime"], + "parent_ref.cwd": ["process.parent.working_directory"], "binary_ref.name": ["process.executable.keyword", "process.parent.executable.keyword"], + "extensions": { + "windows-process-ext": { + "window_title": ["process.title"] + } + }, + "x_exit_code": ["process.exit_code"], "x_ttp_tags": ["tags"], - "x_unique_id": ["process.entity_id.keyword", "process.parent.entity_id.keyword"] - } - }, - "x-ecs-process": { - "fields": { - "args": ["process.args.keyword"], - "args_count": ["process.args_count"], - "executable": ["process.executable.keyword"], - "exit_code": ["process.exit_code"], - "thread.id": ["process.thread.id"], - "thread.name": ["process.thread.name"], - "title": ["process.title.keyword"], - "uptime": ["process.uptime"], - "working_directory": ["process.working_directory.keyword"], - "parent.args": ["process.parent.args.keyword"], - "parent.args_count": ["process.parent.args_count"], - "parent.exit_code": ["process.parent.exit_code"], - "parent.pgid": ["process.parent.pgid"], - "parent.thread.id": ["process.parent.thread.id"], - "parent.thread.name": ["process.parent.thread.name"], - "parent.title": ["process.parent.title"], - "parent.uptime": ["process.parent.uptime"], - "parent.working_directory": ["process.parent.working_directory"], - "pe.company": ["process.pe.company.keyword"], - "pe.description": ["process.pe.description.keyword"], - "pe.file_version": ["process.pe.file_version.keyword"], - "pe.original_file_name": ["process.pe.original_file_name.keyword"], - "pe.product": ["process.pe.product.keyword"], - "code_signature.exists": ["process.code_signature.exists"], - "code_signature.status": ["process.code_signature.status"], - "code_signature.subject_name": ["process.code_signature.subject_name"], - "code_signature.trusted": ["process.code_signature.trusted"], - "code_signature.valid": ["process.code_signature.valid"], - "parent.code_signature.exists": ["process.parent.code_signature.exists"], - "parent.code_signature.status": ["process.parent.code_signature.status"], - "parent.code_signature.subject_name": ["process.parent.code_signature.subject_name"], - "parent.code_signature.trusted": ["process.parent.code_signature.trusted"], - "parent.code_signature.valid": ["process.parent.code_signature.valid"] + "x_unique_id": ["process.entity_id.keyword", "process.parent.entity_id.keyword"], + "x_uptime": ["process.uptime"] } }, "url": { From 4acd337fdfe9026f8ed179b17d835aed8fa87359 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 8 Feb 2023 17:15:30 -0500 Subject: [PATCH 02/32] WIP: eliminate x-ecs-process from ECS connector mapping --- .../stix_translation/json/from_stix_map.json | 59 +++++++------------ 1 file changed, 21 insertions(+), 38 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index d0860d88b..1f813c947 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -108,55 +108,38 @@ "group_name": ["user.group.name"] } }, + "x-oca-thread" :{ + "process_ref": ["process.pid"], + "id": ["process.thread.id"], + "name": ["process.thread.name"] + }, "process" : { "fields": { "command_line": ["process.command_line", "powershell.command.value"], "created": ["process.start"], + "cwd": ["process.working_directory"], "pid": ["process.pid", "process.ppid", "process.parent.pid", "process.parent.ppid"], "name": ["process.name", "process.parent.name"], "creator_user_ref.user_id": ["user.name"], "parent_ref.pid": ["process.ppid", "process.parent.ppid"], "parent_ref.name": ["process.parent.name"], + "parent_ref.exit_code": ["process.parent.exit_code"], + "parent_ref.pgid": ["process.parent.pgid"], + "parent_ref.title": ["process.parent.title"], + "parent_ref.thread.id": ["process.parent.thread.id"], + "parent_ref.thread.name": ["process.parent.thread.name"], + "parent_ref.uptime": ["process.parent.uptime"], + "parent_ref.cwd": ["process.parent.working_directory"], "binary_ref.name": ["process.executable", "process.parent.executable"], + "extensions": { + "windows-process-ext": { + "window_title": ["process.title"] + } + }, + "x_exit_code": ["process.exit_code"], "x_ttp_tags": ["tags"], - "x_unique_id": ["process.entity_id", "process.parent.entity_id"] - } - }, - "x-ecs-process": { - "fields": { - "args": ["process.args"], - "args_count": ["process.args_count"], - "executable": ["process.executable"], - "exit_code": ["process.exit_code"], - "thread.id": ["process.thread.id"], - "thread.name": ["process.thread.name"], - "title": ["process.title"], - "uptime": ["process.uptime"], - "working_directory": ["process.working_directory"], - "parent.args": ["process.parent.args"], - "parent.args_count": ["process.parent.args_count"], - "parent.exit_code": ["process.parent.exit_code"], - "parent.pgid": ["process.parent.pgid"], - "parent.thread.id": ["process.parent.thread.id"], - "parent.thread.name": ["process.parent.thread.name"], - "parent.title": ["process.parent.title"], - "parent.uptime": ["process.parent.uptime"], - "parent.working_directory": ["process.parent.working_directory"], - "pe.company": ["process.pe.company"], - "pe.description": ["process.pe.description"], - "pe.file_version": ["process.pe.file_version"], - "pe.original_file_name": ["process.pe.original_file_name"], - "pe.product": ["process.pe.product"], - "code_signature.exists": ["process.code_signature.exists"], - "code_signature.status": ["process.code_signature.status"], - "code_signature.subject_name": ["process.code_signature.subject_name"], - "code_signature.trusted": ["process.code_signature.trusted"], - "code_signature.valid": ["process.code_signature.valid"], - "parent.code_signature.exists": ["process.parent.code_signature.exists"], - "parent.code_signature.status": ["process.parent.code_signature.status"], - "parent.code_signature.subject_name": ["process.parent.code_signature.subject_name"], - "parent.code_signature.trusted": ["process.parent.code_signature.trusted"], - "parent.code_signature.valid": ["process.parent.code_signature.valid"] + "x_unique_id": ["process.entity_id", "process.parent.entity_id"], + "x_uptime": ["process.uptime"] } }, "url": { From 0e1a32190ea58d01e8c9a7635dd9bf74fa934d1a Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 8 Feb 2023 17:15:42 -0500 Subject: [PATCH 03/32] WIP: eliminate x-ecs-process from ECS connector mapping --- .../stix_translation/json/to_stix_map.json | 106 +++++------------- 1 file changed, 29 insertions(+), 77 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index c502ec5ca..6a56ff048 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -782,16 +782,6 @@ "references": "process" } ], - "code_signature": { - "exists": { - "key": "x-ecs-process.code_signature_exists", - "object": "x_process" - }, - "subject_name": { - "key": "x-ecs-process.code_signature_subject_name", - "object": "x_process" - } - }, "ppid": [ { "key": "process.pid", @@ -804,36 +794,6 @@ "references": "process_parent" } ], - "pe": { - "company": { - "key": "x-ecs-process.pe_company", - "object": "x_process" - }, - "description": { - "key": "x-ecs-process.pe_description", - "object": "x_process" - }, - "file_version": { - "key": "x-ecs-process.pe_file_version", - "object": "x_process" - }, - "original_file_name": { - "key": "x-ecs-process.pe_original_file_name", - "object": "x_process" - }, - "product": { - "key": "x-ecs-process.pe_product", - "object": "x_process" - } - }, - "args": { - "key": "x-ecs-process.args", - "object": "x_process" - }, - "args_count": { - "key": "x-ecs-process.args_count", - "object": "x_process" - }, "command_line": { "key": "process.command_line", "object": "process" @@ -870,18 +830,10 @@ "object": "process" }, "exit_code": { - "key": "x-ecs-process.exit_code", - "object": "x_process" + "key": "process.x_exit_code", + "object": "process" }, "parent": { - "args": { - "key": "x-ecs-process.parent_args", - "object": "x_process" - }, - "args_count": { - "key": "x-ecs-process.parent_args_count", - "object": "x_process" - }, "command_line": { "key": "process.command_line", "object": "process_parent" @@ -891,8 +843,8 @@ "object": "process_parent" }, "exit_code": { - "key": "x-ecs-process.parent_exit_code", - "object": "x_process" + "key": "process.parent_exit_code", + "object": "process" }, "name": [ { @@ -911,8 +863,8 @@ } ], "pgid": { - "key": "x-ecs-process.parent_pgid", - "object": "x_process" + "key": "process.parent_ref.pgid", + "object": "process" }, "pid": [ { @@ -932,8 +884,8 @@ } ], "ppid": { - "key": "x-ecs-process.parent_ppid", - "object": "x_process", + "key": "process.parent_ref.ppid", + "object": "process", "transformer": "ToInteger" }, "start": { @@ -942,25 +894,25 @@ }, "thread": { "id": { - "key": "x-ecs-process.parent_thread_id", - "object": "x_process" + "key": "process.parent_ref.thread.id", + "object": "process" }, "name": { - "key": "x-ecs-process.parent_thread_name", - "object": "x_process" + "key": "process.parent_ref.thread.name", + "object": "process" } }, "title": { - "key": "x-ecs-process.parent_title", - "object": "x_process" + "key": "process.parent_ref.title", + "object": "process" }, "uptime": { - "key": "x-ecs-process.parent_uptime", - "object": "x_process" + "key": "process.parent_ref.uptime", + "object": "process" }, "working_directory": { - "key": "x-ecs-process.parent_working_directory", - "object": "x_process" + "key": "process.parent_ref.cwd", + "object": "process" }, "executable": [ { @@ -991,30 +943,30 @@ ] }, "pgid": { - "key": "x-ecs-process.exit_code", - "object": "x_process" + "key": "process.x_exit_code", + "object": "process" }, "thread": { "id": { - "key": "x-ecs-process.thread_id", + "key": "x-oca-thread.id", "object": "x_process" }, "name": { - "key": "x-ecs-process.thread_name", + "key": "x-oca-thread.name", "object": "x_process" } }, "title": { - "key": "x-ecs-process.title", - "object": "x_process" + "key": "process.windows-process-ext.window_title", + "object": "process" }, "uptime": { - "key": "x-ecs-process.uptime", - "object": "x_process" + "key": "process.x_uptime", + "object": "process" }, - "working_directory": { - "key": "x-ecs-process.working_directory", - "object": "x_process" + "cwd": { + "key": "process.cwd", + "object": "process" } }, "user": { From d2b7181c080b04c972bde92e63355ca5faaed82d Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Thu, 9 Feb 2023 17:45:02 -0500 Subject: [PATCH 04/32] WIP: eliminate x-ecs-process from ECS connector mapping --- .../stix_translation/json/beats_from_stix_map.json | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 3b12bddba..a9d6edd89 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -108,11 +108,6 @@ "group_name": ["user.group.name"] } }, - "x-oca-thread" :{ - "process_ref": ["process.pid"], - "id": ["process.thread.id"], - "name": ["process.thread.name.keyword"] - }, "process" : { "fields": { "command_line": ["process.command_line.keyword", "powershell.command.value"], @@ -126,8 +121,7 @@ "parent_ref.exit_code": ["process.parent.exit_code"], "parent_ref.pgid": ["process.parent.pgid"], "parent_ref.title": ["process.parent.title.keyword"], - "parent_ref.thread.id": ["process.parent.thread.id"], - "parent_ref.thread.name": ["process.parent.thread.name.keyword"], + "parent_ref.thread_id": ["process.parent.thread.id"], "parent_ref.uptime": ["process.parent.uptime"], "parent_ref.cwd": ["process.parent.working_directory"], "binary_ref.name": ["process.executable.keyword", "process.parent.executable.keyword"], @@ -137,6 +131,7 @@ } }, "x_exit_code": ["process.exit_code"], + "x_thread_id": ["process.thread.id"], "x_ttp_tags": ["tags"], "x_unique_id": ["process.entity_id.keyword", "process.parent.entity_id.keyword"], "x_uptime": ["process.uptime"] From 4d8b9c1f7fa0e32c8d6ce9022e0a2f81f47dd3a0 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Thu, 9 Feb 2023 17:45:33 -0500 Subject: [PATCH 05/32] WIP: eliminate x-ecs-process from ECS connector mapping --- .../elastic_ecs/stix_translation/json/from_stix_map.json | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index 1f813c947..2f25ad8d3 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -108,11 +108,6 @@ "group_name": ["user.group.name"] } }, - "x-oca-thread" :{ - "process_ref": ["process.pid"], - "id": ["process.thread.id"], - "name": ["process.thread.name"] - }, "process" : { "fields": { "command_line": ["process.command_line", "powershell.command.value"], @@ -126,8 +121,7 @@ "parent_ref.exit_code": ["process.parent.exit_code"], "parent_ref.pgid": ["process.parent.pgid"], "parent_ref.title": ["process.parent.title"], - "parent_ref.thread.id": ["process.parent.thread.id"], - "parent_ref.thread.name": ["process.parent.thread.name"], + "parent_ref.thread_id": ["process.parent.thread.id"], "parent_ref.uptime": ["process.parent.uptime"], "parent_ref.cwd": ["process.parent.working_directory"], "binary_ref.name": ["process.executable", "process.parent.executable"], @@ -137,6 +131,7 @@ } }, "x_exit_code": ["process.exit_code"], + "x_thread_id": ["process.thread.id"], "x_ttp_tags": ["tags"], "x_unique_id": ["process.entity_id", "process.parent.entity_id"], "x_uptime": ["process.uptime"] From 59fce7fa0be6cf3824b8bcdb4c3a27de20779cb7 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Thu, 9 Feb 2023 17:45:50 -0500 Subject: [PATCH 06/32] WIP: eliminate x-ecs-process from ECS connector mapping --- .../stix_translation/json/to_stix_map.json | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 6a56ff048..3f1ef0771 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -794,6 +794,28 @@ "references": "process_parent" } ], + "pe": { + "company": { + "key": "file.x_pe.company", + "object": "file" + }, + "description": { + "key": "file.x_pe.description", + "object": "file" + }, + "file_version": { + "key": "file.x_pe.file_version", + "object": "file" + }, + "original_file_name": { + "key": "file.x_pe.original_file_name", + "object": "file" + }, + "product": { + "key": "file.x_pe.product", + "object": "file" + } + }, "command_line": { "key": "process.command_line", "object": "process" @@ -894,11 +916,7 @@ }, "thread": { "id": { - "key": "process.parent_ref.thread.id", - "object": "process" - }, - "name": { - "key": "process.parent_ref.thread.name", + "key": "process.parent_ref.thread_id", "object": "process" } }, @@ -948,12 +966,8 @@ }, "thread": { "id": { - "key": "x-oca-thread.id", - "object": "x_process" - }, - "name": { - "key": "x-oca-thread.name", - "object": "x_process" + "key": "process.x_thread_id", + "object": "process" } }, "title": { From 82ea9ac984ddce614167a018d2500f14cef17db8 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Mon, 13 Feb 2023 13:48:44 -0500 Subject: [PATCH 07/32] WIP: consolidate file objects for ECS Connector mapping --- .../json/beats_from_stix_map.json | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index a9d6edd89..460cb0217 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -47,43 +47,39 @@ "fields": { "name": ["file.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword"], "created": ["file.created"], + "accessed": ["file.accessed"], "size": ["file.size"], + "mime_type": ["file.mime_type"], "hashes.MD5": ["file.hash.md5"], "hashes.'SHA-1'": ["file.hash.sha1"], "hashes.'SHA-256'": ["file.hash.sha256"], "hashes.'SHA-512'": ["file.hash.sha512"], - "parent_directory_ref.path": ["file.directory"] - } - }, - "x-ecs-file": { - "fields": { - "accessed": ["file.accessed"], - "attributes": ["file.attributes"], - "ctime": ["file.ctime"], - "device": ["file.device"], - "drive_letter": ["file.drive_letter"], - "extension": ["file.extension"], - "gid": ["file.gid"], - "group": ["file.group"], - "inode": ["file.inode"], - "mime_type": ["file.mime_type"], - "mode": ["file.mode"], - "mtime": ["file.mtime"], - "owner": ["file.owner"], - "path": ["file.path"], - "target_path": ["file.target_path"], - "type": ["file.type"], - "uid": ["file.uid"], - "pe.company": ["file.pe.company"], - "pe.description": ["file.pe.description"], - "pe.file_version": ["file.pe.file_version"], - "pe.original_file_name": ["file.pe.original_file_name"], - "pe.product": ["file.pe.product"], - "code_signature.exists": ["file.code_signature.exists"], - "code_signature.status": ["file.code_signature.status"], - "code_signature.subject_name": ["file.code_signature.subject_name"], - "code_signature.trusted": ["file.code_signature.trusted"], - "code_signature.valid": ["file.code_signature.valid"] + "parent_directory_ref.path": ["file.directory"], + "x_attributes": ["file.attributes"], + "x_ctime": ["file.ctime"], + "x_device": ["file.device"], + "x_drive_letter": ["file.drive_letter"], + "x_extension": ["file.extension"], + "x_gid": ["file.gid"], + "x_group": ["file.group"], + "x_inode": ["file.inode"], + "x_mode": ["file.mode"], + "x_mtime": ["file.mtime"], + "x_owner": ["file.owner"], + "x_path": ["file.path"], + "x_target_path": ["file.target_path"], + "x_type": ["file.type"], + "x_uid": ["file.uid"], + "x_pe.company": ["file.pe.company"], + "x_pe.description": ["file.pe.description"], + "x_pe.file_version": ["file.pe.file_version"], + "x_pe.original_file_name": ["file.pe.original_file_name"], + "x_pe.product": ["file.pe.product"], + "x_code_signature.exists": ["file.code_signature.exists"], + "x_code_signature.status": ["file.code_signature.status"], + "x_code_signature.subject_name": ["file.code_signature.subject_name"], + "x_code_signature.trusted": ["file.code_signature.trusted"], + "x_code_signature.valid": ["file.code_signature.valid"] } }, "directory": { From dca9da6b77f162824776c443243f26b718e45734 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Mon, 13 Feb 2023 13:49:08 -0500 Subject: [PATCH 08/32] WIP: consolidate file objects for ECS Connector mapping --- .../stix_translation/json/from_stix_map.json | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index 2f25ad8d3..49ce1f892 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -47,43 +47,39 @@ "fields": { "name": ["file.name", "file.path", "process.name", "process.executable", "process.parent.name", "process.parent.executable"], "created": ["file.created"], + "accessed": ["file.accessed"], "size": ["file.size"], + "mime_type": ["file.mime_type"], "hashes.MD5": ["file.hash.md5"], "hashes.'SHA-1'": ["file.hash.sha1"], "hashes.'SHA-256'": ["file.hash.sha256"], "hashes.'SHA-512'": ["file.hash.sha512"], - "parent_directory_ref.path": ["file.directory"] - } - }, - "x-ecs-file": { - "fields": { - "accessed": ["file.accessed"], - "attributes": ["file.attributes"], - "ctime": ["file.ctime"], - "device": ["file.device"], - "drive_letter": ["file.drive_letter"], - "extension": ["file.extension"], - "gid": ["file.gid"], - "group": ["file.group"], - "inode": ["file.inode"], - "mime_type": ["file.mime_type"], - "mode": ["file.mode"], - "mtime": ["file.mtime"], - "owner": ["file.owner"], - "path": ["file.path"], - "target_path": ["file.target_path"], - "type": ["file.type"], - "uid": ["file.uid"], - "pe.company": ["file.pe.company"], - "pe.description": ["file.pe.description"], - "pe.file_version": ["file.pe.file_version"], - "pe.original_file_name": ["file.pe.original_file_name"], - "pe.product": ["file.pe.product"], - "code_signature.exists": ["file.code_signature.exists"], - "code_signature.status": ["file.code_signature.status"], - "code_signature.subject_name": ["file.code_signature.subject_name"], - "code_signature.trusted": ["file.code_signature.trusted"], - "code_signature.valid": ["file.code_signature.valid"] + "parent_directory_ref.path": ["file.directory"], + "x_attributes": ["file.attributes"], + "x_ctime": ["file.ctime"], + "x_device": ["file.device"], + "x_drive_letter": ["file.drive_letter"], + "x_extension": ["file.extension"], + "x_gid": ["file.gid"], + "x_group": ["file.group"], + "x_inode": ["file.inode"], + "x_mode": ["file.mode"], + "x_mtime": ["file.mtime"], + "x_owner": ["file.owner"], + "x_path": ["file.path"], + "x_target_path": ["file.target_path"], + "x_type": ["file.type"], + "x_uid": ["file.uid"], + "x_pe.company": ["file.pe.company"], + "x_pe.description": ["file.pe.description"], + "x_pe.file_version": ["file.pe.file_version"], + "x_pe.original_file_name": ["file.pe.original_file_name"], + "x_pe.product": ["file.pe.product"], + "x_code_signature.exists": ["file.code_signature.exists"], + "x_code_signature.status": ["file.code_signature.status"], + "x_code_signature.subject_name": ["file.code_signature.subject_name"], + "x_code_signature.trusted": ["file.code_signature.trusted"], + "x_code_signature.valid": ["file.code_signature.valid"] } }, "directory": { From aa52e383db48bc7fb7295c85e25d343e0f2f2f60 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Mon, 13 Feb 2023 13:49:33 -0500 Subject: [PATCH 09/32] WIP: consolidate file objects for ECS Connector mapping --- .../stix_translation/json/to_stix_map.json | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 3f1ef0771..175eaef28 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -1386,103 +1386,103 @@ }, "pe": { "company": { - "key": "x-ecs-file.pe_company", - "object": "x_file" + "key": "file.x_pe_company", + "object": "file" }, "description": { - "key": "x-ecs-file.pe_description", - "object": "x_file" + "key": "file.x_pe_description", + "object": "file" }, "file_version": { - "key": "x-ecs-file.pe_file_version", - "object": "x_file" + "key": "file.x_pe_file_version", + "object": "file" }, "original_file_name": { - "key": "x-ecs-file.pe_original_file_name", - "object": "x_file" + "key": "file.x_pe_original_file_name", + "object": "file" }, "product": { - "key": "x-ecs-file.pe_product", - "object": "x_file" + "key": "file.x_pe_product", + "object": "file" } }, "code_signature": { "exists": { - "key": "x-ecs-file.code_signature_exists", - "object": "x_file" + "key": "file.x_code_signature_exists", + "object": "file" }, "subject_name": { - "key": "x-ecs-file.code_signature_subject_name", - "object": "x_file" + "key": "file.x_code_signature_subject_name", + "object": "file" } }, "accessed": { - "key": "x-ecs-file.accessed", - "object": "x_file" + "key": "file.accessed", + "object": "file" }, "attributes": { - "key": "x-ecs-file.attributes", - "object": "x_file" + "key": "file.x_attributes", + "object": "file" }, "ctime": { - "key": "x-ecs-file.ctime", - "object": "x_file" + "key": "file.x_ctime", + "object": "file" }, "device": { - "key": "x-ecs-file.device", - "object": "x_file" + "key": "file.x_device", + "object": "file" }, "drive_letter": { - "key": "x-ecs-file.drive_letter", - "object": "x_file" + "key": "file.x_drive_letter", + "object": "file" }, "extension": { - "key": "x-ecs-file.extension", - "object": "x_file" + "key": "file.x_extension", + "object": "file" }, "gid": { - "key": "x-ecs-file.gid", - "object": "x_file" + "key": "file.x_gid", + "object": "file" }, "group": { - "key": "x-ecs-file.group", - "object": "x_file" + "key": "file.x_group", + "object": "file" }, "inode": { - "key": "x-ecs-file.inode", - "object": "x_file" + "key": "file.x_inode", + "object": "file" }, "mime_type": { - "key": "x-ecs-file.mime_type", - "object": "x_file" + "key": "file.mime_type", + "object": "file" }, "mode": { - "key": "x-ecs-file.mode", - "object": "x_file" + "key": "file.x_mode", + "object": "file" }, "mtime": { - "key": "x-ecs-file.mtime", - "object": "x_file" + "key": "file.x_mtime", + "object": "file" }, "owner": { - "key": "x-ecs-file.owner", - "object": "x_file" + "key": "file.x_owner", + "object": "file" }, "path": { - "key": "x-ecs-file.path", - "object": "x_file" + "key": "file.x_path", + "object": "file" }, "target_path": { - "key": "x-ecs-file.target_path", - "object": "x_file" + "key": "file.x_target_path", + "object": "file" }, "type": { - "key": "x-ecs-file.type", - "object": "x_file" + "key": "file.x_type", + "object": "file" }, "uid": { - "key": "x-ecs-file.uid", - "object": "x_file" + "key": "file.x_uid", + "object": "file" } }, "group": { From 80ff776402deb4ced3eaeff17bf69f7c3d8aa54b Mon Sep 17 00:00:00 2001 From: harmedox Date: Mon, 13 Feb 2023 16:04:11 -0500 Subject: [PATCH 10/32] WIP: reconcile mtime and ctime in file object for ECS Connector mapping --- .../stix_translation/json/beats_from_stix_map.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 460cb0217..f9d9a0626 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -46,7 +46,8 @@ "file": { "fields": { "name": ["file.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword"], - "created": ["file.created"], + "created": ["file.created", "file.ctime"], + "modified": ["file.mtime"], "accessed": ["file.accessed"], "size": ["file.size"], "mime_type": ["file.mime_type"], @@ -56,7 +57,6 @@ "hashes.'SHA-512'": ["file.hash.sha512"], "parent_directory_ref.path": ["file.directory"], "x_attributes": ["file.attributes"], - "x_ctime": ["file.ctime"], "x_device": ["file.device"], "x_drive_letter": ["file.drive_letter"], "x_extension": ["file.extension"], @@ -64,7 +64,6 @@ "x_group": ["file.group"], "x_inode": ["file.inode"], "x_mode": ["file.mode"], - "x_mtime": ["file.mtime"], "x_owner": ["file.owner"], "x_path": ["file.path"], "x_target_path": ["file.target_path"], From 01d9cf3fe208e2c077dc287e511cfcc3e9a4b51b Mon Sep 17 00:00:00 2001 From: harmedox Date: Mon, 13 Feb 2023 16:05:06 -0500 Subject: [PATCH 11/32] WIP: reconcile mtime and ctime in file object for ECS Connector mapping --- .../elastic_ecs/stix_translation/json/from_stix_map.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index 49ce1f892..669cd5157 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -46,7 +46,8 @@ "file": { "fields": { "name": ["file.name", "file.path", "process.name", "process.executable", "process.parent.name", "process.parent.executable"], - "created": ["file.created"], + "created": ["file.created", "file.ctime"], + "modified": ["file.mtime"], "accessed": ["file.accessed"], "size": ["file.size"], "mime_type": ["file.mime_type"], @@ -56,7 +57,6 @@ "hashes.'SHA-512'": ["file.hash.sha512"], "parent_directory_ref.path": ["file.directory"], "x_attributes": ["file.attributes"], - "x_ctime": ["file.ctime"], "x_device": ["file.device"], "x_drive_letter": ["file.drive_letter"], "x_extension": ["file.extension"], @@ -64,7 +64,6 @@ "x_group": ["file.group"], "x_inode": ["file.inode"], "x_mode": ["file.mode"], - "x_mtime": ["file.mtime"], "x_owner": ["file.owner"], "x_path": ["file.path"], "x_target_path": ["file.target_path"], From 8485d409df3d63959d403757f4e689210e4e1948 Mon Sep 17 00:00:00 2001 From: harmedox Date: Mon, 13 Feb 2023 16:05:26 -0500 Subject: [PATCH 12/32] WIP: reconcile mtime and ctime in file object for ECS Connector mapping --- .../elastic_ecs/stix_translation/json/to_stix_map.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 175eaef28..778c50cd2 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -1425,7 +1425,7 @@ "object": "file" }, "ctime": { - "key": "file.x_ctime", + "key": "file.created", "object": "file" }, "device": { @@ -1461,7 +1461,7 @@ "object": "file" }, "mtime": { - "key": "file.x_mtime", + "key": "file.modified", "object": "file" }, "owner": { From 4872c51eff12fc18d5ce40179b3313bb161eff0d Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Mon, 13 Feb 2023 18:00:12 -0500 Subject: [PATCH 13/32] Add missing (parent_ref).binary_ref mappings --- .../elastic_ecs/stix_translation/json/from_stix_map.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index 669cd5157..d749c7aa6 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -119,7 +119,14 @@ "parent_ref.thread_id": ["process.parent.thread.id"], "parent_ref.uptime": ["process.parent.uptime"], "parent_ref.cwd": ["process.parent.working_directory"], + "parent_ref.binary_ref.path": ["process.parent.executable"], + "parent_ref.binary_ref.parent_directory_ref.path": ["process.parent.executable"], "binary_ref.name": ["process.executable", "process.parent.executable"], + "binary_ref.parent_directory_ref.path": ["process.executable", "process.parent.executable"], + "binary_ref.hashes.MD5": ["process.hash.md5"], + "binary_ref.hashes.'SHA-1'": ["process.hash.sha1"], + "binary_ref.hashes.'SHA-256'": ["process.hash.sha256"], + "binary_ref.hashes.'SHA-512'": ["process.hash.sha512"], "extensions": { "windows-process-ext": { "window_title": ["process.title"] From 00c1cd1a0e7b37c2975f365a09d591cb71a593e5 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Mon, 13 Feb 2023 18:00:26 -0500 Subject: [PATCH 14/32] Add missing (parent_ref).binary_ref mappings --- .../stix_translation/json/beats_from_stix_map.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index f9d9a0626..3e4a46474 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -119,7 +119,14 @@ "parent_ref.thread_id": ["process.parent.thread.id"], "parent_ref.uptime": ["process.parent.uptime"], "parent_ref.cwd": ["process.parent.working_directory"], + "parent_ref.binary_ref.path": ["process.parent.executable"], + "parent_ref.binary_ref.parent_directory_ref.path": ["process.parent.executable"], "binary_ref.name": ["process.executable.keyword", "process.parent.executable.keyword"], + "binary_ref.parent_directory_ref.path": ["process.executable", "process.parent.executable"], + "binary_ref.hashes.MD5": ["process.hash.md5"], + "binary_ref.hashes.'SHA-1'": ["process.hash.sha1"], + "binary_ref.hashes.'SHA-256'": ["process.hash.sha256"], + "binary_ref.hashes.'SHA-512'": ["process.hash.sha512"], "extensions": { "windows-process-ext": { "window_title": ["process.title"] From e498db4e0eb8b9d3c37223b6e95633efe5022c19 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Mon, 13 Feb 2023 18:21:48 -0500 Subject: [PATCH 15/32] Add process.hash mappings --- .../stix_translation/json/to_stix_map.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 778c50cd2..f02b38eed 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -981,6 +981,24 @@ "cwd": { "key": "process.cwd", "object": "process" + }, + "hash": { + "md5": { + "key": "file.hashes.MD5", + "object": "process_executable" + }, + "sha1": { + "key": "file.hashes.SHA-1", + "object": "process_executable" + }, + "sha256": { + "key": "file.hashes.SHA-256", + "object": "process_executable" + }, + "sha512": { + "key": "file.hashes.SHA-512", + "object": "process_executable" + } } }, "user": { From d224fdb9df1c6bf3548c2992fb1ef20debd42f8c Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Tue, 14 Feb 2023 11:50:23 -0500 Subject: [PATCH 16/32] Fixed mapping for 'pe' structs --- .../elastic_ecs/stix_translation/json/to_stix_map.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index f02b38eed..f4e4001db 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -797,23 +797,23 @@ "pe": { "company": { "key": "file.x_pe.company", - "object": "file" + "object": "process_executable" }, "description": { "key": "file.x_pe.description", - "object": "file" + "object": "process_executable" }, "file_version": { "key": "file.x_pe.file_version", - "object": "file" + "object": "process_executable" }, "original_file_name": { "key": "file.x_pe.original_file_name", - "object": "file" + "object": "process_executable" }, "product": { "key": "file.x_pe.product", - "object": "file" + "object": "process_executable" } }, "command_line": { From ab41b44147e4c0aceee042f45f9ecf19dd5177fb Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Tue, 14 Feb 2023 12:32:49 -0500 Subject: [PATCH 17/32] Added test data - process creation event --- .../test_elastic_ecs_json_to_stix.py | 154 +++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index 0a3a6d4bc..70fb0555d 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -172,6 +172,159 @@ } } +ecs_event_data = { + "event" : { + "category" : [ + "process" + ], + "type" : [ + "start", + "process_start" + ], + "provider" : "Microsoft-Windows-Sysmon", + "code" : 1, + "action" : "Process Create (rule: ProcessCreate)", + "created" : "2021-10-24T23:58:21.586Z", + "kind" : "event", + "module" : "sysmon" + }, + "winlog" : { + "task" : "Process Create (rule: ProcessCreate)", + "provider_name" : "Microsoft-Windows-Sysmon", + "event_id" : 1, + "channel" : "Microsoft-Windows-Sysmon/Operational", + "event_data" : { + "FileVersion" : "10.0.17763.2145 (WinBuild.160101.0800)", + "Company" : "Microsoft Corporation", + "Product" : "Microsoft® Windows® Operating System", + "Description" : "DSREG commandline tool", + "RuleName" : "-", + "TerminalSessionId" : "0", + "LogonId" : "0x3e7", + "LogonGuid" : "{8dfc401c-1ef5-6175-e703-000000000000}", + "IntegrityLevel" : "System" + }, + "api" : "wineventlog", + "user" : { + "domain" : "NT AUTHORITY", + "name" : "SYSTEM", + "type" : "User", + "identifier" : "S-1-5-18" + }, + "provider_guid" : "{5770385f-c22a-43e0-bf4c-06f5698ffbd9}", + "opcode" : "Info", + "computer_name" : "win-server1.example.com", + "process" : { + "pid" : 3684, + "thread" : { + "id" : 4616 + } + }, + "record_id" : 1778428, + "version" : 5 + }, + "message" : "Process Create:\nRuleName: -\nUtcTime: 2021-10-24 23:58:20.569\nProcessGuid: {8dfc401c-f31c-6175-5715-000000001b00}\nProcessId: 5244\nImage: C:\\Windows\\System32\\dsregcmd.exe\nFileVersion: 10.0.17763.2145 (WinBuild.160101.0800)\nDescription: DSREG commandline tool\nProduct: Microsoft® Windows® Operating System\nCompany: Microsoft Corporation\nOriginalFileName: dsregcmd.exe\nCommandLine: C:\\Windows\\System32\\dsregcmd.exe $(Arg0) $(Arg1) $(Arg2)\nCurrentDirectory: C:\\Windows\\system32\\\nUser: NT AUTHORITY\\SYSTEM\nLogonGuid: {8dfc401c-1ef5-6175-e703-000000000000}\nLogonId: 0x3E7\nTerminalSessionId: 0\nIntegrityLevel: System\nHashes: MD5=D6957ACEDA86DE523AF0157800AA3C73,SHA256=BA79462455B6E216D0E7CD6FE36BF0EFF8A0D9DD06358D1C97B1014016256618,IMPHASH=382C77BFA0EEE2BA2BA8671D108AD9A3\nParentProcessGuid: {8dfc401c-1ef7-6175-2900-000000001b00}\nParentProcessId: 2244\nParentImage: C:\\Windows\\System32\\svchost.exe\nParentCommandLine: C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule", + "tags" : [ + "beats_input_codec_plain_applied" + ], + "hash" : { + "imphash" : "382c77bfa0eee2ba2ba8671d108ad9a3", + "sha256" : "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618", + "md5" : "d6957aceda86de523af0157800aa3c73" + }, + "host" : { + "architecture" : "x86_64", + "hostname" : "win-server1", + "os" : { + "kernel" : "10.0.17763.2237 (WinBuild.160101.0800)", + "build" : "17763.2237", + "family" : "windows", + "name" : "Windows Server 2019 Standard", + "version" : "10.0", + "platform" : "windows" + }, + "id" : "8dfc401c-b042-4f41-b427-91a9dc0b61ac", + "name" : "win-server1.example.com", + "mac" : [ + "06:07:08:09:0a:0b" + ], + "ip" : [ + "fedc::ba98:7654:3210:1234", + "9.10.11.12", + "10.11.12.13" + ] + }, + "log" : { + "level" : "information" + }, + "user" : { + "domain" : "NT AUTHORITY", + "name" : "SYSTEM" + }, + "@timestamp" : "2021-10-24T23:58:20.569Z", + "ecs" : { + "version" : "1.7.0" + }, + "agent" : { + "type" : "winlogbeat", + "ephemeral_id" : "c5c31d91-f913-4f23-9609-3f92e83e4cb7", + "hostname" : "win-server1", + "id" : "50a12d7e-a002-4a69-a5e8-f3b07afbfeb7", + "name" : "win-server1", + "version" : "7.11.2" + }, + "process" : { + "pid" : 5244, + "parent" : { + "pid" : 2244, + "entity_id" : "{8dfc401c-1ef7-6175-2900-000000001b00}", + "executable" : "C:\\Windows\\System32\\svchost.exe", + "command_line" : "C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule", + "name" : "svchost.exe", + "args" : [ + "C:\\Windows\\system32\\svchost.exe", + "-k", + "netsvcs", + "-p", + "-s", + "Schedule" + ] + }, + "pe" : { + "company" : "Microsoft Corporation", + "file_version" : "10.0.17763.2145 (WinBuild.160101.0800)", + "description" : "DSREG commandline tool", + "imphash" : "382c77bfa0eee2ba2ba8671d108ad9a3", + "original_file_name" : "dsregcmd.exe", + "product" : "Microsoft® Windows® Operating System" + }, + "entity_id" : "{8dfc401c-f31c-6175-5715-000000001b00}", + "executable" : "C:\\Windows\\System32\\dsregcmd.exe", + "working_directory" : "C:\\Windows\\system32\\", + "command_line" : "C:\\Windows\\System32\\dsregcmd.exe $(Arg0) $(Arg1) $(Arg2)", + "args" : [ + "C:\\Windows\\System32\\dsregcmd.exe", + "$(Arg0)", + "$(Arg1)", + "$(Arg2)" + ], + "hash" : { + "md5" : "d6957aceda86de523af0157800aa3c73", + "sha256" : "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618" + }, + "name" : "dsregcmd.exe" + }, + "@version" : "1", + "related" : { + "hash" : [ + "d6957aceda86de523af0157800aa3c73", + "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618", + "382c77bfa0eee2ba2ba8671d108ad9a3" + ], + "user" : "SYSTEM" + } +} + class TestElasticEcsTransform(unittest.TestCase, object): @staticmethod def get_first(itr, constraint): @@ -549,4 +702,3 @@ def test_unmapped_attribute_alone(self): assert('objects' in observed_data) objects = observed_data['objects'] assert(objects == {}) - From 4670f6f738790f88d7cc07f323335ee649b3a25d Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Tue, 14 Feb 2023 14:13:39 -0500 Subject: [PATCH 18/32] Added translation of ecs_event_data and test for the translation --- .../test_elastic_ecs_json_to_stix.py | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index 70fb0555d..552ba9c5f 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -702,3 +702,217 @@ def test_unmapped_attribute_alone(self): assert('objects' in observed_data) objects = observed_data['objects'] assert(objects == {}) + + + """ This is the translation of ecs_event_data: +{ + "type": "bundle", + "id": "bundle--93346d23-3d0c-4081-ae22-f5c1ce7a8503", + "objects": [ + { + "type": "identity", + "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "name": "ElasticEcs", + "identity_class": "events" + }, + { + "id": "observed-data--e30db2a1-f35a-4654-a1bb-1b1f156487f8", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2023-02-14T17:23:03.684Z", + "modified": "2023-02-14T17:23:03.684Z", + "objects": { + "0": { + "type": "x-oca-event", + "category": [ + "process" + ], + "event_type": [ + "start", + "process_start" + ], + "provider": "Microsoft-Windows-Sysmon", + "code": 1, + "action": "Process Create (rule: ProcessCreate)", + "created": "2021-10-24T23:58:21.586Z", + "kind": "event", + "module": "sysmon", + "host_ref": "2", + "user_ref": "9", + "agent": "win-server1", + "process_ref": "1", + "parent_process_ref": "12" + }, + "1": { + "type": "process", + "x_ttp_tags": [ + "beats_input_codec_plain_applied" + ], + "creator_user_ref": "9", + "pid": 5244, + "parent_ref": "12", + "x_unique_id": "{8dfc401c-f31c-6175-5715-000000001b00}", + "binary_ref": "15", + "command_line": "C:\\Windows\\System32\\dsregcmd.exe $(Arg0) $(Arg1) $(Arg2)", + "name": "dsregcmd.exe" + }, + "2": { + "type": "x-oca-asset", + "architecture": "x86_64", + "hostname": "win-server1", + "os_name": "Windows Server 2019 Standard", + "os_version": "10.0", + "os_platform": "windows", + "host_id": "8dfc401c-b042-4f41-b427-91a9dc0b61ac", + "name": "win-server1.example.com", + "mac_refs": [ + "3" + ], + "ip_refs": [ + "4", + "5", + "6" + ] + }, + "3": { + "type": "mac-addr", + "value": "06:07:08:09:0a:0b" + }, + "4": { + "type": "ipv4-addr", + "value": "9.10.11.12" + }, + "5": { + "type": "ipv4-addr", + "value": "10.11.12.13" + }, + "6": { + "type": "ipv6-addr", + "value": "fedc::ba98:7654:3210:1234" + }, + "7": { + "type": "x-ecs-log", + "level": "information" + }, + "8": { + "type": "x-ecs-user", + "domain": "NT AUTHORITY" + }, + "9": { + "type": "user-account", + "user_id": "SYSTEM", + "account_login": "SYSTEM" + }, + "10": { + "type": "x-ecs", + "version": "1.7.0" + }, + "11": { + "type": "software", + "vendor": "winlogbeat", + "name": "win-server1", + "version": "7.11.2" + }, + "12": { + "type": "process", + "pid": 2244, + "x_unique_id": "{8dfc401c-1ef7-6175-2900-000000001b00}", + "binary_ref": "13", + "command_line": "C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule", + "name": "svchost.exe" + }, + "13": { + "type": "file", + "name": "svchost.exe", + "parent_directory_ref": "14" + }, + "14": { + "type": "directory", + "path": "C:\\Windows\\System32" + }, + "15": { + "type": "file", + "x_pe": { + "company": "Microsoft Corporation", + "file_version": "10.0.17763.2145 (WinBuild.160101.0800)", + "description": "DSREG commandline tool", + "original_file_name": "dsregcmd.exe", + "product": "Microsoft\u00ae Windows\u00ae Operating System" + }, + "name": "dsregcmd.exe", + "parent_directory_ref": "16", + "hashes": { + "MD5": "d6957aceda86de523af0157800aa3c73", + "SHA-256": "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618" + } + }, + "16": { + "type": "directory", + "path": "C:\\Windows\\System32" + }, + "17": { + "type": "x-ecs-related", + "hash": [ + "d6957aceda86de523af0157800aa3c73", + "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618", + "382c77bfa0eee2ba2ba8671d108ad9a3" + ], + "user": "SYSTEM" + } + }, + "first_observed": "2021-10-24T23:58:20.569Z", + "last_observed": "2021-10-24T23:58:20.569Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} + """ + def test_x_ecs_event(self): + result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([ecs_event_data])) + assert (result_bundle['type'] == 'bundle') + translation_objects = result_bundle.get('objects') + assert (translation_objects and len(translation_objects) == 2) + observed_data = translation_objects[1] + stix_objects = observed_data.get("objects") + assert (stix_objects and (stix_objects.__class__ is dict) and (len(stix_objects) > 10)) + x_oca_event = stix_objects.get("0") + assert (x_oca_event and x_oca_event.get("type") == "x-oca-event") + main_process = stix_objects.get("1") + assert ( + main_process and + main_process.get("type") == "process" + and main_process.get("x_unique_id") == "{8dfc401c-f31c-6175-5715-000000001b00}" + ) + parent_process_key = main_process.get("parent_ref") + executable_file_key = main_process.get("binary_ref") + parent_process = stix_objects.get(parent_process_key) + assert ( + parent_process and + parent_process.get("type") == "process" and + parent_process.get("pid") == 2244 and + parent_process.get("x_unique_id") == "{8dfc401c-1ef7-6175-2900-000000001b00}" and + parent_process.get("command_line") == "C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule" and + parent_process.get("name") == "svchost.exe" + ) + executable_file = stix_objects.get(executable_file_key) + assert ( + executable_file and + executable_file.get("type") == "file" and + executable_file.get("name") == "dsregcmd.exe" + ) + exec_file_pe_info = executable_file.get("x_pe") + assert ( + exec_file_pe_info and + exec_file_pe_info.get("company") == "Microsoft Corporation" and + exec_file_pe_info.get("file_version") == "10.0.17763.2145 (WinBuild.160101.0800)" and + exec_file_pe_info.get("description") == "DSREG commandline tool" and + exec_file_pe_info.get("original_file_name") == "dsregcmd.exe" and + exec_file_pe_info.get("product") == "Microsoft\u00ae Windows\u00ae Operating System" + ) + exec_file_hashes = executable_file.get("hashes") + assert ( + exec_file_hashes and + exec_file_hashes.get("MD5") == "d6957aceda86de523af0157800aa3c73" and + exec_file_hashes.get("SHA-256") == "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618" + ) \ No newline at end of file From 2af0c0e6657e6857b679d6b5887309a7055968c8 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 15 Feb 2023 12:28:34 -0500 Subject: [PATCH 19/32] Refactored file entity --- .../json/beats_from_stix_map.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 3e4a46474..1b280405a 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -57,18 +57,18 @@ "hashes.'SHA-512'": ["file.hash.sha512"], "parent_directory_ref.path": ["file.directory"], "x_attributes": ["file.attributes"], - "x_device": ["file.device"], - "x_drive_letter": ["file.drive_letter"], "x_extension": ["file.extension"], - "x_gid": ["file.gid"], - "x_group": ["file.group"], - "x_inode": ["file.inode"], - "x_mode": ["file.mode"], - "x_owner": ["file.owner"], "x_path": ["file.path"], "x_target_path": ["file.target_path"], "x_type": ["file.type"], - "x_uid": ["file.uid"], + "x_unix.device": ["file.device"], + "x_unix.group_id": ["file.gid"], + "x_unix.group": ["file.group"], + "x_unix.inode": ["file.inode"], + "x_unix.mode": ["file.mode"], + "x_user_ref.user_id": ["file.uid"], + "x_user_ref.account_login": ["file.owner"], + "x_win_drive_letter": ["file.drive_letter"], "x_pe.company": ["file.pe.company"], "x_pe.description": ["file.pe.description"], "x_pe.file_version": ["file.pe.file_version"], From 9f9b65842fde9a737ff5dff4416ce07ed14d7f13 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 15 Feb 2023 12:28:42 -0500 Subject: [PATCH 20/32] Refactored file entity --- .../stix_translation/json/from_stix_map.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index d749c7aa6..b97793611 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -57,18 +57,18 @@ "hashes.'SHA-512'": ["file.hash.sha512"], "parent_directory_ref.path": ["file.directory"], "x_attributes": ["file.attributes"], - "x_device": ["file.device"], - "x_drive_letter": ["file.drive_letter"], "x_extension": ["file.extension"], - "x_gid": ["file.gid"], - "x_group": ["file.group"], - "x_inode": ["file.inode"], - "x_mode": ["file.mode"], - "x_owner": ["file.owner"], "x_path": ["file.path"], "x_target_path": ["file.target_path"], "x_type": ["file.type"], - "x_uid": ["file.uid"], + "x_unix.device": ["file.device"], + "x_unix.group_id": ["file.gid"], + "x_unix.group": ["file.group"], + "x_unix.inode": ["file.inode"], + "x_unix.mode": ["file.mode"], + "x_user_ref.user_id": ["file.uid"], + "x_user_ref.account_login": ["file.owner"], + "x_win_drive_letter": ["file.drive_letter"], "x_pe.company": ["file.pe.company"], "x_pe.description": ["file.pe.description"], "x_pe.file_version": ["file.pe.file_version"], From 0977196376d22963c8747ce2a74c1a62be393a86 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 15 Feb 2023 12:29:17 -0500 Subject: [PATCH 21/32] Adjusted mappings for refactored file entity --- .../stix_translation/json/to_stix_map.json | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index f4e4001db..58de73dc3 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -1016,6 +1016,11 @@ "object": "process", "references": "user" }, + { + "key": "file.x_user_ref", + "object": "file", + "references": "user" + }, { "key": "x-oca-event.user_ref", "object": "event", @@ -1059,6 +1064,11 @@ "object": "process", "references": "user" }, + { + "key": "file.x_user_ref", + "object": "file", + "references": "user" + }, { "key": "x-oca-event.user_ref", "object": "event", @@ -1404,29 +1414,29 @@ }, "pe": { "company": { - "key": "file.x_pe_company", + "key": "file.x_pe.company", "object": "file" }, "description": { - "key": "file.x_pe_description", + "key": "file.x_pe.description", "object": "file" }, "file_version": { - "key": "file.x_pe_file_version", + "key": "file.x_pe.file_version", "object": "file" }, "original_file_name": { - "key": "file.x_pe_original_file_name", + "key": "file.x_pe.original_file_name", "object": "file" }, "product": { - "key": "file.x_pe_product", + "key": "file.x_pe.product", "object": "file" } }, "code_signature": { "exists": { - "key": "file.x_code_signature_exists", + "key": "file.x_code_signature.exists", "object": "file" }, "subject_name": { @@ -1447,11 +1457,11 @@ "object": "file" }, "device": { - "key": "file.x_device", + "key": "file.x_unix.device", "object": "file" }, "drive_letter": { - "key": "file.x_drive_letter", + "key": "file.x_win_drive_letter", "object": "file" }, "extension": { @@ -1459,15 +1469,15 @@ "object": "file" }, "gid": { - "key": "file.x_gid", + "key": "file.x_unix.group_id", "object": "file" }, "group": { - "key": "file.x_group", + "key": "file.x_unix.group", "object": "file" }, "inode": { - "key": "file.x_inode", + "key": "file.x_unix.inode", "object": "file" }, "mime_type": { @@ -1475,7 +1485,7 @@ "object": "file" }, "mode": { - "key": "file.x_mode", + "key": "file.x_unix.mode", "object": "file" }, "mtime": { @@ -1499,7 +1509,7 @@ "object": "file" }, "uid": { - "key": "file.x_uid", + "key": "file.x_unix.user_id", "object": "file" } }, From 51f535669c2141dba99e06b1442bfc323b9dce76 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 15 Feb 2023 12:30:00 -0500 Subject: [PATCH 22/32] Changed tests to account for refactored file entity --- .../tests/stix_translation/test_elastic_ecs_json_to_stix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index 552ba9c5f..ee58c47a0 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -561,7 +561,7 @@ def test_x_ibm_event(self): assert(process_obj['command_line'] == "C:\\WINDOWS\\system32\\wbem\\unsecapp.exe -Embedding") binary_obj = objects[process_obj['binary_ref']] assert(binary_obj is not None), "process binary ref not found" - assert(binary_obj.keys() == {'type', 'name', 'parent_directory_ref'}) + assert(binary_obj.keys() == {'type', 'name', 'parent_directory_ref', 'hashes'}) assert(binary_obj['type'] == "file") assert(binary_obj['name'] == "unsecapp.exe") binary_parent_dir_obj = objects[binary_obj['parent_directory_ref']] @@ -590,7 +590,7 @@ def test_x_ibm_event(self): file_ref = event_object['file_ref'] assert(file_ref in objects), f"file_ref with key {event_object['file_ref']} not found" file_obj = objects[file_ref] - assert(file_obj.keys() == {'type', 'name', 'parent_directory_ref'}) + assert(file_obj.keys() == {'type', 'name', 'parent_directory_ref', 'x_user_ref'}) assert(file_obj['type'] == 'file') assert(file_obj['name'] == "example.png") parent_obj = objects[file_obj['parent_directory_ref']] @@ -671,7 +671,7 @@ def test_file_prop(self): file_object = TestElasticEcsTransform.get_first(objects.values(), lambda o: type(o) == dict and o.get('type') == 'file' and o.get('name') == 'example.png') assert (file_object is not None), 'file object type not found' - assert (file_object.keys() == {'type', 'name', 'parent_directory_ref'}) + assert (file_object.keys() == {'type', 'name', 'parent_directory_ref', 'x_user_ref'}) assert (file_object['type'] == 'file') assert (file_object['name'] == 'example.png') parent_directory_ref = file_object['parent_directory_ref'] From 491d42016e0e6b11b96dd82607177660ed089cdc Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Wed, 15 Feb 2023 16:27:09 -0500 Subject: [PATCH 23/32] Replace file.x_user_ref with file.x_owner_ref --- .../stix_translation/json/beats_from_stix_map.json | 4 ++-- .../elastic_ecs/stix_translation/json/from_stix_map.json | 4 ++-- .../elastic_ecs/stix_translation/json/to_stix_map.json | 4 ++-- .../tests/stix_translation/test_elastic_ecs_json_to_stix.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 1b280405a..2a3ce427a 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -66,8 +66,8 @@ "x_unix.group": ["file.group"], "x_unix.inode": ["file.inode"], "x_unix.mode": ["file.mode"], - "x_user_ref.user_id": ["file.uid"], - "x_user_ref.account_login": ["file.owner"], + "x_owner_ref.user_id": ["file.uid"], + "x_owner_ref.account_login": ["file.owner"], "x_win_drive_letter": ["file.drive_letter"], "x_pe.company": ["file.pe.company"], "x_pe.description": ["file.pe.description"], diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index b97793611..be4c85b2d 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -66,8 +66,8 @@ "x_unix.group": ["file.group"], "x_unix.inode": ["file.inode"], "x_unix.mode": ["file.mode"], - "x_user_ref.user_id": ["file.uid"], - "x_user_ref.account_login": ["file.owner"], + "x_owner_ref.user_id": ["file.uid"], + "x_owner_ref.account_login": ["file.owner"], "x_win_drive_letter": ["file.drive_letter"], "x_pe.company": ["file.pe.company"], "x_pe.description": ["file.pe.description"], diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 58de73dc3..27cd2f8e1 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -1017,7 +1017,7 @@ "references": "user" }, { - "key": "file.x_user_ref", + "key": "file.x_owner_ref", "object": "file", "references": "user" }, @@ -1065,7 +1065,7 @@ "references": "user" }, { - "key": "file.x_user_ref", + "key": "file.x_owner_ref", "object": "file", "references": "user" }, diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index ee58c47a0..4cb2c1f5c 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -590,7 +590,7 @@ def test_x_ibm_event(self): file_ref = event_object['file_ref'] assert(file_ref in objects), f"file_ref with key {event_object['file_ref']} not found" file_obj = objects[file_ref] - assert(file_obj.keys() == {'type', 'name', 'parent_directory_ref', 'x_user_ref'}) + assert(file_obj.keys() == {'type', 'name', 'parent_directory_ref', 'x_owner_ref'}) assert(file_obj['type'] == 'file') assert(file_obj['name'] == "example.png") parent_obj = objects[file_obj['parent_directory_ref']] @@ -671,7 +671,7 @@ def test_file_prop(self): file_object = TestElasticEcsTransform.get_first(objects.values(), lambda o: type(o) == dict and o.get('type') == 'file' and o.get('name') == 'example.png') assert (file_object is not None), 'file object type not found' - assert (file_object.keys() == {'type', 'name', 'parent_directory_ref', 'x_user_ref'}) + assert (file_object.keys() == {'type', 'name', 'parent_directory_ref', 'x_owner_ref'}) assert (file_object['type'] == 'file') assert (file_object['name'] == 'example.png') parent_directory_ref = file_object['parent_directory_ref'] From fd2aec2d36598cd10de03cda45cf7c74b9b5f341 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Thu, 16 Feb 2023 10:09:06 -0500 Subject: [PATCH 24/32] Adjust process mappings --- .../json/beats_from_stix_map.json | 14 ++++------- .../stix_translation/json/from_stix_map.json | 14 ++++------- .../stix_translation/json/to_stix_map.json | 24 +++++++++---------- .../test_elastic_ecs_json_to_stix.py | 2 +- 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json index 2a3ce427a..5598a7a59 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json @@ -113,11 +113,11 @@ "creator_user_ref.user_id": ["user.name.keyword"], "parent_ref.pid": ["process.ppid", "process.parent.ppid"], "parent_ref.name": ["process.parent.name.keyword"], - "parent_ref.exit_code": ["process.parent.exit_code"], + "parent_ref.x_exit_code": ["process.parent.exit_code"], "parent_ref.pgid": ["process.parent.pgid"], - "parent_ref.title": ["process.parent.title.keyword"], - "parent_ref.thread_id": ["process.parent.thread.id"], - "parent_ref.uptime": ["process.parent.uptime"], + "parent_ref.x_window_title": ["process.parent.title.keyword"], + "parent_ref.x_thread_id": ["process.parent.thread.id"], + "parent_ref.x_uptime": ["process.parent.uptime"], "parent_ref.cwd": ["process.parent.working_directory"], "parent_ref.binary_ref.path": ["process.parent.executable"], "parent_ref.binary_ref.parent_directory_ref.path": ["process.parent.executable"], @@ -127,11 +127,7 @@ "binary_ref.hashes.'SHA-1'": ["process.hash.sha1"], "binary_ref.hashes.'SHA-256'": ["process.hash.sha256"], "binary_ref.hashes.'SHA-512'": ["process.hash.sha512"], - "extensions": { - "windows-process-ext": { - "window_title": ["process.title"] - } - }, + "x_window_title": ["process.title"], "x_exit_code": ["process.exit_code"], "x_thread_id": ["process.thread.id"], "x_ttp_tags": ["tags"], diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index be4c85b2d..12c362478 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -113,11 +113,11 @@ "creator_user_ref.user_id": ["user.name"], "parent_ref.pid": ["process.ppid", "process.parent.ppid"], "parent_ref.name": ["process.parent.name"], - "parent_ref.exit_code": ["process.parent.exit_code"], + "parent_ref.x_exit_code": ["process.parent.exit_code"], "parent_ref.pgid": ["process.parent.pgid"], - "parent_ref.title": ["process.parent.title"], - "parent_ref.thread_id": ["process.parent.thread.id"], - "parent_ref.uptime": ["process.parent.uptime"], + "parent_ref.x_window_title": ["process.parent.title"], + "parent_ref.x_thread_id": ["process.parent.thread.id"], + "parent_ref.x_uptime": ["process.parent.uptime"], "parent_ref.cwd": ["process.parent.working_directory"], "parent_ref.binary_ref.path": ["process.parent.executable"], "parent_ref.binary_ref.parent_directory_ref.path": ["process.parent.executable"], @@ -127,11 +127,7 @@ "binary_ref.hashes.'SHA-1'": ["process.hash.sha1"], "binary_ref.hashes.'SHA-256'": ["process.hash.sha256"], "binary_ref.hashes.'SHA-512'": ["process.hash.sha512"], - "extensions": { - "windows-process-ext": { - "window_title": ["process.title"] - } - }, + "x_window_title": ["process.title"], "x_exit_code": ["process.exit_code"], "x_thread_id": ["process.thread.id"], "x_ttp_tags": ["tags"], diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 27cd2f8e1..cabca6e32 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -865,8 +865,8 @@ "object": "process_parent" }, "exit_code": { - "key": "process.parent_exit_code", - "object": "process" + "key": "process.x_exit_code", + "object": "process_parent" }, "name": [ { @@ -916,21 +916,21 @@ }, "thread": { "id": { - "key": "process.parent_ref.thread_id", - "object": "process" + "key": "process.x_thread_id", + "object": "process_parent" } }, "title": { - "key": "process.parent_ref.title", - "object": "process" + "key": "process.x_window_title", + "object": "process_parent" }, "uptime": { - "key": "process.parent_ref.uptime", - "object": "process" + "key": "process.x_uptime", + "object": "process_parent" }, "working_directory": { - "key": "process.parent_ref.cwd", - "object": "process" + "key": "process.cwd", + "object": "process_parent" }, "executable": [ { @@ -971,14 +971,14 @@ } }, "title": { - "key": "process.windows-process-ext.window_title", + "key": "process.x_window_title", "object": "process" }, "uptime": { "key": "process.x_uptime", "object": "process" }, - "cwd": { + "working_directory": { "key": "process.cwd", "object": "process" }, diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index 4cb2c1f5c..10d819cd9 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -508,7 +508,7 @@ def test_process_prop(self): proc_object = TestElasticEcsTransform.get_first_of_type(objects.values(), 'process') assert (proc_object is not None), 'process object type not found' assert (proc_object.keys() == - {'type', 'pid', 'name', 'created', 'opened_connection_refs', 'creator_user_ref', 'binary_ref', 'parent_ref', 'x_unique_id'}) + {'type', 'pid', 'name', 'created', 'opened_connection_refs', 'creator_user_ref', 'binary_ref', 'parent_ref', 'x_unique_id', 'cwd'}) assert (proc_object['type'] == 'process') assert (proc_object['pid'] == 609) assert (proc_object['created'] == '2019-04-10T11:33:57.571Z') From fef735411a5ac4f1203cd0143b80e382f9ce5a9c Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Thu, 16 Feb 2023 17:56:27 -0500 Subject: [PATCH 25/32] Added tests for more process mappings --- .../test_elastic_ecs_json_to_stix.py | 180 ++---------------- 1 file changed, 14 insertions(+), 166 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index 10d819cd9..dabd8f1e1 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -288,7 +288,13 @@ "-p", "-s", "Schedule" - ] + ], + "exit_code": 0, + "title": "Just for testing", + "thread": { + "id": 3333 + }, + "uptime": 100 }, "pe" : { "company" : "Microsoft Corporation", @@ -704,172 +710,10 @@ def test_unmapped_attribute_alone(self): assert(objects == {}) - """ This is the translation of ecs_event_data: -{ - "type": "bundle", - "id": "bundle--93346d23-3d0c-4081-ae22-f5c1ce7a8503", - "objects": [ - { - "type": "identity", - "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", - "name": "ElasticEcs", - "identity_class": "events" - }, - { - "id": "observed-data--e30db2a1-f35a-4654-a1bb-1b1f156487f8", - "type": "observed-data", - "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", - "created": "2023-02-14T17:23:03.684Z", - "modified": "2023-02-14T17:23:03.684Z", - "objects": { - "0": { - "type": "x-oca-event", - "category": [ - "process" - ], - "event_type": [ - "start", - "process_start" - ], - "provider": "Microsoft-Windows-Sysmon", - "code": 1, - "action": "Process Create (rule: ProcessCreate)", - "created": "2021-10-24T23:58:21.586Z", - "kind": "event", - "module": "sysmon", - "host_ref": "2", - "user_ref": "9", - "agent": "win-server1", - "process_ref": "1", - "parent_process_ref": "12" - }, - "1": { - "type": "process", - "x_ttp_tags": [ - "beats_input_codec_plain_applied" - ], - "creator_user_ref": "9", - "pid": 5244, - "parent_ref": "12", - "x_unique_id": "{8dfc401c-f31c-6175-5715-000000001b00}", - "binary_ref": "15", - "command_line": "C:\\Windows\\System32\\dsregcmd.exe $(Arg0) $(Arg1) $(Arg2)", - "name": "dsregcmd.exe" - }, - "2": { - "type": "x-oca-asset", - "architecture": "x86_64", - "hostname": "win-server1", - "os_name": "Windows Server 2019 Standard", - "os_version": "10.0", - "os_platform": "windows", - "host_id": "8dfc401c-b042-4f41-b427-91a9dc0b61ac", - "name": "win-server1.example.com", - "mac_refs": [ - "3" - ], - "ip_refs": [ - "4", - "5", - "6" - ] - }, - "3": { - "type": "mac-addr", - "value": "06:07:08:09:0a:0b" - }, - "4": { - "type": "ipv4-addr", - "value": "9.10.11.12" - }, - "5": { - "type": "ipv4-addr", - "value": "10.11.12.13" - }, - "6": { - "type": "ipv6-addr", - "value": "fedc::ba98:7654:3210:1234" - }, - "7": { - "type": "x-ecs-log", - "level": "information" - }, - "8": { - "type": "x-ecs-user", - "domain": "NT AUTHORITY" - }, - "9": { - "type": "user-account", - "user_id": "SYSTEM", - "account_login": "SYSTEM" - }, - "10": { - "type": "x-ecs", - "version": "1.7.0" - }, - "11": { - "type": "software", - "vendor": "winlogbeat", - "name": "win-server1", - "version": "7.11.2" - }, - "12": { - "type": "process", - "pid": 2244, - "x_unique_id": "{8dfc401c-1ef7-6175-2900-000000001b00}", - "binary_ref": "13", - "command_line": "C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule", - "name": "svchost.exe" - }, - "13": { - "type": "file", - "name": "svchost.exe", - "parent_directory_ref": "14" - }, - "14": { - "type": "directory", - "path": "C:\\Windows\\System32" - }, - "15": { - "type": "file", - "x_pe": { - "company": "Microsoft Corporation", - "file_version": "10.0.17763.2145 (WinBuild.160101.0800)", - "description": "DSREG commandline tool", - "original_file_name": "dsregcmd.exe", - "product": "Microsoft\u00ae Windows\u00ae Operating System" - }, - "name": "dsregcmd.exe", - "parent_directory_ref": "16", - "hashes": { - "MD5": "d6957aceda86de523af0157800aa3c73", - "SHA-256": "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618" - } - }, - "16": { - "type": "directory", - "path": "C:\\Windows\\System32" - }, - "17": { - "type": "x-ecs-related", - "hash": [ - "d6957aceda86de523af0157800aa3c73", - "ba79462455b6e216d0e7cd6fe36bf0eff8a0d9dd06358d1c97b1014016256618", - "382c77bfa0eee2ba2ba8671d108ad9a3" - ], - "user": "SYSTEM" - } - }, - "first_observed": "2021-10-24T23:58:20.569Z", - "last_observed": "2021-10-24T23:58:20.569Z", - "number_observed": 1 - } - ], - "spec_version": "2.0" -} - """ def test_x_ecs_event(self): result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([ecs_event_data])) + with open('/tmp/test_ecs.json', 'w') as f: + f.write(json.dumps(result_bundle, indent=2)) assert (result_bundle['type'] == 'bundle') translation_objects = result_bundle.get('objects') assert (translation_objects and len(translation_objects) == 2) @@ -893,7 +737,11 @@ def test_x_ecs_event(self): parent_process.get("pid") == 2244 and parent_process.get("x_unique_id") == "{8dfc401c-1ef7-6175-2900-000000001b00}" and parent_process.get("command_line") == "C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule" and - parent_process.get("name") == "svchost.exe" + parent_process.get("name") == "svchost.exe" and + parent_process.get("x_exit_code") == 0 and + parent_process.get("x_window_title") == "Just for testing" and + parent_process.get("x_thread_id") == 3333 and + parent_process.get("x_uptime") == 100 ) executable_file = stix_objects.get(executable_file_key) assert ( From e498d33525bc40de70382694d0ee76dba701a9e3 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Wed, 22 Feb 2023 15:48:36 -0600 Subject: [PATCH 26/32] consolidate file and process objects in elastic_ecs mapping for STIX v2.1 --- .../json/stix_2_1/beats_from_stix_map.json | 109 +++++++----------- 1 file changed, 39 insertions(+), 70 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json index 22acc8ee8..39e564d38 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json @@ -48,42 +48,37 @@ "name": ["file.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword"], "ctime": ["file.created"], "size": ["file.size"], + "atime": ["file.accessed"], + "mime_type": ["file.mime_type"], + "mtime": ["file.mtime"], "hashes.MD5": ["file.hash.md5"], "hashes.'SHA-1'": ["file.hash.sha1"], "hashes.'SHA-256'": ["file.hash.sha256"], "hashes.'SHA-512'": ["file.hash.sha512"], - "parent_directory_ref.path": ["file.directory"] - } - }, - "x-ecs-file": { - "fields": { - "accessed": ["file.accessed"], - "attributes": ["file.attributes"], - "ctime": ["file.ctime"], - "device": ["file.device"], - "drive_letter": ["file.drive_letter"], - "extension": ["file.extension"], - "gid": ["file.gid"], - "group": ["file.group"], - "inode": ["file.inode"], - "mime_type": ["file.mime_type"], - "mode": ["file.mode"], - "mtime": ["file.mtime"], - "owner": ["file.owner"], - "path": ["file.path"], - "target_path": ["file.target_path"], - "type": ["file.type"], - "uid": ["file.uid"], - "pe.company": ["file.pe.company"], - "pe.description": ["file.pe.description"], - "pe.file_version": ["file.pe.file_version"], - "pe.original_file_name": ["file.pe.original_file_name"], - "pe.product": ["file.pe.product"], - "code_signature.exists": ["file.code_signature.exists"], - "code_signature.status": ["file.code_signature.status"], - "code_signature.subject_name": ["file.code_signature.subject_name"], - "code_signature.trusted": ["file.code_signature.trusted"], - "code_signature.valid": ["file.code_signature.valid"] + "parent_directory_ref.path": ["file.directory"], + "x_attributes": ["file.attributes"], + "x_extension": ["file.extension"], + "x_path": ["file.path"], + "x_target_path": ["file.target_path"], + "x_type": ["file.type"], + "x_unix.device": ["file.device"], + "x_unix.gid": ["file.gid"], + "x_unix.group": ["file.group"], + "x_unix.inode": ["file.inode"], + "x_unix.mode": ["file.mode"], + "x_owner_ref.user_id": ["file.uid"], + "x_owner_ref.account_login": ["file.owner"], + "x_win_drive_letter": ["file.drive_letter"], + "x_pe.company": ["file.pe.company"], + "x_pe.description": ["file.pe.description"], + "x_pe.file_version": ["file.pe.file_version"], + "x_pe.original_file_name": ["file.pe.original_file_name"], + "x_pe.product": ["file.pe.product"], + "x_code_signature.exists": ["file.code_signature.exists"], + "x_code_signature.status": ["file.code_signature.status"], + "x_code_signature.subject_name": ["file.code_signature.subject_name"], + "x_code_signature.trusted": ["file.code_signature.trusted"], + "x_code_signature.valid": ["file.code_signature.valid"] } }, "directory": { @@ -113,49 +108,23 @@ "command_line": ["process.command_line.keyword", "powershell.command.value"], "created_time": ["process.start"], "pid": ["process.pid", "process.ppid", "process.parent.pid", "process.parent.ppid"], + "cwd": ["process.working_directory.keyword"], "creator_user_ref.user_id": ["user.name.keyword"], "parent_ref.pid": ["process.ppid", "process.parent.ppid"], "parent_ref.name": ["process.parent.name.keyword"], + "parent_ref.cwd": ["process.parent.working_directory"], + "parent_ref.x_exit_code": ["process.parent.exit_code"], + "parent_ref.x_pgid": ["process.parent.pgid"], + "parent_ref.x_thread_id": ["process.parent.thread.id"], + "parent_ref.x_window_title": ["process.parent.title"], + "parent_ref.x_uptime": ["process.parent.uptime"], "image_ref.name": ["process.executable.keyword", "process.parent.executable.keyword"], "x_ttp_tags": ["tags"], - "x_unique_id": ["process.entity_id.keyword", "process.parent.entity_id.keyword"] - } - }, - "x-ecs-process": { - "fields": { - "args": ["process.args.keyword"], - "args_count": ["process.args_count"], - "executable": ["process.executable.keyword"], - "exit_code": ["process.exit_code"], - "thread.thread_id": ["process.thread.id"], - "thread.name": ["process.thread.name"], - "title": ["process.title.keyword"], - "uptime": ["process.uptime"], - "working_directory": ["process.working_directory.keyword"], - "parent.args": ["process.parent.args.keyword"], - "parent.args_count": ["process.parent.args_count"], - "parent.exit_code": ["process.parent.exit_code"], - "parent.pgid": ["process.parent.pgid"], - "parent.thread.thread_id": ["process.parent.thread.id"], - "parent.thread.name": ["process.parent.thread.name"], - "parent.title": ["process.parent.title"], - "parent.uptime": ["process.parent.uptime"], - "parent.working_directory": ["process.parent.working_directory"], - "pe.company": ["process.pe.company.keyword"], - "pe.description": ["process.pe.description.keyword"], - "pe.file_version": ["process.pe.file_version.keyword"], - "pe.original_file_name": ["process.pe.original_file_name.keyword"], - "pe.product": ["process.pe.product.keyword"], - "code_signature.exists": ["process.code_signature.exists"], - "code_signature.status": ["process.code_signature.status"], - "code_signature.subject_name": ["process.code_signature.subject_name"], - "code_signature.trusted": ["process.code_signature.trusted"], - "code_signature.valid": ["process.code_signature.valid"], - "parent.code_signature.exists": ["process.parent.code_signature.exists"], - "parent.code_signature.status": ["process.parent.code_signature.status"], - "parent.code_signature.subject_name": ["process.parent.code_signature.subject_name"], - "parent.code_signature.trusted": ["process.parent.code_signature.trusted"], - "parent.code_signature.valid": ["process.parent.code_signature.valid"] + "x_unique_id": ["process.entity_id.keyword", "process.parent.entity_id.keyword"], + "x_exit_code": ["process.exit_code"], + "x_thread_id": ["process.thread.id"], + "x_window_title": ["process.title.keyword"], + "x_uptime": ["process.uptime"] } }, "url": { From fbc30c7581b5137293d91d7802ab190c65fc14e0 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Wed, 22 Feb 2023 15:48:57 -0600 Subject: [PATCH 27/32] consolidate file and process objects in elastic_ecs mapping for STIX v2.1 --- .../json/stix_2_1/from_stix_map.json | 109 +++++++----------- 1 file changed, 39 insertions(+), 70 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json index c0b89a021..bdacd0910 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json @@ -48,42 +48,37 @@ "name": ["file.name", "file.path", "process.name", "process.executable", "process.parent.name", "process.parent.executable"], "ctime": ["file.created"], "size": ["file.size"], + "atime": ["file.accessed"], + "mime_type": ["file.mime_type"], + "mtime": ["file.mtime"], "hashes.MD5": ["file.hash.md5"], "hashes.'SHA-1'": ["file.hash.sha1"], "hashes.'SHA-256'": ["file.hash.sha256"], "hashes.'SHA-512'": ["file.hash.sha512"], - "parent_directory_ref.path": ["file.directory"] - } - }, - "x-ecs-file": { - "fields": { - "accessed": ["file.accessed"], - "attributes": ["file.attributes"], - "ctime": ["file.ctime"], - "device": ["file.device"], - "drive_letter": ["file.drive_letter"], - "extension": ["file.extension"], - "gid": ["file.gid"], - "group": ["file.group"], - "inode": ["file.inode"], - "mime_type": ["file.mime_type"], - "mode": ["file.mode"], - "mtime": ["file.mtime"], - "owner": ["file.owner"], - "path": ["file.path"], - "target_path": ["file.target_path"], - "type": ["file.type"], - "uid": ["file.uid"], - "pe.company": ["file.pe.company"], - "pe.description": ["file.pe.description"], - "pe.file_version": ["file.pe.file_version"], - "pe.original_file_name": ["file.pe.original_file_name"], - "pe.product": ["file.pe.product"], - "code_signature.exists": ["file.code_signature.exists"], - "code_signature.status": ["file.code_signature.status"], - "code_signature.subject_name": ["file.code_signature.subject_name"], - "code_signature.trusted": ["file.code_signature.trusted"], - "code_signature.valid": ["file.code_signature.valid"] + "parent_directory_ref.path": ["file.directory"], + "x_attributes": ["file.attributes"], + "x_extension": ["file.extension"], + "x_path": ["file.path"], + "x_target_path": ["file.target_path"], + "x_type": ["file.type"], + "x_unix.device": ["file.device"], + "x_unix.gid": ["file.gid"], + "x_unix.group": ["file.group"], + "x_unix.inode": ["file.inode"], + "x_unix.mode": ["file.mode"], + "x_owner_ref.user_id": ["file.uid"], + "x_owner_ref.account_login": ["file.owner"], + "x_win_drive_letter": ["file.drive_letter"], + "x_pe.company": ["file.pe.company"], + "x_pe.description": ["file.pe.description"], + "x_pe.file_version": ["file.pe.file_version"], + "x_pe.original_file_name": ["file.pe.original_file_name"], + "x_pe.product": ["file.pe.product"], + "x_code_signature.exists": ["file.code_signature.exists"], + "x_code_signature.status": ["file.code_signature.status"], + "x_code_signature.subject_name": ["file.code_signature.subject_name"], + "x_code_signature.trusted": ["file.code_signature.trusted"], + "x_code_signature.valid": ["file.code_signature.valid"] } }, "directory": { @@ -113,49 +108,23 @@ "command_line": ["process.command_line", "powershell.command.value"], "created_time": ["process.start"], "pid": ["process.pid", "process.ppid", "process.parent.pid", "process.parent.ppid"], + "cwd": ["process.working_directory"], "creator_user_ref.user_id": ["user.name"], "parent_ref.pid": ["process.ppid", "process.parent.ppid"], "parent_ref.name": ["process.parent.name"], + "parent_ref.cwd": ["process.parent.working_directory"], + "parent_ref.x_exit_code": ["process.parent.exit_code"], + "parent_ref.x_pgid": ["process.parent.pgid"], + "parent_ref.x_thread_id": ["process.parent.thread.id"], + "parent_ref.x_window_title": ["process.parent.title"], + "parent_ref.x_uptime": ["process.parent.uptime"], "image_ref.name": ["process.executable", "process.parent.executable"], "x_ttp_tags": ["tags"], - "x_unique_id": ["process.entity_id", "process.parent.entity_id"] - } - }, - "x-ecs-process": { - "fields": { - "args": ["process.args"], - "args_count": ["process.args_count"], - "executable": ["process.executable"], - "exit_code": ["process.exit_code"], - "thread.thread_id": ["process.thread.id"], - "thread.name": ["process.thread.name"], - "title": ["process.title"], - "uptime": ["process.uptime"], - "working_directory": ["process.working_directory"], - "parent.args": ["process.parent.args"], - "parent.args_count": ["process.parent.args_count"], - "parent.exit_code": ["process.parent.exit_code"], - "parent.pgid": ["process.parent.pgid"], - "parent.thread.thread_id": ["process.parent.thread.id"], - "parent.thread.name": ["process.parent.thread.name"], - "parent.title": ["process.parent.title"], - "parent.uptime": ["process.parent.uptime"], - "parent.working_directory": ["process.parent.working_directory"], - "pe.company": ["process.pe.company"], - "pe.description": ["process.pe.description"], - "pe.file_version": ["process.pe.file_version"], - "pe.original_file_name": ["process.pe.original_file_name"], - "pe.product": ["process.pe.product"], - "code_signature.exists": ["process.code_signature.exists"], - "code_signature.status": ["process.code_signature.status"], - "code_signature.subject_name": ["process.code_signature.subject_name"], - "code_signature.trusted": ["process.code_signature.trusted"], - "code_signature.valid": ["process.code_signature.valid"], - "parent.code_signature.exists": ["process.parent.code_signature.exists"], - "parent.code_signature.status": ["process.parent.code_signature.status"], - "parent.code_signature.subject_name": ["process.parent.code_signature.subject_name"], - "parent.code_signature.trusted": ["process.parent.code_signature.trusted"], - "parent.code_signature.valid": ["process.parent.code_signature.valid"] + "x_unique_id": ["process.entity_id", "process.parent.entity_id"], + "x_exit_code": ["process.exit_code"], + "x_thread_id": ["process.thread.id"], + "x_window_title": ["process.title"], + "x_uptime": ["process.uptime"] } }, "url": { From 8564036e8d77d117ad7d20a423f946e6e79866a6 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Wed, 22 Feb 2023 15:49:20 -0600 Subject: [PATCH 28/32] consolidate file and process objects in elastic_ecs mapping for STIX v2.1 --- .../json/stix_2_1/to_stix_map.json | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json index f6cf0f7af..a91b1304e 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json @@ -1393,103 +1393,103 @@ }, "pe": { "company": { - "key": "x-ecs-file.pe_company", - "object": "x_file" + "key": "file.x_pe.company", + "object": "file" }, "description": { - "key": "x-ecs-file.pe_description", - "object": "x_file" + "key": "file.x_pe.description", + "object": "file" }, "file_version": { - "key": "x-ecs-file.pe_file_version", - "object": "x_file" + "key": "file.x_pe.file_version", + "object": "file" }, "original_file_name": { - "key": "x-ecs-file.pe_original_file_name", - "object": "x_file" + "key": "file.x_pe.original_file_name", + "object": "file" }, "product": { - "key": "x-ecs-file.pe_product", - "object": "x_file" + "key": "file.x_pe.product", + "object": "file" } }, "code_signature": { "exists": { - "key": "x-ecs-file.code_signature_exists", - "object": "x_file" + "key": "file.x_code_signature.exists", + "object": "file" }, "subject_name": { - "key": "x-ecs-file.code_signature_subject_name", - "object": "x_file" + "key": "file.x_code_signature.subject_name", + "object": "file" } }, "accessed": { - "key": "x-ecs-file.accessed", - "object": "x_file" + "key": "file.atime", + "object": "file" }, "attributes": { - "key": "x-ecs-file.attributes", - "object": "x_file" + "key": "file.x_attributes", + "object": "file" }, "ctime": { - "key": "x-ecs-file.ctime", - "object": "x_file" + "key": "file.ctime", + "object": "file" }, "device": { - "key": "x-ecs-file.device", - "object": "x_file" + "key": "file.x_unix.device", + "object": "file" }, "drive_letter": { - "key": "x-ecs-file.drive_letter", - "object": "x_file" + "key": "file.x_win_drive_letter", + "object": "file" }, "extension": { - "key": "x-ecs-file.extension", - "object": "x_file" + "key": "file.x_extension", + "object": "file" }, "gid": { - "key": "x-ecs-file.gid", - "object": "x_file" + "key": "file.x_unix.gid", + "object": "file" }, "group": { - "key": "x-ecs-file.group", - "object": "x_file" + "key": "file.x_unix.group", + "object": "file" }, "inode": { - "key": "x-ecs-file.inode", - "object": "x_file" + "key": "file.x_unix.inode", + "object": "file" }, "mime_type": { - "key": "x-ecs-file.mime_type", - "object": "x_file" + "key": "file.mime_type", + "object": "file" }, "mode": { - "key": "x-ecs-file.mode", - "object": "x_file" + "key": "file.x_unix.mode", + "object": "file" }, "mtime": { - "key": "x-ecs-file.mtime", - "object": "x_file" + "key": "file.mtime", + "object": "file" }, "owner": { - "key": "x-ecs-file.owner", - "object": "x_file" + "key": "file.x_owner_ref.account_login", + "object": "file" }, "path": { - "key": "x-ecs-file.path", - "object": "x_file" + "key": "file.x_path", + "object": "file" }, "target_path": { - "key": "x-ecs-file.target_path", - "object": "x_file" + "key": "file.x_target_path", + "object": "file" }, "type": { - "key": "x-ecs-file.type", - "object": "x_file" + "key": "file.x_type", + "object": "file" }, "uid": { - "key": "x-ecs-file.uid", - "object": "x_file" + "key": "file.x_owner_ref.user_id", + "object": "file" } }, "group": { From 53a8ee09cdef63b6629cf72dc451481e4c87b5a8 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Wed, 22 Feb 2023 16:18:52 -0600 Subject: [PATCH 29/32] enhance process object mapping in elastic_ecs connector for STIX v2.1 --- .../stix_translation/json/stix_2_1/beats_from_stix_map.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json index 39e564d38..a96d9fcf7 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/beats_from_stix_map.json @@ -119,6 +119,11 @@ "parent_ref.x_window_title": ["process.parent.title"], "parent_ref.x_uptime": ["process.parent.uptime"], "image_ref.name": ["process.executable.keyword", "process.parent.executable.keyword"], + "image_ref.parent_directory_ref.path": ["process.executable", "process.parent.executable"], + "image_ref.hashes.MD5": ["process.hash.md5"], + "image_ref.hashes.'SHA-1'": ["process.hash.sha1"], + "image_ref.hashes.'SHA-256'": ["process.hash.sha256"], + "image_ref.hashes.'SHA-512'": ["process.hash.sha512"], "x_ttp_tags": ["tags"], "x_unique_id": ["process.entity_id.keyword", "process.parent.entity_id.keyword"], "x_exit_code": ["process.exit_code"], From 25f75d106ce67b4386c81b92e39835c66befa162 Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Wed, 22 Feb 2023 16:19:21 -0600 Subject: [PATCH 30/32] enhance process object mapping in elastic_ecs connector for STIX v2.1 --- .../stix_translation/json/stix_2_1/from_stix_map.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json index bdacd0910..a85ac404d 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/from_stix_map.json @@ -119,6 +119,11 @@ "parent_ref.x_window_title": ["process.parent.title"], "parent_ref.x_uptime": ["process.parent.uptime"], "image_ref.name": ["process.executable", "process.parent.executable"], + "image_ref.parent_directory_ref.path": ["process.executable", "process.parent.executable"], + "image_ref.hashes.MD5": ["process.hash.md5"], + "image_ref.hashes.'SHA-1'": ["process.hash.sha1"], + "image_ref.hashes.'SHA-256'": ["process.hash.sha256"], + "image_ref.hashes.'SHA-512'": ["process.hash.sha512"], "x_ttp_tags": ["tags"], "x_unique_id": ["process.entity_id", "process.parent.entity_id"], "x_exit_code": ["process.exit_code"], From 54bae2945584243d22586336b5081a637c5ff92a Mon Sep 17 00:00:00 2001 From: hamid-adebayo Date: Wed, 22 Feb 2023 16:19:34 -0600 Subject: [PATCH 31/32] enhance process object mapping in elastic_ecs connector for STIX v2.1 --- .../json/stix_2_1/to_stix_map.json | 105 ++++-------------- 1 file changed, 22 insertions(+), 83 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json index a91b1304e..279d9cd6b 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/stix_2_1/to_stix_map.json @@ -771,16 +771,6 @@ "references": "process" } ], - "code_signature": { - "exists": { - "key": "x-ecs-process.code_signature_exists", - "object": "x_process" - }, - "subject_name": { - "key": "x-ecs-process.code_signature_subject_name", - "object": "x_process" - } - }, "ppid": [ { "key": "process.pid", @@ -793,36 +783,6 @@ "references": "process_parent" } ], - "pe": { - "company": { - "key": "x-ecs-process.pe_company", - "object": "x_process" - }, - "description": { - "key": "x-ecs-process.pe_description", - "object": "x_process" - }, - "file_version": { - "key": "x-ecs-process.pe_file_version", - "object": "x_process" - }, - "original_file_name": { - "key": "x-ecs-process.pe_original_file_name", - "object": "x_process" - }, - "product": { - "key": "x-ecs-process.pe_product", - "object": "x_process" - } - }, - "args": { - "key": "x-ecs-process.args", - "object": "x_process" - }, - "args_count": { - "key": "x-ecs-process.args_count", - "object": "x_process" - }, "command_line": { "key": "process.command_line", "object": "process" @@ -859,18 +819,10 @@ "object": "process" }, "exit_code": { - "key": "x-ecs-process.exit_code", - "object": "x_process" + "key": "process.x_exit_code", + "object": "process" }, "parent": { - "args": { - "key": "x-ecs-process.parent_args", - "object": "x_process" - }, - "args_count": { - "key": "x-ecs-process.parent_args_count", - "object": "x_process" - }, "command_line": { "key": "process.command_line", "object": "process_parent" @@ -880,12 +832,12 @@ "object": "process_parent" }, "exit_code": { - "key": "x-ecs-process.parent_exit_code", - "object": "x_process" + "key": "process.parent_ref.x_exit_code", + "object": "process" }, "pgid": { - "key": "x-ecs-process.parent_pgid", - "object": "x_process" + "key": "process.parent_ref.x_pgid", + "object": "process" }, "pid": [ { @@ -904,36 +856,27 @@ "references": "process_parent" } ], - "ppid": { - "key": "x-ecs-process.parent_ppid", - "object": "x_process", - "transformer": "ToInteger" - }, "start": { "key": "process.created_time", "object": "process_parent" }, "thread": { "id": { - "key": "x-ecs-process.parent_thread_id", - "object": "x_process" - }, - "name": { - "key": "x-ecs-process.parent_thread_name", - "object": "x_process" + "key": "process.parent_ref.x_thread_id", + "object": "process" } }, "title": { - "key": "x-ecs-process.parent_title", - "object": "x_process" + "key": "process.parent_ref.x_window_title", + "object": "process" }, "uptime": { - "key": "x-ecs-process.parent_uptime", - "object": "x_process" + "key": "process.parent_ref.x_uptime", + "object": "process" }, "working_directory": { - "key": "x-ecs-process.parent_working_directory", - "object": "x_process" + "key": "process.parent_ref.cwd", + "object": "process" }, "executable": [ { @@ -964,30 +907,26 @@ ] }, "pgid": { - "key": "x-ecs-process.exit_code", + "key": "process.x_exit_code", "object": "x_process" }, "thread": { "id": { - "key": "x-ecs-process.thread_id", - "object": "x_process" - }, - "name": { - "key": "x-ecs-process.thread_name", - "object": "x_process" + "key": "process.x_thread_id", + "object": "process" } }, "title": { - "key": "x-ecs-process.title", - "object": "x_process" + "key": "process.x_window_title", + "object": "process" }, "uptime": { - "key": "x-ecs-process.uptime", + "key": "process.x_uptime", "object": "x_process" }, "working_directory": { - "key": "x-ecs-process.working_directory", - "object": "x_process" + "key": "process.cwd", + "object": "process" } }, "user": { From 88d782b9025543a73e49f293be3167a708b8fac8 Mon Sep 17 00:00:00 2001 From: Constantin Adam Date: Fri, 24 Feb 2023 13:59:09 -0500 Subject: [PATCH 32/32] Remove write to /tmp file --- .../tests/stix_translation/test_elastic_ecs_json_to_stix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py index dabd8f1e1..3b5e8982c 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py @@ -712,8 +712,6 @@ def test_unmapped_attribute_alone(self): def test_x_ecs_event(self): result_bundle = entry_point.translate_results(json.dumps(data_source), json.dumps([ecs_event_data])) - with open('/tmp/test_ecs.json', 'w') as f: - f.write(json.dumps(result_bundle, indent=2)) assert (result_bundle['type'] == 'bundle') translation_objects = result_bundle.get('objects') assert (translation_objects and len(translation_objects) == 2)