From 7815b3f9f316aa4f179d08c1f74e24d804ad4886 Mon Sep 17 00:00:00 2001 From: Jonhnathan <26856693+w0rk3r@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:35:32 -0300 Subject: [PATCH 01/46] Add .caseless subfield to process.name & process.executable (#2341) Adds a subfield to the process.name and process.executable fields to improve the compatibility of data sources like System, Sysmon, etc., with our Elastic Defend data, which enables us to handle language limitations in KQL more effectively. --- CHANGELOG.next.md | 2 + docs/fields/field-details.asciidoc | 6 ++ experimental/generated/beats/fields.ecs.yml | 46 ++++++++++++++++ experimental/generated/csv/fields.csv | 11 ++++ experimental/generated/ecs/ecs_flat.yml | 55 +++++++++++++++++++ experimental/generated/ecs/ecs_nested.yml | 55 +++++++++++++++++++ .../composable/component/process.json | 55 +++++++++++++++++++ .../elasticsearch/legacy/template.json | 55 +++++++++++++++++++ generated/beats/fields.ecs.yml | 46 ++++++++++++++++ generated/csv/fields.csv | 11 ++++ generated/ecs/ecs_flat.yml | 55 +++++++++++++++++++ generated/ecs/ecs_nested.yml | 55 +++++++++++++++++++ .../composable/component/process.json | 55 +++++++++++++++++++ generated/elasticsearch/legacy/template.json | 55 +++++++++++++++++++ schemas/process.yml | 8 +++ 15 files changed, 570 insertions(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 9613fb89e6..85035153ba 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -25,6 +25,8 @@ Thanks, you're awesome :-) --> #### Improvements +* Added `.caseless` subfield to `process.name` and `process.executable`. #2341 + #### Deprecated ### Tooling and Artifact Changes diff --git a/docs/fields/field-details.asciidoc b/docs/fields/field-details.asciidoc index 31273d8c4b..3d817d46ce 100644 --- a/docs/fields/field-details.asciidoc +++ b/docs/fields/field-details.asciidoc @@ -8128,6 +8128,9 @@ type: keyword Multi-fields: +* process.executable.caseless (type: keyword) + + * process.executable.text (type: match_only_text) @@ -8343,6 +8346,9 @@ type: keyword Multi-fields: +* process.name.caseless (type: keyword) + + * process.name.text (type: match_only_text) diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index bc95a6db22..7f63fa34e9 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -5175,6 +5175,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5213,6 +5217,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -5482,6 +5490,11 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 + default_field: false - name: text type: match_only_text default_field: false @@ -5560,6 +5573,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5598,6 +5615,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -6012,6 +6033,11 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 + default_field: false - name: text type: match_only_text default_field: false @@ -6401,6 +6427,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -6644,6 +6674,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -7230,6 +7264,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7345,6 +7383,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7383,6 +7425,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 292ac5f917..5c51865b5d 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -648,11 +648,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.entry_leader.entry_meta.source.ip,ip,core,,,IP address of the source. 8.12.0-dev+exp,true,process,process.entry_leader.entry_meta.type,keyword,extended,,,The entry type for the entry session leader. 8.12.0-dev+exp,true,process,process.entry_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev+exp,true,process,process.entry_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.entry_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.entry_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.entry_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.entry_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev+exp,true,process,process.entry_leader.name,keyword,extended,,ssh,Process name. +8.12.0-dev+exp,true,process,process.entry_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.entry_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.entry_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.entry_leader.parent.pid,long,core,,4242,Process id. @@ -688,6 +690,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.entry_leader.working_directory.text,match_only_text,extended,,/home/alice,The working directory of the process. 8.12.0-dev+exp,true,process,process.env_vars,keyword,extended,array,"[""PATH=/usr/local/bin:/usr/bin"", ""USER=ubuntu""]",Array of environment variable bindings. 8.12.0-dev+exp,true,process,process.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev+exp,true,process,process.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev+exp,true,process,process.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -698,11 +701,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.group_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev+exp,true,process,process.group_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.group_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev+exp,true,process,process.group_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.group_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.group_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.group_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.group_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev+exp,true,process,process.group_leader.name,keyword,extended,,ssh,Process name. +8.12.0-dev+exp,true,process,process.group_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.group_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.group_leader.pid,long,core,,4242,Process id. 8.12.0-dev+exp,true,process,process.group_leader.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -762,6 +767,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev+exp,true,process,process.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev+exp,true,process,process.name,keyword,extended,,ssh,Process name. +8.12.0-dev+exp,true,process,process.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev+exp,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. @@ -817,6 +823,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.end,date,extended,,2016-05-23T08:05:34.853Z,The time the process ended. 8.12.0-dev+exp,true,process,process.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.parent.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev+exp,true,process,process.parent.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.parent.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.parent.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev+exp,true,process,process.parent.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -850,6 +857,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev+exp,true,process,process.parent.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev+exp,true,process,process.parent.name,keyword,extended,,ssh,Process name. +8.12.0-dev+exp,true,process,process.parent.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.parent.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. 8.12.0-dev+exp,true,process,process.parent.pe.company,keyword,extended,,Microsoft Corporation,"Internal company name of the file, provided at compile-time." @@ -933,6 +941,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.previous.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev+exp,true,process,process.previous.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.previous.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev+exp,true,process,process.previous.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.previous.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.real_group.name,keyword,extended,,,Name of the group. @@ -950,11 +959,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.session_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev+exp,true,process,process.session_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.session_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev+exp,true,process,process.session_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.session_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.session_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.session_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.session_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev+exp,true,process,process.session_leader.name,keyword,extended,,ssh,Process name. +8.12.0-dev+exp,true,process,process.session_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.session_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.session_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.session_leader.parent.pid,long,core,,4242,Process id. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 02b972886f..65448363e9 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -8426,6 +8426,11 @@ process.entry_leader.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -8487,6 +8492,11 @@ process.entry_leader.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -8910,6 +8920,11 @@ process.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -9029,6 +9044,11 @@ process.group_leader.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -9090,6 +9110,11 @@ process.group_leader.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -9779,6 +9804,11 @@ process.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -10440,6 +10470,11 @@ process.parent.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -10849,6 +10884,11 @@ process.parent.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -11833,6 +11873,11 @@ process.previous.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.previous.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -12018,6 +12063,11 @@ process.session_leader.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -12079,6 +12129,11 @@ process.session_leader.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index f600ab293a..9c39b3b5ae 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -10636,6 +10636,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -10697,6 +10702,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -11120,6 +11130,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -11239,6 +11254,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -11300,6 +11320,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -11993,6 +12018,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -12655,6 +12685,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -13065,6 +13100,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -14051,6 +14091,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.previous.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -14236,6 +14281,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -14297,6 +14347,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/experimental/generated/elasticsearch/composable/component/process.json b/experimental/generated/elasticsearch/composable/component/process.json index f4dd52c1ce..21bc93af56 100644 --- a/experimental/generated/elasticsearch/composable/component/process.json +++ b/experimental/generated/elasticsearch/composable/component/process.json @@ -275,6 +275,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -299,6 +304,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -471,6 +481,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -516,6 +531,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -540,6 +560,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -799,6 +824,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1014,6 +1044,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1148,6 +1183,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1504,6 +1544,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1594,6 +1639,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1618,6 +1668,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } diff --git a/experimental/generated/elasticsearch/legacy/template.json b/experimental/generated/elasticsearch/legacy/template.json index 18386e190c..4e7c38235a 100644 --- a/experimental/generated/elasticsearch/legacy/template.json +++ b/experimental/generated/elasticsearch/legacy/template.json @@ -2996,6 +2996,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3020,6 +3025,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3192,6 +3202,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3237,6 +3252,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3261,6 +3281,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3520,6 +3545,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3735,6 +3765,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3869,6 +3904,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -4225,6 +4265,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -4315,6 +4360,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -4339,6 +4389,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index fa0007884b..bd55fe7727 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -5125,6 +5125,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5163,6 +5167,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -5432,6 +5440,11 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 + default_field: false - name: text type: match_only_text default_field: false @@ -5510,6 +5523,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5548,6 +5565,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -5962,6 +5983,11 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 + default_field: false - name: text type: match_only_text default_field: false @@ -6351,6 +6377,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -6594,6 +6624,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -7180,6 +7214,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7295,6 +7333,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7333,6 +7375,10 @@ type: keyword ignore_above: 1024 multi_fields: + - name: caseless + type: keyword + normalizer: lowercase + ignore_above: 1024 - name: text type: match_only_text description: 'Process name. diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index c31a8de31c..299f4aa1a7 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -641,11 +641,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.entry_leader.entry_meta.source.ip,ip,core,,,IP address of the source. 8.12.0-dev,true,process,process.entry_leader.entry_meta.type,keyword,extended,,,The entry type for the entry session leader. 8.12.0-dev,true,process,process.entry_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev,true,process,process.entry_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.entry_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.entry_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.entry_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.entry_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev,true,process,process.entry_leader.name,keyword,extended,,ssh,Process name. +8.12.0-dev,true,process,process.entry_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.entry_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.entry_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.entry_leader.parent.pid,long,core,,4242,Process id. @@ -681,6 +683,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.entry_leader.working_directory.text,match_only_text,extended,,/home/alice,The working directory of the process. 8.12.0-dev,true,process,process.env_vars,keyword,extended,array,"[""PATH=/usr/local/bin:/usr/bin"", ""USER=ubuntu""]",Array of environment variable bindings. 8.12.0-dev,true,process,process.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev,true,process,process.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev,true,process,process.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -691,11 +694,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.group_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev,true,process,process.group_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.group_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev,true,process,process.group_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.group_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.group_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.group_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.group_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev,true,process,process.group_leader.name,keyword,extended,,ssh,Process name. +8.12.0-dev,true,process,process.group_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.group_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.group_leader.pid,long,core,,4242,Process id. 8.12.0-dev,true,process,process.group_leader.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -755,6 +760,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev,true,process,process.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev,true,process,process.name,keyword,extended,,ssh,Process name. +8.12.0-dev,true,process,process.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. @@ -810,6 +816,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.end,date,extended,,2016-05-23T08:05:34.853Z,The time the process ended. 8.12.0-dev,true,process,process.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.parent.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev,true,process,process.parent.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.parent.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.parent.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev,true,process,process.parent.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -843,6 +850,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev,true,process,process.parent.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev,true,process,process.parent.name,keyword,extended,,ssh,Process name. +8.12.0-dev,true,process,process.parent.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.parent.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. 8.12.0-dev,true,process,process.parent.pe.company,keyword,extended,,Microsoft Corporation,"Internal company name of the file, provided at compile-time." @@ -926,6 +934,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.previous.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev,true,process,process.previous.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.previous.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev,true,process,process.previous.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.previous.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.real_group.name,keyword,extended,,,Name of the group. @@ -943,11 +952,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.session_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev,true,process,process.session_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.session_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. +8.12.0-dev,true,process,process.session_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.session_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.session_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.session_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.session_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev,true,process,process.session_leader.name,keyword,extended,,ssh,Process name. +8.12.0-dev,true,process,process.session_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.session_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.session_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.session_leader.parent.pid,long,core,,4242,Process id. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 2022bddaf4..2ff10f67c0 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -8357,6 +8357,11 @@ process.entry_leader.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -8418,6 +8423,11 @@ process.entry_leader.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -8841,6 +8851,11 @@ process.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -8960,6 +8975,11 @@ process.group_leader.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -9021,6 +9041,11 @@ process.group_leader.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -9710,6 +9735,11 @@ process.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -10371,6 +10401,11 @@ process.parent.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -10780,6 +10815,11 @@ process.parent.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -11764,6 +11804,11 @@ process.previous.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.previous.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -11949,6 +11994,11 @@ process.session_leader.executable: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -12010,6 +12060,11 @@ process.session_leader.name: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 8057eeed15..cc11243d59 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -10556,6 +10556,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -10617,6 +10622,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.entry_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -11040,6 +11050,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -11159,6 +11174,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -11220,6 +11240,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.group_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -11913,6 +11938,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -12575,6 +12605,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -12985,6 +13020,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.parent.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -13971,6 +14011,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.previous.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -14156,6 +14201,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.executable.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -14217,6 +14267,11 @@ process: ignore_above: 1024 level: extended multi_fields: + - flat_name: process.session_leader.name.caseless + ignore_above: 1024 + name: caseless + normalizer: lowercase + type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/generated/elasticsearch/composable/component/process.json b/generated/elasticsearch/composable/component/process.json index 6cc1382d11..a4b1e5e1f0 100644 --- a/generated/elasticsearch/composable/component/process.json +++ b/generated/elasticsearch/composable/component/process.json @@ -275,6 +275,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -299,6 +304,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -471,6 +481,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -516,6 +531,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -540,6 +560,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -799,6 +824,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1014,6 +1044,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1148,6 +1183,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1504,6 +1544,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1594,6 +1639,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -1618,6 +1668,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } diff --git a/generated/elasticsearch/legacy/template.json b/generated/elasticsearch/legacy/template.json index a6b67033e2..1d343b5050 100644 --- a/generated/elasticsearch/legacy/template.json +++ b/generated/elasticsearch/legacy/template.json @@ -2954,6 +2954,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -2978,6 +2983,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3150,6 +3160,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3195,6 +3210,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3219,6 +3239,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3478,6 +3503,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3693,6 +3723,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -3827,6 +3862,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -4183,6 +4223,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -4273,6 +4318,11 @@ }, "executable": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } @@ -4297,6 +4347,11 @@ }, "name": { "fields": { + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + }, "text": { "type": "match_only_text" } diff --git a/schemas/process.yml b/schemas/process.yml index 91c8ef98ef..b1acf8945c 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -112,6 +112,10 @@ Sometimes called program name or similar. example: ssh multi_fields: + - name: caseless + ignore_above: 1024 + normalizer: lowercase + type: keyword - type: match_only_text name: text @@ -171,6 +175,10 @@ Absolute path to the process executable. example: /usr/bin/ssh multi_fields: + - name: caseless + ignore_above: 1024 + normalizer: lowercase + type: keyword - type: match_only_text name: text From 146c96aae530af83b4d3f7283cf2f081fb51f442 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 23 Jul 2024 15:17:29 -0400 Subject: [PATCH 02/46] Revert "Add .caseless subfield to process.name & process.executable" (#2350) This reverts commit 7815b3f9f316aa4f179d08c1f74e24d804ad4886 from #2341. This is being reverted due to storage concerns. The goal will be to advance the native querying capabilities (ES|QL, KQL) of the Elastic stack such that this extra normalized multi-field is not necessary. In the meantime, localized overrides of the ECS field definition will be used to add the additional multi-field where needed. The downside of localized overrides are that it creates inconsistency across usages of the this field. --- CHANGELOG.next.md | 2 - docs/fields/field-details.asciidoc | 6 -- experimental/generated/beats/fields.ecs.yml | 46 ---------------- experimental/generated/csv/fields.csv | 11 ---- experimental/generated/ecs/ecs_flat.yml | 55 ------------------- experimental/generated/ecs/ecs_nested.yml | 55 ------------------- .../composable/component/process.json | 55 ------------------- .../elasticsearch/legacy/template.json | 55 ------------------- generated/beats/fields.ecs.yml | 46 ---------------- generated/csv/fields.csv | 11 ---- generated/ecs/ecs_flat.yml | 55 ------------------- generated/ecs/ecs_nested.yml | 55 ------------------- .../composable/component/process.json | 55 ------------------- generated/elasticsearch/legacy/template.json | 55 ------------------- schemas/process.yml | 8 --- 15 files changed, 570 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 85035153ba..9613fb89e6 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -25,8 +25,6 @@ Thanks, you're awesome :-) --> #### Improvements -* Added `.caseless` subfield to `process.name` and `process.executable`. #2341 - #### Deprecated ### Tooling and Artifact Changes diff --git a/docs/fields/field-details.asciidoc b/docs/fields/field-details.asciidoc index 3d817d46ce..31273d8c4b 100644 --- a/docs/fields/field-details.asciidoc +++ b/docs/fields/field-details.asciidoc @@ -8128,9 +8128,6 @@ type: keyword Multi-fields: -* process.executable.caseless (type: keyword) - - * process.executable.text (type: match_only_text) @@ -8346,9 +8343,6 @@ type: keyword Multi-fields: -* process.name.caseless (type: keyword) - - * process.name.text (type: match_only_text) diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 7f63fa34e9..bc95a6db22 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -5175,10 +5175,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5217,10 +5213,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -5490,11 +5482,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - default_field: false - name: text type: match_only_text default_field: false @@ -5573,10 +5560,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5615,10 +5598,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -6033,11 +6012,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - default_field: false - name: text type: match_only_text default_field: false @@ -6427,10 +6401,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -6674,10 +6644,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -7264,10 +7230,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7383,10 +7345,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7425,10 +7383,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 5c51865b5d..292ac5f917 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -648,13 +648,11 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.entry_leader.entry_meta.source.ip,ip,core,,,IP address of the source. 8.12.0-dev+exp,true,process,process.entry_leader.entry_meta.type,keyword,extended,,,The entry type for the entry session leader. 8.12.0-dev+exp,true,process,process.entry_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev+exp,true,process,process.entry_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.entry_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.entry_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.entry_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.entry_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev+exp,true,process,process.entry_leader.name,keyword,extended,,ssh,Process name. -8.12.0-dev+exp,true,process,process.entry_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.entry_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.entry_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.entry_leader.parent.pid,long,core,,4242,Process id. @@ -690,7 +688,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.entry_leader.working_directory.text,match_only_text,extended,,/home/alice,The working directory of the process. 8.12.0-dev+exp,true,process,process.env_vars,keyword,extended,array,"[""PATH=/usr/local/bin:/usr/bin"", ""USER=ubuntu""]",Array of environment variable bindings. 8.12.0-dev+exp,true,process,process.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev+exp,true,process,process.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev+exp,true,process,process.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -701,13 +698,11 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.group_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev+exp,true,process,process.group_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.group_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev+exp,true,process,process.group_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.group_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.group_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.group_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.group_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev+exp,true,process,process.group_leader.name,keyword,extended,,ssh,Process name. -8.12.0-dev+exp,true,process,process.group_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.group_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.group_leader.pid,long,core,,4242,Process id. 8.12.0-dev+exp,true,process,process.group_leader.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -767,7 +762,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev+exp,true,process,process.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev+exp,true,process,process.name,keyword,extended,,ssh,Process name. -8.12.0-dev+exp,true,process,process.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev+exp,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. @@ -823,7 +817,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.end,date,extended,,2016-05-23T08:05:34.853Z,The time the process ended. 8.12.0-dev+exp,true,process,process.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.parent.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev+exp,true,process,process.parent.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.parent.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.parent.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev+exp,true,process,process.parent.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -857,7 +850,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev+exp,true,process,process.parent.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev+exp,true,process,process.parent.name,keyword,extended,,ssh,Process name. -8.12.0-dev+exp,true,process,process.parent.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.parent.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. 8.12.0-dev+exp,true,process,process.parent.pe.company,keyword,extended,,Microsoft Corporation,"Internal company name of the file, provided at compile-time." @@ -941,7 +933,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.previous.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev+exp,true,process,process.previous.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.previous.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev+exp,true,process,process.previous.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.previous.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.real_group.name,keyword,extended,,,Name of the group. @@ -959,13 +950,11 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.session_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev+exp,true,process,process.session_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.session_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev+exp,true,process,process.session_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.session_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev+exp,true,process,process.session_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev+exp,true,process,process.session_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev+exp,true,process,process.session_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev+exp,true,process,process.session_leader.name,keyword,extended,,ssh,Process name. -8.12.0-dev+exp,true,process,process.session_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.session_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev+exp,true,process,process.session_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev+exp,true,process,process.session_leader.parent.pid,long,core,,4242,Process id. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 65448363e9..02b972886f 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -8426,11 +8426,6 @@ process.entry_leader.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -8492,11 +8487,6 @@ process.entry_leader.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -8920,11 +8910,6 @@ process.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -9044,11 +9029,6 @@ process.group_leader.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -9110,11 +9090,6 @@ process.group_leader.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -9804,11 +9779,6 @@ process.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -10470,11 +10440,6 @@ process.parent.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -10884,11 +10849,6 @@ process.parent.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -11873,11 +11833,6 @@ process.previous.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.previous.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -12063,11 +12018,6 @@ process.session_leader.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -12129,11 +12079,6 @@ process.session_leader.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 9c39b3b5ae..f600ab293a 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -10636,11 +10636,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -10702,11 +10697,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -11130,11 +11120,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -11254,11 +11239,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -11320,11 +11300,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -12018,11 +11993,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -12685,11 +12655,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -13100,11 +13065,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -14091,11 +14051,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.previous.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -14281,11 +14236,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -14347,11 +14297,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/experimental/generated/elasticsearch/composable/component/process.json b/experimental/generated/elasticsearch/composable/component/process.json index 21bc93af56..f4dd52c1ce 100644 --- a/experimental/generated/elasticsearch/composable/component/process.json +++ b/experimental/generated/elasticsearch/composable/component/process.json @@ -275,11 +275,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -304,11 +299,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -481,11 +471,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -531,11 +516,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -560,11 +540,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -824,11 +799,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1044,11 +1014,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1183,11 +1148,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1544,11 +1504,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1639,11 +1594,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1668,11 +1618,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } diff --git a/experimental/generated/elasticsearch/legacy/template.json b/experimental/generated/elasticsearch/legacy/template.json index 4e7c38235a..18386e190c 100644 --- a/experimental/generated/elasticsearch/legacy/template.json +++ b/experimental/generated/elasticsearch/legacy/template.json @@ -2996,11 +2996,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3025,11 +3020,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3202,11 +3192,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3252,11 +3237,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3281,11 +3261,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3545,11 +3520,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3765,11 +3735,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3904,11 +3869,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -4265,11 +4225,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -4360,11 +4315,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -4389,11 +4339,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index bd55fe7727..fa0007884b 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -5125,10 +5125,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5167,10 +5163,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -5440,11 +5432,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - default_field: false - name: text type: match_only_text default_field: false @@ -5523,10 +5510,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -5565,10 +5548,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -5983,11 +5962,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - default_field: false - name: text type: match_only_text default_field: false @@ -6377,10 +6351,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -6624,10 +6594,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. @@ -7214,10 +7180,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7333,10 +7295,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: Absolute path to the process executable. @@ -7375,10 +7333,6 @@ type: keyword ignore_above: 1024 multi_fields: - - name: caseless - type: keyword - normalizer: lowercase - ignore_above: 1024 - name: text type: match_only_text description: 'Process name. diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 299f4aa1a7..c31a8de31c 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -641,13 +641,11 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.entry_leader.entry_meta.source.ip,ip,core,,,IP address of the source. 8.12.0-dev,true,process,process.entry_leader.entry_meta.type,keyword,extended,,,The entry type for the entry session leader. 8.12.0-dev,true,process,process.entry_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev,true,process,process.entry_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.entry_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.entry_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.entry_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.entry_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev,true,process,process.entry_leader.name,keyword,extended,,ssh,Process name. -8.12.0-dev,true,process,process.entry_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.entry_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.entry_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.entry_leader.parent.pid,long,core,,4242,Process id. @@ -683,7 +681,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.entry_leader.working_directory.text,match_only_text,extended,,/home/alice,The working directory of the process. 8.12.0-dev,true,process,process.env_vars,keyword,extended,array,"[""PATH=/usr/local/bin:/usr/bin"", ""USER=ubuntu""]",Array of environment variable bindings. 8.12.0-dev,true,process,process.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev,true,process,process.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev,true,process,process.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -694,13 +691,11 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.group_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev,true,process,process.group_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.group_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev,true,process,process.group_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.group_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.group_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.group_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.group_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev,true,process,process.group_leader.name,keyword,extended,,ssh,Process name. -8.12.0-dev,true,process,process.group_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.group_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.group_leader.pid,long,core,,4242,Process id. 8.12.0-dev,true,process,process.group_leader.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -760,7 +755,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev,true,process,process.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev,true,process,process.name,keyword,extended,,ssh,Process name. -8.12.0-dev,true,process,process.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. @@ -816,7 +810,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.end,date,extended,,2016-05-23T08:05:34.853Z,The time the process ended. 8.12.0-dev,true,process,process.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.parent.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev,true,process,process.parent.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.parent.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.parent.exit_code,long,extended,,137,The exit code of the process. 8.12.0-dev,true,process,process.parent.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. @@ -850,7 +843,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.macho.sections.virtual_size,long,extended,,,Mach-O Section List virtual size. This is always the same as `physical_size`. 8.12.0-dev,true,process,process.parent.macho.symhash,keyword,extended,,d3ccf195b62a9279c3c19af1080497ec,A hash of the imports in a Mach-O file. 8.12.0-dev,true,process,process.parent.name,keyword,extended,,ssh,Process name. -8.12.0-dev,true,process,process.parent.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.parent.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. 8.12.0-dev,true,process,process.parent.pe.company,keyword,extended,,Microsoft Corporation,"Internal company name of the file, provided at compile-time." @@ -934,7 +926,6 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.previous.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments. 8.12.0-dev,true,process,process.previous.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.previous.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev,true,process,process.previous.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.previous.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.real_group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.real_group.name,keyword,extended,,,Name of the group. @@ -952,13 +943,11 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.session_leader.command_line.text,match_only_text,extended,,/usr/bin/ssh -l user 10.0.0.16,Full command line that started the process. 8.12.0-dev,true,process,process.session_leader.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.session_leader.executable,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. -8.12.0-dev,true,process,process.session_leader.executable.caseless,keyword,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.session_leader.executable.text,match_only_text,extended,,/usr/bin/ssh,Absolute path to the process executable. 8.12.0-dev,true,process,process.session_leader.group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 8.12.0-dev,true,process,process.session_leader.group.name,keyword,extended,,,Name of the group. 8.12.0-dev,true,process,process.session_leader.interactive,boolean,extended,,True,Whether the process is connected to an interactive shell. 8.12.0-dev,true,process,process.session_leader.name,keyword,extended,,ssh,Process name. -8.12.0-dev,true,process,process.session_leader.name.caseless,keyword,extended,,ssh,Process name. 8.12.0-dev,true,process,process.session_leader.name.text,match_only_text,extended,,ssh,Process name. 8.12.0-dev,true,process,process.session_leader.parent.entity_id,keyword,extended,,c2c455d9f99375d,Unique identifier for the process. 8.12.0-dev,true,process,process.session_leader.parent.pid,long,core,,4242,Process id. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 2ff10f67c0..2022bddaf4 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -8357,11 +8357,6 @@ process.entry_leader.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -8423,11 +8418,6 @@ process.entry_leader.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -8851,11 +8841,6 @@ process.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -8975,11 +8960,6 @@ process.group_leader.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -9041,11 +9021,6 @@ process.group_leader.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -9735,11 +9710,6 @@ process.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -10401,11 +10371,6 @@ process.parent.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -10815,11 +10780,6 @@ process.parent.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -11804,11 +11764,6 @@ process.previous.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.previous.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -11994,11 +11949,6 @@ process.session_leader.executable: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -12060,11 +12010,6 @@ process.session_leader.name: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index cc11243d59..8057eeed15 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -10556,11 +10556,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.executable.text name: text type: match_only_text @@ -10622,11 +10617,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.entry_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.entry_leader.name.text name: text type: match_only_text @@ -11050,11 +11040,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.executable.text name: text type: match_only_text @@ -11174,11 +11159,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.executable.text name: text type: match_only_text @@ -11240,11 +11220,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.group_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.group_leader.name.text name: text type: match_only_text @@ -11938,11 +11913,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.name.text name: text type: match_only_text @@ -12605,11 +12575,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.executable.text name: text type: match_only_text @@ -13020,11 +12985,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.parent.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.parent.name.text name: text type: match_only_text @@ -14011,11 +13971,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.previous.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.previous.executable.text name: text type: match_only_text @@ -14201,11 +14156,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.executable.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.executable.text name: text type: match_only_text @@ -14267,11 +14217,6 @@ process: ignore_above: 1024 level: extended multi_fields: - - flat_name: process.session_leader.name.caseless - ignore_above: 1024 - name: caseless - normalizer: lowercase - type: keyword - flat_name: process.session_leader.name.text name: text type: match_only_text diff --git a/generated/elasticsearch/composable/component/process.json b/generated/elasticsearch/composable/component/process.json index a4b1e5e1f0..6cc1382d11 100644 --- a/generated/elasticsearch/composable/component/process.json +++ b/generated/elasticsearch/composable/component/process.json @@ -275,11 +275,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -304,11 +299,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -481,11 +471,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -531,11 +516,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -560,11 +540,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -824,11 +799,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1044,11 +1014,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1183,11 +1148,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1544,11 +1504,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1639,11 +1594,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -1668,11 +1618,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } diff --git a/generated/elasticsearch/legacy/template.json b/generated/elasticsearch/legacy/template.json index 1d343b5050..a6b67033e2 100644 --- a/generated/elasticsearch/legacy/template.json +++ b/generated/elasticsearch/legacy/template.json @@ -2954,11 +2954,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -2983,11 +2978,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3160,11 +3150,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3210,11 +3195,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3239,11 +3219,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3503,11 +3478,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3723,11 +3693,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -3862,11 +3827,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -4223,11 +4183,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -4318,11 +4273,6 @@ }, "executable": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } @@ -4347,11 +4297,6 @@ }, "name": { "fields": { - "caseless": { - "ignore_above": 1024, - "normalizer": "lowercase", - "type": "keyword" - }, "text": { "type": "match_only_text" } diff --git a/schemas/process.yml b/schemas/process.yml index b1acf8945c..91c8ef98ef 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -112,10 +112,6 @@ Sometimes called program name or similar. example: ssh multi_fields: - - name: caseless - ignore_above: 1024 - normalizer: lowercase - type: keyword - type: match_only_text name: text @@ -175,10 +171,6 @@ Absolute path to the process executable. example: /usr/bin/ssh multi_fields: - - name: caseless - ignore_above: 1024 - normalizer: lowercase - type: keyword - type: match_only_text name: text From fa370236b9237764e8b493ddef05875911571b14 Mon Sep 17 00:00:00 2001 From: Thijs Xhaflaire Date: Tue, 13 Aug 2024 20:48:13 +0200 Subject: [PATCH 03/46] [RFC] Apple Platform specific fields (#2338) Adds RFS stage 0 --------- Co-authored-by: Alexandra Konrad Co-authored-by: Michael Wolf --- ...0044-add-apple-platform-specific-fields.md | 119 ++++++++++++++++++ rfcs/text/0044/code_signature.yml | 10 ++ rfcs/text/0044/device.yml | 10 ++ rfcs/text/0044/hash.yml | 9 ++ rfcs/text/0044/process.yml | 36 ++++++ 5 files changed, 184 insertions(+) create mode 100644 rfcs/text/0044-add-apple-platform-specific-fields.md create mode 100644 rfcs/text/0044/code_signature.yml create mode 100644 rfcs/text/0044/device.yml create mode 100644 rfcs/text/0044/hash.yml create mode 100644 rfcs/text/0044/process.yml diff --git a/rfcs/text/0044-add-apple-platform-specific-fields.md b/rfcs/text/0044-add-apple-platform-specific-fields.md new file mode 100644 index 0000000000..8b14a79d5d --- /dev/null +++ b/rfcs/text/0044-add-apple-platform-specific-fields.md @@ -0,0 +1,119 @@ +# 0000: Name of RFC + + +- Stage: **0 (strawperson)** +- Date: **TBD** + + +### Summary +This RFC proposes the addition of Apple platform-specific fields to the ECS schema. This enhancement will enable security software vendors to more accurately map out data, particularly for Apple platforms. + +The following feelds needs to be considered being added: + +## Fields + +##### Proposed New Fields for Process object + +Field | Type | Example | Description +--- | --- | --- | --- +responsible | keyword | Terminal.app | The responsible process on macOS, from an ancestry perspective, is the process that originally launched or spawned a given process. +platform_binary | boolean | true | Indicates wethether this process executable is a default platform binary shipped with the operating system. +endpoint_security_client | boolean | true | Indicates wethether this process executable is an Endpoint Security client. + +##### Proposed New Fields for Code Signature object + +Field | Type | Example | Description +--- | --- | --- | --- +flags | string | 570522385 | The flags used to sign the process. + +##### Proposed New Fields for Hash object + +Field | Type | Example | Description +--- | --- | --- | --- +cdhash | keyword | 3783b4052fd474dbe30676b45c329e7a6d44acd9 | The Code Directory (CD) hash of an executable + +##### Proposed New Fields for Device object + +Field | Type | Example | Description +--- | --- | --- | --- +serial_number | keyword | DJGAQS4CW5 | The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication. + +### Motivation + +As the number of Apple endpoints in enterprises grows, having the right fields to map data becomes increasingly valuable. This enables security researchers using Elastic, particularly those focusing on macOS, to query data more effectively by leveraging enriched data sets. + +## Usage + +As a developer at Jamf, working on the Elastic integration for Jamf Protect, our goal is to map as many fields as possible, especially as Jamf specializes in Apple platform security. While developing the integration, we've identified some gaps related to mapping events to ECS. + +These new fields offer versatile methods. For instance, they facilitate querying process executions by platform binaries or endpoint security clients without requiring specific identifiers. The added hash fields are particularly valuable for tracking the hash of an application bundle alongside the hash of the executable in the directory itself, while the others are self-explanatory. + +## Source data + +This data originates from Endpoint Security software operating on a macOS host and can be transmitted through various methods, including an Elastic Agent and as example the use of the Jamf Protect integration, which supports AWS S3 or HTTPs. + + + + + + + + + + + + + +## People + +The following are the people that consulted on the contents of this RFC. + +* txhaflaire | author + + + + +## References + + + +### RFC Pull Requests + + + +* Stage 0: https://github.com/elastic/ecs/pull/2338 + + diff --git a/rfcs/text/0044/code_signature.yml b/rfcs/text/0044/code_signature.yml new file mode 100644 index 0000000000..091339048d --- /dev/null +++ b/rfcs/text/0044/code_signature.yml @@ -0,0 +1,10 @@ +--- +- name: code_signature + fields: + - name: flags + level: extended + type: string + short: Code signing flags of the process + description: > + The flags used to sign the process. + example: 570522385 \ No newline at end of file diff --git a/rfcs/text/0044/device.yml b/rfcs/text/0044/device.yml new file mode 100644 index 0000000000..5bfdcdb161 --- /dev/null +++ b/rfcs/text/0044/device.yml @@ -0,0 +1,10 @@ +--- +- name: device + fields: + - name: serial_number + level: core + type: keyword + short: Serial Number of the device + description: > + The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication. + example: DJGAQS4CW5 \ No newline at end of file diff --git a/rfcs/text/0044/hash.yml b/rfcs/text/0044/hash.yml new file mode 100644 index 0000000000..9d76c4957c --- /dev/null +++ b/rfcs/text/0044/hash.yml @@ -0,0 +1,9 @@ +--- +- name: file + fields: + - name: cdhash + level: extended + type: keyword + short: The Code Directory (CD) hash of an executable. + description: Code directory hash, utilized to uniquely identify and authenticate the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 \ No newline at end of file diff --git a/rfcs/text/0044/process.yml b/rfcs/text/0044/process.yml new file mode 100644 index 0000000000..d9e8691439 --- /dev/null +++ b/rfcs/text/0044/process.yml @@ -0,0 +1,36 @@ +--- +- name: process + title: Process + group: 2 + short: These fields contain information about a process. + description: > + These fields contain information about a process. + + These fields can help you correlate metrics information with a process id/name + from a log message. The `process.pid` often stays in the metric itself and is + copied to the global field for correlation. + type: group + reusable: + top_level: true + expected: + - at: process + as: responsible + short_override: Information about the responsible process. + +- name: process + fields: + - name: platform_binary + level: extended + type: boolean + short: Indicates whether this process executable is a default platform binary shipped with the operating system. + description: > + Binaries that are shipped by the operating system are defined as platform binaries, this value is then set to true. + example: true + + - name: endpoint_security_client + level: extended + type: boolean + short: Indicates whether this process executable is an Endpoint Security client. + description: > + Processes that have an endpoint security client must have the com.apple.endpointsecurity entitlement and the value is set to true in the message. + example: true \ No newline at end of file From a664f225c07188beb2ec8739cfd6a71e9e927e4a Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:39:17 -0700 Subject: [PATCH 04/46] Add renovate.json (#2352) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- renovate.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000000..36a1298021 --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "local>elastic/renovate-config" + ] +} From e3f0f0e4e628a17cd7e95504ae43be921a4871a1 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 19 Aug 2024 08:40:35 -0700 Subject: [PATCH 05/46] Update template fields (#2354) Update some templated fields that were missed before merging the RFC --- rfcs/text/0044-add-apple-platform-specific-fields.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rfcs/text/0044-add-apple-platform-specific-fields.md b/rfcs/text/0044-add-apple-platform-specific-fields.md index 8b14a79d5d..68d0da3214 100644 --- a/rfcs/text/0044-add-apple-platform-specific-fields.md +++ b/rfcs/text/0044-add-apple-platform-specific-fields.md @@ -1,8 +1,8 @@ -# 0000: Name of RFC +# 0044: Apple Platform specific fields - Stage: **0 (strawperson)** -- Date: **TBD** +- Date: **2024-08-13** ### Summary From 86791b1500b4d9fbf00324998330d21fbaa6ca16 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:32:59 -0700 Subject: [PATCH 06/46] Pin dependencies (#2355) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/docs-preview-comment.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/test.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-preview-comment.yml b/.github/workflows/docs-preview-comment.yml index 34371e767b..7edee4d477 100644 --- a/.github/workflows/docs-preview-comment.yml +++ b/.github/workflows/docs-preview-comment.yml @@ -10,7 +10,7 @@ jobs: doc-preview: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v6 + - uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6 name: Add doc preview links with: script: | diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7d2d2e6526..20bd20e379 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v4 + uses: actions/stale@a20b814fb01b71def3bd6f56e7494d667ddf28da # v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 60 days with no activity.' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70ae6ad400..c61715c973 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,8 +7,8 @@ jobs: runs-on: ubuntu-20.04 name: Unit Tests steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 with: python-version: '3.x' - run: git fetch --prune --unshallow --tags From 529cca7983257d9efc5fc5c32bf6c50710a95138 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:11:54 -0700 Subject: [PATCH 07/46] Update dependency PyYAML to v6.0.2 (#2356) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index e703c6ad50..b3ca161f29 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,6 +1,6 @@ pip # License: MIT -PyYAML==6.0.1 +PyYAML==6.0.2 # License: BSD gitpython==3.1.41 # License: BSD From 33662b6879207e7833457594ef4f7b64c8a0701b Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:59:11 -0700 Subject: [PATCH 08/46] Update dependency gitpython to v3.1.43 (#2358) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index b3ca161f29..106d1de1b0 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -2,6 +2,6 @@ pip # License: MIT PyYAML==6.0.2 # License: BSD -gitpython==3.1.41 +gitpython==3.1.43 # License: BSD Jinja2==3.1.4 From 229312ae834b0dc673dfa866458a092081ab46f6 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:44:50 -0700 Subject: [PATCH 09/46] Update dependency yamllint to v1.35.1 (#2361) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- scripts/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index cb5626221a..6c73c2552e 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -3,4 +3,4 @@ autopep8==1.6.0 # License: BSD mock==4.0.3 # License: GPLv3 -yamllint==1.26.3 +yamllint==1.35.1 From 22a03bce8e9eb28fb374077c3e9e026bc8b0f1a1 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 20 Aug 2024 02:12:57 -0700 Subject: [PATCH 10/46] Update stale PR message (#2369) Add a friendlier stale PR message, based from the [Beats stale message](https://github.com/elastic/beats/blob/main/.github/stale.yml#L63-L74). This will hopefully also prompt contributors to respond, so we'll be better able to track PRs people are still interested in contributing. --- .github/workflows/stale.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 20bd20e379..b2f5e26547 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -18,7 +18,18 @@ jobs: uses: actions/stale@a20b814fb01b71def3bd6f56e7494d667ddf28da # v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: 'This PR is stale because it has been open for 60 days with no activity.' + stale-pr-message: | + Hi! + + We just realized that we haven't looked into this PR in a while. We're + sorry! + + We're labeling this PR as `Stale` to make it hit our filters and + make sure we get back to it as soon as possible. In the meantime, it'd + be extremely helpful if you could take a look at it as well and confirm its + relevance. A simple comment with a nice emoji will be enough `:+1`. + + Thank you for your contribution! stale-pr-label: 'stale' ascending: true days-before-pr-stale: 60 From 70f4bcae7e29b2448d29596eea2a828f208db760 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 10:49:31 -0700 Subject: [PATCH 11/46] Update actions/checkout action to v4 (#2362) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c61715c973..62b7a5d214 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-20.04 name: Unit Tests steps: - - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 with: python-version: '3.x' From c097a0e014ef05fdb702c12d89ce7f88c1ddf17e Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:32:21 -0700 Subject: [PATCH 12/46] Update actions/github-script action to v7 (#2363) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/docs-preview-comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-comment.yml b/.github/workflows/docs-preview-comment.yml index 7edee4d477..52c5ca16a8 100644 --- a/.github/workflows/docs-preview-comment.yml +++ b/.github/workflows/docs-preview-comment.yml @@ -10,7 +10,7 @@ jobs: doc-preview: runs-on: ubuntu-latest steps: - - uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6 + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 name: Add doc preview links with: script: | From 22d5d35609de2e3d9c49e8396a9abc95612c2c90 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:15:59 -0700 Subject: [PATCH 13/46] Update actions/setup-python action to v5 (#2364) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 62b7a5d214..f4969c765b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: name: Unit Tests steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 with: python-version: '3.x' - run: git fetch --prune --unshallow --tags From 07ffbd1e730be7e309e9a8db8211a1a7a89900d4 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:30:05 -0700 Subject: [PATCH 14/46] Update actions/stale action to v9 (#2365) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b2f5e26547..1bd28c751e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@a20b814fb01b71def3bd6f56e7494d667ddf28da # v4 + uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: | From 71a5e5dddb16430a761063db6356c448f7713c4e Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:07:17 -0700 Subject: [PATCH 15/46] Update dependency mock to v5 (#2367) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- scripts/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index 6c73c2552e..9a9650f135 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -1,6 +1,6 @@ # License: MIT autopep8==1.6.0 # License: BSD -mock==4.0.3 +mock==5.1.0 # License: GPLv3 yamllint==1.35.1 From 53765702d7036414a520b87bbe7b8a12b1e7be30 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:26:47 -0700 Subject: [PATCH 16/46] Update dependency ubuntu to v22 (#2368) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4969c765b..a804fb2b06 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 name: Unit Tests steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 From a793bb2f62e4e7e25233acf3c73859a030bfd14a Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:16:27 -0700 Subject: [PATCH 17/46] Update dependency autopep8 to v1.7.0 (#2359) Update dependency autopep8 to v1.7.0 --------- Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- scripts/requirements-dev.txt | 2 +- scripts/schema/subset_filter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index 9a9650f135..35c40a6198 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -1,5 +1,5 @@ # License: MIT -autopep8==1.6.0 +autopep8==1.7.0 # License: BSD mock==5.1.0 # License: GPLv3 diff --git a/scripts/schema/subset_filter.py b/scripts/schema/subset_filter.py index 8b5f0d1762..9f72205777 100644 --- a/scripts/schema/subset_filter.py +++ b/scripts/schema/subset_filter.py @@ -121,7 +121,7 @@ def remove_docs_only_entries(paths: List[str], fields: Dict[str, FieldEntry]) -> split_path = path.split('.') field_set = split_path[0] field = split_path[1] - del(fields[field_set]['fields'][field]) + del (fields[field_set]['fields'][field]) return fields From e3e73de6d26c48f22126f6fbbcebdf07032044a9 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:34:37 -0700 Subject: [PATCH 18/46] Update dependency autopep8 to v2 (#2366) * Update dependency autopep8 to v2 --------- Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- scripts/requirements-dev.txt | 2 +- scripts/tests/test_asciidoc_fields.py | 2 +- scripts/tests/test_ecs_helpers.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index 35c40a6198..d9d2c538a8 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -1,5 +1,5 @@ # License: MIT -autopep8==1.7.0 +autopep8==2.3.1 # License: BSD mock==5.1.0 # License: GPLv3 diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index 0516ac3093..1e0388c6c2 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -186,7 +186,7 @@ def dummy_nested_event_fieldset(self): def test_validate_sort_fieldset(self): sorted_foo_fields = asciidoc_fields.sort_fields(self.foo_fieldset) - #import pdb;pdb.set_trace() + # import pdb;pdb.set_trace() self.assertIsInstance(sorted_foo_fields, list) # `allowed_value_names` always present diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index a1b2af4fc0..8ad616c4f4 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -144,7 +144,7 @@ def test_glob_yaml_files(self): self.assertEqual(ecs_helpers.glob_yaml_files('non_existent_wildcard.*'), []) self.assertEqual(ecs_helpers.glob_yaml_files('schemas/base.yml'), ['schemas/base.yml']) self.assertEqual(ecs_helpers.glob_yaml_files(['schemas/base.yml']), ['schemas/base.yml']) - # convert to set as element order is not being tested + #  convert to set as element order is not being tested self.assertEqual(set(ecs_helpers.glob_yaml_files( ['schemas/base.yml', 'schemas/log.yml'])), {'schemas/base.yml', 'schemas/log.yml'}) self.assertTrue(set(ecs_helpers.glob_yaml_files('schemas/b*.yml')).intersection({'schemas/base.yml'}) != set()) From 3f3ff683c9be28ffaa4bd4f6fc1c3667524635e8 Mon Sep 17 00:00:00 2001 From: Stefan Bischof Date: Fri, 6 Sep 2024 02:05:16 +0200 Subject: [PATCH 19/46] add license header (#2377) --- schemas/device.yml | 16 ++++++++++++++++ schemas/email.yml | 17 +++++++++++++++++ schemas/risk.yml | 16 ++++++++++++++++ schemas/subsets/main.yml | 16 ++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/schemas/device.yml b/schemas/device.yml index 9e9e7613d6..38fe123937 100644 --- a/schemas/device.yml +++ b/schemas/device.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: device title: Device diff --git a/schemas/email.yml b/schemas/email.yml index 1982edb7d9..82bfd5b219 100644 --- a/schemas/email.yml +++ b/schemas/email.yml @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- - name: email title: Email group: 2 diff --git a/schemas/risk.yml b/schemas/risk.yml index b70640c473..84835f08e2 100644 --- a/schemas/risk.yml +++ b/schemas/risk.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- - name: risk title: Risk information diff --git a/schemas/subsets/main.yml b/schemas/subsets/main.yml index ebefde9e2e..63b44449fc 100644 --- a/schemas/subsets/main.yml +++ b/schemas/subsets/main.yml @@ -1,3 +1,19 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- name: main fields: From 93453f477a25b1f77a9106c7c822927c362f5ebb Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:46:53 -0700 Subject: [PATCH 20/46] Update actions/setup-python digest to f677139 (#2374) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Michael Wolf --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a804fb2b06..b618b7394b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: name: Unit Tests steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 with: python-version: '3.x' - run: git fetch --prune --unshallow --tags From 029cf00ada5e63375fe9967632f80ba4cb20efeb Mon Sep 17 00:00:00 2001 From: Smriti <152067238+smriti0321@users.noreply.github.com> Date: Tue, 10 Sep 2024 12:24:36 +0200 Subject: [PATCH 21/46] [RFC] Stage 0: Introducing new field in rule namespace (#2330) * Update 0000-rfc-template.md Updating the temaplate for RFC Stage 0 for adding 2 new rule fields: rule.tags and rule.remediation * Update 0000-rfc-template.md Incorporating review comments. * Renaming the template file with recommended name * Resolving conflicts * Removing Tag Field * Resolving comments from @trisch-me * Moving file to rfcs/text folder as per @trisch-me comment. using next number in series. * I saw number 44 was used in a recent RFC, using next number in series --------- Co-authored-by: Eric Beahan Co-authored-by: Alexandra Konrad --- .../0046-additional-rule-field.md} | 44 +++++++------------ 1 file changed, 16 insertions(+), 28 deletions(-) rename rfcs/{0000-rfc-template.md => text/0046-additional-rule-field.md} (63%) diff --git a/rfcs/0000-rfc-template.md b/rfcs/text/0046-additional-rule-field.md similarity index 63% rename from rfcs/0000-rfc-template.md rename to rfcs/text/0046-additional-rule-field.md index 1ac7c95052..f9354ce2f2 100644 --- a/rfcs/0000-rfc-template.md +++ b/rfcs/text/0046-additional-rule-field.md @@ -1,35 +1,27 @@ -# 0000: Name of RFC +# 0000: Additional Rule Field - Stage: **0 (strawperson)** - Date: **TBD** - + +This RFC proposes addition of 1 new field (rule.remediation) in rule fieldset to the Elastic Common Schema (ECS). The goal of this field is to provide more context to the users in the rule fieldset, rule.remediation will be used to capture the remediation instructions associated with rules, it is generally provided by the benchmark or framework from which the rule is published. - - ## Fields - +The `rule` fields being proposed are as follows: + +Field | Type | Example | Description/Usage +-- | -- | -- | -- +rule.remediation | array | Enable encryption on all S3 buckets | Used to capture remediation instructions that come from the benchmark / framework the rule is from + - ## Usage @@ -79,25 +71,21 @@ Stage 3: Document resolutions for all existing concerns. Any new concerns should The following are the people that consulted on the contents of this RFC. -* TBD | author +* @smriti0321 | author +* @tinnytintin10 | Product Manager +* @oren-zohar | Engineering Manager +* @orouz | Engineer +* @trisch-me | Security ECS team - ## References + + ### RFC Pull Requests From 149a4cc023462acc25c49ae94ad921a813b99ee4 Mon Sep 17 00:00:00 2001 From: Thijs Xhaflaire Date: Wed, 11 Sep 2024 23:28:06 +0200 Subject: [PATCH 22/46] [RFC] Stage 2: Adding Apple Platform specific fields (#2370) Updating the RFC and moving it to stage two. --- docs/fields/field-details.asciidoc | 64 +++++ experimental/generated/beats/fields.ecs.yml | 100 ++++++++ experimental/generated/csv/fields.csv | 14 ++ experimental/generated/ecs/ecs_flat.yml | 183 +++++++++++++++ experimental/generated/ecs/ecs_nested.yml | 219 ++++++++++++++++++ .../composable/component/device.json | 4 + .../composable/component/dll.json | 7 + .../composable/component/email.json | 4 + .../composable/component/file.json | 7 + .../composable/component/process.json | 14 ++ .../composable/component/threat.json | 14 ++ .../elasticsearch/legacy/template.json | 50 ++++ generated/beats/fields.ecs.yml | 100 ++++++++ generated/csv/fields.csv | 14 ++ generated/ecs/ecs_flat.yml | 183 +++++++++++++++ generated/ecs/ecs_nested.yml | 219 ++++++++++++++++++ .../composable/component/device.json | 4 + .../composable/component/dll.json | 7 + .../composable/component/email.json | 4 + .../composable/component/file.json | 7 + .../composable/component/process.json | 14 ++ .../composable/component/threat.json | 14 ++ generated/elasticsearch/legacy/template.json | 50 ++++ ...0044-add-apple-platform-specific-fields.md | 24 +- schemas/code_signature.yml | 9 + schemas/device.yml | 10 +- schemas/hash.yml | 8 + schemas/process.yml | 20 ++ 28 files changed, 1363 insertions(+), 4 deletions(-) diff --git a/docs/fields/field-details.asciidoc b/docs/fields/field-details.asciidoc index 31273d8c4b..4d7f9f959b 100644 --- a/docs/fields/field-details.asciidoc +++ b/docs/fields/field-details.asciidoc @@ -865,6 +865,24 @@ example: `true` // =============================================================== +| +[[field-code-signature-flags]] +<> + +a| beta:[ This field is beta and subject to change. ] + +The flags used to sign the process. + +type: string + + + +example: `570522385` + +| extended + +// =============================================================== + | [[field-code-signature-signing-id]] <> @@ -1693,6 +1711,24 @@ example: `Samsung Galaxy S6` // =============================================================== +| +[[field-device-serial-number]] +<> + +a| beta:[ This field is beta and subject to change. ] + +The unique serial number serves as a distinct identifier for each device, aiding in inventory management and device authentication. + +type: keyword + + + +example: `DJGAQS4CW5` + +| core + +// =============================================================== + |===== @@ -4811,6 +4847,24 @@ Note that this fieldset is used for common hashes that may be computed over a ra // =============================================================== +| +[[field-hash-cdhash]] +<> + +a| beta:[ This field is beta and subject to change. ] + +Code directory hash, utilized to uniquely identify and authenticate the integrity of the executable code. + +type: keyword + + + +example: `3783b4052fd474dbe30676b45c329e7a6d44acd9` + +| extended + +// =============================================================== + | [[field-hash-md5]] <> @@ -8685,6 +8739,8 @@ The `process` fields are expected to be nested at: * `process.previous` +* `process.responsible` + * `process.session_leader` * `process.session_leader.parent` @@ -8839,6 +8895,14 @@ Note: this reuse should contain an array of process field set objects. // =============================================================== +| `process.responsible.*` +| <>| beta:[ This field is beta and subject to change.] + +Responsible process in macOS tracks the originating process of an app, key for understanding permissions and hierarchy. + +// =============================================================== + + | `process.saved_group.*` | <> | The saved group (sgid). diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index bc95a6db22..f39b2c42e0 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1227,6 +1227,14 @@ description: The human readable marketing name of the device model. example: Samsung Galaxy S6 default_field: false + - name: serial_number + level: core + type: keyword + ignore_above: 1024 + description: The unique serial number serves as a distinct identifier for each + device, aiding in inventory management and device authentication. + example: DJGAQS4CW5 + default_field: false - name: dll title: DLL group: 2 @@ -1261,6 +1269,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: code_signature.signing_id level: extended type: keyword @@ -1323,6 +1337,14 @@ Leave unpopulated if a certificate was unchecked.' example: 'true' default_field: false + - name: hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: hash.md5 level: extended type: keyword @@ -1760,6 +1782,14 @@ description: Attachment file extension, excluding the leading dot. example: txt default_field: false + - name: attachments.file.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: attachments.file.hash.md5 level: extended type: keyword @@ -2405,6 +2435,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: code_signature.signing_id level: extended type: keyword @@ -2789,6 +2825,14 @@ ignore_above: 1024 description: Primary group name of the file. example: alice + - name: hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: hash.md5 level: extended type: keyword @@ -4745,6 +4789,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: code_signature.signing_id level: extended type: keyword @@ -5774,6 +5824,14 @@ description: The working directory of the process. example: /home/alice default_field: false + - name: hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: hash.md5 level: extended type: keyword @@ -6055,6 +6113,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: parent.code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: parent.code_signature.signing_id level: extended type: keyword @@ -6466,6 +6530,14 @@ the process exists within.' example: 4242 default_field: false + - name: parent.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: parent.hash.md5 level: extended type: keyword @@ -9101,6 +9173,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: enrichments.indicator.file.code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: enrichments.indicator.file.code_signature.signing_id level: extended type: keyword @@ -9492,6 +9570,14 @@ description: Primary group name of the file. example: alice default_field: false + - name: enrichments.indicator.file.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: enrichments.indicator.file.hash.md5 level: extended type: keyword @@ -10708,6 +10794,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: indicator.file.code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: indicator.file.code_signature.signing_id level: extended type: keyword @@ -11099,6 +11191,14 @@ description: Primary group name of the file. example: alice default_field: false + - name: indicator.file.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: indicator.file.hash.md5 level: extended type: keyword diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 292ac5f917..85f24dce13 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -146,8 +146,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,device,device.manufacturer,keyword,extended,,Samsung,The vendor name of the device manufacturer. 8.12.0-dev+exp,true,device,device.model.identifier,keyword,extended,,SM-G920F,The machine readable identifier of the device model. 8.12.0-dev+exp,true,device,device.model.name,keyword,extended,,Samsung Galaxy S6,The human readable marketing name of the device model. +8.12.0-dev+exp,true,device,device.serial_number,keyword,core,,DJGAQS4CW5,Serial Number of the device 8.12.0-dev+exp,true,dll,dll.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,dll,dll.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev+exp,true,dll,dll.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,dll,dll.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,dll,dll.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,dll,dll.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -155,6 +157,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,dll,dll.code_signature.timestamp,date,extended,,2021-01-01T12:10:30Z,When the signature was generated and signed. 8.12.0-dev+exp,true,dll,dll.code_signature.trusted,boolean,extended,,true,Stores the trust status of the certificate chain. 8.12.0-dev+exp,true,dll,dll.code_signature.valid,boolean,extended,,true,Boolean to capture if the digital signature is verified against the binary content. +8.12.0-dev+exp,true,dll,dll.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,dll,dll.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,dll,dll.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,dll,dll.hash.sha256,keyword,extended,,,SHA256 hash. @@ -208,6 +211,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,ecs,ecs.version,keyword,core,,1.0.0,ECS version this event conforms to. 8.12.0-dev+exp,true,email,email.attachments,nested,extended,array,,List of objects describing the attachments. 8.12.0-dev+exp,true,email,email.attachments.file.extension,keyword,extended,,txt,Attachment file extension. +8.12.0-dev+exp,true,email,email.attachments.file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,email,email.attachments.file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,email,email.attachments.file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,email,email.attachments.file.hash.sha256,keyword,extended,,,SHA256 hash. @@ -276,6 +280,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,file,file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev+exp,true,file,file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,file,file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev+exp,true,file,file.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,file,file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,file,file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,file,file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -330,6 +335,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,file,file.fork_name,keyword,extended,,Zone.Identifer,A fork is additional data associated with a filesystem object. 8.12.0-dev+exp,true,file,file.gid,keyword,extended,,1001,Primary group ID (GID) of the file. 8.12.0-dev+exp,true,file,file.group,keyword,extended,,alice,Primary group name of the file. +8.12.0-dev+exp,true,file,file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,file,file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,file,file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,file,file.hash.sha256,keyword,extended,,,SHA256 hash. @@ -587,6 +593,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,process,process.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev+exp,true,process,process.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,process,process.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,process,process.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,process,process.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -728,6 +735,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.group_leader.vpid,long,core,,4242,Virtual process id. 8.12.0-dev+exp,true,process,process.group_leader.working_directory,keyword,extended,,/home/alice,The working directory of the process. 8.12.0-dev+exp,true,process,process.group_leader.working_directory.text,match_only_text,extended,,/home/alice,The working directory of the process. +8.12.0-dev+exp,true,process,process.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,process,process.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,process,process.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,process,process.hash.sha256,keyword,extended,,,SHA256 hash. @@ -767,6 +775,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.parent.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,process,process.parent.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev+exp,true,process,process.parent.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,process,process.parent.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,process,process.parent.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,process,process.parent.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -825,6 +834,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.group_leader.pid,long,core,,4242,Process id. 8.12.0-dev+exp,true,process,process.parent.group_leader.start,date,extended,,2016-05-23T08:05:34.853Z,The time the process started. 8.12.0-dev+exp,true,process,process.parent.group_leader.vpid,long,core,,4242,Virtual process id. +8.12.0-dev+exp,true,process,process.parent.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,process,process.parent.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,process,process.parent.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,process,process.parent.hash.sha256,keyword,extended,,,SHA256 hash. @@ -1152,6 +1162,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1206,6 +1217,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.fork_name,keyword,extended,,Zone.Identifer,A fork is additional data associated with a filesystem object. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.gid,keyword,extended,,1001,Primary group ID (GID) of the file. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.group,keyword,extended,,alice,Primary group name of the file. +8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.hash.sha256,keyword,extended,,,SHA256 hash. @@ -1369,6 +1381,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,threat,threat.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1423,6 +1436,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,threat,threat.indicator.file.fork_name,keyword,extended,,Zone.Identifer,A fork is additional data associated with a filesystem object. 8.12.0-dev+exp,true,threat,threat.indicator.file.gid,keyword,extended,,1001,Primary group ID (GID) of the file. 8.12.0-dev+exp,true,threat,threat.indicator.file.group,keyword,extended,,alice,Primary group name of the file. +8.12.0-dev+exp,true,threat,threat.indicator.file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev+exp,true,threat,threat.indicator.file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev+exp,true,threat,threat.indicator.file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev+exp,true,threat,threat.indicator.file.hash.sha256,keyword,extended,,,SHA256 hash. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 02b972886f..131fbdeb3e 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -1759,6 +1759,19 @@ device.model.name: normalize: [] short: The human readable marketing name of the device model. type: keyword +device.serial_number: + beta: This field is beta and subject to change. + dashed_name: device-serial-number + description: The unique serial number serves as a distinct identifier for each device, + aiding in inventory management and device authentication. + example: DJGAQS4CW5 + flat_name: device.serial_number + ignore_above: 1024 + level: core + name: serial_number + normalize: [] + short: Serial Number of the device + type: keyword dll.code_signature.digest_algorithm: dashed_name: dll-code-signature-digest-algorithm description: 'The hashing algorithm used to sign the process. @@ -1785,6 +1798,18 @@ dll.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +dll.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: dll-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: dll.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -1883,6 +1908,20 @@ dll.code_signature.valid: short: Boolean to capture if the digital signature is verified against the binary content. type: boolean +dll.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: dll-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: dll.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword dll.hash.md5: dashed_name: dll-hash-md5 description: MD5 hash. @@ -2566,6 +2605,20 @@ email.attachments.file.extension: normalize: [] short: Attachment file extension. type: keyword +email.attachments.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: email-attachments-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: email.attachments.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword email.attachments.file.hash.md5: dashed_name: email-attachments-file-hash-md5 description: MD5 hash. @@ -3896,6 +3949,18 @@ file.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -4555,6 +4620,20 @@ file.group: normalize: [] short: Primary group name of the file. type: keyword +file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword file.hash.md5: dashed_name: file-hash-md5 description: MD5 hash. @@ -7700,6 +7779,18 @@ process.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +process.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -9372,6 +9463,20 @@ process.group_leader.working_directory: original_fieldset: process short: The working directory of the process. type: keyword +process.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.hash.md5: dashed_name: process-hash-md5 description: MD5 hash. @@ -9843,6 +9948,18 @@ process.parent.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +process.parent.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-parent-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.parent.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -10543,6 +10660,20 @@ process.parent.group_leader.vpid: original_fieldset: process short: Virtual process id. type: long +process.parent.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-parent-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.parent.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.parent.hash.md5: dashed_name: process-parent-hash-md5 description: MD5 hash. @@ -14643,6 +14774,18 @@ threat.enrichments.indicator.file.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +threat.enrichments.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.enrichments.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -15311,6 +15454,20 @@ threat.enrichments.indicator.file.group: original_fieldset: file short: Primary group name of the file. type: keyword +threat.enrichments.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.enrichments.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.enrichments.indicator.file.hash.md5: dashed_name: threat-enrichments-indicator-file-hash-md5 description: MD5 hash. @@ -17353,6 +17510,18 @@ threat.indicator.file.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +threat.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -18021,6 +18190,20 @@ threat.indicator.file.group: original_fieldset: file short: Primary group name of the file. type: keyword +threat.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.indicator.file.hash.md5: dashed_name: threat-indicator-file-hash-md5 description: MD5 hash. diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index f600ab293a..c7bb8e8af7 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1320,6 +1320,17 @@ code_signature: normalize: [] short: Boolean to capture if a signature is present. type: boolean + code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: code_signature.flags + level: extended + name: flags + normalize: [] + short: Code signing flags of the process + type: string code_signature.signing_id: dashed_name: code-signature-signing-id description: 'The identifier used to sign the process. @@ -2211,6 +2222,19 @@ device: normalize: [] short: The human readable marketing name of the device model. type: keyword + device.serial_number: + beta: This field is beta and subject to change. + dashed_name: device-serial-number + description: The unique serial number serves as a distinct identifier for each + device, aiding in inventory management and device authentication. + example: DJGAQS4CW5 + flat_name: device.serial_number + ignore_above: 1024 + level: core + name: serial_number + normalize: [] + short: Serial Number of the device + type: keyword group: 2 name: device prefix: device. @@ -2258,6 +2282,18 @@ dll: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + dll.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: dll-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: dll.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -2356,6 +2392,20 @@ dll: short: Boolean to capture if the digital signature is verified against the binary content. type: boolean + dll.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: dll-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: dll.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword dll.hash.md5: dashed_name: dll-hash-md5 description: MD5 hash. @@ -3518,6 +3568,20 @@ email: normalize: [] short: Attachment file extension. type: keyword + email.attachments.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: email-attachments-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: email.attachments.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword email.attachments.file.hash.md5: dashed_name: email-attachments-file-hash-md5 description: MD5 hash. @@ -4929,6 +4993,18 @@ file: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -5589,6 +5665,20 @@ file: normalize: [] short: Primary group name of the file. type: keyword + file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword file.hash.md5: dashed_name: file-hash-md5 description: MD5 hash. @@ -6886,6 +6976,19 @@ hash: range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed in the fieldsets to which they relate (tls and pe, respectively).' fields: + hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + short: The Code Directory (CD) hash of an executable. + type: keyword hash.md5: dashed_name: hash-md5 description: MD5 hash. @@ -9909,6 +10012,18 @@ process: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + process.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -11582,6 +11697,20 @@ process: original_fieldset: process short: The working directory of the process. type: keyword + process.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.hash.md5: dashed_name: process-hash-md5 description: MD5 hash. @@ -12057,6 +12186,18 @@ process: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + process.parent.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-parent-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.parent.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -12758,6 +12899,20 @@ process: original_fieldset: process short: Virtual process id. type: long + process.parent.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-parent-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.parent.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.parent.hash.md5: dashed_name: process-parent-hash-md5 description: MD5 hash. @@ -14947,6 +15102,7 @@ process: - process.previous - process.real_group - process.real_user + - process.responsible - process.saved_group - process.saved_user - process.session_leader @@ -15008,6 +15164,12 @@ process: - array short_override: An array of previous executions for the process, including the initial fork. Only executable and args are set. + - as: responsible + at: process + beta: This field is beta and subject to change. + full: process.responsible + short_override: Responsible process in macOS tracks the originating process + of an app, key for understanding permissions and hierarchy. top_level: true reused_here: - full: process.group @@ -15105,6 +15267,11 @@ process: schema_name: process short: An array of previous executions for the process, including the initial fork. Only executable and args are set. + - beta: This field is beta and subject to change. + full: process.responsible + schema_name: process + short: Responsible process in macOS tracks the originating process of an app, + key for understanding permissions and hierarchy. short: These fields contain information about a process. title: Process type: group @@ -17307,6 +17474,18 @@ threat: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + threat.enrichments.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.enrichments.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17976,6 +18155,20 @@ threat: original_fieldset: file short: Primary group name of the file. type: keyword + threat.enrichments.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.enrichments.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.enrichments.indicator.file.hash.md5: dashed_name: threat-enrichments-indicator-file-hash-md5 description: MD5 hash. @@ -20023,6 +20216,18 @@ threat: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + threat.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -20692,6 +20897,20 @@ threat: original_fieldset: file short: Primary group name of the file. type: keyword + threat.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.indicator.file.hash.md5: dashed_name: threat-indicator-file-hash-md5 description: MD5 hash. diff --git a/experimental/generated/elasticsearch/composable/component/device.json b/experimental/generated/elasticsearch/composable/component/device.json index cf66d72b06..215d046175 100644 --- a/experimental/generated/elasticsearch/composable/component/device.json +++ b/experimental/generated/elasticsearch/composable/component/device.json @@ -27,6 +27,10 @@ "type": "keyword" } } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" } } } diff --git a/experimental/generated/elasticsearch/composable/component/dll.json b/experimental/generated/elasticsearch/composable/component/dll.json index 2de113a6ea..55e2246263 100644 --- a/experimental/generated/elasticsearch/composable/component/dll.json +++ b/experimental/generated/elasticsearch/composable/component/dll.json @@ -17,6 +17,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -46,6 +49,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/experimental/generated/elasticsearch/composable/component/email.json b/experimental/generated/elasticsearch/composable/component/email.json index 83863c9c0c..5de733e5f7 100644 --- a/experimental/generated/elasticsearch/composable/component/email.json +++ b/experimental/generated/elasticsearch/composable/component/email.json @@ -18,6 +18,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/experimental/generated/elasticsearch/composable/component/file.json b/experimental/generated/elasticsearch/composable/component/file.json index a04643e7d9..adb9d1d8ec 100644 --- a/experimental/generated/elasticsearch/composable/component/file.json +++ b/experimental/generated/elasticsearch/composable/component/file.json @@ -24,6 +24,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -233,6 +236,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/experimental/generated/elasticsearch/composable/component/process.json b/experimental/generated/elasticsearch/composable/component/process.json index f4dd52c1ce..de0be5f249 100644 --- a/experimental/generated/elasticsearch/composable/component/process.json +++ b/experimental/generated/elasticsearch/composable/component/process.json @@ -24,6 +24,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -674,6 +677,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -824,6 +831,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -1055,6 +1065,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/experimental/generated/elasticsearch/composable/component/threat.json b/experimental/generated/elasticsearch/composable/component/threat.json index 7f002d5bb7..cdcbbd7ae6 100644 --- a/experimental/generated/elasticsearch/composable/component/threat.json +++ b/experimental/generated/elasticsearch/composable/component/threat.json @@ -66,6 +66,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -275,6 +278,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -987,6 +994,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -1196,6 +1206,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/experimental/generated/elasticsearch/legacy/template.json b/experimental/generated/elasticsearch/legacy/template.json index 18386e190c..6b9172fe34 100644 --- a/experimental/generated/elasticsearch/legacy/template.json +++ b/experimental/generated/elasticsearch/legacy/template.json @@ -782,6 +782,10 @@ "type": "keyword" } } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -796,6 +800,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -825,6 +832,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -1050,6 +1061,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -1360,6 +1375,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -1569,6 +1587,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -2745,6 +2767,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -3395,6 +3420,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -3545,6 +3574,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -3776,6 +3808,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -5283,6 +5319,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -5492,6 +5531,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -6204,6 +6247,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -6413,6 +6459,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index fa0007884b..bbe911f6ce 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1177,6 +1177,14 @@ description: The human readable marketing name of the device model. example: Samsung Galaxy S6 default_field: false + - name: serial_number + level: core + type: keyword + ignore_above: 1024 + description: The unique serial number serves as a distinct identifier for each + device, aiding in inventory management and device authentication. + example: DJGAQS4CW5 + default_field: false - name: dll title: DLL group: 2 @@ -1211,6 +1219,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: code_signature.signing_id level: extended type: keyword @@ -1273,6 +1287,14 @@ Leave unpopulated if a certificate was unchecked.' example: 'true' default_field: false + - name: hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: hash.md5 level: extended type: keyword @@ -1710,6 +1732,14 @@ description: Attachment file extension, excluding the leading dot. example: txt default_field: false + - name: attachments.file.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: attachments.file.hash.md5 level: extended type: keyword @@ -2355,6 +2385,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: code_signature.signing_id level: extended type: keyword @@ -2739,6 +2775,14 @@ ignore_above: 1024 description: Primary group name of the file. example: alice + - name: hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: hash.md5 level: extended type: keyword @@ -4695,6 +4739,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: code_signature.signing_id level: extended type: keyword @@ -5724,6 +5774,14 @@ description: The working directory of the process. example: /home/alice default_field: false + - name: hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: hash.md5 level: extended type: keyword @@ -6005,6 +6063,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: parent.code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: parent.code_signature.signing_id level: extended type: keyword @@ -6416,6 +6480,14 @@ the process exists within.' example: 4242 default_field: false + - name: parent.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: parent.hash.md5 level: extended type: keyword @@ -9051,6 +9123,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: enrichments.indicator.file.code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: enrichments.indicator.file.code_signature.signing_id level: extended type: keyword @@ -9442,6 +9520,14 @@ description: Primary group name of the file. example: alice default_field: false + - name: enrichments.indicator.file.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: enrichments.indicator.file.hash.md5 level: extended type: keyword @@ -10658,6 +10744,12 @@ description: Boolean to capture if a signature is present. example: 'true' default_field: false + - name: indicator.file.code_signature.flags + level: extended + type: string + description: The flags used to sign the process. + example: 570522385 + default_field: false - name: indicator.file.code_signature.signing_id level: extended type: keyword @@ -11049,6 +11141,14 @@ description: Primary group name of the file. example: alice default_field: false + - name: indicator.file.hash.cdhash + level: extended + type: keyword + ignore_above: 1024 + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + default_field: false - name: indicator.file.hash.md5 level: extended type: keyword diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index c31a8de31c..a7210ad73b 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -139,8 +139,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,device,device.manufacturer,keyword,extended,,Samsung,The vendor name of the device manufacturer. 8.12.0-dev,true,device,device.model.identifier,keyword,extended,,SM-G920F,The machine readable identifier of the device model. 8.12.0-dev,true,device,device.model.name,keyword,extended,,Samsung Galaxy S6,The human readable marketing name of the device model. +8.12.0-dev,true,device,device.serial_number,keyword,core,,DJGAQS4CW5,Serial Number of the device 8.12.0-dev,true,dll,dll.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,dll,dll.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev,true,dll,dll.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,dll,dll.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,dll,dll.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,dll,dll.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -148,6 +150,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,dll,dll.code_signature.timestamp,date,extended,,2021-01-01T12:10:30Z,When the signature was generated and signed. 8.12.0-dev,true,dll,dll.code_signature.trusted,boolean,extended,,true,Stores the trust status of the certificate chain. 8.12.0-dev,true,dll,dll.code_signature.valid,boolean,extended,,true,Boolean to capture if the digital signature is verified against the binary content. +8.12.0-dev,true,dll,dll.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,dll,dll.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,dll,dll.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,dll,dll.hash.sha256,keyword,extended,,,SHA256 hash. @@ -201,6 +204,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,ecs,ecs.version,keyword,core,,1.0.0,ECS version this event conforms to. 8.12.0-dev,true,email,email.attachments,nested,extended,array,,List of objects describing the attachments. 8.12.0-dev,true,email,email.attachments.file.extension,keyword,extended,,txt,Attachment file extension. +8.12.0-dev,true,email,email.attachments.file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,email,email.attachments.file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,email,email.attachments.file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,email,email.attachments.file.hash.sha256,keyword,extended,,,SHA256 hash. @@ -269,6 +273,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,file,file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev,true,file,file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,file,file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev,true,file,file.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,file,file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,file,file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,file,file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -323,6 +328,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,file,file.fork_name,keyword,extended,,Zone.Identifer,A fork is additional data associated with a filesystem object. 8.12.0-dev,true,file,file.gid,keyword,extended,,1001,Primary group ID (GID) of the file. 8.12.0-dev,true,file,file.group,keyword,extended,,alice,Primary group name of the file. +8.12.0-dev,true,file,file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,file,file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,file,file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,file,file.hash.sha256,keyword,extended,,,SHA256 hash. @@ -580,6 +586,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,process,process.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev,true,process,process.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,process,process.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,process,process.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,process,process.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -721,6 +728,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.group_leader.vpid,long,core,,4242,Virtual process id. 8.12.0-dev,true,process,process.group_leader.working_directory,keyword,extended,,/home/alice,The working directory of the process. 8.12.0-dev,true,process,process.group_leader.working_directory.text,match_only_text,extended,,/home/alice,The working directory of the process. +8.12.0-dev,true,process,process.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,process,process.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,process,process.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,process,process.hash.sha256,keyword,extended,,,SHA256 hash. @@ -760,6 +768,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.parent.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,process,process.parent.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev,true,process,process.parent.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,process,process.parent.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,process,process.parent.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,process,process.parent.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -818,6 +827,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.group_leader.pid,long,core,,4242,Process id. 8.12.0-dev,true,process,process.parent.group_leader.start,date,extended,,2016-05-23T08:05:34.853Z,The time the process started. 8.12.0-dev,true,process,process.parent.group_leader.vpid,long,core,,4242,Virtual process id. +8.12.0-dev,true,process,process.parent.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,process,process.parent.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,process,process.parent.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,process,process.parent.hash.sha256,keyword,extended,,,SHA256 hash. @@ -1145,6 +1155,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,threat,threat.enrichments.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1199,6 +1210,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,threat,threat.enrichments.indicator.file.fork_name,keyword,extended,,Zone.Identifer,A fork is additional data associated with a filesystem object. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.gid,keyword,extended,,1001,Primary group ID (GID) of the file. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.group,keyword,extended,,alice,Primary group name of the file. +8.12.0-dev,true,threat,threat.enrichments.indicator.file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.hash.sha256,keyword,extended,,,SHA256 hash. @@ -1362,6 +1374,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,threat,threat.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. +8.12.0-dev,true,threat,threat.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,threat,threat.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1416,6 +1429,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,threat,threat.indicator.file.fork_name,keyword,extended,,Zone.Identifer,A fork is additional data associated with a filesystem object. 8.12.0-dev,true,threat,threat.indicator.file.gid,keyword,extended,,1001,Primary group ID (GID) of the file. 8.12.0-dev,true,threat,threat.indicator.file.group,keyword,extended,,alice,Primary group name of the file. +8.12.0-dev,true,threat,threat.indicator.file.hash.cdhash,keyword,extended,,3783b4052fd474dbe30676b45c329e7a6d44acd9,The Code Directory (CD) hash of an executable. 8.12.0-dev,true,threat,threat.indicator.file.hash.md5,keyword,extended,,,MD5 hash. 8.12.0-dev,true,threat,threat.indicator.file.hash.sha1,keyword,extended,,,SHA1 hash. 8.12.0-dev,true,threat,threat.indicator.file.hash.sha256,keyword,extended,,,SHA256 hash. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 2022bddaf4..28f3e7d65c 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1690,6 +1690,19 @@ device.model.name: normalize: [] short: The human readable marketing name of the device model. type: keyword +device.serial_number: + beta: This field is beta and subject to change. + dashed_name: device-serial-number + description: The unique serial number serves as a distinct identifier for each device, + aiding in inventory management and device authentication. + example: DJGAQS4CW5 + flat_name: device.serial_number + ignore_above: 1024 + level: core + name: serial_number + normalize: [] + short: Serial Number of the device + type: keyword dll.code_signature.digest_algorithm: dashed_name: dll-code-signature-digest-algorithm description: 'The hashing algorithm used to sign the process. @@ -1716,6 +1729,18 @@ dll.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +dll.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: dll-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: dll.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -1814,6 +1839,20 @@ dll.code_signature.valid: short: Boolean to capture if the digital signature is verified against the binary content. type: boolean +dll.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: dll-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: dll.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword dll.hash.md5: dashed_name: dll-hash-md5 description: MD5 hash. @@ -2497,6 +2536,20 @@ email.attachments.file.extension: normalize: [] short: Attachment file extension. type: keyword +email.attachments.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: email-attachments-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: email.attachments.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword email.attachments.file.hash.md5: dashed_name: email-attachments-file-hash-md5 description: MD5 hash. @@ -3827,6 +3880,18 @@ file.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -4486,6 +4551,20 @@ file.group: normalize: [] short: Primary group name of the file. type: keyword +file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword file.hash.md5: dashed_name: file-hash-md5 description: MD5 hash. @@ -7631,6 +7710,18 @@ process.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +process.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -9303,6 +9394,20 @@ process.group_leader.working_directory: original_fieldset: process short: The working directory of the process. type: keyword +process.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.hash.md5: dashed_name: process-hash-md5 description: MD5 hash. @@ -9774,6 +9879,18 @@ process.parent.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +process.parent.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-parent-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.parent.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -10474,6 +10591,20 @@ process.parent.group_leader.vpid: original_fieldset: process short: Virtual process id. type: long +process.parent.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-parent-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.parent.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.parent.hash.md5: dashed_name: process-parent-hash-md5 description: MD5 hash. @@ -14574,6 +14705,18 @@ threat.enrichments.indicator.file.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +threat.enrichments.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.enrichments.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -15242,6 +15385,20 @@ threat.enrichments.indicator.file.group: original_fieldset: file short: Primary group name of the file. type: keyword +threat.enrichments.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.enrichments.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.enrichments.indicator.file.hash.md5: dashed_name: threat-enrichments-indicator-file-hash-md5 description: MD5 hash. @@ -17284,6 +17441,18 @@ threat.indicator.file.code_signature.exists: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean +threat.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17952,6 +18121,20 @@ threat.indicator.file.group: original_fieldset: file short: Primary group name of the file. type: keyword +threat.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.indicator.file.hash.md5: dashed_name: threat-indicator-file-hash-md5 description: MD5 hash. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 8057eeed15..fe512c7afd 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1240,6 +1240,17 @@ code_signature: normalize: [] short: Boolean to capture if a signature is present. type: boolean + code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: code_signature.flags + level: extended + name: flags + normalize: [] + short: Code signing flags of the process + type: string code_signature.signing_id: dashed_name: code-signature-signing-id description: 'The identifier used to sign the process. @@ -2131,6 +2142,19 @@ device: normalize: [] short: The human readable marketing name of the device model. type: keyword + device.serial_number: + beta: This field is beta and subject to change. + dashed_name: device-serial-number + description: The unique serial number serves as a distinct identifier for each + device, aiding in inventory management and device authentication. + example: DJGAQS4CW5 + flat_name: device.serial_number + ignore_above: 1024 + level: core + name: serial_number + normalize: [] + short: Serial Number of the device + type: keyword group: 2 name: device prefix: device. @@ -2178,6 +2202,18 @@ dll: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + dll.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: dll-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: dll.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -2276,6 +2312,20 @@ dll: short: Boolean to capture if the digital signature is verified against the binary content. type: boolean + dll.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: dll-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: dll.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword dll.hash.md5: dashed_name: dll-hash-md5 description: MD5 hash. @@ -3438,6 +3488,20 @@ email: normalize: [] short: Attachment file extension. type: keyword + email.attachments.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: email-attachments-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: email.attachments.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword email.attachments.file.hash.md5: dashed_name: email-attachments-file-hash-md5 description: MD5 hash. @@ -4849,6 +4913,18 @@ file: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -5509,6 +5585,20 @@ file: normalize: [] short: Primary group name of the file. type: keyword + file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword file.hash.md5: dashed_name: file-hash-md5 description: MD5 hash. @@ -6806,6 +6896,19 @@ hash: range of generic bytes. Entity-specific hashes such as ja3 or imphash are placed in the fieldsets to which they relate (tls and pe, respectively).' fields: + hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + short: The Code Directory (CD) hash of an executable. + type: keyword hash.md5: dashed_name: hash-md5 description: MD5 hash. @@ -9829,6 +9932,18 @@ process: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + process.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -11502,6 +11617,20 @@ process: original_fieldset: process short: The working directory of the process. type: keyword + process.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.hash.md5: dashed_name: process-hash-md5 description: MD5 hash. @@ -11977,6 +12106,18 @@ process: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + process.parent.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: process-parent-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: process.parent.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -12678,6 +12819,20 @@ process: original_fieldset: process short: Virtual process id. type: long + process.parent.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: process-parent-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: process.parent.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword process.parent.hash.md5: dashed_name: process-parent-hash-md5 description: MD5 hash. @@ -14867,6 +15022,7 @@ process: - process.previous - process.real_group - process.real_user + - process.responsible - process.saved_group - process.saved_user - process.session_leader @@ -14928,6 +15084,12 @@ process: - array short_override: An array of previous executions for the process, including the initial fork. Only executable and args are set. + - as: responsible + at: process + beta: This field is beta and subject to change. + full: process.responsible + short_override: Responsible process in macOS tracks the originating process + of an app, key for understanding permissions and hierarchy. top_level: true reused_here: - full: process.group @@ -15025,6 +15187,11 @@ process: schema_name: process short: An array of previous executions for the process, including the initial fork. Only executable and args are set. + - beta: This field is beta and subject to change. + full: process.responsible + schema_name: process + short: Responsible process in macOS tracks the originating process of an app, + key for understanding permissions and hierarchy. short: These fields contain information about a process. title: Process type: group @@ -17227,6 +17394,18 @@ threat: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + threat.enrichments.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.enrichments.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17896,6 +18075,20 @@ threat: original_fieldset: file short: Primary group name of the file. type: keyword + threat.enrichments.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-enrichments-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.enrichments.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.enrichments.indicator.file.hash.md5: dashed_name: threat-enrichments-indicator-file-hash-md5 description: MD5 hash. @@ -19943,6 +20136,18 @@ threat: original_fieldset: code_signature short: Boolean to capture if a signature is present. type: boolean + threat.indicator.file.code_signature.flags: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-code-signature-flags + description: The flags used to sign the process. + example: 570522385 + flat_name: threat.indicator.file.code_signature.flags + level: extended + name: flags + normalize: [] + original_fieldset: code_signature + short: Code signing flags of the process + type: string threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -20612,6 +20817,20 @@ threat: original_fieldset: file short: Primary group name of the file. type: keyword + threat.indicator.file.hash.cdhash: + beta: This field is beta and subject to change. + dashed_name: threat-indicator-file-hash-cdhash + description: Code directory hash, utilized to uniquely identify and authenticate + the integrity of the executable code. + example: 3783b4052fd474dbe30676b45c329e7a6d44acd9 + flat_name: threat.indicator.file.hash.cdhash + ignore_above: 1024 + level: extended + name: cdhash + normalize: [] + original_fieldset: hash + short: The Code Directory (CD) hash of an executable. + type: keyword threat.indicator.file.hash.md5: dashed_name: threat-indicator-file-hash-md5 description: MD5 hash. diff --git a/generated/elasticsearch/composable/component/device.json b/generated/elasticsearch/composable/component/device.json index e03f268c86..741cf82323 100644 --- a/generated/elasticsearch/composable/component/device.json +++ b/generated/elasticsearch/composable/component/device.json @@ -27,6 +27,10 @@ "type": "keyword" } } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" } } } diff --git a/generated/elasticsearch/composable/component/dll.json b/generated/elasticsearch/composable/component/dll.json index d3561dd742..b5f52995c5 100644 --- a/generated/elasticsearch/composable/component/dll.json +++ b/generated/elasticsearch/composable/component/dll.json @@ -17,6 +17,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -46,6 +49,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/composable/component/email.json b/generated/elasticsearch/composable/component/email.json index 94e8c70084..4046e33558 100644 --- a/generated/elasticsearch/composable/component/email.json +++ b/generated/elasticsearch/composable/component/email.json @@ -18,6 +18,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/composable/component/file.json b/generated/elasticsearch/composable/component/file.json index d055adf323..cc12f10be1 100644 --- a/generated/elasticsearch/composable/component/file.json +++ b/generated/elasticsearch/composable/component/file.json @@ -24,6 +24,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -233,6 +236,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/composable/component/process.json b/generated/elasticsearch/composable/component/process.json index 6cc1382d11..610ff74848 100644 --- a/generated/elasticsearch/composable/component/process.json +++ b/generated/elasticsearch/composable/component/process.json @@ -24,6 +24,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -674,6 +677,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -824,6 +831,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -1055,6 +1065,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/composable/component/threat.json b/generated/elasticsearch/composable/component/threat.json index 17d9b1e77f..c9030c4167 100644 --- a/generated/elasticsearch/composable/component/threat.json +++ b/generated/elasticsearch/composable/component/threat.json @@ -66,6 +66,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -275,6 +278,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -987,6 +994,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -1196,6 +1206,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/legacy/template.json b/generated/elasticsearch/legacy/template.json index a6b67033e2..6725cae44d 100644 --- a/generated/elasticsearch/legacy/template.json +++ b/generated/elasticsearch/legacy/template.json @@ -740,6 +740,10 @@ "type": "keyword" } } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -754,6 +758,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -783,6 +790,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -1008,6 +1019,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -1318,6 +1333,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -1527,6 +1545,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -2703,6 +2725,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -3353,6 +3378,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -3503,6 +3532,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -3734,6 +3766,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -5241,6 +5277,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -5450,6 +5489,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" @@ -6162,6 +6205,9 @@ "exists": { "type": "boolean" }, + "flags": { + "type": "string" + }, "signing_id": { "ignore_above": 1024, "type": "keyword" @@ -6371,6 +6417,10 @@ }, "hash": { "properties": { + "cdhash": { + "ignore_above": 1024, + "type": "keyword" + }, "md5": { "ignore_above": 1024, "type": "keyword" diff --git a/rfcs/text/0044-add-apple-platform-specific-fields.md b/rfcs/text/0044-add-apple-platform-specific-fields.md index 68d0da3214..1c58a32b94 100644 --- a/rfcs/text/0044-add-apple-platform-specific-fields.md +++ b/rfcs/text/0044-add-apple-platform-specific-fields.md @@ -1,8 +1,8 @@ # 0044: Apple Platform specific fields -- Stage: **0 (strawperson)** -- Date: **2024-08-13** +- Stage: **2 (Candidate)** +- Date: **2024-09-11** ### Summary @@ -60,7 +60,11 @@ Stage 2: Included a real world example source document. Ideally this example com Stage 3: Add more real world example source documents so we have at least 2 total, but ideally 3. Format as described in stage 2. --> - +https://developer.apple.com/documentation/endpointsecurity/es_process_t/3228978-is_es_client + +https://developer.apple.com/documentation/endpointsecurity/es_process_t/3228979-is_platform_binary + +https://developer.apple.com/documentation/endpointsecurity/es_process_t/3684982-responsible_audit_token + +https://developer.apple.com/documentation/endpointsecurity/es_process_t/3334987-codesigning_flags + +https://developer.apple.com/documentation/endpointsecurity/es_process_t/3228976-cdhash + ### RFC Pull Requests * Stage 0: https://github.com/elastic/ecs/pull/2338 +* Stage 2: https://github.com/elastic/ecs/pull/2370 + +- Stage: **0 (strawperson)** +- Date: **TBD** + + + + +This RFC proposes to expand the vulnerability fieldset to include more fields, the proposal takes into consideration various customer feedbacks provided to Security integration team, inputs from Infosec team managing vulnerabilities across Elastic and other companies. This will benefit our customers and internal product teams to provide more effective vulnerability management experience to end user. to come up with the list of fields, extensive research was done across various Vulnerability management products and schemas like OSV. It is a continuation of one of the previous RFC on similar topic- https://github.com/elastic/ecs/issues/1685 + + + + + +## Fields +The `vulnerabilities` fields being proposed are as follows: + +| Field | Type | Description / Use Case | +| ----- | ---- | ---------------------- | +| `vulnerability.vendor.id` | keyword | A vulnerability doesn't have necessary a CVE associated with it. It makes sense to seperate vulnerability ID (like CVEs) to the vendor/detection IDs. | +| `vulnerability.title` | keyword | Title/Name/Short Description for vulnerability, to be used in flyout and dashboards. | +| `vulnerability.mitigation` | text | Explains user how to fix or mitigate the problem, could be usefd to store resolution from the scanner vendor or document mitigation in place | +| `vulnerability.published` | date | The “published” field indicates the date when information about a specific vulnerability was publicly disclosed or made available.It represents the moment when details about the vulnerability were shared with the security community, vendors, and the public.This field helps security professionals track the timeline of vulnerability awareness, in ISO 8601 format - YYYY-MM-DD | +| `vulnerability.patch.*` | object | - | +| `vulnerability.patch.exists` | boolean | The “patch” field refers to whether a security fix or update (commonly known as a patch) is available to address the identified vulnerability. It indicates whether the software vendor or developer has released a solution to mitigate the vulnerability. | +| `vulnerability.patch.name` | text | Name of the patch | +| `vulnerability.patch.code` | keyword | Associated patch code for example ESA-2020-13 | +| `vulnerability.evidence` | text | A demonstration of the validity of a vulnerability claim, e.g. app.any.run replaying the exploitation of the vulnerability. | +| `vulnerability.status` | keyword | The status field helps security teams track vulnerabilities, prioritize actions, and communicate their progress effectively. Examples- open/ignored/patched/mitigated/false_positive/risk_accepted/reopened..| +| `vulnerability.tags` | keyword | This is different from cloud provider assigned resource tags, this is specifically for vulnerability. Vulnerability tags serve as a way to add custom metadata to vulnerabilities, enhancing their context and aiding in search and automation. | +| `vulnerability.first_found` | date | First time a vulnerability was found on the asset, in ISO 8601 format: 2016-05-23T08:05:34.853Z | +| `vulnerability.last_found` | date | Last time a vulnerability was found on the asset, in ISO 8601 format: 2016-05-23T08:05:34.853Z | +| `vulnerability.last_scanned` | date | Last time a scan was performed on the asset. It's important as some companies are scanning on a quarterly basis. If last_found and last_scanned are close, it means it's still an active vulnerability, in ISO 8601 format: 2016-05-23T08:05:34.853Z| +| `vulnerability.age` | long | Numbers of days since the vulnerability is active. It should be dynamically calculated (runtime fields, ingest, ...). It could either be then difference between the last_found date and the published date (preferred). It could also be the difference between the first_found and last_found dates. | +| `vulnerability.uid` | keyword | It's extremely important to be able to deduplicate different scans. It's often that we have different scanners showing the same vulnerability on the same asset. | +| `vulnerability.type` | keyword | To conclude if the vulnerability is confirmed or potential. | +| `vulnerability.exploitability.*` | object | - | +| `vulnerability.exploitability.exploited` | boolean | To indicate if the vulnerability has been exploited or not. | +| `vulnerability.exploitability.reference` | keyword | Exploitability databse for example CSA-KEV. | +| `vulnerability.exploitability.confidence` | keyword | Confidence measure the credibility of existence and exploitability. | +| `vulnerability.exploitability.first_seen` | date | First time of exploitability, in ISO 8601 format: 2016-05-23T08:05:34.853Z | +| `vulnerability.exploitability.last_seen` | date | Last time of exploitability, in ISO 8601 format: 2016-05-23T08:05:34.853Z | +| `vulnerability.affected.*` | object | The affected field is a JSON array containing objects that describes the affected package versions, meaning those that contain the vulnerability. | +| `vulnerability.affected.package` | array | Package field is a JSON object identifying the affected code library or command provided by the package. | +| `vulnerability.affected.severity` | array | This field applies to a specific package, in cases where affected packages have differing severities for the same vulnerability. | +| `vulnerability.affected.versions` | array | Affected version in whatever version syntax is used by the given package ecosystem. | + + + + + + +## Usage + + + +## Source data + + + + + + + +## Scope of impact + + + +## Concerns + + + + + + + +## People + +The following are the people that consulted on the contents of this RFC. + +* @smriti0321 | author +* @tinnytintin10 | Product Manager Cloud Security +* @oren-zohar | Engineering Manager Cloud Security +* @orouz | Engineer +* @clement-fouque | Information Security Analyst + + + +## References + + +previous RFC - https://github.com/elastic/ecs/issues/1685 +https://ossf.github.io/osv-schema/#affected-fields + +### RFC Pull Requests + + + +* Stage 0: https://github.com/elastic/ecs/pull/2331 + + From 220ecee12b6829515d2935e331d08c0f1579d012 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 23 Sep 2024 11:19:19 -0700 Subject: [PATCH 26/46] Fix type in code signature (#2382) Change the type of code_signature.flags to keyword, which is what it should be. Also add a unit test that will verify all types are valid. --- docs/fields/field-details.asciidoc | 2 +- experimental/generated/beats/fields.ecs.yml | 18 ++++++---- experimental/generated/csv/fields.csv | 12 +++---- experimental/generated/ecs/ecs_flat.yml | 18 ++++++---- experimental/generated/ecs/ecs_nested.yml | 21 +++++++---- .../composable/component/dll.json | 3 +- .../composable/component/file.json | 3 +- .../composable/component/process.json | 6 ++-- .../composable/component/threat.json | 6 ++-- .../elasticsearch/legacy/template.json | 18 ++++++---- generated/beats/fields.ecs.yml | 18 ++++++---- generated/csv/fields.csv | 12 +++---- generated/ecs/ecs_flat.yml | 18 ++++++---- generated/ecs/ecs_nested.yml | 21 +++++++---- .../composable/component/dll.json | 3 +- .../composable/component/file.json | 3 +- .../composable/component/process.json | 6 ++-- .../composable/component/threat.json | 6 ++-- generated/elasticsearch/legacy/template.json | 18 ++++++---- rfcs/text/0044/code_signature.yml | 4 +-- schemas/code_signature.yml | 2 +- scripts/tests/test_ecs_spec.py | 35 +++++++++++++++++++ 22 files changed, 175 insertions(+), 78 deletions(-) diff --git a/docs/fields/field-details.asciidoc b/docs/fields/field-details.asciidoc index f2259fb87e..23ae02e99a 100644 --- a/docs/fields/field-details.asciidoc +++ b/docs/fields/field-details.asciidoc @@ -873,7 +873,7 @@ a| beta:[ This field is beta and subject to change. ] The flags used to sign the process. -type: string +type: keyword diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 625206235f..ee0ecb5e3b 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1273,7 +1273,8 @@ default_field: false - name: code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -2439,7 +2440,8 @@ default_field: false - name: code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -4793,7 +4795,8 @@ default_field: false - name: code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -6117,7 +6120,8 @@ default_field: false - name: parent.code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -9177,7 +9181,8 @@ default_field: false - name: enrichments.indicator.file.code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -10798,7 +10803,8 @@ default_field: false - name: indicator.file.code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 85f24dce13..be5ee33461 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -149,7 +149,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,device,device.serial_number,keyword,core,,DJGAQS4CW5,Serial Number of the device 8.12.0-dev+exp,true,dll,dll.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,dll,dll.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev+exp,true,dll,dll.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev+exp,true,dll,dll.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,dll,dll.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,dll,dll.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,dll,dll.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -280,7 +280,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,file,file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev+exp,true,file,file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,file,file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev+exp,true,file,file.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev+exp,true,file,file.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,file,file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,file,file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,file,file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -593,7 +593,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,process,process.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev+exp,true,process,process.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev+exp,true,process,process.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,process,process.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,process,process.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,process,process.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -775,7 +775,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev+exp,true,process,process.parent.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,process,process.parent.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev+exp,true,process,process.parent.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev+exp,true,process,process.parent.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,process,process.parent.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,process,process.parent.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,process,process.parent.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1162,7 +1162,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,threat,threat.enrichments.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1381,7 +1381,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev+exp,true,threat,threat.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev+exp,true,threat,threat.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 6e09b7f52f..e529df5f93 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -1806,12 +1806,13 @@ dll.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: dll.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -3957,12 +3958,13 @@ file.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -7787,12 +7789,13 @@ process.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: process.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -9956,12 +9959,13 @@ process.parent.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: process.parent.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -14782,12 +14786,13 @@ threat.enrichments.indicator.file.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: threat.enrichments.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17518,12 +17523,13 @@ threat.indicator.file.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: threat.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 1f7f9648b7..f4a2844515 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1326,11 +1326,12 @@ code_signature: description: The flags used to sign the process. example: 570522385 flat_name: code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] short: Code signing flags of the process - type: string + type: keyword code_signature.signing_id: dashed_name: code-signature-signing-id description: 'The identifier used to sign the process. @@ -2290,12 +2291,13 @@ dll: description: The flags used to sign the process. example: 570522385 flat_name: dll.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -5001,12 +5003,13 @@ file: description: The flags used to sign the process. example: 570522385 flat_name: file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -10020,12 +10023,13 @@ process: description: The flags used to sign the process. example: 570522385 flat_name: process.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -12194,12 +12198,13 @@ process: description: The flags used to sign the process. example: 570522385 flat_name: process.parent.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17482,12 +17487,13 @@ threat: description: The flags used to sign the process. example: 570522385 flat_name: threat.enrichments.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -20224,12 +20230,13 @@ threat: description: The flags used to sign the process. example: 570522385 flat_name: threat.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. diff --git a/experimental/generated/elasticsearch/composable/component/dll.json b/experimental/generated/elasticsearch/composable/component/dll.json index 55e2246263..e59687764d 100644 --- a/experimental/generated/elasticsearch/composable/component/dll.json +++ b/experimental/generated/elasticsearch/composable/component/dll.json @@ -18,7 +18,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/experimental/generated/elasticsearch/composable/component/file.json b/experimental/generated/elasticsearch/composable/component/file.json index adb9d1d8ec..175a0cbab7 100644 --- a/experimental/generated/elasticsearch/composable/component/file.json +++ b/experimental/generated/elasticsearch/composable/component/file.json @@ -25,7 +25,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/experimental/generated/elasticsearch/composable/component/process.json b/experimental/generated/elasticsearch/composable/component/process.json index de0be5f249..76b8983a3b 100644 --- a/experimental/generated/elasticsearch/composable/component/process.json +++ b/experimental/generated/elasticsearch/composable/component/process.json @@ -25,7 +25,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -832,7 +833,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/experimental/generated/elasticsearch/composable/component/threat.json b/experimental/generated/elasticsearch/composable/component/threat.json index cdcbbd7ae6..32056d1507 100644 --- a/experimental/generated/elasticsearch/composable/component/threat.json +++ b/experimental/generated/elasticsearch/composable/component/threat.json @@ -67,7 +67,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -995,7 +996,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/experimental/generated/elasticsearch/legacy/template.json b/experimental/generated/elasticsearch/legacy/template.json index 6b9172fe34..bc7f446065 100644 --- a/experimental/generated/elasticsearch/legacy/template.json +++ b/experimental/generated/elasticsearch/legacy/template.json @@ -801,7 +801,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -1376,7 +1377,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -2768,7 +2770,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -3575,7 +3578,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -5320,7 +5324,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -6248,7 +6253,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 77f9536d95..3883c5b045 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1223,7 +1223,8 @@ default_field: false - name: code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -2389,7 +2390,8 @@ default_field: false - name: code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -4743,7 +4745,8 @@ default_field: false - name: code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -6067,7 +6070,8 @@ default_field: false - name: parent.code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -9127,7 +9131,8 @@ default_field: false - name: enrichments.indicator.file.code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false @@ -10748,7 +10753,8 @@ default_field: false - name: indicator.file.code_signature.flags level: extended - type: string + type: keyword + ignore_above: 1024 description: The flags used to sign the process. example: 570522385 default_field: false diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index a7210ad73b..8af3fac81a 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -142,7 +142,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,device,device.serial_number,keyword,core,,DJGAQS4CW5,Serial Number of the device 8.12.0-dev,true,dll,dll.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,dll,dll.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev,true,dll,dll.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev,true,dll,dll.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,dll,dll.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,dll,dll.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,dll,dll.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -273,7 +273,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,file,file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev,true,file,file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,file,file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev,true,file,file.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev,true,file,file.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,file,file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,file,file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,file,file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -586,7 +586,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,process,process.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev,true,process,process.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev,true,process,process.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,process,process.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,process,process.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,process,process.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -768,7 +768,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. 8.12.0-dev,true,process,process.parent.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,process,process.parent.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev,true,process,process.parent.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev,true,process,process.parent.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,process,process.parent.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,process,process.parent.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,process,process.parent.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1155,7 +1155,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,threat,threat.enrichments.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,threat,threat.enrichments.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer @@ -1374,7 +1374,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 8.12.0-dev,true,threat,threat.indicator.file.attributes,keyword,extended,array,"[""readonly"", ""system""]",Array of file attributes. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.digest_algorithm,keyword,extended,,sha256,Hashing algorithm used to sign the process. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. -8.12.0-dev,true,threat,threat.indicator.file.code_signature.flags,string,extended,,570522385,Code signing flags of the process +8.12.0-dev,true,threat,threat.indicator.file.code_signature.flags,keyword,extended,,570522385,Code signing flags of the process 8.12.0-dev,true,threat,threat.indicator.file.code_signature.signing_id,keyword,extended,,com.apple.xpc.proxy,The identifier used to sign the process. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. 8.12.0-dev,true,threat,threat.indicator.file.code_signature.subject_name,keyword,core,,Microsoft Corporation,Subject name of the code signer diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 7e504589db..bad8611fa7 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1737,12 +1737,13 @@ dll.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: dll.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -3888,12 +3889,13 @@ file.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -7718,12 +7720,13 @@ process.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: process.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -9887,12 +9890,13 @@ process.parent.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: process.parent.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -14713,12 +14717,13 @@ threat.enrichments.indicator.file.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: threat.enrichments.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17449,12 +17454,13 @@ threat.indicator.file.code_signature.flags: description: The flags used to sign the process. example: 570522385 flat_name: threat.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index b08955b69b..a401fa7b0a 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -1246,11 +1246,12 @@ code_signature: description: The flags used to sign the process. example: 570522385 flat_name: code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] short: Code signing flags of the process - type: string + type: keyword code_signature.signing_id: dashed_name: code-signature-signing-id description: 'The identifier used to sign the process. @@ -2210,12 +2211,13 @@ dll: description: The flags used to sign the process. example: 570522385 flat_name: dll.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword dll.code_signature.signing_id: dashed_name: dll-code-signature-signing-id description: 'The identifier used to sign the process. @@ -4921,12 +4923,13 @@ file: description: The flags used to sign the process. example: 570522385 flat_name: file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword file.code_signature.signing_id: dashed_name: file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -9940,12 +9943,13 @@ process: description: The flags used to sign the process. example: 570522385 flat_name: process.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.code_signature.signing_id: dashed_name: process-code-signature-signing-id description: 'The identifier used to sign the process. @@ -12114,12 +12118,13 @@ process: description: The flags used to sign the process. example: 570522385 flat_name: process.parent.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword process.parent.code_signature.signing_id: dashed_name: process-parent-code-signature-signing-id description: 'The identifier used to sign the process. @@ -17402,12 +17407,13 @@ threat: description: The flags used to sign the process. example: 570522385 flat_name: threat.enrichments.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.enrichments.indicator.file.code_signature.signing_id: dashed_name: threat-enrichments-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. @@ -20144,12 +20150,13 @@ threat: description: The flags used to sign the process. example: 570522385 flat_name: threat.indicator.file.code_signature.flags + ignore_above: 1024 level: extended name: flags normalize: [] original_fieldset: code_signature short: Code signing flags of the process - type: string + type: keyword threat.indicator.file.code_signature.signing_id: dashed_name: threat-indicator-file-code-signature-signing-id description: 'The identifier used to sign the process. diff --git a/generated/elasticsearch/composable/component/dll.json b/generated/elasticsearch/composable/component/dll.json index b5f52995c5..7c76d1ed0d 100644 --- a/generated/elasticsearch/composable/component/dll.json +++ b/generated/elasticsearch/composable/component/dll.json @@ -18,7 +18,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/generated/elasticsearch/composable/component/file.json b/generated/elasticsearch/composable/component/file.json index cc12f10be1..c032c0a53c 100644 --- a/generated/elasticsearch/composable/component/file.json +++ b/generated/elasticsearch/composable/component/file.json @@ -25,7 +25,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/generated/elasticsearch/composable/component/process.json b/generated/elasticsearch/composable/component/process.json index 610ff74848..d48a4eddab 100644 --- a/generated/elasticsearch/composable/component/process.json +++ b/generated/elasticsearch/composable/component/process.json @@ -25,7 +25,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -832,7 +833,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/generated/elasticsearch/composable/component/threat.json b/generated/elasticsearch/composable/component/threat.json index c9030c4167..40f98ec195 100644 --- a/generated/elasticsearch/composable/component/threat.json +++ b/generated/elasticsearch/composable/component/threat.json @@ -67,7 +67,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -995,7 +996,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/generated/elasticsearch/legacy/template.json b/generated/elasticsearch/legacy/template.json index 6725cae44d..66b302cebd 100644 --- a/generated/elasticsearch/legacy/template.json +++ b/generated/elasticsearch/legacy/template.json @@ -759,7 +759,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -1334,7 +1335,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -2726,7 +2728,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -3533,7 +3536,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -5278,7 +5282,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, @@ -6206,7 +6211,8 @@ "type": "boolean" }, "flags": { - "type": "string" + "ignore_above": 1024, + "type": "keyword" }, "signing_id": { "ignore_above": 1024, diff --git a/rfcs/text/0044/code_signature.yml b/rfcs/text/0044/code_signature.yml index 091339048d..d16b64b6e4 100644 --- a/rfcs/text/0044/code_signature.yml +++ b/rfcs/text/0044/code_signature.yml @@ -3,8 +3,8 @@ fields: - name: flags level: extended - type: string + type: keyword short: Code signing flags of the process description: > The flags used to sign the process. - example: 570522385 \ No newline at end of file + example: 570522385 diff --git a/schemas/code_signature.yml b/schemas/code_signature.yml index 67e6a5da4a..e5808e6e3d 100644 --- a/schemas/code_signature.yml +++ b/schemas/code_signature.yml @@ -37,7 +37,7 @@ - name: flags level: extended - type: string + type: keyword short: Code signing flags of the process description: > The flags used to sign the process. diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index edb0b8420c..a55d7bbb68 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -130,6 +130,41 @@ def test_normalize_always_array(self): for (field_name, field) in self.ecs_fields.items(): self.assertIsInstance(field.get('normalize'), list, field_name) + def test_valid_type(self): + valid_types = ['binary', + 'boolean', + 'keyword', + 'constant_keyword', + 'wildcard', + 'long', + 'integer', + 'short', + 'byte', + 'double', + 'float', + 'half_float', + 'scaled_float', + 'unsigned_long', + 'date', + 'date_nanos', + 'alias', + 'object', + 'flattened', + 'nested', + 'join', + 'long_range', + 'double_range', + 'date_range', + 'ip', + 'text', + 'match_only_text', + 'geo_point', + 'geo_shape', + 'point', + 'shape'] + for (field_name, field) in self.ecs_fields.items(): + self.assertIn(field.get('type'), valid_types, field_name) + if __name__ == '__main__': unittest.main() From e78c4247394c8440dbdfa4224d68ca433742fcea Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Tue, 24 Sep 2024 01:22:33 -0700 Subject: [PATCH 27/46] Enforce yamllint in CI (#2381) Start running and enforcing yamllint checks in CI. --- .github/workflows/test.yml | 2 +- Makefile | 4 ++-- schemas/email.yml | 4 ++-- schemas/pe.yml | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b618b7394b..cc2d5276c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,4 +12,4 @@ jobs: with: python-version: '3.x' - run: git fetch --prune --unshallow --tags - - run: make check + - run: make check yamllint diff --git a/Makefile b/Makefile index d6e65d4a45..5aaeebcebd 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ misspell: fi ./build/misspell/bin/misspell -error README.md CONTRIBUTING.md schemas/* docs/* experimental/schemas/* -# Warn re misspell removal +# Warn re misspell removal .PHONY: misspell_warn misspell_warn: @echo "Warning: due to lack of cross-platform support, misspell is no longer included in this task and may be deprecated in future\n" @@ -110,4 +110,4 @@ build/ve/bin/activate: scripts/requirements.txt scripts/requirements-dev.txt # Check YAML syntax (currently not enforced). .PHONY: yamllint yamllint: ve - build/ve/bin/yamllint schemas/*.yml + build/ve/bin/yamllint -d '{extends: default, rules: {line-length: disable}}' schemas/*.yml diff --git a/schemas/email.yml b/schemas/email.yml index 82bfd5b219..9c8b6ff390 100644 --- a/schemas/email.yml +++ b/schemas/email.yml @@ -180,8 +180,8 @@ A brief summary of the topic of the message. example: "Please see this important message." multi_fields: - - type: match_only_text - name: text + - type: match_only_text + name: text - name: to.address level: extended diff --git a/schemas/pe.yml b/schemas/pe.yml index f2a5f1561e..d201544658 100644 --- a/schemas/pe.yml +++ b/schemas/pe.yml @@ -208,4 +208,3 @@ format: string type: long level: extended - From 68fd03838cc2e167d08cbc1c7f534d34a8df4246 Mon Sep 17 00:00:00 2001 From: Michal Stanek <75310947+stanek-michal@users.noreply.github.com> Date: Fri, 27 Sep 2024 20:11:23 +0200 Subject: [PATCH 28/46] Add Stage0 RFC for new fields for fileless execution on Linux (#2322) --- rfcs/text/0047-fileless-execution-linux.md | 132 +++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 rfcs/text/0047-fileless-execution-linux.md diff --git a/rfcs/text/0047-fileless-execution-linux.md b/rfcs/text/0047-fileless-execution-linux.md new file mode 100644 index 0000000000..8ab161511d --- /dev/null +++ b/rfcs/text/0047-fileless-execution-linux.md @@ -0,0 +1,132 @@ +# 0047: Fileless execution on Linux + + +- Stage: **0 (strawperson)** +- Date: **2024-09-26** + + + + + +This RFC proposes adding new fields and event types to enhance the detection of fileless malware execution and related malicious activities on Linux systems. + +The new fields include: + * file.is_memfd - Indicates if the file is an anonymous file descriptor (memfd) created using the memfd_create system call. + * file.is_shmem - Indicates if the file is a POSIX shared memory object created using the shm_open system call, typically located in /dev/shm. + * process.is_setuid - Indicates if the process has the setuid bit set, allowing it to run with the privileges of its owner. + * process.is_setgid - Indicates if the process has the setgid bit set, allowing it to run with the privileges of its group. + * process.is_memfd - Indicates if the process was executed from a memory file descriptor (memfd). + * process.inode_nlink - Number of links to the inode of the process executable file, obtained from the i_nlink variable in the inode structure. + +New process event types: + * memfd_create + * shmget (SystemV shared memory API) + * ptrace + * load_module + +New file event types: + * memfd_open + * shmem_open + +These additions will enable the detection and investigation of various malware execution techniques, such as executing code from memory file descriptors (memfd), hiding malicious binaries in shared memory objects (shm_open and shmget), debugging other processes for code injection (ptrace), and loading kernel modules for rootkits (load_module). The proposed fields also cover privilege escalation using setuid/setgid binaries. + + + + + +## Fields + + + + + +## Usage + + + +## Source data + +The data can be collected by monitoring system calls and events on Linux hosts using kernel instrumentation techniques like eBPF or kprobes. + + + + + + + +## Scope of impact + + + +## Concerns + + + + + + + +## People + +The following are the people that consulted on the contents of this RFC. + + * @stanek-michal | author + + + + +## References + + + +### RFC Pull Requests + + + + From 2a8f82e558fe89e900ac6c2b9b8373a9948e3a0a Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:55:20 -0400 Subject: [PATCH 29/46] Add support for settings --- scripts/_types/schema_fields.py | 4 + scripts/generators/es_template.py | 247 ++++++++++++++++-------------- scripts/schema/loader.py | 180 ++++++++++++---------- 3 files changed, 237 insertions(+), 194 deletions(-) diff --git a/scripts/_types/schema_fields.py b/scripts/_types/schema_fields.py index 9dd849533e..598ab4b08b 100644 --- a/scripts/_types/schema_fields.py +++ b/scripts/_types/schema_fields.py @@ -53,6 +53,7 @@ class Field(TypedDict, total=False): name: str node_name: str normalize: List[str] + # normalizer: str norms: bool required: bool short: str @@ -75,6 +76,8 @@ class SchemaDetails(TypedDict, total=False): prefix: str reusable: Reuseable root: bool + # root_type: str + settings: Dict[str, str] title: str @@ -92,6 +95,7 @@ class FieldNestedEntry(TypedDict, total=False): name: str node_name: str prefix: str + settings: Dict[str, str] short: str title: str type: str diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index fa9fdda9c0..13570346fc 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -17,12 +17,7 @@ import json import sys -from typing import ( - Dict, - List, - Optional, - Union -) +from typing import Dict, List, Optional, Union from os.path import join @@ -40,38 +35,59 @@ def generate( ecs_version: str, out_dir: str, mapping_settings_file: str, - template_settings_file: str + template_settings_file: str, ) -> None: """This generates all artifacts for the composable template approach""" all_component_templates(ecs_nested, ecs_version, out_dir) component_names = component_name_convention(ecs_version, ecs_nested) - save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file) - - -def save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file): + save_composable_template( + ecs_version, + component_names, + out_dir, + mapping_settings_file, + template_settings_file, + ) + + +def save_composable_template( + ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file +): mappings_section = mapping_settings(mapping_settings_file) - template = template_settings(ecs_version, mappings_section, template_settings_file, component_names=component_names) + template = template_settings( + ecs_version, + mappings_section, + template_settings_file, + component_names=component_names, + ) filename = join(out_dir, "elasticsearch/composable/template.json") save_json(filename, template) def all_component_templates( - ecs_nested: Dict[str, FieldNestedEntry], - ecs_version: str, - out_dir: str + ecs_nested: Dict[str, FieldNestedEntry], ecs_version: str, out_dir: str ) -> None: """Generate one component template per field set""" - component_dir: str = join(out_dir, 'elasticsearch/composable/component') + component_dir: str = join(out_dir, "elasticsearch/composable/component") ecs_helpers.make_dirs(component_dir) - for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items(): + for fieldset_name, fieldset in ecs_helpers.remove_top_level_reusable_false( + ecs_nested + ).items(): field_mappings = {} - for (flat_name, field) in fieldset['fields'].items(): - name_parts = flat_name.split('.') + for flat_name, field in fieldset["fields"].items(): + name_parts = flat_name.split(".") dict_add_nested(field_mappings, name_parts, entry_for(field)) + fieldset_settings = fieldset.get("settings", None) - save_component_template(fieldset_name, field['level'], ecs_version, component_dir, field_mappings) + save_component_template( + fieldset_name, + field["level"], + ecs_version, + component_dir, + field_mappings, + fieldset_settings, + ) def save_component_template( @@ -79,32 +95,38 @@ def save_component_template( field_level: str, ecs_version: str, out_dir: str, - field_mappings: Dict + field_mappings: Dict, + fieldset_settings: [Dict, None], ) -> None: filename: str = join(out_dir, template_name) + ".json" - reference_url: str = "https://www.elastic.co/guide/en/ecs/current/ecs-{}.html".format(template_name) + reference_url: str = ( + "https://www.elastic.co/guide/en/ecs/current/ecs-{}.html".format(template_name) + ) template: Dict = { - 'template': {'mappings': {'properties': field_mappings}}, - '_meta': { - 'ecs_version': ecs_version, - } + "template": {"mappings": {"properties": field_mappings}}, + "_meta": { + "ecs_version": ecs_version, + }, } + if fieldset_settings is not None: + template["template"]["settings"] = fieldset_settings """Only generate a documentation link for ECS fields""" - if (field_level != 'custom'): - template['_meta']['documentation'] = reference_url + if field_level != "custom": + template["_meta"]["documentation"] = reference_url save_json(filename, template) def component_name_convention( - ecs_version: str, - ecs_nested: Dict[str, FieldNestedEntry] + ecs_version: str, ecs_nested: Dict[str, FieldNestedEntry] ) -> List[str]: - version: str = ecs_version.replace('+', '-') + version: str = ecs_version.replace("+", "-") names: List[str] = [] - for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items(): + for fieldset_name, fieldset in ecs_helpers.remove_top_level_reusable_false( + ecs_nested + ).items(): names.append("ecs_{}_{}".format(version, fieldset_name.lower())) return names @@ -117,29 +139,30 @@ def generate_legacy( ecs_version: str, out_dir: str, mapping_settings_file: str, - template_settings_file: str + template_settings_file: str, ) -> None: """Generate the legacy index template""" field_mappings = {} for flat_name in sorted(ecs_flat): field = ecs_flat[flat_name] - name_parts = flat_name.split('.') + name_parts = flat_name.split(".") dict_add_nested(field_mappings, name_parts, entry_for(field)) mappings_section: Dict = mapping_settings(mapping_settings_file) - mappings_section['properties'] = field_mappings + mappings_section["properties"] = field_mappings - generate_legacy_template_version(ecs_version, mappings_section, out_dir, template_settings_file) + generate_legacy_template_version( + ecs_version, mappings_section, out_dir, template_settings_file + ) def generate_legacy_template_version( - ecs_version: str, - mappings_section: Dict, - out_dir: str, - template_settings_file: str + ecs_version: str, mappings_section: Dict, out_dir: str, template_settings_file: str ) -> None: - ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', "legacy")) - template: Dict = template_settings(ecs_version, mappings_section, template_settings_file, is_legacy=True) + ecs_helpers.make_dirs(join(out_dir, "elasticsearch", "legacy")) + template: Dict = template_settings( + ecs_version, mappings_section, template_settings_file, is_legacy=True + ) filename: str = join(out_dir, "elasticsearch/legacy/template.json") save_json(filename, template) @@ -148,64 +171,63 @@ def generate_legacy_template_version( # Common helpers -def dict_add_nested( - dct: Dict, - name_parts: List[str], - value: Dict -) -> None: +def dict_add_nested(dct: Dict, name_parts: List[str], value: Dict) -> None: current_nesting: str = name_parts[0] rest_name_parts: List[str] = name_parts[1:] if len(rest_name_parts) > 0: dct.setdefault(current_nesting, {}) - dct[current_nesting].setdefault('properties', {}) + dct[current_nesting].setdefault("properties", {}) - dict_add_nested( - dct[current_nesting]['properties'], - rest_name_parts, - value) + dict_add_nested(dct[current_nesting]["properties"], rest_name_parts, value) else: - if current_nesting in dct and 'type' in value and 'object' == value['type']: + if current_nesting in dct and "type" in value and "object" == value["type"]: return dct[current_nesting] = value def entry_for(field: Field) -> Dict: - field_entry: Dict = {'type': field['type']} + field_entry: Dict = {"type": field["type"]} try: - if field['type'] == 'object' or field['type'] == 'nested': - if 'enabled' in field and not field['enabled']: - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['enabled']) + if field["type"] == "object" or field["type"] == "nested": + if "enabled" in field and not field["enabled"]: + ecs_helpers.dict_copy_existing_keys(field, field_entry, ["enabled"]) # the index field is only valid for field types that are not object and nested - elif 'index' in field and not field['index']: - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['index', 'doc_values']) - - if field['type'] == 'keyword' or field['type'] == 'flattened': - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['ignore_above']) - elif field['type'] == 'constant_keyword': - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['value']) - elif field['type'] == 'text': - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['norms']) - elif field['type'] == 'alias': - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['path']) - elif field['type'] == 'scaled_float': - ecs_helpers.dict_copy_existing_keys(field, field_entry, ['scaling_factor']) - - if 'multi_fields' in field: - field_entry['fields'] = {} - for mf in field['multi_fields']: - mf_type = mf['type'] - mf_entry = {'type': mf_type} - if mf_type == 'keyword': - ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['normalizer', 'ignore_above']) - elif mf_type == 'text': - ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['norms', 'analyzer']) - if 'parameters' in mf: - mf_entry.update(mf['parameters']) - field_entry['fields'][mf['name']] = mf_entry - - if 'parameters' in field: - field_entry.update(field['parameters']) + elif "index" in field and not field["index"]: + ecs_helpers.dict_copy_existing_keys( + field, field_entry, ["index", "doc_values"] + ) + + if field["type"] == "keyword" or field["type"] == "flattened": + ecs_helpers.dict_copy_existing_keys(field, field_entry, ["ignore_above"]) + elif field["type"] == "constant_keyword": + ecs_helpers.dict_copy_existing_keys(field, field_entry, ["value"]) + elif field["type"] == "text": + ecs_helpers.dict_copy_existing_keys(field, field_entry, ["norms"]) + elif field["type"] == "alias": + ecs_helpers.dict_copy_existing_keys(field, field_entry, ["path"]) + elif field["type"] == "scaled_float": + ecs_helpers.dict_copy_existing_keys(field, field_entry, ["scaling_factor"]) + + if "multi_fields" in field: + field_entry["fields"] = {} + for mf in field["multi_fields"]: + mf_type = mf["type"] + mf_entry = {"type": mf_type} + if mf_type == "keyword": + ecs_helpers.dict_copy_existing_keys( + mf, mf_entry, ["normalizer", "ignore_above"] + ) + elif mf_type == "text": + ecs_helpers.dict_copy_existing_keys( + mf, mf_entry, ["norms", "analyzer"] + ) + if "parameters" in mf: + mf_entry.update(mf["parameters"]) + field_entry["fields"][mf["name"]] = mf_entry + + if "parameters" in field: + field_entry.update(field["parameters"]) except KeyError as ex: print("Exception {} occurred for field {}".format(ex, field)) @@ -227,7 +249,7 @@ def template_settings( mappings_section: Dict, template_settings_file: Union[str, None], is_legacy: Optional[bool] = False, - component_names: Optional[List[str]] = None + component_names: Optional[List[str]] = None, ) -> Dict: if template_settings_file: with open(template_settings_file) as f: @@ -238,7 +260,9 @@ def template_settings( else: template = default_template_settings(ecs_version) - finalize_template(template, ecs_version, is_legacy, mappings_section, component_names) + finalize_template( + template, ecs_version, is_legacy, mappings_section, component_names + ) return template @@ -248,23 +272,23 @@ def finalize_template( ecs_version: str, is_legacy: bool, mappings_section: Dict, - component_names: List[str] + component_names: List[str], ) -> None: if is_legacy: if mappings_section: - template['mappings'] = mappings_section + template["mappings"] = mappings_section # _meta can't be at template root in legacy templates, so moving back to mappings section # if present - if '_meta' in template: - mappings_section['_meta'] = template.pop('_meta') + if "_meta" in template: + mappings_section["_meta"] = template.pop("_meta") else: - template['template']['mappings'] = mappings_section - template['composed_of'] = component_names - template['_meta'] = { + template["template"]["mappings"] = mappings_section + template["composed_of"] = component_names + template["_meta"] = { "ecs_version": ecs_version, - "description": "Sample composable template that includes all ECS fields" + "description": "Sample composable template that includes all ECS fields", } @@ -274,7 +298,7 @@ def save_json(file: str, data: Dict) -> None: open_mode = "w" with open(file, open_mode) as jsonfile: json.dump(data, jsonfile, indent=2, sort_keys=True) - jsonfile.write('\n') + jsonfile.write("\n") def default_template_settings(ecs_version: str) -> Dict: @@ -282,21 +306,17 @@ def default_template_settings(ecs_version: str) -> Dict: "index_patterns": ["try-ecs-*"], "_meta": { "ecs_version": ecs_version, - "description": "Sample composable template that includes all ECS fields" + "description": "Sample composable template that includes all ECS fields", }, "priority": 1, # Very low, as this is a sample template "template": { "settings": { "index": { "codec": "best_compression", - "mapping": { - "total_fields": { - "limit": 2000 - } - } + "mapping": {"total_fields": {"limit": 2000}}, } }, - } + }, } @@ -307,14 +327,10 @@ def default_legacy_template_settings(ecs_version: str) -> Dict: "order": 1, "settings": { "index": { - "mapping": { - "total_fields": { - "limit": 10000 - } - }, - "refresh_interval": "5s" + "mapping": {"total_fields": {"limit": 10000}}, + "refresh_interval": "5s", } - } + }, } @@ -324,12 +340,9 @@ def default_mapping_settings() -> Dict: "dynamic_templates": [ { "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" + "mapping": {"ignore_above": 1024, "type": "keyword"}, + "match_mapping_type": "string", } } - ] + ], } diff --git a/scripts/schema/loader.py b/scripts/schema/loader.py index ef73805e5e..8fb028c4af 100644 --- a/scripts/schema/loader.py +++ b/scripts/schema/loader.py @@ -72,32 +72,39 @@ # Examples of this are 'dns.answers', 'observer.egress'. -EXPERIMENTAL_SCHEMA_DIR = 'experimental/schemas' +EXPERIMENTAL_SCHEMA_DIR = "experimental/schemas" def load_schemas( - ref: Optional[str] = None, - included_files: Optional[List[str]] = [] + ref: Optional[str] = None, included_files: Optional[List[str]] = [] ) -> Dict[str, FieldEntry]: """Loads ECS and custom schemas. They are returned deeply nested and merged.""" # ECS fields (from git ref or not) - schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git( - ref) if ref else load_schema_files(ecs_helpers.ecs_files()) + schema_files_raw: Dict[str, FieldNestedEntry] = ( + load_schemas_from_git(ref) + if ref + else load_schema_files(ecs_helpers.ecs_files()) + ) fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw) # Custom additional files if included_files and len(included_files) > 0: - print('Loading user defined schemas: {0}'.format(included_files)) + print("Loading user defined schemas: {0}".format(included_files)) # If --ref provided and --include loading experimental schemas if ref and EXPERIMENTAL_SCHEMA_DIR in included_files: exp_schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git( - ref, target_dir=EXPERIMENTAL_SCHEMA_DIR) - exp_fields: Dict[str, FieldEntry] = deep_nesting_representation(exp_schema_files_raw) + ref, target_dir=EXPERIMENTAL_SCHEMA_DIR + ) + exp_fields: Dict[str, FieldEntry] = deep_nesting_representation( + exp_schema_files_raw + ) fields = merge_fields(fields, exp_fields) included_files.remove(EXPERIMENTAL_SCHEMA_DIR) # Remaining additional custom files (never from git ref) custom_files: List[str] = ecs_helpers.glob_yaml_files(included_files) - custom_fields: Dict[str, FieldEntry] = deep_nesting_representation(load_schema_files(custom_files)) + custom_fields: Dict[str, FieldEntry] = deep_nesting_representation( + load_schema_files(custom_files) + ) fields = merge_fields(fields, custom_fields) return fields @@ -111,8 +118,7 @@ def load_schema_files(files: List[str]) -> Dict[str, FieldNestedEntry]: def load_schemas_from_git( - ref: str, - target_dir: Optional[str] = 'schemas' + ref: str, target_dir: Optional[str] = "schemas" ) -> Dict[str, FieldNestedEntry]: tree: git.objects.tree.Tree = ecs_helpers.get_tree_by_ref(ref) fields_nested: Dict[str, FieldNestedEntry] = {} @@ -120,11 +126,13 @@ def load_schemas_from_git( # Handles case if target dir doesn't exists in git ref if ecs_helpers.path_exists_in_git_tree(tree, target_dir): for blob in tree[target_dir].blobs: - if blob.name.endswith('.yml'): + if blob.name.endswith(".yml"): new_fields: Dict[str, FieldNestedEntry] = read_schema_blob(blob, ref) fields_nested = ecs_helpers.safe_merge_dicts(fields_nested, new_fields) else: - raise KeyError(f"Target directory './{target_dir}' not present in git ref '{ref}'!") + raise KeyError( + f"Target directory './{target_dir}' not present in git ref '{ref}'!" + ) return fields_nested @@ -136,17 +144,18 @@ def read_schema_file(file_name: str) -> Dict[str, FieldNestedEntry]: def read_schema_blob( - blob: git.objects.blob.Blob, - ref: str + blob: git.objects.blob.Blob, ref: str ) -> Dict[str, FieldNestedEntry]: """Read a raw schema yml git blob into a dict.""" - content: str = blob.data_stream.read().decode('utf-8') + content: str = blob.data_stream.read().decode("utf-8") raw: List[FieldNestedEntry] = yaml.safe_load(content) file_name: str = "{} (git ref {})".format(blob.name, ref) return nest_schema(raw, file_name) -def nest_schema(raw: List[FieldNestedEntry], file_name: str) -> Dict[str, FieldNestedEntry]: +def nest_schema( + raw: List[FieldNestedEntry], file_name: str +) -> Dict[str, FieldNestedEntry]: """ Raw schema files are an array of schema details: [{'name': 'base', ...}] @@ -155,80 +164,86 @@ def nest_schema(raw: List[FieldNestedEntry], file_name: str) -> Dict[str, FieldN """ fields: Dict[str, FieldNestedEntry] = {} for schema in raw: - if 'name' not in schema: - raise ValueError("Schema file {} is missing mandatory attribute 'name'".format(file_name)) - fields[schema['name']] = schema + if "name" not in schema: + raise ValueError( + "Schema file {} is missing mandatory attribute 'name'".format(file_name) + ) + fields[schema["name"]] = schema return fields -def deep_nesting_representation(fields: Dict[str, FieldNestedEntry]) -> Dict[str, FieldEntry]: +def deep_nesting_representation( + fields: Dict[str, FieldNestedEntry], +) -> Dict[str, FieldEntry]: deeply_nested: Dict[str, FieldEntry] = {} - for (name, flat_schema) in fields.items(): - + for name, flat_schema in fields.items(): # We destructively select what goes into schema_details and child fields. # The rest is 'field_details'. flat_schema = flat_schema.copy() - flat_schema['node_name'] = flat_schema['name'] + flat_schema["node_name"] = flat_schema["name"] # Schema-only details. Not present on other nested field groups. schema_details: SchemaDetails = {} - for schema_key in ['root', 'group', 'reusable', 'title']: + for schema_key in ["root", "group", "reusable", "title", "settings"]: if schema_key in flat_schema: schema_details[schema_key] = flat_schema.pop(schema_key) - nested_schema = nest_fields(flat_schema.pop('fields', [])) + nested_schema = nest_fields(flat_schema.pop("fields", [])) # Re-assemble new structure deeply_nested[name] = { - 'schema_details': schema_details, + "schema_details": schema_details, # What's still in flat_schema is the field_details for the field set itself - 'field_details': flat_schema, - 'fields': nested_schema['fields'] + "field_details": flat_schema, + "fields": nested_schema["fields"], } return deeply_nested def nest_fields(field_array: List[Field]) -> Dict[str, Dict[str, FieldEntry]]: - schema_root: Dict[str, Dict[str, FieldEntry]] = {'fields': {}} + schema_root: Dict[str, Dict[str, FieldEntry]] = {"fields": {}} for field in field_array: - nested_levels: List[str] = field['name'].split('.') + nested_levels: List[str] = field["name"].split(".") parent_fields: List[str] = nested_levels[:-1] leaf_field: str = nested_levels[-1] # "nested_schema" is a cursor we move within the schema_root structure we're building. # Here we reset the cursor for this new field. - nested_schema = schema_root['fields'] + nested_schema = schema_root["fields"] current_path = [] for idx, level in enumerate(parent_fields): nested_schema.setdefault(level, {}) # Where nested fields will live - nested_schema[level].setdefault('fields', {}) + nested_schema[level].setdefault("fields", {}) # Make type:object explicit for intermediate parent fields - nested_schema[level].setdefault('field_details', {}) - field_details = nested_schema[level]['field_details'] - field_details['node_name'] = level + nested_schema[level].setdefault("field_details", {}) + field_details = nested_schema[level]["field_details"] + field_details["node_name"] = level # Respect explicitly defined object fields - if 'type' in field_details and field_details['type'] in ['object', 'nested']: - field_details.setdefault('intermediate', False) + if "type" in field_details and field_details["type"] in [ + "object", + "nested", + ]: + field_details.setdefault("intermediate", False) else: - field_details.setdefault('type', 'object') - field_details.setdefault('name', '.'.join(parent_fields[:idx + 1])) - field_details.setdefault('intermediate', True) + field_details.setdefault("type", "object") + field_details.setdefault("name", ".".join(parent_fields[: idx + 1])) + field_details.setdefault("intermediate", True) # moving the nested_schema cursor deeper current_path.extend([level]) - nested_schema = nested_schema[level]['fields'] + nested_schema = nested_schema[level]["fields"] nested_schema.setdefault(leaf_field, {}) # Overwrite 'name' with the leaf field's name. The flat_name is already computed. - field['node_name'] = leaf_field - nested_schema[leaf_field]['field_details'] = field + field["node_name"] = leaf_field + nested_schema[leaf_field]["field_details"] = field return schema_root def array_of_maps_to_map(array_vals: List[MultiField]) -> Dict[str, MultiField]: ret_map: Dict[str, MultiField] = {} for map_val in array_vals: - name: str = map_val['name'] + name: str = map_val["name"] # if multiple name fields exist in the same custom definition this will take the last one ret_map[name] = map_val return ret_map @@ -238,16 +253,20 @@ def map_of_maps_to_array(map_vals: Dict[str, MultiField]) -> List[MultiField]: ret_list: List[MultiField] = [] for key in map_vals: ret_list.append(map_vals[key]) - return sorted(ret_list, key=lambda k: k['name']) + return sorted(ret_list, key=lambda k: k["name"]) -def dedup_and_merge_lists(list_a: List[MultiField], list_b: List[MultiField]) -> List[MultiField]: +def dedup_and_merge_lists( + list_a: List[MultiField], list_b: List[MultiField] +) -> List[MultiField]: list_a_map: Dict[str, MultiField] = array_of_maps_to_map(list_a) list_a_map.update(array_of_maps_to_map(list_b)) return map_of_maps_to_array(list_a_map) -def merge_fields(a: Dict[str, FieldEntry], b: Dict[str, FieldEntry]) -> Dict[str, FieldEntry]: +def merge_fields( + a: Dict[str, FieldEntry], b: Dict[str, FieldEntry] +) -> Dict[str, FieldEntry]: """Merge ECS field sets with custom field sets.""" a = copy.deepcopy(a) b = copy.deepcopy(b) @@ -256,39 +275,46 @@ def merge_fields(a: Dict[str, FieldEntry], b: Dict[str, FieldEntry]) -> Dict[str a[key] = b[key] continue # merge field details - if 'normalize' in b[key]['field_details']: - a[key].setdefault('field_details', {}) - a[key]['field_details'].setdefault('normalize', []) - a[key]['field_details']['normalize'].extend(b[key]['field_details'].pop('normalize')) - if 'multi_fields' in b[key]['field_details']: - a[key].setdefault('field_details', {}) - a[key]['field_details'].setdefault('multi_fields', []) - a[key]['field_details']['multi_fields'] = dedup_and_merge_lists( - a[key]['field_details']['multi_fields'], b[key]['field_details']['multi_fields']) + if "normalize" in b[key]["field_details"]: + a[key].setdefault("field_details", {}) + a[key]["field_details"].setdefault("normalize", []) + a[key]["field_details"]["normalize"].extend( + b[key]["field_details"].pop("normalize") + ) + if "multi_fields" in b[key]["field_details"]: + a[key].setdefault("field_details", {}) + a[key]["field_details"].setdefault("multi_fields", []) + a[key]["field_details"]["multi_fields"] = dedup_and_merge_lists( + a[key]["field_details"]["multi_fields"], + b[key]["field_details"]["multi_fields"], + ) # if we don't do this then the update call below will overwrite a's field_details, with the original # contents of b, which undoes our merging the multi_fields - del b[key]['field_details']['multi_fields'] - a[key]['field_details'].update(b[key]['field_details']) + del b[key]["field_details"]["multi_fields"] + a[key]["field_details"].update(b[key]["field_details"]) # merge schema details - if 'schema_details' in b[key]: - asd = a[key]['schema_details'] - bsd = b[key]['schema_details'] - if 'reusable' in b[key]['schema_details']: - asd.setdefault('reusable', {}) - if 'top_level' in bsd['reusable']: - asd['reusable']['top_level'] = bsd['reusable']['top_level'] + if "schema_details" in b[key]: + asd = a[key]["schema_details"] + bsd = b[key]["schema_details"] + if "reusable" in b[key]["schema_details"]: + asd.setdefault("reusable", {}) + if "top_level" in bsd["reusable"]: + asd["reusable"]["top_level"] = bsd["reusable"]["top_level"] else: - asd['reusable'].setdefault('top_level', True) - if 'order' in bsd['reusable']: - asd['reusable']['order'] = bsd['reusable']['order'] - asd['reusable'].setdefault('expected', []) - asd['reusable']['expected'].extend(bsd['reusable']['expected']) - bsd.pop('reusable') + asd["reusable"].setdefault("top_level", True) + if "order" in bsd["reusable"]: + asd["reusable"]["order"] = bsd["reusable"]["order"] + asd["reusable"].setdefault("expected", []) + asd["reusable"]["expected"].extend(bsd["reusable"]["expected"]) + bsd.pop("reusable") + asd.setdefault("settings", {}) + if "settings" in bsd["settings"]: + asd["settings"] = merge_fields(asd["settings"], bsd["settings"]) asd.update(bsd) # merge nested fields - if 'fields' in b[key]: - a[key].setdefault('fields', {}) - a[key]['fields'] = merge_fields(a[key]['fields'], b[key]['fields']) + if "fields" in b[key]: + a[key].setdefault("fields", {}) + a[key]["fields"] = merge_fields(a[key]["fields"], b[key]["fields"]) return a @@ -306,8 +332,8 @@ def eval_globs(globs): """Accepts an array of glob patterns or file names, returns the array of actual files""" all_files = [] for g in globs: - if g.endswith('/'): - g += '*' + if g.endswith("/"): + g += "*" new_files = glob.glob(g) if len(new_files) == 0: warn("{} did not match any files".format(g)) From 8e10ca5a5e138c2523af26a1de2a2cee97b5170d Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:13:49 -0400 Subject: [PATCH 30/46] Fix settings merging --- scripts/schema/loader.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/schema/loader.py b/scripts/schema/loader.py index 8fb028c4af..031aed9566 100644 --- a/scripts/schema/loader.py +++ b/scripts/schema/loader.py @@ -296,7 +296,7 @@ def merge_fields( if "schema_details" in b[key]: asd = a[key]["schema_details"] bsd = b[key]["schema_details"] - if "reusable" in b[key]["schema_details"]: + if "reusable" in bsd: asd.setdefault("reusable", {}) if "top_level" in bsd["reusable"]: asd["reusable"]["top_level"] = bsd["reusable"]["top_level"] @@ -308,8 +308,11 @@ def merge_fields( asd["reusable"]["expected"].extend(bsd["reusable"]["expected"]) bsd.pop("reusable") asd.setdefault("settings", {}) - if "settings" in bsd["settings"]: - asd["settings"] = merge_fields(asd["settings"], bsd["settings"]) + if "settings" in bsd: + asd.setdefault("settings", {}) + asd["settings"] = merge_fields(asd["settings"], bsd["settings"]) + # Prevents bsd["settings"] overwritting the merging we just did in the update below + del bsd["settings"] asd.update(bsd) # merge nested fields if "fields" in b[key]: From cc0994c89ba223317de5febdaf1e9af129521a0b Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:15:53 -0400 Subject: [PATCH 31/46] Restrict test workflow --- .github/workflows/test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc2d5276c5..8d9c6a266d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,14 @@ name: Tests -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + paths: + - "!docker/**" + branches: + - main jobs: tests: From 3b191873ee0f05cfe4a44386b94545efce8067ad Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:24:10 -0400 Subject: [PATCH 32/46] Fix merge conflicts --- scripts/tests/test_ecs_helpers.py | 202 +++-- scripts/tests/unit/test_schema_loader.py | 996 +++++++++++------------ 2 files changed, 588 insertions(+), 610 deletions(-) diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index 8ad616c4f4..54999a36b0 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -19,156 +19,194 @@ import sys import unittest -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from scripts.generators import ecs_helpers class TestECSHelpers(unittest.TestCase): - def test_is_intermediate_field(self): - pseudo_field = {'field_details': {}} + pseudo_field = {"field_details": {}} self.assertEqual(ecs_helpers.is_intermediate(pseudo_field), False) - pseudo_field['field_details']['intermediate'] = False + pseudo_field["field_details"]["intermediate"] = False self.assertEqual(ecs_helpers.is_intermediate(pseudo_field), False) - pseudo_field['field_details']['intermediate'] = True + pseudo_field["field_details"]["intermediate"] = True self.assertEqual(ecs_helpers.is_intermediate(pseudo_field), True) # dict_copy_existing_keys def test_dict_copy_existing_keys(self): - source = {'key1': 'value1'} + source = {"key1": "value1"} destination = {} - ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'missingkey']) - self.assertEqual(destination, {'key1': 'value1'}) + ecs_helpers.dict_copy_existing_keys(source, destination, ["key1", "missingkey"]) + self.assertEqual(destination, {"key1": "value1"}) def test_dict_copy_existing_keys_overwrites(self): - source = {'key1': 'new_value'} - destination = {'key1': 'overwritten', 'untouched': 'untouched'} - ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'untouched']) - self.assertEqual(destination, {'key1': 'new_value', 'untouched': 'untouched'}) + source = {"key1": "new_value"} + destination = {"key1": "overwritten", "untouched": "untouched"} + ecs_helpers.dict_copy_existing_keys(source, destination, ["key1", "untouched"]) + self.assertEqual(destination, {"key1": "new_value", "untouched": "untouched"}) # dict_sorted_by_keys def test_sorted_by_one_key(self): dict = { - 'message': {'name': 'message'}, - 'labels': {'name': 'labels'}, - '@timestamp': {'name': '@timestamp'}, - 'tags': {'name': 'tags'} + "message": {"name": "message"}, + "labels": {"name": "labels"}, + "@timestamp": {"name": "@timestamp"}, + "tags": {"name": "tags"}, } expected = [ - {'name': '@timestamp'}, - {'name': 'labels'}, - {'name': 'message'}, - {'name': 'tags'} + {"name": "@timestamp"}, + {"name": "labels"}, + {"name": "message"}, + {"name": "tags"}, ] - result = ecs_helpers.dict_sorted_by_keys(dict, 'name') + result = ecs_helpers.dict_sorted_by_keys(dict, "name") self.assertEqual(result, expected) - result = ecs_helpers.dict_sorted_by_keys(dict, ['name']) + result = ecs_helpers.dict_sorted_by_keys(dict, ["name"]) self.assertEqual(result, expected) def test_sorted_by_multiple_keys(self): dict = { - 'cloud': {'group': 2, 'name': 'cloud'}, - 'agent': {'group': 2, 'name': 'agent'}, - 'base': {'group': 1, 'name': 'base'}, + "cloud": {"group": 2, "name": "cloud"}, + "agent": {"group": 2, "name": "agent"}, + "base": {"group": 1, "name": "base"}, } expected = [ - {'group': 1, 'name': 'base'}, - {'group': 2, 'name': 'agent'}, - {'group': 2, 'name': 'cloud'} + {"group": 1, "name": "base"}, + {"group": 2, "name": "agent"}, + {"group": 2, "name": "cloud"}, ] - result = ecs_helpers.dict_sorted_by_keys(dict, ['group', 'name']) + result = ecs_helpers.dict_sorted_by_keys(dict, ["group", "name"]) self.assertEqual(result, expected) def test_merge_dicts(self): a = { - 'cloud': {'group': 2, 'name': 'cloud'}, - 'agent': {'group': 2, 'name': 'agent'}, + "cloud": {"group": 2, "name": "cloud"}, + "agent": {"group": 2, "name": "agent"}, } - b = {'base': {'group': 1, 'name': 'base'}} + b = {"base": {"group": 1, "name": "base"}} result = ecs_helpers.safe_merge_dicts(a, b) - self.assertEqual(result, - { - 'cloud': {'group': 2, 'name': 'cloud'}, - 'agent': {'group': 2, 'name': 'agent'}, - 'base': {'group': 1, 'name': 'base'} - }) + self.assertEqual( + result, + { + "cloud": {"group": 2, "name": "cloud"}, + "agent": {"group": 2, "name": "agent"}, + "base": {"group": 1, "name": "base"}, + }, + ) def test_merge_dicts_raises_if_duplicate_key_added(self): - a = {'cloud': {'group': 2, 'name': 'cloud'}} - b = {'cloud': {'group': 9, 'name': 'bazbar'}} + a = {"cloud": {"group": 2, "name": "cloud"}} + b = {"cloud": {"group": 9, "name": "bazbar"}} with self.assertRaises(ValueError): ecs_helpers.safe_merge_dicts(a, b) def test_clean_string_values(self): - dict = {'dirty': ' space, the final frontier ', 'clean': 'val', 'int': 1} + dict = {"dirty": " space, the final frontier ", "clean": "val", "int": 1} ecs_helpers.dict_clean_string_values(dict) - self.assertEqual(dict, {'dirty': 'space, the final frontier', 'clean': 'val', 'int': 1}) + self.assertEqual( + dict, {"dirty": "space, the final frontier", "clean": "val", "int": 1} + ) # List helper tests def test_list_subtract(self): - self.assertEqual(ecs_helpers.list_subtract(['a', 'b'], ['a']), ['b']) - self.assertEqual(ecs_helpers.list_subtract(['a', 'b'], ['a', 'c']), ['b']) + self.assertEqual(ecs_helpers.list_subtract(["a", "b"], ["a"]), ["b"]) + self.assertEqual(ecs_helpers.list_subtract(["a", "b"], ["a", "c"]), ["b"]) # git helper tests - def test_get_tree_by_ref(self): - ref = 'v1.5.0' - tree = ecs_helpers.get_tree_by_ref(ref) - self.assertEqual(tree.hexsha, '4449df245f6930d59bcd537a5958891261a9476b') - - def test_path_exists_in_git_tree(self): - ref = 'v1.6.0' - tree = ecs_helpers.get_tree_by_ref(ref) - self.assertFalse(ecs_helpers.path_exists_in_git_tree(tree, 'nonexistant')) - self.assertTrue(ecs_helpers.path_exists_in_git_tree(tree, 'schemas')) + # def test_get_tree_by_ref(self): + # ref = 'v1.5.0' + # tree = ecs_helpers.get_tree_by_ref(ref) + # self.assertEqual(tree.hexsha, '4449df245f6930d59bcd537a5958891261a9476b') + # + # def test_path_exists_in_git_tree(self): + # ref = 'v1.6.0' + # tree = ecs_helpers.get_tree_by_ref(ref) + # self.assertFalse(ecs_helpers.path_exists_in_git_tree(tree, 'nonexistant')) + # self.assertTrue(ecs_helpers.path_exists_in_git_tree(tree, 'schemas')) # file helpers def test_is_yaml(self): - self.assertTrue(ecs_helpers.is_yaml('./schemas/base.yml')) - self.assertTrue(ecs_helpers.is_yaml('./build/docs/conf.yaml')) - self.assertFalse(ecs_helpers.is_yaml('./README.md')) - self.assertFalse(ecs_helpers.is_yaml('./schemas/')) - self.assertFalse(ecs_helpers.is_yaml('./build')) + self.assertTrue(ecs_helpers.is_yaml("./schemas/base.yml")) + self.assertTrue(ecs_helpers.is_yaml("./build/docs/conf.yaml")) + self.assertFalse(ecs_helpers.is_yaml("./README.md")) + self.assertFalse(ecs_helpers.is_yaml("./schemas/")) + self.assertFalse(ecs_helpers.is_yaml("./build")) def test_glob_yaml_files(self): - self.assertEqual(ecs_helpers.glob_yaml_files('non_existent_file'), []) - self.assertEqual(ecs_helpers.glob_yaml_files('non_existent_directory/'), []) - self.assertEqual(ecs_helpers.glob_yaml_files('non_existent_wildcard.*'), []) - self.assertEqual(ecs_helpers.glob_yaml_files('schemas/base.yml'), ['schemas/base.yml']) - self.assertEqual(ecs_helpers.glob_yaml_files(['schemas/base.yml']), ['schemas/base.yml']) - #  convert to set as element order is not being tested - self.assertEqual(set(ecs_helpers.glob_yaml_files( - ['schemas/base.yml', 'schemas/log.yml'])), {'schemas/base.yml', 'schemas/log.yml'}) - self.assertTrue(set(ecs_helpers.glob_yaml_files('schemas/b*.yml')).intersection({'schemas/base.yml'}) != set()) - self.assertTrue(set(ecs_helpers.glob_yaml_files( - 'schemas/[bl]*.yml')).intersection({'schemas/base.yml', 'schemas/log.yml'}) != set()) + self.assertEqual(ecs_helpers.glob_yaml_files("non_existent_file"), []) + self.assertEqual(ecs_helpers.glob_yaml_files("non_existent_directory/"), []) + self.assertEqual(ecs_helpers.glob_yaml_files("non_existent_wildcard.*"), []) + self.assertEqual( + ecs_helpers.glob_yaml_files("schemas/base.yml"), ["schemas/base.yml"] + ) + self.assertEqual( + ecs_helpers.glob_yaml_files(["schemas/base.yml"]), ["schemas/base.yml"] + ) + # convert to set as element order is not being tested + self.assertEqual( + set(ecs_helpers.glob_yaml_files(["schemas/base.yml", "schemas/log.yml"])), + {"schemas/base.yml", "schemas/log.yml"}, + ) + self.assertTrue( + set(ecs_helpers.glob_yaml_files("schemas/b*.yml")).intersection( + {"schemas/base.yml"} + ) + != set() + ) + self.assertTrue( + set(ecs_helpers.glob_yaml_files("schemas/[bl]*.yml")).intersection( + {"schemas/base.yml", "schemas/log.yml"} + ) + != set() + ) min_schema_count = 46 - self.assertTrue(len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files('schemas'))) >= min_schema_count) - self.assertTrue(len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files('schemas/'))) >= min_schema_count) - self.assertTrue(len(ecs_helpers.glob_yaml_files( - ecs_helpers.glob_yaml_files('schemas/*.yml'))) >= min_schema_count) - self.assertEqual(len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files('schemas/*.yaml'))), 0) + self.assertTrue( + len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files("schemas"))) + >= min_schema_count + ) + self.assertTrue( + len(ecs_helpers.glob_yaml_files(ecs_helpers.glob_yaml_files("schemas/"))) + >= min_schema_count + ) + self.assertTrue( + len( + ecs_helpers.glob_yaml_files( + ecs_helpers.glob_yaml_files("schemas/*.yml") + ) + ) + >= min_schema_count + ) + self.assertEqual( + len( + ecs_helpers.glob_yaml_files( + ecs_helpers.glob_yaml_files("schemas/*.yaml") + ) + ), + 0, + ) # Remove top_level:false field sets helper def test_remove_top_level_false_field_sets(self): nested_schema_original = { - 'as': {'group': 2, 'name': 'as', 'reusable': {'top_level': False}}, - 'agent': {'group': 2, 'name': 'agent'}, - } - nested_schema_expected = { - 'agent': {'group': 2, 'name': 'agent'} + "as": {"group": 2, "name": "as", "reusable": {"top_level": False}}, + "agent": {"group": 2, "name": "agent"}, } - self.assertEqual(ecs_helpers.remove_top_level_reusable_false(nested_schema_original), nested_schema_expected) + nested_schema_expected = {"agent": {"group": 2, "name": "agent"}} + self.assertEqual( + ecs_helpers.remove_top_level_reusable_false(nested_schema_original), + nested_schema_expected, + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/unit/test_schema_loader.py b/scripts/tests/unit/test_schema_loader.py index 2146b18468..175f1f8678 100644 --- a/scripts/tests/unit/test_schema_loader.py +++ b/scripts/tests/unit/test_schema_loader.py @@ -21,93 +21,96 @@ import sys import unittest -sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) +sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) from schema import loader class TestSchemaLoader(unittest.TestCase): - def setUp(self): self.maxDiff = None - @mock.patch('schema.loader.warn') + @mock.patch("schema.loader.warn") def test_eval_globs(self, mock_warn): - files = loader.eval_globs(['schemas/*.yml', 'missing*']) - self.assertTrue(mock_warn.called, "a warning should have been printed for missing*") - self.assertIn('schemas/base.yml', files) - self.assertEqual(list(filter(lambda f: f.startswith('missing'), files)), [], - "The 'missing*' pattern should not show up in the resulting files") + files = loader.eval_globs(["schemas/*.yml", "missing*"]) + self.assertTrue( + mock_warn.called, "a warning should have been printed for missing*" + ) + self.assertIn("schemas/base.yml", files) + self.assertEqual( + list(filter(lambda f: f.startswith("missing"), files)), + [], + "The 'missing*' pattern should not show up in the resulting files", + ) - @mock.patch('schema.loader.warn') + @mock.patch("schema.loader.warn") def test_eval_globs_pattern(self, mock_warn): - files = loader.eval_globs(['schemas/a*.yml']) - self.assertEqual(['schemas/agent.yml', 'schemas/as.yml'].sort(), files.sort(), - "glob should load all files matching 'schemas/a*.yml'") + files = loader.eval_globs(["schemas/a*.yml"]) + self.assertEqual( + ["schemas/agent.yml", "schemas/as.yml"].sort(), + files.sort(), + "glob should load all files matching 'schemas/a*.yml'", + ) - @mock.patch('schema.loader.warn') + @mock.patch("schema.loader.warn") def test_eval_globs_filenames(self, mock_warn): - files = loader.eval_globs(['schemas/agent.yml', 'schemas/x509.yml']) - self.assertEqual(['schemas/agent.yml', 'schemas/x509.yml'], files, - "glob should load all files matching '['schemas/agent.yml', 'schemas/x509.yml']'") + files = loader.eval_globs(["schemas/agent.yml", "schemas/x509.yml"]) + self.assertEqual( + ["schemas/agent.yml", "schemas/x509.yml"], + files, + "glob should load all files matching '['schemas/agent.yml', 'schemas/x509.yml']'", + ) - @mock.patch('schema.loader.warn') + @mock.patch("schema.loader.warn") def test_eval_globs_folder(self, mock_warn): - files = loader.eval_globs(['schemas/']) - self.assertIn('schemas/base.yml', files, - "glob should load all files in folder 'schemas' including 'schemas/base.yml']'") + files = loader.eval_globs(["schemas/"]) + self.assertIn( + "schemas/base.yml", + files, + "glob should load all files in folder 'schemas' including 'schemas/base.yml']'", + ) - @mock.patch('schema.loader.warn') + @mock.patch("schema.loader.warn") def test_eval_globs_folders(self, mock_warn): - files = loader.eval_globs(['schemas/', 'usage-example/fields/custom/']) - self.assertIn('usage-example/fields/custom/acme.yml', files, - "glob should load all files in folders ['schemas/', 'usage-example/fields/custom/'] including 'usage-example/fields/custom/acme.yml''") + files = loader.eval_globs(["schemas/", "usage-example/fields/custom/"]) + self.assertIn( + "usage-example/fields/custom/acme.yml", + files, + "glob should load all files in folders ['schemas/', 'usage-example/fields/custom/'] including 'usage-example/fields/custom/acme.yml''", + ) # Pseudo-fixtures def schema_base(self): return { - 'base': { - 'schema_details': {'root': True}, - 'field_details': {'name': 'base', 'type': 'group'}, - 'fields': { - 'message': { - 'field_details': { - 'name': 'message', - 'type': 'keyword' - } - } - } + "base": { + "schema_details": {"root": True}, + "field_details": {"name": "base", "type": "group"}, + "fields": { + "message": {"field_details": {"name": "message", "type": "keyword"}} + }, } } def schema_process(self): return { - 'process': { - 'schema_details': {}, - 'field_details': { - 'name': 'process', - 'type': 'group' - }, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'pid', - 'type': 'keyword' - } - }, - 'parent': { - 'field_details': {'type': 'object'}, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'parent.pid', - 'type': 'keyword' + "process": { + "schema_details": {}, + "field_details": {"name": "process", "type": "group"}, + "fields": { + "pid": {"field_details": {"name": "pid", "type": "keyword"}}, + "parent": { + "field_details": {"type": "object"}, + "fields": { + "pid": { + "field_details": { + "name": "parent.pid", + "type": "keyword", } } - } - } - } + }, + }, + }, } } @@ -116,146 +119,133 @@ def schema_process(self): def test_load_schemas_no_custom(self): fields = loader.load_schemas([]) self.assertEqual( - ['field_details', 'fields', 'schema_details'], - sorted(fields['process'].keys()), - "Schemas should have 'field_details', 'fields' and 'schema_details' subkeys") + ["field_details", "fields", "schema_details"], + sorted(fields["process"].keys()), + "Schemas should have 'field_details', 'fields' and 'schema_details' subkeys", + ) self.assertEqual( - ['field_details'], - list(fields['process']['fields']['pid'].keys()), - "Leaf fields should have only the 'field_details' subkey") + ["field_details"], + list(fields["process"]["fields"]["pid"].keys()), + "Leaf fields should have only the 'field_details' subkey", + ) self.assertIn( - 'fields', - fields['process']['fields']['thread'].keys(), - "Fields containing nested fields should at least have the 'fields' subkey") - - def test_load_schemas_git_ref(self): - fields = loader.load_schemas(ref='v1.6.0') - self.assertEqual( - ['field_details', 'fields', 'schema_details'], - sorted(fields['process'].keys()), - "Schemas should have 'field_details', 'fields' and 'schema_details' subkeys") - self.assertEqual( - ['field_details'], - list(fields['process']['fields']['pid'].keys()), - "Leaf fields should have only the 'field_details' subkey") - self.assertIn( - 'fields', - fields['process']['fields']['thread'].keys(), - "Fields containing nested fields should at least have the 'fields' subkey") - - @mock.patch('schema.loader.read_schema_file') - def test_load_schemas_fail_on_accidental_fieldset_redefinition(self, mock_read_schema): + "fields", + fields["process"]["fields"]["thread"].keys(), + "Fields containing nested fields should at least have the 'fields' subkey", + ) + + # def test_load_schemas_git_ref(self): + # fields = loader.load_schemas(ref="v1.6.0") + # self.assertEqual( + # ["field_details", "fields", "schema_details"], + # sorted(fields["process"].keys()), + # "Schemas should have 'field_details', 'fields' and 'schema_details' subkeys", + # ) + # self.assertEqual( + # ["field_details"], + # list(fields["process"]["fields"]["pid"].keys()), + # "Leaf fields should have only the 'field_details' subkey", + # ) + # self.assertIn( + # "fields", + # fields["process"]["fields"]["thread"].keys(), + # "Fields containing nested fields should at least have the 'fields' subkey", + # ) + + @mock.patch("schema.loader.read_schema_file") + def test_load_schemas_fail_on_accidental_fieldset_redefinition( + self, mock_read_schema + ): mock_read_schema.side_effect = [ - { - 'file': { - 'name': 'file', - 'type': 'keyword' - } - }, - { - 'file': { - 'name': 'file', - 'type': 'text' - } - } + {"file": {"name": "file", "type": "keyword"}}, + {"file": {"name": "file", "type": "text"}}, ] with self.assertRaises(ValueError): - loader.load_schema_files(['a.yml', 'b.yml']) + loader.load_schema_files(["a.yml", "b.yml"]) - @mock.patch('schema.loader.read_schema_file') + @mock.patch("schema.loader.read_schema_file") def test_load_schemas_allows_unique_fieldsets(self, mock_read_schema): - file_map = { - 'file': { - 'name': 'file', - 'type': 'keyword' - } - } - host_map = { - 'host': { - 'name': 'host', - 'type': 'text' - } - } + file_map = {"file": {"name": "file", "type": "keyword"}} + host_map = {"host": {"name": "host", "type": "text"}} mock_read_schema.side_effect = [file_map, host_map] - exp = { - 'file': file_map['file'], - 'host': host_map['host'] - } - res = loader.load_schema_files(['a.yml', 'b.yml']) + exp = {"file": file_map["file"], "host": host_map["host"]} + res = loader.load_schema_files(["a.yml", "b.yml"]) self.assertEqual(res, exp) def test_nest_schema_raises_on_missing_schema_name(self): - with self.assertRaisesRegex(ValueError, 'incomplete.yml'): - loader.nest_schema([{'description': 'just a description'}], 'incomplete.yml') - - def test_load_schemas_from_git(self): - fields = loader.load_schemas_from_git('v1.0.0', target_dir='schemas') - self.assertEqual( - ['agent', - 'base', - 'client', - 'cloud', - 'container', - 'destination', - 'ecs', - 'error', - 'event', - 'file', - 'geo', - 'group', - 'host', - 'http', - 'log', - 'network', - 'observer', - 'organization', - 'os', - 'process', - 'related', - 'server', - 'service', - 'source', - 'url', - 'user', - 'user_agent'], - sorted(fields.keys()), - "Raw schema fields should have expected fieldsets for v1.0.0") - - def test_load_schemas_from_git_missing_target_directory(self): - with self.assertRaisesRegex(KeyError, "not present in git ref 'v1.5.0'"): - loader.load_schemas_from_git('v1.5.0', target_dir='experimental') + with self.assertRaisesRegex(ValueError, "incomplete.yml"): + loader.nest_schema( + [{"description": "just a description"}], "incomplete.yml" + ) + + # def test_load_schemas_from_git(self): + # fields = loader.load_schemas_from_git('v1.0.0', target_dir='schemas') + # self.assertEqual( + # ['agent', + # 'base', + # 'client', + # 'cloud', + # 'container', + # 'destination', + # 'ecs', + # 'error', + # 'event', + # 'file', + # 'geo', + # 'group', + # 'host', + # 'http', + # 'log', + # 'network', + # 'observer', + # 'organization', + # 'os', + # 'process', + # 'related', + # 'server', + # 'service', + # 'source', + # 'url', + # 'user', + # 'user_agent'], + # sorted(fields.keys()), + # "Raw schema fields should have expected fieldsets for v1.0.0") + + # def test_load_schemas_from_git_missing_target_directory(self): + # with self.assertRaisesRegex(KeyError, "not present in git ref 'v1.5.0'"): + # loader.load_schemas_from_git("v1.5.0", target_dir="experimental") # nesting stuff def test_nest_fields(self): process_fields = [ - {'name': 'pid'}, - {'name': 'parent.pid'}, + {"name": "pid"}, + {"name": "parent.pid"}, ] expected_nested_fields = { - 'fields': { - 'pid': { - 'field_details': { - 'name': 'pid', - 'node_name': 'pid', + "fields": { + "pid": { + "field_details": { + "name": "pid", + "node_name": "pid", } }, - 'parent': { - 'field_details': { - 'name': 'parent', - 'node_name': 'parent', - 'type': 'object', - 'intermediate': True, + "parent": { + "field_details": { + "name": "parent", + "node_name": "parent", + "type": "object", + "intermediate": True, }, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'parent.pid', - 'node_name': 'pid', + "fields": { + "pid": { + "field_details": { + "name": "parent.pid", + "node_name": "pid", } } - } - } + }, + }, } } nested_fields = loader.nest_fields(process_fields) @@ -263,80 +253,80 @@ def test_nest_fields(self): def test_nest_fields_recognizes_explicitly_defined_object_fields(self): dns_fields = [ - {'name': 'question.name', 'type': 'keyword'}, - {'name': 'answers', 'type': 'object'}, - {'name': 'answers.data', 'type': 'keyword'}, + {"name": "question.name", "type": "keyword"}, + {"name": "answers", "type": "object"}, + {"name": "answers.data", "type": "keyword"}, ] expected_nested_fields = { - 'fields': { - 'answers': { - 'field_details': { - 'name': 'answers', - 'node_name': 'answers', - 'type': 'object', - 'intermediate': False, + "fields": { + "answers": { + "field_details": { + "name": "answers", + "node_name": "answers", + "type": "object", + "intermediate": False, }, - 'fields': { - 'data': { - 'field_details': { - 'name': 'answers.data', - 'node_name': 'data', - 'type': 'keyword', + "fields": { + "data": { + "field_details": { + "name": "answers.data", + "node_name": "data", + "type": "keyword", } } - } + }, }, - 'question': { - 'field_details': { - 'name': 'question', - 'node_name': 'question', - 'type': 'object', - 'intermediate': True, + "question": { + "field_details": { + "name": "question", + "node_name": "question", + "type": "object", + "intermediate": True, }, - 'fields': { - 'name': { - 'field_details': { - 'name': 'question.name', - 'node_name': 'name', - 'type': 'keyword', + "fields": { + "name": { + "field_details": { + "name": "question.name", + "node_name": "name", + "type": "keyword", } } - } - } + }, + }, } } nested_fields = loader.nest_fields(dns_fields) self.assertEqual(nested_fields, expected_nested_fields) def test_nest_fields_multiple_intermediate_fields(self): - log_fields = [{'name': 'origin.file.name'}] + log_fields = [{"name": "origin.file.name"}] expected_nested_fields = { - 'fields': { - 'origin': { - 'field_details': { - 'name': 'origin', - 'node_name': 'origin', - 'type': 'object', - 'intermediate': True, + "fields": { + "origin": { + "field_details": { + "name": "origin", + "node_name": "origin", + "type": "object", + "intermediate": True, }, - 'fields': { - 'file': { - 'field_details': { - 'name': 'origin.file', - 'node_name': 'file', - 'type': 'object', - 'intermediate': True, + "fields": { + "file": { + "field_details": { + "name": "origin.file", + "node_name": "file", + "type": "object", + "intermediate": True, }, - 'fields': { - 'name': { - 'field_details': { - 'name': 'origin.file.name', - 'node_name': 'name', + "fields": { + "name": { + "field_details": { + "name": "origin.file.name", + "node_name": "name", } } - } + }, } - } + }, } } } @@ -345,213 +335,190 @@ def test_nest_fields_multiple_intermediate_fields(self): def test_deep_nesting_representation(self): all_schemas = { - 'base': { - 'name': 'base', - 'title': 'Base', - 'root': True, - 'type': 'group', - 'fields': [ - {'name': 'message', 'type': 'keyword'} - ] + "base": { + "name": "base", + "title": "Base", + "root": True, + "type": "group", + "fields": [{"name": "message", "type": "keyword"}], + }, + "process": { + "name": "process", + "type": "group", + "fields": [ + {"name": "pid", "type": "keyword"}, + {"name": "parent.pid", "type": "keyword"}, + ], }, - 'process': { - 'name': 'process', - 'type': 'group', - 'fields': [ - {'name': 'pid', 'type': 'keyword'}, - {'name': 'parent.pid', 'type': 'keyword'}, - ] - } } deeply_nested = loader.deep_nesting_representation(all_schemas) expected_deeply_nested = { - 'base': { - 'schema_details': { - 'root': True, - 'title': 'Base', + "base": { + "schema_details": { + "root": True, + "title": "Base", }, - 'field_details': { - 'name': 'base', - 'node_name': 'base', - 'type': 'group', + "field_details": { + "name": "base", + "node_name": "base", + "type": "group", }, - 'fields': { - 'message': { - 'field_details': { - 'name': 'message', - 'node_name': 'message', - 'type': 'keyword', + "fields": { + "message": { + "field_details": { + "name": "message", + "node_name": "message", + "type": "keyword", } } - } + }, }, - 'process': { - 'schema_details': {}, - 'field_details': { - 'name': 'process', - 'node_name': 'process', - 'type': 'group' + "process": { + "schema_details": {}, + "field_details": { + "name": "process", + "node_name": "process", + "type": "group", }, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'pid', - 'node_name': 'pid', - 'type': 'keyword', + "fields": { + "pid": { + "field_details": { + "name": "pid", + "node_name": "pid", + "type": "keyword", } }, - 'parent': { - 'field_details': { + "parent": { + "field_details": { # These are made explicit for intermediate fields - 'name': 'parent', - 'node_name': 'parent', - 'type': 'object', - 'intermediate': True, + "name": "parent", + "node_name": "parent", + "type": "object", + "intermediate": True, }, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'parent.pid', - 'node_name': 'pid', - 'type': 'keyword', + "fields": { + "pid": { + "field_details": { + "name": "parent.pid", + "node_name": "pid", + "type": "keyword", } } - } - } - } - } + }, + }, + }, + }, } - process_fields = deeply_nested['process']['fields'] - self.assertEqual(process_fields['parent']['field_details']['intermediate'], True) + process_fields = deeply_nested["process"]["fields"] + self.assertEqual( + process_fields["parent"]["field_details"]["intermediate"], True + ) self.assertEqual(deeply_nested, expected_deeply_nested) # Merging def test_merge_new_schema(self): custom = { - 'custom': { - 'schema_details': {}, - 'field_details': { - 'name': 'custom', - 'type': 'group' - }, - 'fields': { - 'my_field': { - 'field_details': { - 'name': 'my_field', - 'type': 'keyword' - } + "custom": { + "schema_details": {}, + "field_details": {"name": "custom", "type": "group"}, + "fields": { + "my_field": { + "field_details": {"name": "my_field", "type": "keyword"} } - } + }, } } expected_fields = {**self.schema_base(), **custom} merged_fields = loader.merge_fields(self.schema_base(), custom) - self.assertEqual(expected_fields, merged_fields, - "New schemas should just be a dictionary merge") + self.assertEqual( + expected_fields, + merged_fields, + "New schemas should just be a dictionary merge", + ) def test_merge_field_within_schema(self): custom = { - 'base': { - 'schema_details': {}, - 'field_details': { - 'name': 'base' - }, - 'fields': { - 'my_field': { - 'field_details': { - 'name': 'my_field', - 'type': 'keyword' - } + "base": { + "schema_details": {}, + "field_details": {"name": "base"}, + "fields": { + "my_field": { + "field_details": {"name": "my_field", "type": "keyword"} } - } + }, } } expected_fields = { - 'base': { - 'schema_details': {'root': True}, - 'field_details': { - 'name': 'base', - 'type': 'group' - }, - 'fields': { - 'message': { - 'field_details': { - 'name': 'message', - 'type': 'keyword' - } + "base": { + "schema_details": {"root": True}, + "field_details": {"name": "base", "type": "group"}, + "fields": { + "message": { + "field_details": {"name": "message", "type": "keyword"} }, - 'my_field': { - 'field_details': { - 'name': 'my_field', - 'type': 'keyword' - } - } - } + "my_field": { + "field_details": {"name": "my_field", "type": "keyword"} + }, + }, } } merged_fields = loader.merge_fields(self.schema_base(), custom) - self.assertEqual(['message', 'my_field'], - sorted(expected_fields['base']['fields'].keys())) - self.assertEqual(expected_fields, merged_fields, - "New fields being merged in existing schemas are merged in the 'fields' dict.") + self.assertEqual( + ["message", "my_field"], sorted(expected_fields["base"]["fields"].keys()) + ) + self.assertEqual( + expected_fields, + merged_fields, + "New fields being merged in existing schemas are merged in the 'fields' dict.", + ) def test_fields_with_subfields_mergeable(self): custom = { - 'process': { - 'schema_details': {}, - 'field_details': { - 'name': 'process' - }, - 'fields': { - 'parent': { - 'field_details': {'type': 'object'}, - 'fields': { - 'name': { - 'field_details': { - 'name': 'parent.name', - 'type': 'keyword' + "process": { + "schema_details": {}, + "field_details": {"name": "process"}, + "fields": { + "parent": { + "field_details": {"type": "object"}, + "fields": { + "name": { + "field_details": { + "name": "parent.name", + "type": "keyword", } } - } + }, } - } + }, } } merged_fields = loader.merge_fields(self.schema_process(), custom) expected_fields = { - 'process': { - 'schema_details': {}, - 'field_details': { - 'name': 'process', - 'type': 'group' - }, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'pid', - 'type': 'keyword' - } - }, - 'parent': { - 'field_details': {'type': 'object'}, - 'fields': { - 'pid': { - 'field_details': { - 'name': 'parent.pid', - 'type': 'keyword' + "process": { + "schema_details": {}, + "field_details": {"name": "process", "type": "group"}, + "fields": { + "pid": {"field_details": {"name": "pid", "type": "keyword"}}, + "parent": { + "field_details": {"type": "object"}, + "fields": { + "pid": { + "field_details": { + "name": "parent.pid", + "type": "keyword", } }, - 'name': { - 'field_details': { - 'name': 'parent.name', - 'type': 'keyword' + "name": { + "field_details": { + "name": "parent.name", + "type": "keyword", } - } - } - } - } + }, + }, + }, + }, } } self.assertEqual(merged_fields, expected_fields) @@ -561,215 +528,188 @@ def test_merge_array_attributes(self): # - schema/reusable.expected # - field/normalize ecs = { - 'foo': { - 'schema_details': { - 'reusable': { - 'top_level': True, - 'expected': ['normal.location'] - } + "foo": { + "schema_details": { + "reusable": {"top_level": True, "expected": ["normal.location"]} }, - 'field_details': {'name': 'foo', 'type': 'group'}, - 'fields': { - 'normalized_field': { - 'field_details': { - 'name': 'normalized_field', - 'type': 'keyword', - 'normalize': ['lowercase'] + "field_details": {"name": "foo", "type": "group"}, + "fields": { + "normalized_field": { + "field_details": { + "name": "normalized_field", + "type": "keyword", + "normalize": ["lowercase"], } }, - 'not_initially_normalized': { - 'field_details': { - 'name': 'not_initially_normalized', - 'type': 'keyword' + "not_initially_normalized": { + "field_details": { + "name": "not_initially_normalized", + "type": "keyword", } - } - } + }, + }, } } custom = { - 'foo': { - 'schema_details': { - 'reusable': { - 'expected': ['a_new.location'] - } - }, - 'field_details': {'name': 'foo', 'type': 'group'}, - 'fields': { - 'normalized_field': { - 'field_details': { - 'name': 'normalized_field', - 'normalize': ['array'] + "foo": { + "schema_details": {"reusable": {"expected": ["a_new.location"]}}, + "field_details": {"name": "foo", "type": "group"}, + "fields": { + "normalized_field": { + "field_details": { + "name": "normalized_field", + "normalize": ["array"], } }, - 'not_initially_normalized': { - 'field_details': { - 'name': 'not_initially_normalized', - 'normalize': ['array'] + "not_initially_normalized": { + "field_details": { + "name": "not_initially_normalized", + "normalize": ["array"], } - } - } + }, + }, } } merged_fields = loader.merge_fields(ecs, custom) expected_fields = { - 'foo': { - 'schema_details': { - 'reusable': { - 'top_level': True, - 'expected': ['normal.location', 'a_new.location'] - } + "foo": { + "schema_details": { + "reusable": { + "top_level": True, + "expected": ["normal.location", "a_new.location"], + }, + "settings": {}, }, - 'field_details': {'name': 'foo', 'type': 'group'}, - 'fields': { - 'normalized_field': { - 'field_details': { - 'name': 'normalized_field', - 'type': 'keyword', - 'normalize': ['lowercase', 'array'] + "field_details": {"name": "foo", "type": "group"}, + "fields": { + "normalized_field": { + "field_details": { + "name": "normalized_field", + "type": "keyword", + "normalize": ["lowercase", "array"], } }, - 'not_initially_normalized': { - 'field_details': { - 'name': 'not_initially_normalized', - 'type': 'keyword', - 'normalize': ['array'] + "not_initially_normalized": { + "field_details": { + "name": "not_initially_normalized", + "type": "keyword", + "normalize": ["array"], } - } - } + }, + }, } } self.assertEqual( - merged_fields['foo']['schema_details']['reusable']['expected'], - ['normal.location', 'a_new.location']) + merged_fields["foo"]["schema_details"]["reusable"]["expected"], + ["normal.location", "a_new.location"], + ) self.assertEqual( - merged_fields['foo']['fields']['normalized_field']['field_details']['normalize'], - ['lowercase', 'array']) + merged_fields["foo"]["fields"]["normalized_field"]["field_details"][ + "normalize" + ], + ["lowercase", "array"], + ) self.assertEqual( - merged_fields['foo']['fields']['not_initially_normalized']['field_details']['normalize'], - ['array']) + merged_fields["foo"]["fields"]["not_initially_normalized"]["field_details"][ + "normalize" + ], + ["array"], + ) self.assertEqual(merged_fields, expected_fields) def test_merge_non_array_attributes(self): custom = { - 'base': { - 'schema_details': { - 'root': False, # Override (not that I'd recommend overriding that) - 'group': 3 # New + "base": { + "schema_details": { + "root": False, # Override (not that I'd recommend overriding that) + "group": 3, # New }, - 'field_details': { - 'type': 'object', # Override - 'example': 'foo' # New + "field_details": { + "type": "object", # Override + "example": "foo", # New }, - 'fields': { - 'message': { - 'field_details': { - 'type': 'wildcard', # Override - 'example': 'wild value' # New + "fields": { + "message": { + "field_details": { + "type": "wildcard", # Override + "example": "wild value", # New } } - } + }, } } merged_fields = loader.merge_fields(self.schema_base(), custom) expected_fields = { - 'base': { - 'schema_details': { - 'root': False, - 'group': 3 - }, - 'field_details': { - 'name': 'base', - 'type': 'object', - 'example': 'foo' - }, - 'fields': { - 'message': { - 'field_details': { - 'name': 'message', - 'type': 'wildcard', - 'example': 'wild value' + "base": { + "schema_details": {"root": False, "group": 3}, + "field_details": {"name": "base", "type": "object", "example": "foo"}, + "fields": { + "message": { + "field_details": { + "name": "message", + "type": "wildcard", + "example": "wild value", } } - } + }, } } self.assertEqual(merged_fields, expected_fields) def test_merge_and_overwrite_multi_fields(self): originalSchema = { - 'overwrite_field': { - 'field_details': { - 'multi_fields': [ - { - 'type': 'text', - 'name': 'text', - 'norms': True - } - ] + "overwrite_field": { + "field_details": { + "multi_fields": [{"type": "text", "name": "text", "norms": True}] }, - 'fields': { - 'message': { - 'field_details': { - 'multi_fields': [ - { - 'type': 'text', - 'name': 'text' - } - ] + "fields": { + "message": { + "field_details": { + "multi_fields": [{"type": "text", "name": "text"}] } } - } + }, } } customSchema = { - 'overwrite_field': { - 'field_details': { - 'multi_fields': [ + "overwrite_field": { + "field_details": { + "multi_fields": [ # this entry will completely overwrite the originalSchema's name text entry - { - 'type': 'text', - 'name': 'text' - } + {"type": "text", "name": "text"} ] }, - 'fields': { - 'message': { - 'field_details': { - 'multi_fields': [ + "fields": { + "message": { + "field_details": { + "multi_fields": [ # this entry will be merged with the originalSchema's multi_fields entries - { - 'type': 'keyword', - 'name': 'a_field' - } + {"type": "keyword", "name": "a_field"} ] } } - } + }, } } merged_fields = loader.merge_fields(originalSchema, customSchema) - expected_overwrite_field_mf = [ - { - 'type': 'text', - 'name': 'text' - } - ] + expected_overwrite_field_mf = [{"type": "text", "name": "text"}] expected_message_mf = [ - { - 'type': 'keyword', - 'name': 'a_field' - }, - { - 'type': 'text', - 'name': 'text' - } + {"type": "keyword", "name": "a_field"}, + {"type": "text", "name": "text"}, ] - self.assertEqual(merged_fields['overwrite_field']['field_details']['multi_fields'], expected_overwrite_field_mf) - self.assertEqual(merged_fields['overwrite_field']['fields']['message']['field_details'] - ['multi_fields'], expected_message_mf) + self.assertEqual( + merged_fields["overwrite_field"]["field_details"]["multi_fields"], + expected_overwrite_field_mf, + ) + self.assertEqual( + merged_fields["overwrite_field"]["fields"]["message"]["field_details"][ + "multi_fields" + ], + expected_message_mf, + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 48b7cd268bf85f9a8f6ac5262ae9c20311b9bf06 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:24:29 -0400 Subject: [PATCH 33/46] Less restrictive --- .github/workflows/test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d9c6a266d..becb9a702d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,10 +5,6 @@ on: branches: - main pull_request: - paths: - - "!docker/**" - branches: - - main jobs: tests: From 956fd87cd50df8791af79e5aa93fc81ef54d7e06 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:23:08 -0400 Subject: [PATCH 34/46] Add docker files and pipeline --- .../build_elastic_common_schema_toolchain.yml | 72 +++++++++++++++++++ docker/Dockerfile | 8 +++ docker/scripts/entry_point.sh | 23 ++++++ 3 files changed, 103 insertions(+) create mode 100644 .github/workflows/build_elastic_common_schema_toolchain.yml create mode 100644 docker/Dockerfile create mode 100644 docker/scripts/entry_point.sh diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml new file mode 100644 index 0000000000..8f32422e0e --- /dev/null +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -0,0 +1,72 @@ +name: Build ECS Toolchain Image +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::068738303278:role/PushPullElasticImages + aws-region: us-east-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + # Setting up Docker Buildx with docker-container driver is required + # Setting up Docker Buildx with docker-container driver is required + # at the moment to be able to use a subdirectory with Git context + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Debug + run: 'echo "Current branch: ${{ github.ref_name }}"' + + - name: Build + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:docker/elastic-common-schema-toolchain" + tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest + cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + build-args: | + BRANCH=${{ github.ref_name }} + + push: + needs: [build] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + if: github.ref_name == 'main' + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::068738303278:role/PushPullElasticImages + aws-region: us-east-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + # Setting up Docker Buildx with docker-container driver is required + # at the moment to be able to use a subdirectory with Git context + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build + uses: docker/build-push-action@v5 + with: + context: "{{defaultContext}}:docker/elastic-common-schema-toolchain" + push: true + tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest + cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + build-args: | + BRANCH=${{ github.ref_name }} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..ed8d7d91cc --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:latest +ARG ECS_VERSION="8.11.0" +ARG BRANCH=main +RUN mkdir /ecs && apt update && apt install -y git python3-pip && apt clean +WORKDIR /ecs +RUN git clone https://github.com/huntresslabs/ecs . && git checkout ${BRANCH} && python3 -m pip install --break-system-packages -r scripts/requirements.txt +COPY scripts/entry_point.sh entry_point.sh +ENTRYPOINT ["/bin/bash", "/ecs/entry_point.sh"] diff --git a/docker/scripts/entry_point.sh b/docker/scripts/entry_point.sh new file mode 100644 index 0000000000..02585abdfd --- /dev/null +++ b/docker/scripts/entry_point.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +python3 scripts/generator.py --subset /data_stream/subset --out /data_stream --include /include /data_stream/include +code=$? +if [ $code -ne 0 ]; then + exit $code +fi + +# Moving this functionality into the ECS tool +# for yaml_file in $(find {"/include","/data_stream/include"} -name '*.yml' -type f); do +# file_name="${yaml_file##*/}" +# if [ "$(yq '.0 | has("settings")' $yaml_file)" == "true" ]; then +# out_file="/data_stream/generated/elasticsearch/composable/component/${file_name%.yml}.json" +# echo "Adding settings from ${file_name} to ${out_file##*/}" +# yq '.0.settings' -o json $yaml_file | jq '.tmp.template.settings = .' | jq '.tmp' > /tmp/settings.json +# jq -s '.[0] * .[1]' $out_file /tmp/settings.json > /tmp/combined.json +# mv /tmp/combined.json $out_file +# else +# echo "$file_name does NOT have settings" +# fi +# done +echo "Opening permissions" +chmod -R 'u=rwX,g=rwX,o=rwX' "/data_stream/generated" From 5678a9993a69d40d59f33ae7106b25e18f2367b5 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:24:36 -0400 Subject: [PATCH 35/46] Make building more restrictive --- .github/workflows/build_elastic_common_schema_toolchain.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index 8f32422e0e..77e477477d 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -1,5 +1,9 @@ name: Build ECS Toolchain Image -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: jobs: build: From eb6d21759185a7e7cfe09e8a80014a1f3e45f792 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:48:58 -0400 Subject: [PATCH 36/46] Simplify build workflow --- .../build_elastic_common_schema_toolchain.yml | 36 +++---------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index 77e477477d..c77b66d80e 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -32,8 +32,9 @@ jobs: run: 'echo "Current branch: ${{ github.ref_name }}"' - name: Build + if: github.ref_name != 'main' uses: docker/build-push-action@v5 - with: + with: &buildArgs context: "{{defaultContext}}:docker/elastic-common-schema-toolchain" tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache @@ -41,36 +42,9 @@ jobs: build-args: | BRANCH=${{ github.ref_name }} - push: - needs: [build] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - if: github.ref_name == 'main' - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::068738303278:role/PushPullElasticImages - aws-region: us-east-1 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - # Setting up Docker Buildx with docker-container driver is required - # at the moment to be able to use a subdirectory with Git context - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build + - name: Build and Push + if: github.ref_name == 'main' uses: docker/build-push-action@v5 with: - context: "{{defaultContext}}:docker/elastic-common-schema-toolchain" + <<: *buildArgs push: true - tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest - cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache - cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache - build-args: | - BRANCH=${{ github.ref_name }} From 50898bb515585360ca10d0c98f915d051faeaaba Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:43:22 -0400 Subject: [PATCH 37/46] Update tagging strategy --- .../build_elastic_common_schema_toolchain.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index c77b66d80e..459a9d564f 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -31,20 +31,22 @@ jobs: - name: Debug run: 'echo "Current branch: ${{ github.ref_name }}"' - - name: Build - if: github.ref_name != 'main' + # Always push with the branch name, this allows for external testing + - name: Build and Push uses: docker/build-push-action@v5 with: &buildArgs - context: "{{defaultContext}}:docker/elastic-common-schema-toolchain" - tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest + context: "{{defaultContext}}:docker" + tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:${{ github.ref_name }} cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + push: true build-args: | BRANCH=${{ github.ref_name }} - - name: Build and Push + # Once it's in main, we want to update to the latest stable version + - name: Push Latest Tag if: github.ref_name == 'main' uses: docker/build-push-action@v5 with: <<: *buildArgs - push: true + tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest From 1627f7ef99336f5dfefa6cb647e47b1a9bbdb2a0 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:45:02 -0400 Subject: [PATCH 38/46] Removing unused variable --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ed8d7d91cc..59a3965c5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,4 @@ FROM ubuntu:latest -ARG ECS_VERSION="8.11.0" ARG BRANCH=main RUN mkdir /ecs && apt update && apt install -y git python3-pip && apt clean WORKDIR /ecs From e3776ad440a61cabf1c9f6e7ff20a1d557dbc06a Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:53:10 -0400 Subject: [PATCH 39/46] Kick? --- .github/workflows/build_elastic_common_schema_toolchain.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index 459a9d564f..08d35ba25b 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -4,6 +4,8 @@ on: branches: - main pull_request: + branches: + - main jobs: build: @@ -28,9 +30,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Debug - run: 'echo "Current branch: ${{ github.ref_name }}"' - # Always push with the branch name, this allows for external testing - name: Build and Push uses: docker/build-push-action@v5 From 5fcc445777877ddb3604df28bef682f75fa75b11 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:54:45 -0400 Subject: [PATCH 40/46] =?UTF-8?q?Anchors=20aren't=20supported=20?= =?UTF-8?q?=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/build_elastic_common_schema_toolchain.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index 08d35ba25b..9f50fdaea1 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -33,7 +33,7 @@ jobs: # Always push with the branch name, this allows for external testing - name: Build and Push uses: docker/build-push-action@v5 - with: &buildArgs + with: context: "{{defaultContext}}:docker" tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:${{ github.ref_name }} cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache @@ -47,5 +47,10 @@ jobs: if: github.ref_name == 'main' uses: docker/build-push-action@v5 with: - <<: *buildArgs + context: "{{defaultContext}}:docker" tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:latest + cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache + push: true + build-args: | + BRANCH=${{ github.ref_name }} From ea9a35d40a8ce0d9a9d2839d9424a95a11b0f61c Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:58:03 -0400 Subject: [PATCH 41/46] Fix role name --- .github/workflows/build_elastic_common_schema_toolchain.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index 9f50fdaea1..c2916af587 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -17,7 +17,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::068738303278:role/PushPullElasticImages + role-to-assume: arn:aws:iam::068738303278:role/PushPullECSToolchainImage aws-region: us-east-1 - name: Login to Amazon ECR From bb823a634467c52dafb004de467138f0cf9be20e Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:02:58 -0400 Subject: [PATCH 42/46] Test branch name --- .../workflows/build_elastic_common_schema_toolchain.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index c2916af587..ae943f167a 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -30,17 +30,20 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Debug + run: 'echo Branch name: "${{ github.head_ref || github.ref_name }}"' + # Always push with the branch name, this allows for external testing - name: Build and Push uses: docker/build-push-action@v5 with: context: "{{defaultContext}}:docker" - tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:${{ github.ref_name }} + tags: 068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:${{ github.head_ref || github.ref_name }} cache-from: type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache cache-to: mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref=068738303278.dkr.ecr.us-east-1.amazonaws.com/elastic-common-schema-toolchain:cache push: true build-args: | - BRANCH=${{ github.ref_name }} + BRANCH=${{ github.head_ref || github.ref_name }} # Once it's in main, we want to update to the latest stable version - name: Push Latest Tag From a46fcd3a8791be0597913737a3ff4c90e98e9c33 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:30:15 -0400 Subject: [PATCH 43/46] Remove extra default update (#3) * Remove extra default update * Fix role name --- .github/workflows/build_elastic_common_schema_toolchain.yml | 2 +- scripts/schema/loader.py | 1 - scripts/tests/unit/test_schema_loader.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_elastic_common_schema_toolchain.yml b/.github/workflows/build_elastic_common_schema_toolchain.yml index ae943f167a..9bdd64540c 100644 --- a/.github/workflows/build_elastic_common_schema_toolchain.yml +++ b/.github/workflows/build_elastic_common_schema_toolchain.yml @@ -17,7 +17,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::068738303278:role/PushPullECSToolchainImage + role-to-assume: arn:aws:iam::068738303278:role/GithubECSRepoPolicy aws-region: us-east-1 - name: Login to Amazon ECR diff --git a/scripts/schema/loader.py b/scripts/schema/loader.py index 031aed9566..dd89ef6634 100644 --- a/scripts/schema/loader.py +++ b/scripts/schema/loader.py @@ -307,7 +307,6 @@ def merge_fields( asd["reusable"].setdefault("expected", []) asd["reusable"]["expected"].extend(bsd["reusable"]["expected"]) bsd.pop("reusable") - asd.setdefault("settings", {}) if "settings" in bsd: asd.setdefault("settings", {}) asd["settings"] = merge_fields(asd["settings"], bsd["settings"]) diff --git a/scripts/tests/unit/test_schema_loader.py b/scripts/tests/unit/test_schema_loader.py index 175f1f8678..30a9d16ff7 100644 --- a/scripts/tests/unit/test_schema_loader.py +++ b/scripts/tests/unit/test_schema_loader.py @@ -578,7 +578,6 @@ def test_merge_array_attributes(self): "top_level": True, "expected": ["normal.location", "a_new.location"], }, - "settings": {}, }, "field_details": {"name": "foo", "type": "group"}, "fields": { From 58e91358f6a4f8124276182e38a1db2c4b7b1a4e Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:11:03 -0400 Subject: [PATCH 44/46] Add support for a top-level type (#4) * Add support for a top-level type * Actually, don't need to be all the complicated --- schemas/README.md | 2 +- scripts/generators/es_template.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/schemas/README.md b/schemas/README.md index bb9e9fce3e..652ec611ff 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -29,7 +29,7 @@ Optional field set attributes: - group (default 2): To sort field sets against one another. For example the "base" field set has group=1 and is the first listed in the documentation. All others have group=2 and are therefore after "base" (sorted alphabetically). -- type (ignored): at this level, should always be `group` +- type: The fieldset type. Can be 'nested' or 'object'. - reusable (optional): Used to identify which field sets are expected to be reused in multiple places. See "Field set reuse" for details. - short_override: Used to override the top-level fieldset's short description when nesting. diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 13570346fc..5d675d4f78 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -79,6 +79,8 @@ def all_component_templates( name_parts = flat_name.split(".") dict_add_nested(field_mappings, name_parts, entry_for(field)) fieldset_settings = fieldset.get("settings", None) + if fieldset.get("type", "group") in ["object", "nested"]: + field_mappings["type"] = fieldset.get("type") save_component_template( fieldset_name, From 81c2e2114cb9a41ea3441a1ad2c3f9de5cdac7f9 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:42:57 -0400 Subject: [PATCH 45/46] Type needs to be nested within the field name (#5) --- scripts/generators/es_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 5d675d4f78..33352eea3d 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -80,7 +80,7 @@ def all_component_templates( dict_add_nested(field_mappings, name_parts, entry_for(field)) fieldset_settings = fieldset.get("settings", None) if fieldset.get("type", "group") in ["object", "nested"]: - field_mappings["type"] = fieldset.get("type") + field_mappings[fieldset_name]["type"] = fieldset.get("type") save_component_template( fieldset_name, From d6b4a624a59deb536e4760615748bf8495206f10 Mon Sep 17 00:00:00 2001 From: Luke Snyder <709836+lksnyder0@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:10:20 -0400 Subject: [PATCH 46/46] Add documention for parameters field (#6) * Add undocumented field argument * Remove the PR template --- .github/PULL_REQUEST_TEMPLATE.md | 14 -------------- schemas/README.md | 1 + 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 5a03ece333..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,14 +0,0 @@ - - -- Have you signed the [contributor license agreement](https://www.elastic.co/contributor-agreement)? -- Have you followed the [contributor guidelines](https://github.com/elastic/ecs/blob/main/CONTRIBUTING.md)? -- For proposing substantial changes or additions to the schema, have you reviewed the [RFC process](https://github.com/elastic/ecs/blob/main/rfcs/README.md)? -- If submitting code/script changes, have you verified all tests pass locally using `make test`? -- If submitting schema/fields updates, have you generated new artifacts by running `make` and committed those changes? -- Is your pull request against main? Unless there is a good reason otherwise, we prefer pull requests against main and will backport as needed. -- Have you added an entry to the [CHANGELOG.next.md](https://github.com/elastic/ecs/blob/main/CHANGELOG.next.md)? diff --git a/schemas/README.md b/schemas/README.md index 652ec611ff..71320b667f 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -170,6 +170,7 @@ Supported keys to describe fields - normalize: Normalization steps that should be applied at ingestion time. Supported values: - array: the content of the field should be an array (even when there's only one value). - beta (optional): Adds a beta marker for the field to the description. The text provided in this attribute is used as content of the beta marker in the documentation. Note that when a whole field set is marked as beta, it is not necessary nor recommended to mark all fields in the field set as beta. Beta notices should not have newlines. +- parameters (optional): An object of arbitrary parameters to include in the field settings. Supported keys to describe expected values for a field