From c899ce8eb78cd07535a6265837143185344ed426 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 21 Mar 2019 08:39:14 -0400 Subject: [PATCH] Backport #347 and #390 to 1.0: Generator for asciidoc representation of all ECS fields (#347) (#392) This code generates both the index page for the fields section, and the detailed page for each fieldset. Also in this PR: - If `short` not set, copy `definition` to it - If `short` contains \n, fail. Beginning of an ECS linter. - Beef up the local file imports in tests. - autopep8 ignore E402, in order to leave local imports alone - Create generators/ecs_helpers.py, to start reusing functionality beween generators --- CHANGELOG.next.md | 1 + Makefile | 2 +- docs/field-details.asciidoc | 2704 ++++++++++++++++++++++ docs/fields-gen.asciidoc | 429 ---- docs/fields.asciidoc | 102 +- fields.yml | 1 + generated/ecs/fields_flat.yml | 283 ++- generated/ecs/fields_nested.yml | 308 ++- schemas/event.yml | 1 + scripts/generator.py | 2 + scripts/generators/asciidoc_fields.py | 246 ++ scripts/generators/beats.py | 38 +- scripts/generators/ecs_helpers.py | 59 + scripts/generators/es_template.py | 14 +- scripts/generators/intermediate_files.py | 9 +- scripts/schema_reader.py | 21 +- scripts/tests/test_ecs_helpers.py | 61 + scripts/tests/test_ecs_spec.py | 6 + scripts/tests/test_es_template.py | 19 +- scripts/tests/test_schema_reader.py | 13 +- 20 files changed, 3769 insertions(+), 550 deletions(-) create mode 100644 docs/field-details.asciidoc delete mode 100644 docs/fields-gen.asciidoc create mode 100644 scripts/generators/asciidoc_fields.py create mode 100644 scripts/generators/ecs_helpers.py create mode 100644 scripts/tests/test_ecs_helpers.py diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 19de990ec6..7282b049fd 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -11,6 +11,7 @@ * New generator that supports reusable fields, for files based on ECS. It generates schema.csv, Elasticsearch 6 and 7 templates, and field documentation for the main website. #336 +* Generator for the asciidoc rendering of field definitions. #347 * Generator for the Beats fields.ecs.yml file. #379 ### Improvements diff --git a/Makefile b/Makefile index 5082bf0867..49a0ab74a8 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ fields: # Format code and files in the repo. .PHONY: fmt fmt: ve - $(FIND) -name '*.py' -exec build/ve/bin/autopep8 --in-place --max-line-length 120 {} \; + $(FIND) -name '*.py' -exec build/ve/bin/autopep8 --ignore E402 --in-place --max-line-length 120 {} \; go get golang.org/x/tools/cmd/goimports goimports -w -l -local github.com/elastic $(shell $(FIND) -name '*.go') diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc new file mode 100644 index 0000000000..ae8b0162f8 --- /dev/null +++ b/docs/field-details.asciidoc @@ -0,0 +1,2704 @@ + +[[ecs-base]] +=== Base Fields + +The `base` field set contains all fields which are on the top level. These fields are common across all types of events. + +==== Base Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| @timestamp +| Date/time when the event originated. +This is the date/time extracted from the event, typically representing when the event was generated by the source. +If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. +Required field for all events. + +type: date + +example: `2016-05-23T08:05:34.853Z` + +| core + +// =============================================================== + +| labels +| Custom key/value pairs. +Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. +Example: `docker` and `k8s` labels. + +type: object + +example: `{'application': 'foo-bar', 'env': 'production'}` + +| core + +// =============================================================== + +| message +| For log events the message field contains the log message, optimized for viewing in a log viewer. +For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. +If multiple messages exist, they can be combined into one message. + +type: text + +example: `Hello World` + +| core + +// =============================================================== + +| tags +| List of keywords used to tag each event. + +type: keyword + +example: `["production", "env2"]` + +| core + +// =============================================================== + +|===== + +[[ecs-agent]] +=== Agent Fields + +The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. +Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. + +==== Agent Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| agent.ephemeral_id +| Ephemeral identifier of this agent (if one exists). +This id normally changes across restarts, but `agent.id` does not. + +type: keyword + +example: `8a4f500f` + +| extended + +// =============================================================== + +| agent.id +| Unique identifier of this agent (if one exists). +Example: For Beats this would be beat.id. + +type: keyword + +example: `8a4f500d` + +| core + +// =============================================================== + +| agent.name +| Custom name of the agent. +This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. +If no name is given, the name is often left empty. + +type: keyword + +example: `foo` + +| core + +// =============================================================== + +| agent.type +| Type of the agent. +The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. + +type: keyword + +example: `filebeat` + +| core + +// =============================================================== + +| agent.version +| Version of the agent. + +type: keyword + +example: `6.0.0-rc2` + +| core + +// =============================================================== + +|===== + +[[ecs-client]] +=== Client Fields + +A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. + +==== Client Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| client.address +| Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. + +type: keyword + + + +| extended + +// =============================================================== + +| client.bytes +| Bytes sent from the client to the server. + +type: long + +example: `184` + +| core + +// =============================================================== + +| client.domain +| Client domain. + +type: keyword + + + +| core + +// =============================================================== + +| client.ip +| IP address of the client. +Can be one or multiple IPv4 or IPv6 addresses. + +type: ip + + + +| core + +// =============================================================== + +| client.mac +| MAC address of the client. + +type: keyword + + + +| core + +// =============================================================== + +| client.packets +| Packets sent from the client to the server. + +type: long + +example: `12` + +| core + +// =============================================================== + +| client.port +| Port of the client. + +type: long + + + +| core + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-client-nestings]] +===== Field sets that can be nested under Client + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[client.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-user.html[client.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-cloud]] +=== Cloud Fields + +Fields related to the cloud or infrastructure the events are coming from. + +==== Cloud Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| cloud.account.id +| The cloud account or organization id used to identify different entities in a multi-tenant environment. +Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. + +type: keyword + +example: `666777888999` + +| extended + +// =============================================================== + +| cloud.availability_zone +| Availability zone in which this host is running. + +type: keyword + +example: `us-east-1c` + +| extended + +// =============================================================== + +| cloud.instance.id +| Instance ID of the host machine. + +type: keyword + +example: `i-1234567890abcdef0` + +| extended + +// =============================================================== + +| cloud.instance.name +| Instance name of the host machine. + +type: keyword + + + +| extended + +// =============================================================== + +| cloud.machine.type +| Machine type of the host machine. + +type: keyword + +example: `t2.medium` + +| extended + +// =============================================================== + +| cloud.provider +| Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + +type: keyword + +example: `aws` + +| extended + +// =============================================================== + +| cloud.region +| Region in which this host is running. + +type: keyword + +example: `us-east-1` + +| extended + +// =============================================================== + +|===== + +[[ecs-container]] +=== Container Fields + +Container fields are used for meta information about the specific container that is the source of information. +These fields help correlate data based containers from any runtime. + +==== Container Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| container.id +| Unique container id. + +type: keyword + + + +| core + +// =============================================================== + +| container.image.name +| Name of the image the container was built on. + +type: keyword + + + +| extended + +// =============================================================== + +| container.image.tag +| Container image tag. + +type: keyword + + + +| extended + +// =============================================================== + +| container.labels +| Image labels. + +type: object + + + +| extended + +// =============================================================== + +| container.name +| Container name. + +type: keyword + + + +| extended + +// =============================================================== + +| container.runtime +| Runtime managing this container. + +type: keyword + +example: `docker` + +| extended + +// =============================================================== + +|===== + +[[ecs-destination]] +=== Destination Fields + +Destination fields describe details about the destination of a packet/event. +Destination fields are usually populated in conjunction with source fields. + +==== Destination Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| destination.address +| Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. + +type: keyword + + + +| extended + +// =============================================================== + +| destination.bytes +| Bytes sent from the destination to the source. + +type: long + +example: `184` + +| core + +// =============================================================== + +| destination.domain +| Destination domain. + +type: keyword + + + +| core + +// =============================================================== + +| destination.ip +| IP address of the destination. +Can be one or multiple IPv4 or IPv6 addresses. + +type: ip + + + +| core + +// =============================================================== + +| destination.mac +| MAC address of the destination. + +type: keyword + + + +| core + +// =============================================================== + +| destination.packets +| Packets sent from the destination to the source. + +type: long + +example: `12` + +| core + +// =============================================================== + +| destination.port +| Port of the destination. + +type: long + + + +| core + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-destination-nestings]] +===== Field sets that can be nested under Destination + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[destination.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-user.html[destination.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-ecs]] +=== ECS Fields + +Meta-information specific to ECS. + +==== ECS Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| ecs.version +| ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. +When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. + +type: keyword + +example: `1.0.0` + +| core + +// =============================================================== + +|===== + +[[ecs-error]] +=== Error Fields + +These fields can represent errors of any kind. +Use them for errors that happen while fetching events or in cases where the event itself contains an error. + +==== Error Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| error.code +| Error code describing the error. + +type: keyword + + + +| core + +// =============================================================== + +| error.id +| Unique identifier for the error. + +type: keyword + + + +| core + +// =============================================================== + +| error.message +| Error message. + +type: text + + + +| core + +// =============================================================== + +|===== + +[[ecs-event]] +=== Event Fields + +The event fields are used for context information about the log or metric event itself. +A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. + +==== Event Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| event.action +| The action captured by the event. +This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. + +type: keyword + +example: `user-password-change` + +| core + +// =============================================================== + +| event.category +| Event category. +This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. + +type: keyword + +example: `user-management` + +| core + +// =============================================================== + +| event.created +| event.created contains the date/time when the event was first read by an agent, or by your pipeline. +This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. +In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. +In case the two timestamps are identical, @timestamp should be used. + +type: date + + + +| core + +// =============================================================== + +| event.dataset +| Name of the dataset. +The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. + +type: keyword + +example: `stats` + +| core + +// =============================================================== + +| event.duration +| Duration of the event in nanoseconds. +If event.start and event.end are known this value should be the difference between the end and start time. + +type: long + + + +| core + +// =============================================================== + +| event.end +| event.end contains the date when the event ended or when the activity was last observed. + +type: date + + + +| extended + +// =============================================================== + +| event.hash +| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. + +type: keyword + +example: `123456789012345678901234567890ABCD` + +| extended + +// =============================================================== + +| event.id +| Unique ID to describe the event. + +type: keyword + +example: `8a4f500d` + +| core + +// =============================================================== + +| event.kind +| The kind of the event. +This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. + +type: keyword + +example: `state` + +| extended + +// =============================================================== + +| event.module +| Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. + +type: keyword + +example: `mysql` + +| core + +// =============================================================== + +| event.original +| Raw text message of entire event. Used to demonstrate log integrity. +This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. + +type: keyword + +example: `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232` + +| core + +// =============================================================== + +| event.outcome +| The outcome of the event. +If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. + +type: keyword + +example: `success` + +| extended + +// =============================================================== + +| event.risk_score +| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. + +type: float + + + +| core + +// =============================================================== + +| event.risk_score_norm +| Normalized risk score or priority of the event, on a scale of 0 to 100. +This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. + +type: float + + + +| extended + +// =============================================================== + +| event.severity +| Severity describes the original severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. + +type: long + +example: `7` + +| core + +// =============================================================== + +| event.start +| event.start contains the date when the event started or when the activity was first observed. + +type: date + + + +| extended + +// =============================================================== + +| event.timezone +| This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. +Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). + +type: keyword + + + +| extended + +// =============================================================== + +| event.type +| Reserved for future usage. +Please avoid using this field for user data. + +type: keyword + + + +| core + +// =============================================================== + +|===== + +[[ecs-file]] +=== File Fields + +A file is defined as a set of information that has been created on, or has existed on a filesystem. +File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. + +==== File Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| file.ctime +| Last time file metadata changed. + +type: date + + + +| extended + +// =============================================================== + +| file.device +| Device that is the source of the file. + +type: keyword + + + +| extended + +// =============================================================== + +| file.extension +| File extension. +This should allow easy filtering by file extensions. + +type: keyword + +example: `png` + +| extended + +// =============================================================== + +| file.gid +| Primary group ID (GID) of the file. + +type: keyword + + + +| extended + +// =============================================================== + +| file.group +| Primary group name of the file. + +type: keyword + + + +| extended + +// =============================================================== + +| file.inode +| Inode representing the file in the filesystem. + +type: keyword + + + +| extended + +// =============================================================== + +| file.mode +| Mode of the file in octal representation. + +type: keyword + +example: `416` + +| extended + +// =============================================================== + +| file.mtime +| Last time file content was modified. + +type: date + + + +| extended + +// =============================================================== + +| file.owner +| File owner's username. + +type: keyword + + + +| extended + +// =============================================================== + +| file.path +| Path to the file. + +type: keyword + + + +| extended + +// =============================================================== + +| file.size +| File size in bytes (field is only added when `type` is `file`). + +type: long + + + +| extended + +// =============================================================== + +| file.target_path +| Target path for symlinks. + +type: keyword + + + +| extended + +// =============================================================== + +| file.type +| File type (file, dir, or symlink). + +type: keyword + + + +| extended + +// =============================================================== + +| file.uid +| The user ID (UID) or security identifier (SID) of the file owner. + +type: keyword + + + +| extended + +// =============================================================== + +|===== + +[[ecs-geo]] +=== Geo Fields + +Geo fields can carry data about a specific location related to an event. +This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. + +==== Geo Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| geo.city_name +| City name. + +type: keyword + +example: `Montreal` + +| core + +// =============================================================== + +| geo.continent_name +| Name of the continent. + +type: keyword + +example: `North America` + +| core + +// =============================================================== + +| geo.country_iso_code +| Country ISO code. + +type: keyword + +example: `CA` + +| core + +// =============================================================== + +| geo.country_name +| Country name. + +type: keyword + +example: `Canada` + +| core + +// =============================================================== + +| geo.location +| Longitude and latitude. + +type: geo_point + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +| core + +// =============================================================== + +| geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + +type: keyword + +example: `boston-dc` + +| extended + +// =============================================================== + +| geo.region_iso_code +| Region ISO code. + +type: keyword + +example: `CA-QC` + +| core + +// =============================================================== + +| geo.region_name +| Region name. + +type: keyword + +example: `Quebec` + +| core + +// =============================================================== + +|===== + +==== Field Reuse + +The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. + +Note also that the `geo` fields are not expected to be used directly at the top level. + + + + +[[ecs-group]] +=== Group Fields + +The group fields are meant to represent groups that are relevant to the event. + +==== Group Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| group.id +| Unique identifier for the group on the system/platform. + +type: keyword + + + +| extended + +// =============================================================== + +| group.name +| Name of the group. + +type: keyword + + + +| extended + +// =============================================================== + +|===== + +==== Field Reuse + +The `group` fields are expected to be nested at: `user.group`. + +Note also that the `group` fields may be used directly at the top level. + + + + +[[ecs-host]] +=== Host Fields + +A host is defined as a general computing instance. +ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. + +==== Host Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| host.architecture +| Operating system architecture. + +type: keyword + +example: `x86_64` + +| core + +// =============================================================== + +| host.hostname +| Hostname of the host. +It normally contains what the `hostname` command returns on the host machine. + +type: keyword + + + +| core + +// =============================================================== + +| host.id +| Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. + +type: keyword + + + +| core + +// =============================================================== + +| host.ip +| Host ip address. + +type: ip + + + +| core + +// =============================================================== + +| host.mac +| Host mac address. + +type: keyword + + + +| core + +// =============================================================== + +| host.name +| Name of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + +type: keyword + + + +| core + +// =============================================================== + +| host.type +| Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. + +type: keyword + + + +| core + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-host-nestings]] +===== Field sets that can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-http]] +=== HTTP Fields + +Fields related to HTTP activity. Use the `url` field set to store the url of the request. + +==== HTTP Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| http.request.body.bytes +| Size in bytes of the request body. + +type: long + +example: `887` + +| extended + +// =============================================================== + +| http.request.body.content +| The full HTTP request body. + +type: keyword + +example: `Hello world` + +| extended + +// =============================================================== + +| http.request.bytes +| Total size in bytes of the request (body and headers). + +type: long + +example: `1437` + +| extended + +// =============================================================== + +| http.request.method +| HTTP request method. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `get, post, put` + +| extended + +// =============================================================== + +| http.request.referrer +| Referrer for this HTTP request. + +type: keyword + +example: `https://blog.example.com/` + +| extended + +// =============================================================== + +| http.response.body.bytes +| Size in bytes of the response body. + +type: long + +example: `887` + +| extended + +// =============================================================== + +| http.response.body.content +| The full HTTP response body. + +type: keyword + +example: `Hello world` + +| extended + +// =============================================================== + +| http.response.bytes +| Total size in bytes of the response (body and headers). + +type: long + +example: `1437` + +| extended + +// =============================================================== + +| http.response.status_code +| HTTP response status code. + +type: long + +example: `404` + +| extended + +// =============================================================== + +| http.version +| HTTP version. + +type: keyword + +example: `1.1` + +| extended + +// =============================================================== + +|===== + +[[ecs-log]] +=== Log Fields + +Fields which are specific to log events. + +==== Log Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| log.level +| Original log level of the log event. +Some examples are `warn`, `error`, `i`. + +type: keyword + +example: `err` + +| core + +// =============================================================== + +| log.original +| This is the original log message and contains the full log message before splitting it up in multiple parts. +In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. +This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. + +type: keyword + +example: `Sep 19 08:26:10 localhost My log` + +| core + +// =============================================================== + +|===== + +[[ecs-network]] +=== Network Fields + +The network is defined as the communication path over which a host or network event happens. +The network.* fields should be populated with details about the network activity associated with an event. + +==== Network Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| network.application +| A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `aim` + +| extended + +// =============================================================== + +| network.bytes +| Total bytes transferred in both directions. +If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. + +type: long + +example: `368` + +| core + +// =============================================================== + +| network.community_id +| A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. +Learn more at https://github.com/corelight/community-id-spec. + +type: keyword + +example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` + +| extended + +// =============================================================== + +| network.direction +| Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * internal + * external + * unknown + +When mapping events from a host-based monitoring context, populate this field from the host's point of view. +When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. + +type: keyword + +example: `inbound` + +| core + +// =============================================================== + +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. + +type: ip + +example: `192.1.1.2` + +| core + +// =============================================================== + +| network.iana_number +| IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. + +type: keyword + +example: `6` + +| extended + +// =============================================================== + +| network.name +| Name given by operators to sections of their network. + +type: keyword + +example: `Guest Wifi` + +| extended + +// =============================================================== + +| network.packets +| Total packets transferred in both directions. +If `source.packets` and `destination.packets` are known, `network.packets` is their sum. + +type: long + +example: `24` + +| core + +// =============================================================== + +| network.protocol +| L7 Network protocol name. ex. http, lumberjack, transport protocol. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `http` + +| core + +// =============================================================== + +| network.transport +| Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `tcp` + +| core + +// =============================================================== + +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `ipv4` + +| core + +// =============================================================== + +|===== + +[[ecs-observer]] +=== Observer Fields + +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. +This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. + +==== Observer Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| observer.hostname +| Hostname of the observer. + +type: keyword + + + +| core + +// =============================================================== + +| observer.ip +| IP address of the observer. + +type: ip + + + +| core + +// =============================================================== + +| observer.mac +| MAC address of the observer + +type: keyword + + + +| core + +// =============================================================== + +| observer.serial_number +| Observer serial number. + +type: keyword + + + +| extended + +// =============================================================== + +| observer.type +| The type of the observer the data is coming from. +There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. + +type: keyword + +example: `firewall` + +| core + +// =============================================================== + +| observer.vendor +| observer vendor information. + +type: keyword + + + +| core + +// =============================================================== + +| observer.version +| Observer version. + +type: keyword + + + +| core + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-observer-nestings]] +===== Field sets that can be nested under Observer + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[observer.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-os.html[observer.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + + +|===== + +[[ecs-organization]] +=== Organization Fields + +The organization fields enrich data with information about the company or entity the data is associated with. +These fields help you arrange or filter data stored in an index by one or multiple organizations. + +==== Organization Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| organization.id +| Unique identifier for the organization. + +type: keyword + + + +| extended + +// =============================================================== + +| organization.name +| Organization name. + +type: keyword + + + +| extended + +// =============================================================== + +|===== + +[[ecs-os]] +=== Operating System Fields + +The OS fields contain information about the operating system. + +==== Operating System Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| os.family +| OS family (such as redhat, debian, freebsd, windows). + +type: keyword + +example: `debian` + +| extended + +// =============================================================== + +| os.full +| Operating system name, including the version or code name. + +type: keyword + +example: `Mac OS Mojave` + +| extended + +// =============================================================== + +| os.kernel +| Operating system kernel version as a raw string. + +type: keyword + +example: `4.4.0-112-generic` + +| extended + +// =============================================================== + +| os.name +| Operating system name, without the version. + +type: keyword + +example: `Mac OS X` + +| extended + +// =============================================================== + +| os.platform +| Operating system platform (such centos, ubuntu, windows). + +type: keyword + +example: `darwin` + +| extended + +// =============================================================== + +| os.version +| Operating system version as a raw string. + +type: keyword + +example: `10.14.1` + +| extended + +// =============================================================== + +|===== + +==== Field Reuse + +The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. + +Note also that the `os` fields are not expected to be used directly at the top level. + + + + +[[ecs-process]] +=== Process Fields + +These fields contain information about a process. +These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. + +==== Process Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| process.args +| Array of process arguments. +May be filtered to protect sensitive information. + +type: keyword + +example: `['ssh', '-l', 'user', '10.0.0.16']` + +| extended + +// =============================================================== + +| process.executable +| Absolute path to the process executable. + +type: keyword + +example: `/usr/bin/ssh` + +| extended + +// =============================================================== + +| process.name +| Process name. +Sometimes called program name or similar. + +type: keyword + +example: `ssh` + +| extended + +// =============================================================== + +| process.pid +| Process id. + +type: long + + + +| core + +// =============================================================== + +| process.ppid +| Process parent id. + +type: long + + + +| extended + +// =============================================================== + +| process.start +| The time the process started. + +type: date + +example: `2016-05-23T08:05:34.853Z` + +| extended + +// =============================================================== + +| process.thread.id +| Thread ID. + +type: long + +example: `4242` + +| extended + +// =============================================================== + +| process.title +| Process title. +The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. + +type: keyword + + + +| extended + +// =============================================================== + +| process.working_directory +| The working directory of the process. + +type: keyword + +example: `/home/alice` + +| extended + +// =============================================================== + +|===== + +[[ecs-related]] +=== Related Fields + +This field set is meant to facilitate pivoting around a piece of data. +Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. +A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. + +==== Related Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| related.ip +| All of the IPs seen on your event. + +type: ip + + + +| extended + +// =============================================================== + +|===== + +[[ecs-server]] +=== Server Fields + +A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. + +==== Server Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| server.address +| Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. + +type: keyword + + + +| extended + +// =============================================================== + +| server.bytes +| Bytes sent from the server to the client. + +type: long + +example: `184` + +| core + +// =============================================================== + +| server.domain +| Server domain. + +type: keyword + + + +| core + +// =============================================================== + +| server.ip +| IP address of the server. +Can be one or multiple IPv4 or IPv6 addresses. + +type: ip + + + +| core + +// =============================================================== + +| server.mac +| MAC address of the server. + +type: keyword + + + +| core + +// =============================================================== + +| server.packets +| Packets sent from the server to the client. + +type: long + +example: `12` + +| core + +// =============================================================== + +| server.port +| Port of the server. + +type: long + + + +| core + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-server-nestings]] +===== Field sets that can be nested under Server + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[server.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-user.html[server.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-service]] +=== Service Fields + +The service fields describe the service for or from which the data was collected. +These fields help you find and correlate logs for a specific service and version. + +==== Service Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| service.ephemeral_id +| Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. + +type: keyword + +example: `8a4f500f` + +| extended + +// =============================================================== + +| service.id +| Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. + +type: keyword + +example: `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` + +| core + +// =============================================================== + +| service.name +| Name of the service data is collected from. +The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. +Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. +In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. + +type: keyword + +example: `elasticsearch-metrics` + +| core + +// =============================================================== + +| service.state +| Current state of the service. + +type: keyword + + + +| core + +// =============================================================== + +| service.type +| The type of the service data is collected from. +The type can be used to group and correlate logs and metrics from one service type. +Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. + +type: keyword + +example: `elasticsearch` + +| core + +// =============================================================== + +| service.version +| Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. + +type: keyword + +example: `3.2.4` + +| core + +// =============================================================== + +|===== + +[[ecs-source]] +=== Source Fields + +Source fields describe details about the source of a packet/event. +Source fields are usually populated in conjunction with destination fields. + +==== Source Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| source.address +| Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. + +type: keyword + + + +| extended + +// =============================================================== + +| source.bytes +| Bytes sent from the source to the destination. + +type: long + +example: `184` + +| core + +// =============================================================== + +| source.domain +| Source domain. + +type: keyword + + + +| core + +// =============================================================== + +| source.ip +| IP address of the source. +Can be one or multiple IPv4 or IPv6 addresses. + +type: ip + + + +| core + +// =============================================================== + +| source.mac +| MAC address of the source. + +type: keyword + + + +| core + +// =============================================================== + +| source.packets +| Packets sent from the source to the destination. + +type: long + +example: `12` + +| core + +// =============================================================== + +| source.port +| Port of the source. + +type: long + + + +| core + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-source-nestings]] +===== Field sets that can be nested under Source + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[source.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-user.html[source.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-url]] +=== URL Fields + +URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. + +==== URL Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| url.domain +| Domain of the url, such as "www.elastic.co". +In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. + +type: keyword + +example: `www.elastic.co` + +| extended + +// =============================================================== + +| url.fragment +| Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. + +type: keyword + + + +| extended + +// =============================================================== + +| url.full +| If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. + +type: keyword + +example: `https://www.elastic.co:443/search?q=elasticsearch#top` + +| extended + +// =============================================================== + +| url.original +| Unmodified original url as seen in the event source. +Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. +This field is meant to represent the URL as it was observed, complete or not. + +type: keyword + +example: `https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch` + +| extended + +// =============================================================== + +| url.password +| Password of the request. + +type: keyword + + + +| extended + +// =============================================================== + +| url.path +| Path of the request, such as "/search". + +type: keyword + + + +| extended + +// =============================================================== + +| url.port +| Port of the request, such as 443. + +type: long + +example: `443` + +| extended + +// =============================================================== + +| url.query +| The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. + +type: keyword + + + +| extended + +// =============================================================== + +| url.scheme +| Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. + +type: keyword + +example: `https` + +| extended + +// =============================================================== + +| url.username +| Username of the request. + +type: keyword + + + +| extended + +// =============================================================== + +|===== + +[[ecs-user]] +=== User Fields + +The user fields describe information about the user that is relevant to the event. +Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. + +==== User Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| user.email +| User email address. + +type: keyword + + + +| extended + +// =============================================================== + +| user.full_name +| User's full name, if available. + +type: keyword + +example: `Albert Einstein` + +| extended + +// =============================================================== + +| user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. + +type: keyword + + + +| extended + +// =============================================================== + +| user.id +| One or multiple unique identifiers of the user. + +type: keyword + + + +| core + +// =============================================================== + +| user.name +| Short name or login of the user. + +type: keyword + +example: `albert` + +| core + +// =============================================================== + +|===== + +==== Field Reuse + +The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. + +Note also that the `user` fields may be used directly at the top level. + + + + +[[ecs-user-nestings]] +===== Field sets that can be nested under User + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-group.html[user.group.*] +| User's group relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-user_agent]] +=== User agent Fields + +The user_agent fields normally come from a browser request. +They often show up in web service logs coming from the parsed user agent string. + +==== User agent Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| user_agent.device.name +| Name of the device. + +type: keyword + +example: `iPhone` + +| extended + +// =============================================================== + +| user_agent.name +| Name of the user agent. + +type: keyword + +example: `Safari` + +| extended + +// =============================================================== + +| user_agent.original +| Unparsed version of the user_agent. + +type: keyword + +example: `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` + +| extended + +// =============================================================== + +| user_agent.version +| Version of the user agent. + +type: keyword + +example: `12.0` + +| extended + +// =============================================================== + +|===== + +==== Field Reuse + + + + +[[ecs-user_agent-nestings]] +===== Field sets that can be nested under User agent + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-os.html[user_agent.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + + +|===== diff --git a/docs/fields-gen.asciidoc b/docs/fields-gen.asciidoc deleted file mode 100644 index 6d33b4febc..0000000000 --- a/docs/fields-gen.asciidoc +++ /dev/null @@ -1,429 +0,0 @@ -[[ecs-base]] -=== Base fields - -The `base` set contains top level fields that are common across all types of events -(such as `@timestamp`, `tags`, `message`, and `labels`). - - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| [[@timestamp]] -| Date/time when the event originated. -For log events this is the date/time when the event was generated, and not when it was read. -Required field for all events. - -type: date - -Example: `2016-05-23T08:05:34.853Z` - -| core - -// =============================================================== - -| tags -| List of keywords used to tag each event. - -type: keyword - -Example:`["production", "env2"]` - -| core - -// =============================================================== - -| labels -| Key/value pairs. -Can be used to add meta information to events. Should not contain nested objects. -All values are stored as keyword. - -type: object - -Examples: `docker` and `k8s` labels. Examples: `{'application': 'foo-bar', 'env': 'production'}` - -| core - -// =============================================================== - -| message -| For log events the message field contains the log message. -In other use cases the message field can be used to concatenate different values -which are then freely searchable. If multiple messages exist, they can be -combined into one message. - -type: text - -Example: `Hello World` - -| core -|===== - - -[[ecs-agent]] -=== Agent fields - -The agent fields contain the data about the agent/client/shipper that created the event. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| agent.version | Version of the agent. | core | keyword | `6.0.0-rc2` -| agent.name | Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty. | core | keyword | `foo` -| agent.type | Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. | core | keyword | `filebeat` -| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | core | keyword | `8a4f500d` -| agent.ephemeral_id | Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not. | extended | keyword | `8a4f500f` -|======================================================================= - -Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it is the agent running in the app/service. The agent information does not change if data is sent through queuing systems like Kafka, Redis, or processing systems such as Logstash or APM Server. - -[[ecs-cloud]] -=== Cloud fields - -Fields related to the cloud or infrastructure the events are coming from. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| cloud.provider | Name of the cloud provider. Example values are ec2, gce, or digitalocean. | extended | keyword | `ec2` -| cloud.availability_zone | Availability zone in which this host is running. | extended | keyword | `us-east-1c` -| cloud.region | Region in which this host is running. | extended | keyword | `us-east-1` -| cloud.instance.id | Instance ID of the host machine. | extended | keyword | `i-1234567890abcdef0` -| cloud.instance.name | Instance name of the host machine. | extended | keyword | -| cloud.machine.type | Machine type of the host machine. | extended | keyword | `t2.medium` -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. - -Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | extended | keyword | `666777888999` -|======================================================================= - -Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on. - -[[ecs-container]] -=== Container fields - -Container fields are used for meta information about the specific container that -is the source of information. These fields help correlate data based containers -from any runtime. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| container.runtime | Runtime managing this container. | extended | keyword | `docker` -| container.id | Unique container id. | core | keyword | -| container.image.name | Name of the image the container was built on. | extended | keyword | -| container.image.tag | Container image tag. | extended | keyword | -| container.name | Container name. | extended | keyword | -| container.labels | Image labels. | extended | object | -|======================================================================= - -[[ecs-destination]] -=== Destination fields - -Destination fields describe details about the destination of a packet/event. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| destination.ip | IP address of the destination. Can be one or multiple IPv4 or IPv6 addresses. | core | ip | -| destination.port | Port of the destination. | core | long | -| destination.mac | MAC address of the destination. | core | keyword | -| destination.domain | Destination domain. | core | keyword | -|======================================================================= - -[[ecs-device]] -=== Device fields - -Device fields are used to provide additional information about the device that -is the source of the information. This could be a firewall, network device, etc. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| device.mac | MAC address of the device | core | keyword | -| device.ip | IP address of the device. | core | ip | -| device.hostname | Hostname of the device. | core | keyword | -| device.vendor | Device vendor information. | core | keyword | -| device.version | Device version. | core | keyword | -| device.serial_number | Device serial number. | extended | keyword | -| device.type | The type of the device the data is coming from. There is no predefined list of device types. Some examples are `endpoint`, `firewall`, `ids`, `ips`, `proxy`. | core | keyword | `firewall` -|======================================================================= - -[[ecs-ecs]] -=== ECS fields - -Meta-information specific to ECS. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| ecs.version | ECS version for this event. `ecs.version` is a required field and must exist in all events. - -Different indices may conform to different ECS versions. -This field helps integrations adjust to the correct schema version for events when you query across multiple indices. -The current version is 1.0.0-beta1 . | core | keyword | `1.0.0-beta1` -|======================================================================= - -// Can we make the version a VARIABLE so that it's always up-to-date? - -[[ecs-error]] -=== Error fields - -These fields can represent errors of any kind. Use them for errors that happen -while fetching events or in cases where the event itself contains an error. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| error.id | Unique identifier for the error. | core | keyword | -| error.message | Error message. | core | text | -| error.code | Error code describing the error. | core | keyword | -|======================================================================= - - -[[ecs-event]] -=== Event fields - -The event fields are used for context information about the data itself. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| event.id | Unique ID to describe the event. | core | keyword | `8a4f500d` -| event.kind | The kind of the event. This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. | extended | keyword | `state` -| event.category | Event category. This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. | core | keyword | `user-management` -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | core | keyword | `user-password-change` -| event.outcome | The outcome of the event. If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. | extended | keyword | `success` -| event.type | Reserved for future usage. Please avoid using this field for user data. | core | keyword | -| event.module | Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash. | core | keyword | `mysql` -| event.dataset | Name of the dataset. The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. | core | keyword | `stats` -| event.severity | Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. | core | long | `7` -| event.original | Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. | core | (not indexed) | `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232` -| event.hash | Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. | extended | keyword | `123456789012345678901234567890ABCD` -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | core | long | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | extended | keyword | -| event.created | event.created contains the date when the event was created. This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. In case the two timestamps are identical, @timestamp should be used. | core | date | -| event.start | event.start contains the date when the event started or when the activity was first observed. | extended | date | -| event.end | event.end contains the date when the event ended or when the activity was last observed. | extended | date | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | core | float | -| event.risk_score_norm | Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. | extended | float | -|======================================================================= - -[[ecs-file]] -=== File fields - -File fields provide details about each file. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| file.path | Path to the file. | extended | keyword | -| file.target_path | Target path for symlinks. | extended | keyword | -| file.extension | File extension. This should allow easy filtering by file extensions. | extended | keyword | `png` -| file.type | File type (file, dir, or symlink). | extended | keyword | -| file.device | Device that is the source of the file. | extended | keyword | -| file.inode | Inode representing the file in the filesystem. | extended | keyword | -| file.uid | The user ID (UID) or security identifier (SID) of the file owner. | extended | keyword | -| file.owner | File owner's username. | extended | keyword | -| file.gid | Primary group ID (GID) of the file. | extended | keyword | -| file.group | Primary group name of the file. | extended | keyword | -| file.mode | Mode of the file in octal representation. | extended | keyword | `416` | -| file.size | File size in bytes (field is only added when `type` is `file`). | extended | long | -| file.mtime | Last time file content was modified. | extended | date | -| file.ctime | Last time file metadata changed. | extended | date | -|======================================================================= - -[[ecs-geo]] -=== Geo fields - -Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. - -The `geo` fields are expected to be nested at: `destination.geo`, `device.geo`, `host.geo`, `source.geo`. - -Note also that the `geo` fields are not expected to be used directly at the top level. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| geo.continent_name | Name of the continent. | core | keyword | `North America` -| geo.country_iso_code | Country ISO code. | core | keyword | `CA` -| geo.location | Longitude and latitude. | core | geo_point | `{ "lon": -73.614830, "lat": 45.505918 }` -| geo.region_name | Region name. | core | keyword | `Quebec` -| geo.city_name | City name. | core | keyword | `Montreal` -|======================================================================= - -[[ecs-host]] -=== Host fields - -Host fields provide information related to a host. A host can be a physical -machine, a virtual machine, or a Docker container. Normally the host information -is related to the machine on which the event was generated/collected, but they -can be used differently if needed. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | core | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | core | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | core | keyword | -| host.ip | Host ip address. | core | ip | -| host.mac | Host mac address. | core | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | core | keyword | -| host.architecture | Operating system architecture. | core | keyword | `x86_64` -|======================================================================= - -[[ecs-log]] -=== Log fields - -Fields which are specific to log events. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| log.level | Log level of the log event. Some examples are `WARN`, `ERR`, `INFO`. | core | keyword | `ERR` | -| log.original | This is the original log message and contains the full log message before splitting it up in multiple parts. In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. | core | keyword | `Sep 19 08:26:10 localhost My log` | -|======================================================================= - -[[ecs-network]] -=== Network fields - -Fields related to network data. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| network.name | Name given by operators to sections of their network. | extended | keyword | `Guest Wifi` -| network.type | In the OSI Model this would be the Network Layer. IPv4, IPv6, IPSec, PIM, etc | core | keyword | `IPv4` -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | extended | keyword | `6` -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (UDP, TCP, IPv6-ICMP, etc.) | core | keyword | `TCP` -| network.application | A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. | extended | keyword | `AIM` -| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol | core | keyword | `http` -| network.direction | Direction of the network traffic. Recommended values are: * inbound * outbound * unknown | core | keyword | `inbound` -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | core | ip | `192.1.1.2` -| network.inbound.bytes | Network inbound bytes. | core | long | `184` -| network.inbound.packets | Network inbound packets. | core | long | `12` -| network.outbound.bytes | Network outbound bytes. | core | long | `184` -| network.outbound.packets | Network outbound packets. | core | long | `12` -| network.total.bytes | Network total bytes. The sum of inbound.bytes + outbound.bytes. | core | long | `368` -| network.total.packets | Network outbound packets. The sum of inbound.packets + outbound.packets | core | long | `24` -|======================================================================= - -[[ecs-organization]] -=== Organization fields - -The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| organization.name | Organization name. | extended | keyword | -| organization.id | Unique identifier for the organization. | extended | keyword | -|======================================================================= - -[[ecs-os]] -=== Operating System fields - -The OS fields contain information about the operating system. - -The `os` fields are expected to be nested at: `device.os`, `host.os`, `user_agent.os`. -Note also that the `os` fields are not expected to be used directly at the top level. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| os.platform | Operating system platform (such centos, ubuntu, windows). | extended | keyword | `darwin` -| os.name | Operating system name. | extended | keyword | `Mac OS X` -| os.family | OS family (such as redhat, debian, freebsd, windows). | extended | keyword | `debian` -| os.version | Operating system version as a raw string. | extended | keyword | `10.12.6-rc2` -| os.kernel | Operating system kernel version as a raw string. | extended | keyword | `4.4.0-112-generic` -|======================================================================= - -[[ecs-process]] -=== Process fields - -These fields contain information about a process. These fields can help you -correlate metrics information with a process id/name from a log message. The -`process.pid` often stays in the metric itself and is copied to the global field -for correlation. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| process.args | Process arguments. May be filtered to protect sensitive information. | extended | keyword | `['-l', 'user', '10.0.0.16']` -| process.name | Process name. Sometimes called program name or similar. | extended | keyword | `ssh` -| process.pid | Process id. | core | long | -| process.ppid | Process parent id. | extended | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | extended | keyword | -|======================================================================= - -[[ecs-service]] -=== Service fields - -The service fields describe the service for or from which the data was -collected. These fields help you find and correlate logs for a specific service -and version. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| service.id | Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. | core | keyword | `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. | core | keyword | `elasticsearch-metrics` -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | core | keyword | `elasticsearch` -| service.state | Current state of the service. | core | keyword | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | core | keyword | `3.2.4` -| service.ephemeral_id | Ephemeral identifier of this service (if one exists). This id normally changes across restarts, but `service.id` does not. | extended | keyword | `8a4f500f` -|======================================================================= - -[[ecs-source]] -=== Source fields - -Source fields describe details about the destination of a packet/event. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| source.ip | IP address of the source. Can be one or multiple IPv4 or IPv6 addresses. | core | ip | -| source.port | Port of the source. | core | long | -| source.mac | MAC address of the source. | core | keyword | -| source.domain | Source domain. | core | keyword | -|======================================================================= - -[[ecs-url]] -=== URL fields - -URL fields provide a complete URL, with scheme, host, and path. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| url.original | Full original url. The field is stored as keyword. | extended | keyword | `https://www.elastic.co:443/search?q=elasticsearch#top` -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | extended | keyword | `https` -| url.domain | Domain of the request, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. | extended | keyword | `www.elastic.co` -| url.port | Port of the request, such as 443. | extended | integer | `443` -| url.path | Path of the request, such as "/search". | extended | keyword | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | extended | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | extended | keyword | -| url.username | Username of the request. | extended | keyword | -| url.password | Password of the request. | extended | keyword | -|======================================================================= - -[[ecs-user]] -=== User fields - -The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. - -The `user` fields are expected to be nested at: `destination.user`, `host.user`, `source.user`. - -Note also that the `user` fields may be used directly at the top level. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| user.id | One or multiple unique identifiers of the user. | core | keyword | -| user.name | Name of the user. The field is a keyword, and will not be tokenized. | core | keyword | -| user.email | User email address. | extended | keyword | -| user.hash | Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used. | extended | keyword | -|======================================================================= - - diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 29d85c04b0..833d1aaf00 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -1,58 +1,68 @@ + [[ecs-fields]] == {ecs} Fields -// Set make to generate short description for use in the Fields overview section. -// Pull in generated field content using `include` statements - [float] -[[ecs-categories]] -=== Field categories +[[ecs-fieldsets]] +=== Field Sets [cols="<,<",options="header",] |======================================================================= -| Fields | Description -| <> | Top level fields that are common across all types of events -(such as `@timestamp`, `tags`, `message`, and `labels`). -| <> | The agent fields contain data about the -agent/client/shipper that created the event. -| <> | Fields related to the cloud or infrastructure the events are - coming from. -| <> | Container fields are used for meta information about the specific container that -is the source of information. These fields help correlate data based containers -from any runtime. -| <> | Destination fields describe details about the destination of a packet/event. -| <> | Device fields are used to provide additional information -about the device that is the source of the information. This could be a firewall, network device, etc. +| Field Set | Description + +| <> | All fields defined directly at the top level + +| <> | Fields about the monitoring agent. + +| <> | Fields about the client side of a network connection, used with server. + +| <> | Fields about the cloud resource. + +| <> | Fields describing the container that generated this event. + +| <> | Fields about the destination side of a network connection, used with source. + | <> | Meta-information specific to ECS. -| <> | These fields can represent errors of any kind. Use them for errors that happen -while fetching events or in cases where the event itself contains an error. -| <> | The event fields are used for context information about the data itself. -| <> | File fields provide details about each file. -| <> | Geo fields can carry data about a specific location related to -an event or geo information derived from an IP field. -The `geo` fields are expected to be nested at: `destination.geo`, `device.geo`, `host.geo`, `source.geo`. -| <> | Host fields provide information related to a host. A host can be a physical -machine, a virtual machine, or a Docker container. + +| <> | Fields about errors of any kind. + +| <> | Fields breaking down the event details. + +| <> | Fields describing files. + +| <> | Fields describing a location. + +| <> | User's group relevant to the event. + +| <> | Fields describing the relevant computing instance. + +| <> | Fields describing an HTTP request. + | <> | Fields which are specific to log events. -| <> | Fields related to network data. -| <> | The organization fields enrich data with - information about the company or entity the data is associated with. These fields help - you arrange or filter data stored in an index by one or multiple organizations. -| <> | The OS fields contain information about the operating system. -| <> | These fields contain information about a process. These fields can help you -correlate metrics information with a process id/name from a log message. The -`process.pid` often stays in the metric itself and is copied to the global field -for correlation. -| <> | The service fields describe the service for or from which the data was -collected. These fields help you find and correlate logs for a specific service -and version. -| <> | Source fields describe details about the destination of a packet/event. -| <> | URL fields provide a complete URL, with scheme, host, and path. -| <> | The user fields describe information about the user - that is relevant to the event. Fields can have one entry or multiple entries. - If a user has more than one id, provide an array that includes all of them. -|======================================================================= -include::fields-gen.asciidoc[] +| <> | Fields describing the communication path over which the event happened. + +| <> | Fields describing an entity observing the event from outside the host. + +| <> | Fields describing the organization or company the event is associated with. + +| <> | OS fields contain information about the operating system. + +| <> | These fields contain information about a process. + +| <> | Fields meant to facilitate pivoting around a piece of data. + +| <> | Fields about the server side of a network connection, used with client. + +| <> | Fields describing the service for or from which the data was collected. + +| <> | Fields about the source side of a network connection, used with destination. + +| <> | Fields that let you store URLs in various forms. + +| <> | Fields to describe the user relevant to the event. +| <> | Fields to describe a browser user_agent string. +|===== +include::field-details.asciidoc[] diff --git a/fields.yml b/fields.yml index 917035411c..75c721c7da 100644 --- a/fields.yml +++ b/fields.yml @@ -526,6 +526,7 @@ - name: module level: core type: keyword + short: Name of the module this data is coming from. description: > Name of the module this data is coming from. diff --git a/generated/ecs/fields_flat.yml b/generated/ecs/fields_flat.yml index 905d06d9a5..b86249ca18 100644 --- a/generated/ecs/fields_flat.yml +++ b/generated/ecs/fields_flat.yml @@ -12,6 +12,7 @@ flat_name: '@timestamp' level: core name: '@timestamp' + order: 0 required: true short: Date/time when the event originated. type: date @@ -24,6 +25,7 @@ agent.ephemeral_id: ignore_above: 1024 level: extended name: ephemeral_id + order: 4 short: Ephemeral identifier of this agent. type: keyword agent.id: @@ -35,6 +37,7 @@ agent.id: ignore_above: 1024 level: core name: id + order: 3 short: Unique identifier of this agent. type: keyword agent.name: @@ -50,6 +53,7 @@ agent.name: ignore_above: 1024 level: core name: name + order: 1 short: Custom name of the agent. type: keyword agent.type: @@ -63,6 +67,7 @@ agent.type: ignore_above: 1024 level: core name: type + order: 2 short: Type of the agent. type: keyword agent.version: @@ -72,6 +77,7 @@ agent.version: ignore_above: 1024 level: core name: version + order: 0 short: Version of the agent. type: keyword client.address: @@ -85,6 +91,7 @@ client.address: ignore_above: 1024 level: extended name: address + order: 0 short: Client network address. type: keyword client.bytes: @@ -93,6 +100,7 @@ client.bytes: flat_name: client.bytes level: core name: bytes + order: 5 short: Bytes sent from the client to the server. type: long client.domain: @@ -101,6 +109,7 @@ client.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Client domain. type: keyword client.geo.city_name: @@ -110,6 +119,7 @@ client.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -120,6 +130,7 @@ client.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -130,6 +141,7 @@ client.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -140,6 +152,7 @@ client.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -149,6 +162,7 @@ client.geo.location: flat_name: client.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -165,6 +179,7 @@ client.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -175,6 +190,7 @@ client.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -185,6 +201,7 @@ client.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -195,6 +212,7 @@ client.ip: flat_name: client.ip level: core name: ip + order: 1 short: IP address of the client. type: ip client.mac: @@ -203,6 +221,7 @@ client.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the client. type: keyword client.packets: @@ -211,6 +230,7 @@ client.packets: flat_name: client.packets level: core name: packets + order: 6 short: Packets sent from the client to the server. type: long client.port: @@ -218,6 +238,7 @@ client.port: flat_name: client.port level: core name: port + order: 2 short: Port of the client. type: long client.user.email: @@ -226,6 +247,7 @@ client.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -236,6 +258,7 @@ client.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -245,6 +268,7 @@ client.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -254,6 +278,7 @@ client.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -267,6 +292,7 @@ client.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -276,6 +302,7 @@ client.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -286,6 +313,7 @@ client.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -299,6 +327,7 @@ cloud.account.id: ignore_above: 1024 level: extended name: account.id + order: 6 short: The cloud account or organization id. type: keyword cloud.availability_zone: @@ -308,6 +337,7 @@ cloud.availability_zone: ignore_above: 1024 level: extended name: availability_zone + order: 1 short: Availability zone in which this host is running. type: keyword cloud.instance.id: @@ -317,6 +347,7 @@ cloud.instance.id: ignore_above: 1024 level: extended name: instance.id + order: 3 short: Instance ID of the host machine. type: keyword cloud.instance.name: @@ -325,6 +356,7 @@ cloud.instance.name: ignore_above: 1024 level: extended name: instance.name + order: 4 short: Instance name of the host machine. type: keyword cloud.machine.type: @@ -334,6 +366,7 @@ cloud.machine.type: ignore_above: 1024 level: extended name: machine.type + order: 5 short: Machine type of the host machine. type: keyword cloud.provider: @@ -344,6 +377,7 @@ cloud.provider: ignore_above: 1024 level: extended name: provider + order: 0 short: Name of the cloud provider. type: keyword cloud.region: @@ -353,6 +387,7 @@ cloud.region: ignore_above: 1024 level: extended name: region + order: 2 short: Region in which this host is running. type: keyword container.id: @@ -361,6 +396,7 @@ container.id: ignore_above: 1024 level: core name: id + order: 1 short: Unique container id. type: keyword container.image.name: @@ -369,6 +405,7 @@ container.image.name: ignore_above: 1024 level: extended name: image.name + order: 2 short: Name of the image the container was built on. type: keyword container.image.tag: @@ -377,6 +414,7 @@ container.image.tag: ignore_above: 1024 level: extended name: image.tag + order: 3 short: Container image tag. type: keyword container.labels: @@ -385,6 +423,7 @@ container.labels: level: extended name: labels object_type: keyword + order: 5 short: Image labels. type: object container.name: @@ -393,6 +432,7 @@ container.name: ignore_above: 1024 level: extended name: name + order: 4 short: Container name. type: keyword container.runtime: @@ -402,6 +442,7 @@ container.runtime: ignore_above: 1024 level: extended name: runtime + order: 0 short: Runtime managing this container. type: keyword destination.address: @@ -415,6 +456,7 @@ destination.address: ignore_above: 1024 level: extended name: address + order: 0 short: Destination network address. type: keyword destination.bytes: @@ -423,6 +465,7 @@ destination.bytes: flat_name: destination.bytes level: core name: bytes + order: 5 short: Bytes sent from the destination to the source. type: long destination.domain: @@ -431,6 +474,7 @@ destination.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Destination domain. type: keyword destination.geo.city_name: @@ -440,6 +484,7 @@ destination.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -450,6 +495,7 @@ destination.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -460,6 +506,7 @@ destination.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -470,6 +517,7 @@ destination.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -479,6 +527,7 @@ destination.geo.location: flat_name: destination.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -495,6 +544,7 @@ destination.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -505,6 +555,7 @@ destination.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -515,6 +566,7 @@ destination.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -525,6 +577,7 @@ destination.ip: flat_name: destination.ip level: core name: ip + order: 1 short: IP address of the destination. type: ip destination.mac: @@ -533,6 +586,7 @@ destination.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the destination. type: keyword destination.packets: @@ -541,6 +595,7 @@ destination.packets: flat_name: destination.packets level: core name: packets + order: 6 short: Packets sent from the destination to the source. type: long destination.port: @@ -548,6 +603,7 @@ destination.port: flat_name: destination.port level: core name: port + order: 2 short: Port of the destination. type: long destination.user.email: @@ -556,6 +612,7 @@ destination.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -566,6 +623,7 @@ destination.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -575,6 +633,7 @@ destination.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -584,6 +643,7 @@ destination.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -597,6 +657,7 @@ destination.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -606,6 +667,7 @@ destination.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -616,6 +678,7 @@ destination.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -631,6 +694,7 @@ ecs.version: ignore_above: 1024 level: core name: version + order: 0 required: true short: ECS version this event conforms to. type: keyword @@ -640,6 +704,7 @@ error.code: ignore_above: 1024 level: core name: code + order: 2 short: Error code describing the error. type: keyword error.id: @@ -648,6 +713,7 @@ error.id: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier for the error. type: keyword error.message: @@ -656,6 +722,7 @@ error.message: level: core name: message norms: false + order: 1 short: Error message. type: text event.action: @@ -669,6 +736,7 @@ event.action: ignore_above: 1024 level: core name: action + order: 3 short: The action captured by the event. type: keyword event.category: @@ -683,6 +751,7 @@ event.category: ignore_above: 1024 level: core name: category + order: 2 short: Event category. type: keyword event.created: @@ -701,6 +770,7 @@ event.created: flat_name: event.created level: core name: created + order: 13 short: Time when the event was first read by an agent or by your pipeline. type: date event.dataset: @@ -714,6 +784,7 @@ event.dataset: ignore_above: 1024 level: core name: dataset + order: 7 short: Name of the dataset. type: keyword event.duration: @@ -724,6 +795,7 @@ event.duration: flat_name: event.duration level: core name: duration + order: 11 short: Duration of the event in nanoseconds. type: long event.end: @@ -732,6 +804,7 @@ event.end: flat_name: event.end level: extended name: end + order: 15 short: event.end contains the date when the event ended or when the activity was last observed. type: date @@ -743,6 +816,7 @@ event.hash: ignore_above: 1024 level: extended name: hash + order: 10 short: Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. type: keyword @@ -753,6 +827,7 @@ event.id: ignore_above: 1024 level: core name: id + order: 0 short: Unique ID to describe the event. type: keyword event.kind: @@ -767,6 +842,7 @@ event.kind: ignore_above: 1024 level: extended name: kind + order: 1 short: The kind of the event. type: keyword event.module: @@ -778,9 +854,8 @@ event.module: ignore_above: 1024 level: core name: module - short: 'Name of the module this data is coming from. - - This information is coming from the modules used in Beats or Logstash.' + order: 6 + short: Name of the module this data is coming from. type: keyword event.original: description: 'Raw text message of entire event. Used to demonstrate log integrity. @@ -795,6 +870,7 @@ event.original: index: false level: core name: original + order: 9 short: Raw text message of entire event. type: keyword event.outcome: @@ -809,6 +885,7 @@ event.outcome: ignore_above: 1024 level: extended name: outcome + order: 4 short: The outcome of the event. type: keyword event.risk_score: @@ -817,6 +894,7 @@ event.risk_score: flat_name: event.risk_score level: core name: risk_score + order: 16 short: Risk score or priority of the event (e.g. security solutions). Use your system's original value here. type: float @@ -829,6 +907,7 @@ event.risk_score_norm: flat_name: event.risk_score_norm level: extended name: risk_score_norm + order: 17 short: Normalized risk score or priority of the event (0-100). type: float event.severity: @@ -839,6 +918,7 @@ event.severity: flat_name: event.severity level: core name: severity + order: 8 short: Original severity of the event. type: long event.start: @@ -847,6 +927,7 @@ event.start: flat_name: event.start level: extended name: start + order: 14 short: event.start contains the date when the event started or when the activity was first observed. type: date @@ -861,6 +942,7 @@ event.timezone: ignore_above: 1024 level: extended name: timezone + order: 12 short: Event time zone. type: keyword event.type: @@ -871,6 +953,7 @@ event.type: ignore_above: 1024 level: core name: type + order: 5 short: Reserved for future usage. type: keyword file.ctime: @@ -878,6 +961,7 @@ file.ctime: flat_name: file.ctime level: extended name: ctime + order: 13 short: Last time file metadata changed. type: date file.device: @@ -886,6 +970,7 @@ file.device: ignore_above: 1024 level: extended name: device + order: 4 short: Device that is the source of the file. type: keyword file.extension: @@ -897,6 +982,7 @@ file.extension: ignore_above: 1024 level: extended name: extension + order: 2 short: File extension. type: keyword file.gid: @@ -905,6 +991,7 @@ file.gid: ignore_above: 1024 level: extended name: gid + order: 8 short: Primary group ID (GID) of the file. type: keyword file.group: @@ -913,6 +1000,7 @@ file.group: ignore_above: 1024 level: extended name: group + order: 9 short: Primary group name of the file. type: keyword file.inode: @@ -921,6 +1009,7 @@ file.inode: ignore_above: 1024 level: extended name: inode + order: 5 short: Inode representing the file in the filesystem. type: keyword file.mode: @@ -930,6 +1019,7 @@ file.mode: ignore_above: 1024 level: extended name: mode + order: 10 short: Mode of the file in octal representation. type: keyword file.mtime: @@ -937,6 +1027,7 @@ file.mtime: flat_name: file.mtime level: extended name: mtime + order: 12 short: Last time file content was modified. type: date file.owner: @@ -945,6 +1036,7 @@ file.owner: ignore_above: 1024 level: extended name: owner + order: 7 short: File owner's username. type: keyword file.path: @@ -953,6 +1045,7 @@ file.path: ignore_above: 1024 level: extended name: path + order: 0 short: Path to the file. type: keyword file.size: @@ -960,6 +1053,7 @@ file.size: flat_name: file.size level: extended name: size + order: 11 short: File size in bytes (field is only added when `type` is `file`). type: long file.target_path: @@ -968,6 +1062,7 @@ file.target_path: ignore_above: 1024 level: extended name: target_path + order: 1 short: Target path for symlinks. type: keyword file.type: @@ -976,6 +1071,7 @@ file.type: ignore_above: 1024 level: extended name: type + order: 3 short: File type (file, dir, or symlink). type: keyword file.uid: @@ -984,6 +1080,7 @@ file.uid: ignore_above: 1024 level: extended name: uid + order: 6 short: The user ID (UID) or security identifier (SID) of the file owner. type: keyword geo.city_name: @@ -993,6 +1090,7 @@ geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 short: City name. type: keyword geo.continent_name: @@ -1002,6 +1100,7 @@ geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 short: Name of the continent. type: keyword geo.country_iso_code: @@ -1011,6 +1110,7 @@ geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 short: Country ISO code. type: keyword geo.country_name: @@ -1020,6 +1120,7 @@ geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 short: Country name. type: keyword geo.location: @@ -1028,6 +1129,7 @@ geo.location: flat_name: geo.location level: core name: location + order: 0 short: Longitude and latitude. type: geo_point geo.name: @@ -1043,6 +1145,7 @@ geo.name: ignore_above: 1024 level: extended name: name + order: 7 short: User-defined description of a location. type: keyword geo.region_iso_code: @@ -1052,6 +1155,7 @@ geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 short: Region ISO code. type: keyword geo.region_name: @@ -1061,6 +1165,7 @@ geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 short: Region name. type: keyword group.id: @@ -1069,6 +1174,7 @@ group.id: ignore_above: 1024 level: extended name: id + order: 0 short: Unique identifier for the group on the system/platform. type: keyword group.name: @@ -1077,6 +1183,7 @@ group.name: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the group. type: keyword host.architecture: @@ -1086,6 +1193,7 @@ host.architecture: ignore_above: 1024 level: core name: architecture + order: 6 short: Operating system architecture. type: keyword host.geo.city_name: @@ -1095,6 +1203,7 @@ host.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1105,6 +1214,7 @@ host.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1115,6 +1225,7 @@ host.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1125,6 +1236,7 @@ host.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1134,6 +1246,7 @@ host.geo.location: flat_name: host.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1150,6 +1263,7 @@ host.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1160,6 +1274,7 @@ host.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1170,6 +1285,7 @@ host.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1181,6 +1297,7 @@ host.hostname: ignore_above: 1024 level: core name: hostname + order: 0 short: Hostname of the host. type: keyword host.id: @@ -1193,6 +1310,7 @@ host.id: ignore_above: 1024 level: core name: id + order: 2 short: Unique host id. type: keyword host.ip: @@ -1200,6 +1318,7 @@ host.ip: flat_name: host.ip level: core name: ip + order: 3 short: Host ip address. type: ip host.mac: @@ -1208,6 +1327,7 @@ host.mac: ignore_above: 1024 level: core name: mac + order: 4 short: Host mac address. type: keyword host.name: @@ -1219,6 +1339,7 @@ host.name: ignore_above: 1024 level: core name: name + order: 1 short: Name of the host. type: keyword host.os.family: @@ -1228,6 +1349,7 @@ host.os.family: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -1238,6 +1360,7 @@ host.os.full: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -1248,6 +1371,7 @@ host.os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -1258,6 +1382,7 @@ host.os.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -1268,6 +1393,7 @@ host.os.platform: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -1278,6 +1404,7 @@ host.os.version: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -1290,6 +1417,7 @@ host.type: ignore_above: 1024 level: core name: type + order: 5 short: Type of host. type: keyword host.user.email: @@ -1298,6 +1426,7 @@ host.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -1308,6 +1437,7 @@ host.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -1317,6 +1447,7 @@ host.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -1326,6 +1457,7 @@ host.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -1339,6 +1471,7 @@ host.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -1348,6 +1481,7 @@ host.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -1358,6 +1492,7 @@ host.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -1367,6 +1502,7 @@ http.request.body.bytes: flat_name: http.request.body.bytes level: extended name: request.body.bytes + order: 7 short: Size in bytes of the request body. type: long http.request.body.content: @@ -1376,6 +1512,7 @@ http.request.body.content: ignore_above: 1024 level: extended name: request.body.content + order: 1 short: The full HTTP request body. type: keyword http.request.bytes: @@ -1384,6 +1521,7 @@ http.request.bytes: flat_name: http.request.bytes level: extended name: request.bytes + order: 6 short: Total size in bytes of the request (body and headers). type: long http.request.method: @@ -1396,6 +1534,7 @@ http.request.method: ignore_above: 1024 level: extended name: request.method + order: 0 short: HTTP request method. type: keyword http.request.referrer: @@ -1405,6 +1544,7 @@ http.request.referrer: ignore_above: 1024 level: extended name: request.referrer + order: 2 short: Referrer for this HTTP request. type: keyword http.response.body.bytes: @@ -1413,6 +1553,7 @@ http.response.body.bytes: flat_name: http.response.body.bytes level: extended name: response.body.bytes + order: 9 short: Size in bytes of the response body. type: long http.response.body.content: @@ -1422,6 +1563,7 @@ http.response.body.content: ignore_above: 1024 level: extended name: response.body.content + order: 4 short: The full HTTP response body. type: keyword http.response.bytes: @@ -1430,6 +1572,7 @@ http.response.bytes: flat_name: http.response.bytes level: extended name: response.bytes + order: 8 short: Total size in bytes of the response (body and headers). type: long http.response.status_code: @@ -1438,6 +1581,7 @@ http.response.status_code: flat_name: http.response.status_code level: extended name: response.status_code + order: 3 short: HTTP response status code. type: long http.version: @@ -1447,6 +1591,7 @@ http.version: ignore_above: 1024 level: extended name: version + order: 5 short: HTTP version. type: keyword labels: @@ -1463,6 +1608,7 @@ labels: level: core name: labels object_type: keyword + order: 2 short: Custom key/value pairs. type: object log.level: @@ -1474,6 +1620,7 @@ log.level: ignore_above: 1024 level: core name: level + order: 0 short: Log level of the log event. type: keyword log.original: @@ -1494,6 +1641,7 @@ log.original: index: false level: core name: original + order: 1 short: Original log message with light interpretation only (encoding, newlines). type: keyword message: @@ -1509,6 +1657,7 @@ message: level: core name: message norms: false + order: 3 short: Log message optimized for viewing in a log viewer. type: text network.application: @@ -1524,6 +1673,7 @@ network.application: ignore_above: 1024 level: extended name: application + order: 4 short: Application level protocol name. type: keyword network.bytes: @@ -1535,6 +1685,7 @@ network.bytes: flat_name: network.bytes level: core name: bytes + order: 9 short: Total bytes transferred in both directions. type: long network.community_id: @@ -1547,6 +1698,7 @@ network.community_id: ignore_above: 1024 level: extended name: community_id + order: 8 short: A hash of source and destination IPs and ports. type: keyword network.direction: @@ -1560,6 +1712,7 @@ network.direction: ignore_above: 1024 level: core name: direction + order: 6 short: Direction of the network traffic. type: keyword network.forwarded_ip: @@ -1568,6 +1721,7 @@ network.forwarded_ip: flat_name: network.forwarded_ip level: core name: forwarded_ip + order: 7 short: Host IP address when the source IP address is the proxy. type: ip network.iana_number: @@ -1579,6 +1733,7 @@ network.iana_number: ignore_above: 1024 level: extended name: iana_number + order: 2 short: IANA Protocol Number. type: keyword network.name: @@ -1588,6 +1743,7 @@ network.name: ignore_above: 1024 level: extended name: name + order: 0 short: Name given by operators to sections of their network. type: keyword network.packets: @@ -1599,6 +1755,7 @@ network.packets: flat_name: network.packets level: core name: packets + order: 10 short: Total packets transferred in both directions. type: long network.protocol: @@ -1611,6 +1768,7 @@ network.protocol: ignore_above: 1024 level: core name: protocol + order: 5 short: L7 Network protocol name. type: keyword network.transport: @@ -1624,6 +1782,7 @@ network.transport: ignore_above: 1024 level: core name: transport + order: 3 short: Protocol Name corresponding to the field `iana_number`. type: keyword network.type: @@ -1637,6 +1796,7 @@ network.type: ignore_above: 1024 level: core name: type + order: 1 short: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc type: keyword @@ -1647,6 +1807,7 @@ observer.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1657,6 +1818,7 @@ observer.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1667,6 +1829,7 @@ observer.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1677,6 +1840,7 @@ observer.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1686,6 +1850,7 @@ observer.geo.location: flat_name: observer.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1702,6 +1867,7 @@ observer.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1712,6 +1878,7 @@ observer.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1722,6 +1889,7 @@ observer.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1731,6 +1899,7 @@ observer.hostname: ignore_above: 1024 level: core name: hostname + order: 2 short: Hostname of the observer. type: keyword observer.ip: @@ -1738,6 +1907,7 @@ observer.ip: flat_name: observer.ip level: core name: ip + order: 1 short: IP address of the observer. type: ip observer.mac: @@ -1746,6 +1916,7 @@ observer.mac: ignore_above: 1024 level: core name: mac + order: 0 short: MAC address of the observer type: keyword observer.os.family: @@ -1755,6 +1926,7 @@ observer.os.family: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -1765,6 +1937,7 @@ observer.os.full: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -1775,6 +1948,7 @@ observer.os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -1785,6 +1959,7 @@ observer.os.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -1795,6 +1970,7 @@ observer.os.platform: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -1805,6 +1981,7 @@ observer.os.version: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -1814,6 +1991,7 @@ observer.serial_number: ignore_above: 1024 level: extended name: serial_number + order: 5 short: Observer serial number. type: keyword observer.type: @@ -1826,6 +2004,7 @@ observer.type: ignore_above: 1024 level: core name: type + order: 6 short: The type of the observer the data is coming from. type: keyword observer.vendor: @@ -1834,6 +2013,7 @@ observer.vendor: ignore_above: 1024 level: core name: vendor + order: 3 short: observer vendor information. type: keyword observer.version: @@ -1842,6 +2022,7 @@ observer.version: ignore_above: 1024 level: core name: version + order: 4 short: Observer version. type: keyword organization.id: @@ -1850,6 +2031,7 @@ organization.id: ignore_above: 1024 level: extended name: id + order: 1 short: Unique identifier for the organization. type: keyword organization.name: @@ -1858,6 +2040,7 @@ organization.name: ignore_above: 1024 level: extended name: name + order: 0 short: Organization name. type: keyword os.family: @@ -1867,6 +2050,7 @@ os.family: ignore_above: 1024 level: extended name: family + order: 3 short: OS family (such as redhat, debian, freebsd, windows). type: keyword os.full: @@ -1876,6 +2060,7 @@ os.full: ignore_above: 1024 level: extended name: full + order: 2 short: Operating system name, including the version or code name. type: keyword os.kernel: @@ -1885,6 +2070,7 @@ os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 short: Operating system kernel version as a raw string. type: keyword os.name: @@ -1894,6 +2080,7 @@ os.name: ignore_above: 1024 level: extended name: name + order: 1 short: Operating system name, without the version. type: keyword os.platform: @@ -1903,6 +2090,7 @@ os.platform: ignore_above: 1024 level: extended name: platform + order: 0 short: Operating system platform (such centos, ubuntu, windows). type: keyword os.version: @@ -1912,6 +2100,7 @@ os.version: ignore_above: 1024 level: extended name: version + order: 4 short: Operating system version as a raw string. type: keyword process.args: @@ -1927,6 +2116,7 @@ process.args: ignore_above: 1024 level: extended name: args + order: 3 short: Array of process arguments. type: keyword process.executable: @@ -1936,6 +2126,7 @@ process.executable: ignore_above: 1024 level: extended name: executable + order: 4 short: Absolute path to the process executable. type: keyword process.name: @@ -1947,6 +2138,7 @@ process.name: ignore_above: 1024 level: extended name: name + order: 1 short: Process name. type: keyword process.pid: @@ -1955,6 +2147,7 @@ process.pid: flat_name: process.pid level: core name: pid + order: 0 short: Process id. type: long process.ppid: @@ -1962,6 +2155,7 @@ process.ppid: flat_name: process.ppid level: extended name: ppid + order: 2 short: Process parent id. type: long process.start: @@ -1970,6 +2164,7 @@ process.start: flat_name: process.start level: extended name: start + order: 7 short: The time the process started. type: date process.thread.id: @@ -1978,6 +2173,7 @@ process.thread.id: flat_name: process.thread.id level: extended name: thread.id + order: 6 short: Thread ID. type: long process.title: @@ -1989,6 +2185,7 @@ process.title: ignore_above: 1024 level: extended name: title + order: 5 short: Process title. type: keyword process.working_directory: @@ -1998,6 +2195,7 @@ process.working_directory: ignore_above: 1024 level: extended name: working_directory + order: 8 short: The working directory of the process. type: keyword related.ip: @@ -2005,6 +2203,7 @@ related.ip: flat_name: related.ip level: extended name: ip + order: 0 short: All of the IPs seen on your event. type: ip server.address: @@ -2018,6 +2217,7 @@ server.address: ignore_above: 1024 level: extended name: address + order: 0 short: Server network address. type: keyword server.bytes: @@ -2026,6 +2226,7 @@ server.bytes: flat_name: server.bytes level: core name: bytes + order: 5 short: Bytes sent from the server to the client. type: long server.domain: @@ -2034,6 +2235,7 @@ server.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Server domain. type: keyword server.geo.city_name: @@ -2043,6 +2245,7 @@ server.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2053,6 +2256,7 @@ server.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2063,6 +2267,7 @@ server.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2073,6 +2278,7 @@ server.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2082,6 +2288,7 @@ server.geo.location: flat_name: server.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2098,6 +2305,7 @@ server.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2108,6 +2316,7 @@ server.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2118,6 +2327,7 @@ server.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2128,6 +2338,7 @@ server.ip: flat_name: server.ip level: core name: ip + order: 1 short: IP address of the server. type: ip server.mac: @@ -2136,6 +2347,7 @@ server.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the server. type: keyword server.packets: @@ -2144,6 +2356,7 @@ server.packets: flat_name: server.packets level: core name: packets + order: 6 short: Packets sent from the server to the client. type: long server.port: @@ -2151,6 +2364,7 @@ server.port: flat_name: server.port level: core name: port + order: 2 short: Port of the server. type: long server.user.email: @@ -2159,6 +2373,7 @@ server.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2169,6 +2384,7 @@ server.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2178,6 +2394,7 @@ server.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2187,6 +2404,7 @@ server.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2200,6 +2418,7 @@ server.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2209,6 +2428,7 @@ server.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2219,6 +2439,7 @@ server.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -2231,6 +2452,7 @@ service.ephemeral_id: ignore_above: 1024 level: extended name: ephemeral_id + order: 5 short: Ephemeral identifier of this service. type: keyword service.id: @@ -2246,6 +2468,7 @@ service.id: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier of the running service. type: keyword service.name: @@ -2266,6 +2489,7 @@ service.name: ignore_above: 1024 level: core name: name + order: 1 short: Name of the service. type: keyword service.state: @@ -2274,6 +2498,7 @@ service.state: ignore_above: 1024 level: core name: state + order: 3 short: Current state of the service. type: keyword service.type: @@ -2289,6 +2514,7 @@ service.type: ignore_above: 1024 level: core name: type + order: 2 short: The type of the service. type: keyword service.version: @@ -2300,6 +2526,7 @@ service.version: ignore_above: 1024 level: core name: version + order: 4 short: Version of the service. type: keyword source.address: @@ -2313,6 +2540,7 @@ source.address: ignore_above: 1024 level: extended name: address + order: 0 short: Source network address. type: keyword source.bytes: @@ -2321,6 +2549,7 @@ source.bytes: flat_name: source.bytes level: core name: bytes + order: 5 short: Bytes sent from the source to the destination. type: long source.domain: @@ -2329,6 +2558,7 @@ source.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Source domain. type: keyword source.geo.city_name: @@ -2338,6 +2568,7 @@ source.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2348,6 +2579,7 @@ source.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2358,6 +2590,7 @@ source.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2368,6 +2601,7 @@ source.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2377,6 +2611,7 @@ source.geo.location: flat_name: source.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2393,6 +2628,7 @@ source.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2403,6 +2639,7 @@ source.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2413,6 +2650,7 @@ source.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2423,6 +2661,7 @@ source.ip: flat_name: source.ip level: core name: ip + order: 1 short: IP address of the source. type: ip source.mac: @@ -2431,6 +2670,7 @@ source.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the source. type: keyword source.packets: @@ -2439,6 +2679,7 @@ source.packets: flat_name: source.packets level: core name: packets + order: 6 short: Packets sent from the source to the destination. type: long source.port: @@ -2446,6 +2687,7 @@ source.port: flat_name: source.port level: core name: port + order: 2 short: Port of the source. type: long source.user.email: @@ -2454,6 +2696,7 @@ source.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2464,6 +2707,7 @@ source.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2473,6 +2717,7 @@ source.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2482,6 +2727,7 @@ source.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2495,6 +2741,7 @@ source.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2504,6 +2751,7 @@ source.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2514,6 +2762,7 @@ source.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -2524,6 +2773,7 @@ tags: ignore_above: 1024 level: core name: tags + order: 1 short: List of keywords used to tag each event. type: keyword url.domain: @@ -2536,6 +2786,7 @@ url.domain: ignore_above: 1024 level: extended name: domain + order: 3 short: Domain of the url. type: keyword url.fragment: @@ -2546,6 +2797,7 @@ url.fragment: ignore_above: 1024 level: extended name: fragment + order: 7 short: Portion of the url after the `#`. type: keyword url.full: @@ -2556,6 +2808,7 @@ url.full: ignore_above: 1024 level: extended name: full + order: 1 short: Full unparsed URL. type: keyword url.original: @@ -2570,6 +2823,7 @@ url.original: ignore_above: 1024 level: extended name: original + order: 0 short: Unmodified original url as seen in the event source. type: keyword url.password: @@ -2578,6 +2832,7 @@ url.password: ignore_above: 1024 level: extended name: password + order: 9 short: Password of the request. type: keyword url.path: @@ -2586,6 +2841,7 @@ url.path: ignore_above: 1024 level: extended name: path + order: 5 short: Path of the request, such as "/search". type: keyword url.port: @@ -2594,6 +2850,7 @@ url.port: flat_name: url.port level: extended name: port + order: 4 short: Port of the request, such as 443. type: long url.query: @@ -2608,6 +2865,7 @@ url.query: ignore_above: 1024 level: extended name: query + order: 6 short: Query string of the request. type: keyword url.scheme: @@ -2619,6 +2877,7 @@ url.scheme: ignore_above: 1024 level: extended name: scheme + order: 2 short: Scheme of the url. type: keyword url.username: @@ -2627,6 +2886,7 @@ url.username: ignore_above: 1024 level: extended name: username + order: 8 short: Username of the request. type: keyword user.email: @@ -2635,6 +2895,7 @@ user.email: ignore_above: 1024 level: extended name: email + order: 3 short: User email address. type: keyword user.full_name: @@ -2644,6 +2905,7 @@ user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 short: User's full name, if available. type: keyword user.group.id: @@ -2652,6 +2914,7 @@ user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: group short: Unique identifier for the group on the system/platform. type: keyword @@ -2661,6 +2924,7 @@ user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: group short: Name of the group. type: keyword @@ -2674,6 +2938,7 @@ user.hash: ignore_above: 1024 level: extended name: hash + order: 4 short: Unique user hash to correlate information for a user in anonymized form. type: keyword user.id: @@ -2682,6 +2947,7 @@ user.id: ignore_above: 1024 level: core name: id + order: 0 short: One or multiple unique identifiers of the user. type: keyword user.name: @@ -2691,6 +2957,7 @@ user.name: ignore_above: 1024 level: core name: name + order: 1 short: Short name or login of the user. type: keyword user_agent.device.name: @@ -2700,6 +2967,7 @@ user_agent.device.name: ignore_above: 1024 level: extended name: device.name + order: 3 short: Name of the device. type: keyword user_agent.name: @@ -2709,6 +2977,7 @@ user_agent.name: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the user agent. type: keyword user_agent.original: @@ -2719,6 +2988,7 @@ user_agent.original: ignore_above: 1024 level: extended name: original + order: 0 short: Unparsed version of the user_agent. type: keyword user_agent.os.family: @@ -2728,6 +2998,7 @@ user_agent.os.family: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -2738,6 +3009,7 @@ user_agent.os.full: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -2748,6 +3020,7 @@ user_agent.os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -2758,6 +3031,7 @@ user_agent.os.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -2768,6 +3042,7 @@ user_agent.os.platform: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -2778,6 +3053,7 @@ user_agent.os.version: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -2788,5 +3064,6 @@ user_agent.version: ignore_above: 1024 level: extended name: version + order: 2 short: Version of the user agent. type: keyword diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index 92331173d5..4621ebb0dd 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -16,6 +16,7 @@ agent: ignore_above: 1024 level: extended name: ephemeral_id + order: 4 short: Ephemeral identifier of this agent. type: keyword id: @@ -27,6 +28,7 @@ agent: ignore_above: 1024 level: core name: id + order: 3 short: Unique identifier of this agent. type: keyword name: @@ -42,6 +44,7 @@ agent: ignore_above: 1024 level: core name: name + order: 1 short: Custom name of the agent. type: keyword type: @@ -55,6 +58,7 @@ agent: ignore_above: 1024 level: core name: type + order: 2 short: Type of the agent. type: keyword version: @@ -64,6 +68,7 @@ agent: ignore_above: 1024 level: core name: version + order: 0 short: Version of the agent. type: keyword footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. @@ -94,6 +99,7 @@ base: flat_name: '@timestamp' level: core name: '@timestamp' + order: 0 required: true short: Date/time when the event originated. type: date @@ -111,6 +117,7 @@ base: level: core name: labels object_type: keyword + order: 2 short: Custom key/value pairs. type: object message: @@ -126,6 +133,7 @@ base: level: core name: message norms: false + order: 3 short: Log message optimized for viewing in a log viewer. type: text tags: @@ -135,6 +143,7 @@ base: ignore_above: 1024 level: core name: tags + order: 1 short: List of keywords used to tag each event. type: keyword group: 1 @@ -172,6 +181,7 @@ client: ignore_above: 1024 level: extended name: address + order: 0 short: Client network address. type: keyword bytes: @@ -180,6 +190,7 @@ client: flat_name: client.bytes level: core name: bytes + order: 5 short: Bytes sent from the client to the server. type: long domain: @@ -188,6 +199,7 @@ client: ignore_above: 1024 level: core name: domain + order: 4 short: Client domain. type: keyword geo.city_name: @@ -197,6 +209,7 @@ client: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -207,6 +220,7 @@ client: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -217,6 +231,7 @@ client: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -227,6 +242,7 @@ client: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -236,6 +252,7 @@ client: flat_name: client.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -252,6 +269,7 @@ client: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -262,6 +280,7 @@ client: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -272,6 +291,7 @@ client: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -282,6 +302,7 @@ client: flat_name: client.ip level: core name: ip + order: 1 short: IP address of the client. type: ip mac: @@ -290,6 +311,7 @@ client: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the client. type: keyword packets: @@ -298,6 +320,7 @@ client: flat_name: client.packets level: core name: packets + order: 6 short: Packets sent from the client to the server. type: long port: @@ -305,6 +328,7 @@ client: flat_name: client.port level: core name: port + order: 2 short: Port of the client. type: long user.email: @@ -313,6 +337,7 @@ client: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -323,6 +348,7 @@ client: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -332,6 +358,7 @@ client: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -341,6 +368,7 @@ client: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -354,6 +382,7 @@ client: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -363,6 +392,7 @@ client: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -373,11 +403,15 @@ client: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword group: 2 name: client + nestings: + - geo + - user prefix: client. short: Fields about the client side of a network connection, used with server. title: Client @@ -396,6 +430,7 @@ cloud: ignore_above: 1024 level: extended name: account.id + order: 6 short: The cloud account or organization id. type: keyword availability_zone: @@ -405,6 +440,7 @@ cloud: ignore_above: 1024 level: extended name: availability_zone + order: 1 short: Availability zone in which this host is running. type: keyword instance.id: @@ -414,6 +450,7 @@ cloud: ignore_above: 1024 level: extended name: instance.id + order: 3 short: Instance ID of the host machine. type: keyword instance.name: @@ -422,6 +459,7 @@ cloud: ignore_above: 1024 level: extended name: instance.name + order: 4 short: Instance name of the host machine. type: keyword machine.type: @@ -431,6 +469,7 @@ cloud: ignore_above: 1024 level: extended name: machine.type + order: 5 short: Machine type of the host machine. type: keyword provider: @@ -441,6 +480,7 @@ cloud: ignore_above: 1024 level: extended name: provider + order: 0 short: Name of the cloud provider. type: keyword region: @@ -450,6 +490,7 @@ cloud: ignore_above: 1024 level: extended name: region + order: 2 short: Region in which this host is running. type: keyword footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from @@ -475,6 +516,7 @@ container: ignore_above: 1024 level: core name: id + order: 1 short: Unique container id. type: keyword image.name: @@ -483,6 +525,7 @@ container: ignore_above: 1024 level: extended name: image.name + order: 2 short: Name of the image the container was built on. type: keyword image.tag: @@ -491,6 +534,7 @@ container: ignore_above: 1024 level: extended name: image.tag + order: 3 short: Container image tag. type: keyword labels: @@ -499,6 +543,7 @@ container: level: extended name: labels object_type: keyword + order: 5 short: Image labels. type: object name: @@ -507,6 +552,7 @@ container: ignore_above: 1024 level: extended name: name + order: 4 short: Container name. type: keyword runtime: @@ -516,6 +562,7 @@ container: ignore_above: 1024 level: extended name: runtime + order: 0 short: Runtime managing this container. type: keyword group: 2 @@ -540,6 +587,7 @@ destination: ignore_above: 1024 level: extended name: address + order: 0 short: Destination network address. type: keyword bytes: @@ -548,6 +596,7 @@ destination: flat_name: destination.bytes level: core name: bytes + order: 5 short: Bytes sent from the destination to the source. type: long domain: @@ -556,6 +605,7 @@ destination: ignore_above: 1024 level: core name: domain + order: 4 short: Destination domain. type: keyword geo.city_name: @@ -565,6 +615,7 @@ destination: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -575,6 +626,7 @@ destination: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -585,6 +637,7 @@ destination: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -595,6 +648,7 @@ destination: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -604,6 +658,7 @@ destination: flat_name: destination.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -620,6 +675,7 @@ destination: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -630,6 +686,7 @@ destination: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -640,6 +697,7 @@ destination: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -650,6 +708,7 @@ destination: flat_name: destination.ip level: core name: ip + order: 1 short: IP address of the destination. type: ip mac: @@ -658,6 +717,7 @@ destination: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the destination. type: keyword packets: @@ -666,6 +726,7 @@ destination: flat_name: destination.packets level: core name: packets + order: 6 short: Packets sent from the destination to the source. type: long port: @@ -673,6 +734,7 @@ destination: flat_name: destination.port level: core name: port + order: 2 short: Port of the destination. type: long user.email: @@ -681,6 +743,7 @@ destination: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -691,6 +754,7 @@ destination: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -700,6 +764,7 @@ destination: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -709,6 +774,7 @@ destination: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -722,6 +788,7 @@ destination: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -731,6 +798,7 @@ destination: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -741,11 +809,15 @@ destination: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword group: 2 name: destination + nestings: + - geo + - user prefix: destination. short: Fields about the destination side of a network connection, used with source. title: Destination @@ -765,12 +837,14 @@ ecs: ignore_above: 1024 level: core name: version + order: 0 required: true short: ECS version this event conforms to. type: keyword group: 2 name: ecs prefix: ecs. + short: Meta-information specific to ECS. title: ECS type: group error: @@ -785,6 +859,7 @@ error: ignore_above: 1024 level: core name: code + order: 2 short: Error code describing the error. type: keyword id: @@ -793,6 +868,7 @@ error: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier for the error. type: keyword message: @@ -801,6 +877,7 @@ error: level: core name: message norms: false + order: 1 short: Error message. type: text group: 2 @@ -833,6 +910,7 @@ event: ignore_above: 1024 level: core name: action + order: 3 short: The action captured by the event. type: keyword category: @@ -847,6 +925,7 @@ event: ignore_above: 1024 level: core name: category + order: 2 short: Event category. type: keyword created: @@ -865,6 +944,7 @@ event: flat_name: event.created level: core name: created + order: 13 short: Time when the event was first read by an agent or by your pipeline. type: date dataset: @@ -878,6 +958,7 @@ event: ignore_above: 1024 level: core name: dataset + order: 7 short: Name of the dataset. type: keyword duration: @@ -888,6 +969,7 @@ event: flat_name: event.duration level: core name: duration + order: 11 short: Duration of the event in nanoseconds. type: long end: @@ -896,6 +978,7 @@ event: flat_name: event.end level: extended name: end + order: 15 short: event.end contains the date when the event ended or when the activity was last observed. type: date @@ -907,6 +990,7 @@ event: ignore_above: 1024 level: extended name: hash + order: 10 short: Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. type: keyword @@ -917,6 +1001,7 @@ event: ignore_above: 1024 level: core name: id + order: 0 short: Unique ID to describe the event. type: keyword kind: @@ -931,6 +1016,7 @@ event: ignore_above: 1024 level: extended name: kind + order: 1 short: The kind of the event. type: keyword module: @@ -942,9 +1028,8 @@ event: ignore_above: 1024 level: core name: module - short: 'Name of the module this data is coming from. - - This information is coming from the modules used in Beats or Logstash.' + order: 6 + short: Name of the module this data is coming from. type: keyword original: description: 'Raw text message of entire event. Used to demonstrate log integrity. @@ -959,6 +1044,7 @@ event: index: false level: core name: original + order: 9 short: Raw text message of entire event. type: keyword outcome: @@ -973,6 +1059,7 @@ event: ignore_above: 1024 level: extended name: outcome + order: 4 short: The outcome of the event. type: keyword risk_score: @@ -981,6 +1068,7 @@ event: flat_name: event.risk_score level: core name: risk_score + order: 16 short: Risk score or priority of the event (e.g. security solutions). Use your system's original value here. type: float @@ -993,6 +1081,7 @@ event: flat_name: event.risk_score_norm level: extended name: risk_score_norm + order: 17 short: Normalized risk score or priority of the event (0-100). type: float severity: @@ -1003,6 +1092,7 @@ event: flat_name: event.severity level: core name: severity + order: 8 short: Original severity of the event. type: long start: @@ -1011,6 +1101,7 @@ event: flat_name: event.start level: extended name: start + order: 14 short: event.start contains the date when the event started or when the activity was first observed. type: date @@ -1025,6 +1116,7 @@ event: ignore_above: 1024 level: extended name: timezone + order: 12 short: Event time zone. type: keyword type: @@ -1035,6 +1127,7 @@ event: ignore_above: 1024 level: core name: type + order: 5 short: Reserved for future usage. type: keyword group: 2 @@ -1057,6 +1150,7 @@ file: flat_name: file.ctime level: extended name: ctime + order: 13 short: Last time file metadata changed. type: date device: @@ -1065,6 +1159,7 @@ file: ignore_above: 1024 level: extended name: device + order: 4 short: Device that is the source of the file. type: keyword extension: @@ -1076,6 +1171,7 @@ file: ignore_above: 1024 level: extended name: extension + order: 2 short: File extension. type: keyword gid: @@ -1084,6 +1180,7 @@ file: ignore_above: 1024 level: extended name: gid + order: 8 short: Primary group ID (GID) of the file. type: keyword group: @@ -1092,6 +1189,7 @@ file: ignore_above: 1024 level: extended name: group + order: 9 short: Primary group name of the file. type: keyword inode: @@ -1100,6 +1198,7 @@ file: ignore_above: 1024 level: extended name: inode + order: 5 short: Inode representing the file in the filesystem. type: keyword mode: @@ -1109,6 +1208,7 @@ file: ignore_above: 1024 level: extended name: mode + order: 10 short: Mode of the file in octal representation. type: keyword mtime: @@ -1116,6 +1216,7 @@ file: flat_name: file.mtime level: extended name: mtime + order: 12 short: Last time file content was modified. type: date owner: @@ -1124,6 +1225,7 @@ file: ignore_above: 1024 level: extended name: owner + order: 7 short: File owner's username. type: keyword path: @@ -1132,6 +1234,7 @@ file: ignore_above: 1024 level: extended name: path + order: 0 short: Path to the file. type: keyword size: @@ -1139,6 +1242,7 @@ file: flat_name: file.size level: extended name: size + order: 11 short: File size in bytes (field is only added when `type` is `file`). type: long target_path: @@ -1147,6 +1251,7 @@ file: ignore_above: 1024 level: extended name: target_path + order: 1 short: Target path for symlinks. type: keyword type: @@ -1155,6 +1260,7 @@ file: ignore_above: 1024 level: extended name: type + order: 3 short: File type (file, dir, or symlink). type: keyword uid: @@ -1163,6 +1269,7 @@ file: ignore_above: 1024 level: extended name: uid + order: 6 short: The user ID (UID) or security identifier (SID) of the file owner. type: keyword group: 2 @@ -1185,6 +1292,7 @@ geo: ignore_above: 1024 level: core name: city_name + order: 4 short: City name. type: keyword continent_name: @@ -1194,6 +1302,7 @@ geo: ignore_above: 1024 level: core name: continent_name + order: 1 short: Name of the continent. type: keyword country_iso_code: @@ -1203,6 +1312,7 @@ geo: ignore_above: 1024 level: core name: country_iso_code + order: 5 short: Country ISO code. type: keyword country_name: @@ -1212,6 +1322,7 @@ geo: ignore_above: 1024 level: core name: country_name + order: 2 short: Country name. type: keyword location: @@ -1220,6 +1331,7 @@ geo: flat_name: geo.location level: core name: location + order: 0 short: Longitude and latitude. type: geo_point name: @@ -1235,6 +1347,7 @@ geo: ignore_above: 1024 level: extended name: name + order: 7 short: User-defined description of a location. type: keyword region_iso_code: @@ -1244,6 +1357,7 @@ geo: ignore_above: 1024 level: core name: region_iso_code + order: 6 short: Region ISO code. type: keyword region_name: @@ -1253,6 +1367,7 @@ geo: ignore_above: 1024 level: core name: region_name + order: 3 short: Region name. type: keyword group: 2 @@ -1280,6 +1395,7 @@ group: ignore_above: 1024 level: extended name: id + order: 0 short: Unique identifier for the group on the system/platform. type: keyword name: @@ -1288,6 +1404,7 @@ group: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the group. type: keyword group: 2 @@ -1314,6 +1431,7 @@ host: ignore_above: 1024 level: core name: architecture + order: 6 short: Operating system architecture. type: keyword geo.city_name: @@ -1323,6 +1441,7 @@ host: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1333,6 +1452,7 @@ host: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1343,6 +1463,7 @@ host: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1353,6 +1474,7 @@ host: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1362,6 +1484,7 @@ host: flat_name: host.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1378,6 +1501,7 @@ host: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1388,6 +1512,7 @@ host: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1398,6 +1523,7 @@ host: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1409,6 +1535,7 @@ host: ignore_above: 1024 level: core name: hostname + order: 0 short: Hostname of the host. type: keyword id: @@ -1421,6 +1548,7 @@ host: ignore_above: 1024 level: core name: id + order: 2 short: Unique host id. type: keyword ip: @@ -1428,6 +1556,7 @@ host: flat_name: host.ip level: core name: ip + order: 3 short: Host ip address. type: ip mac: @@ -1436,6 +1565,7 @@ host: ignore_above: 1024 level: core name: mac + order: 4 short: Host mac address. type: keyword name: @@ -1448,6 +1578,7 @@ host: ignore_above: 1024 level: core name: name + order: 1 short: Name of the host. type: keyword os.family: @@ -1457,6 +1588,7 @@ host: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -1467,6 +1599,7 @@ host: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -1477,6 +1610,7 @@ host: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -1487,6 +1621,7 @@ host: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -1497,6 +1632,7 @@ host: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -1507,6 +1643,7 @@ host: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -1520,6 +1657,7 @@ host: ignore_above: 1024 level: core name: type + order: 5 short: Type of host. type: keyword user.email: @@ -1528,6 +1666,7 @@ host: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -1538,6 +1677,7 @@ host: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -1547,6 +1687,7 @@ host: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -1556,6 +1697,7 @@ host: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -1569,6 +1711,7 @@ host: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -1578,6 +1721,7 @@ host: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -1588,11 +1732,16 @@ host: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword group: 2 name: host + nestings: + - geo + - os + - user prefix: host. short: Fields describing the relevant computing instance. title: Host @@ -1607,6 +1756,7 @@ http: flat_name: http.request.body.bytes level: extended name: request.body.bytes + order: 7 short: Size in bytes of the request body. type: long request.body.content: @@ -1616,6 +1766,7 @@ http: ignore_above: 1024 level: extended name: request.body.content + order: 1 short: The full HTTP request body. type: keyword request.bytes: @@ -1624,6 +1775,7 @@ http: flat_name: http.request.bytes level: extended name: request.bytes + order: 6 short: Total size in bytes of the request (body and headers). type: long request.method: @@ -1636,6 +1788,7 @@ http: ignore_above: 1024 level: extended name: request.method + order: 0 short: HTTP request method. type: keyword request.referrer: @@ -1645,6 +1798,7 @@ http: ignore_above: 1024 level: extended name: request.referrer + order: 2 short: Referrer for this HTTP request. type: keyword response.body.bytes: @@ -1653,6 +1807,7 @@ http: flat_name: http.response.body.bytes level: extended name: response.body.bytes + order: 9 short: Size in bytes of the response body. type: long response.body.content: @@ -1662,6 +1817,7 @@ http: ignore_above: 1024 level: extended name: response.body.content + order: 4 short: The full HTTP response body. type: keyword response.bytes: @@ -1670,6 +1826,7 @@ http: flat_name: http.response.bytes level: extended name: response.bytes + order: 8 short: Total size in bytes of the response (body and headers). type: long response.status_code: @@ -1678,6 +1835,7 @@ http: flat_name: http.response.status_code level: extended name: response.status_code + order: 3 short: HTTP response status code. type: long version: @@ -1687,6 +1845,7 @@ http: ignore_above: 1024 level: extended name: version + order: 5 short: HTTP version. type: keyword group: 2 @@ -1707,6 +1866,7 @@ log: ignore_above: 1024 level: core name: level + order: 0 short: Log level of the log event. type: keyword original: @@ -1727,11 +1887,13 @@ log: index: false level: core name: original + order: 1 short: Original log message with light interpretation only (encoding, newlines). type: keyword group: 2 name: log prefix: log. + short: Fields which are specific to log events. title: Log type: group network: @@ -1755,6 +1917,7 @@ network: ignore_above: 1024 level: extended name: application + order: 4 short: Application level protocol name. type: keyword bytes: @@ -1766,6 +1929,7 @@ network: flat_name: network.bytes level: core name: bytes + order: 9 short: Total bytes transferred in both directions. type: long community_id: @@ -1779,6 +1943,7 @@ network: ignore_above: 1024 level: extended name: community_id + order: 8 short: A hash of source and destination IPs and ports. type: keyword direction: @@ -1793,6 +1958,7 @@ network: ignore_above: 1024 level: core name: direction + order: 6 short: Direction of the network traffic. type: keyword forwarded_ip: @@ -1801,6 +1967,7 @@ network: flat_name: network.forwarded_ip level: core name: forwarded_ip + order: 7 short: Host IP address when the source IP address is the proxy. type: ip iana_number: @@ -1812,6 +1979,7 @@ network: ignore_above: 1024 level: extended name: iana_number + order: 2 short: IANA Protocol Number. type: keyword name: @@ -1821,6 +1989,7 @@ network: ignore_above: 1024 level: extended name: name + order: 0 short: Name given by operators to sections of their network. type: keyword packets: @@ -1832,6 +2001,7 @@ network: flat_name: network.packets level: core name: packets + order: 10 short: Total packets transferred in both directions. type: long protocol: @@ -1844,6 +2014,7 @@ network: ignore_above: 1024 level: core name: protocol + order: 5 short: L7 Network protocol name. type: keyword transport: @@ -1857,6 +2028,7 @@ network: ignore_above: 1024 level: core name: transport + order: 3 short: Protocol Name corresponding to the field `iana_number`. type: keyword type: @@ -1870,6 +2042,7 @@ network: ignore_above: 1024 level: core name: type + order: 1 short: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc type: keyword @@ -1900,6 +2073,7 @@ observer: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1910,6 +2084,7 @@ observer: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1920,6 +2095,7 @@ observer: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1930,6 +2106,7 @@ observer: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1939,6 +2116,7 @@ observer: flat_name: observer.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1955,6 +2133,7 @@ observer: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1965,6 +2144,7 @@ observer: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1975,6 +2155,7 @@ observer: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1984,6 +2165,7 @@ observer: ignore_above: 1024 level: core name: hostname + order: 2 short: Hostname of the observer. type: keyword ip: @@ -1991,6 +2173,7 @@ observer: flat_name: observer.ip level: core name: ip + order: 1 short: IP address of the observer. type: ip mac: @@ -1999,6 +2182,7 @@ observer: ignore_above: 1024 level: core name: mac + order: 0 short: MAC address of the observer type: keyword os.family: @@ -2008,6 +2192,7 @@ observer: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -2018,6 +2203,7 @@ observer: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -2028,6 +2214,7 @@ observer: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -2038,6 +2225,7 @@ observer: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -2048,6 +2236,7 @@ observer: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -2058,6 +2247,7 @@ observer: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -2067,6 +2257,7 @@ observer: ignore_above: 1024 level: extended name: serial_number + order: 5 short: Observer serial number. type: keyword type: @@ -2079,6 +2270,7 @@ observer: ignore_above: 1024 level: core name: type + order: 6 short: The type of the observer the data is coming from. type: keyword vendor: @@ -2087,6 +2279,7 @@ observer: ignore_above: 1024 level: core name: vendor + order: 3 short: observer vendor information. type: keyword version: @@ -2095,10 +2288,14 @@ observer: ignore_above: 1024 level: core name: version + order: 4 short: Observer version. type: keyword group: 2 name: observer + nestings: + - geo + - os prefix: observer. short: Fields describing an entity observing the event from outside the host. title: Observer @@ -2116,6 +2313,7 @@ organization: ignore_above: 1024 level: extended name: id + order: 1 short: Unique identifier for the organization. type: keyword name: @@ -2124,6 +2322,7 @@ organization: ignore_above: 1024 level: extended name: name + order: 0 short: Organization name. type: keyword group: 2 @@ -2142,6 +2341,7 @@ os: ignore_above: 1024 level: extended name: family + order: 3 short: OS family (such as redhat, debian, freebsd, windows). type: keyword full: @@ -2151,6 +2351,7 @@ os: ignore_above: 1024 level: extended name: full + order: 2 short: Operating system name, including the version or code name. type: keyword kernel: @@ -2160,6 +2361,7 @@ os: ignore_above: 1024 level: extended name: kernel + order: 5 short: Operating system kernel version as a raw string. type: keyword name: @@ -2169,6 +2371,7 @@ os: ignore_above: 1024 level: extended name: name + order: 1 short: Operating system name, without the version. type: keyword platform: @@ -2178,6 +2381,7 @@ os: ignore_above: 1024 level: extended name: platform + order: 0 short: Operating system platform (such centos, ubuntu, windows). type: keyword version: @@ -2187,6 +2391,7 @@ os: ignore_above: 1024 level: extended name: version + order: 4 short: Operating system version as a raw string. type: keyword group: 2 @@ -2221,6 +2426,7 @@ process: ignore_above: 1024 level: extended name: args + order: 3 short: Array of process arguments. type: keyword executable: @@ -2230,6 +2436,7 @@ process: ignore_above: 1024 level: extended name: executable + order: 4 short: Absolute path to the process executable. type: keyword name: @@ -2241,6 +2448,7 @@ process: ignore_above: 1024 level: extended name: name + order: 1 short: Process name. type: keyword pid: @@ -2249,6 +2457,7 @@ process: flat_name: process.pid level: core name: pid + order: 0 short: Process id. type: long ppid: @@ -2256,6 +2465,7 @@ process: flat_name: process.ppid level: extended name: ppid + order: 2 short: Process parent id. type: long start: @@ -2264,6 +2474,7 @@ process: flat_name: process.start level: extended name: start + order: 7 short: The time the process started. type: date thread.id: @@ -2272,6 +2483,7 @@ process: flat_name: process.thread.id level: extended name: thread.id + order: 6 short: Thread ID. type: long title: @@ -2283,6 +2495,7 @@ process: ignore_above: 1024 level: extended name: title + order: 5 short: Process title. type: keyword working_directory: @@ -2292,6 +2505,7 @@ process: ignore_above: 1024 level: extended name: working_directory + order: 8 short: The working directory of the process. type: keyword group: 2 @@ -2317,6 +2531,7 @@ related: flat_name: related.ip level: extended name: ip + order: 0 short: All of the IPs seen on your event. type: ip group: 2 @@ -2353,6 +2568,7 @@ server: ignore_above: 1024 level: extended name: address + order: 0 short: Server network address. type: keyword bytes: @@ -2361,6 +2577,7 @@ server: flat_name: server.bytes level: core name: bytes + order: 5 short: Bytes sent from the server to the client. type: long domain: @@ -2369,6 +2586,7 @@ server: ignore_above: 1024 level: core name: domain + order: 4 short: Server domain. type: keyword geo.city_name: @@ -2378,6 +2596,7 @@ server: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2388,6 +2607,7 @@ server: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2398,6 +2618,7 @@ server: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2408,6 +2629,7 @@ server: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2417,6 +2639,7 @@ server: flat_name: server.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2433,6 +2656,7 @@ server: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2443,6 +2667,7 @@ server: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2453,6 +2678,7 @@ server: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2463,6 +2689,7 @@ server: flat_name: server.ip level: core name: ip + order: 1 short: IP address of the server. type: ip mac: @@ -2471,6 +2698,7 @@ server: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the server. type: keyword packets: @@ -2479,6 +2707,7 @@ server: flat_name: server.packets level: core name: packets + order: 6 short: Packets sent from the server to the client. type: long port: @@ -2486,6 +2715,7 @@ server: flat_name: server.port level: core name: port + order: 2 short: Port of the server. type: long user.email: @@ -2494,6 +2724,7 @@ server: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2504,6 +2735,7 @@ server: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2513,6 +2745,7 @@ server: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2522,6 +2755,7 @@ server: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2535,6 +2769,7 @@ server: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2544,6 +2779,7 @@ server: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2554,11 +2790,15 @@ server: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword group: 2 name: server + nestings: + - geo + - user prefix: server. short: Fields about the server side of a network connection, used with client. title: Server @@ -2578,6 +2818,7 @@ service: ignore_above: 1024 level: extended name: ephemeral_id + order: 5 short: Ephemeral identifier of this service. type: keyword id: @@ -2593,6 +2834,7 @@ service: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier of the running service. type: keyword name: @@ -2613,6 +2855,7 @@ service: ignore_above: 1024 level: core name: name + order: 1 short: Name of the service. type: keyword state: @@ -2621,6 +2864,7 @@ service: ignore_above: 1024 level: core name: state + order: 3 short: Current state of the service. type: keyword type: @@ -2636,6 +2880,7 @@ service: ignore_above: 1024 level: core name: type + order: 2 short: The type of the service. type: keyword version: @@ -2647,6 +2892,7 @@ service: ignore_above: 1024 level: core name: version + order: 4 short: Version of the service. type: keyword group: 2 @@ -2671,6 +2917,7 @@ source: ignore_above: 1024 level: extended name: address + order: 0 short: Source network address. type: keyword bytes: @@ -2679,6 +2926,7 @@ source: flat_name: source.bytes level: core name: bytes + order: 5 short: Bytes sent from the source to the destination. type: long domain: @@ -2687,6 +2935,7 @@ source: ignore_above: 1024 level: core name: domain + order: 4 short: Source domain. type: keyword geo.city_name: @@ -2696,6 +2945,7 @@ source: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2706,6 +2956,7 @@ source: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2716,6 +2967,7 @@ source: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2726,6 +2978,7 @@ source: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2735,6 +2988,7 @@ source: flat_name: source.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2751,6 +3005,7 @@ source: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2761,6 +3016,7 @@ source: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2771,6 +3027,7 @@ source: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2781,6 +3038,7 @@ source: flat_name: source.ip level: core name: ip + order: 1 short: IP address of the source. type: ip mac: @@ -2789,6 +3047,7 @@ source: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the source. type: keyword packets: @@ -2797,6 +3056,7 @@ source: flat_name: source.packets level: core name: packets + order: 6 short: Packets sent from the source to the destination. type: long port: @@ -2804,6 +3064,7 @@ source: flat_name: source.port level: core name: port + order: 2 short: Port of the source. type: long user.email: @@ -2812,6 +3073,7 @@ source: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2822,6 +3084,7 @@ source: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2831,6 +3094,7 @@ source: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2840,6 +3104,7 @@ source: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2853,6 +3118,7 @@ source: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2862,6 +3128,7 @@ source: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2872,11 +3139,15 @@ source: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword group: 2 name: source + nestings: + - geo + - user prefix: source. short: Fields about the source side of a network connection, used with destination. title: Source @@ -2895,6 +3166,7 @@ url: ignore_above: 1024 level: extended name: domain + order: 3 short: Domain of the url. type: keyword fragment: @@ -2905,6 +3177,7 @@ url: ignore_above: 1024 level: extended name: fragment + order: 7 short: Portion of the url after the `#`. type: keyword full: @@ -2916,6 +3189,7 @@ url: ignore_above: 1024 level: extended name: full + order: 1 short: Full unparsed URL. type: keyword original: @@ -2930,6 +3204,7 @@ url: ignore_above: 1024 level: extended name: original + order: 0 short: Unmodified original url as seen in the event source. type: keyword password: @@ -2938,6 +3213,7 @@ url: ignore_above: 1024 level: extended name: password + order: 9 short: Password of the request. type: keyword path: @@ -2946,6 +3222,7 @@ url: ignore_above: 1024 level: extended name: path + order: 5 short: Path of the request, such as "/search". type: keyword port: @@ -2954,6 +3231,7 @@ url: flat_name: url.port level: extended name: port + order: 4 short: Port of the request, such as 443. type: long query: @@ -2968,6 +3246,7 @@ url: ignore_above: 1024 level: extended name: query + order: 6 short: Query string of the request. type: keyword scheme: @@ -2979,6 +3258,7 @@ url: ignore_above: 1024 level: extended name: scheme + order: 2 short: Scheme of the url. type: keyword username: @@ -2987,6 +3267,7 @@ url: ignore_above: 1024 level: extended name: username + order: 8 short: Username of the request. type: keyword group: 2 @@ -3008,6 +3289,7 @@ user: ignore_above: 1024 level: extended name: email + order: 3 short: User email address. type: keyword full_name: @@ -3017,6 +3299,7 @@ user: ignore_above: 1024 level: extended name: full_name + order: 2 short: User's full name, if available. type: keyword group.id: @@ -3025,6 +3308,7 @@ user: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: group short: Unique identifier for the group on the system/platform. type: keyword @@ -3034,6 +3318,7 @@ user: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: group short: Name of the group. type: keyword @@ -3047,6 +3332,7 @@ user: ignore_above: 1024 level: extended name: hash + order: 4 short: Unique user hash to correlate information for a user in anonymized form. type: keyword id: @@ -3055,6 +3341,7 @@ user: ignore_above: 1024 level: core name: id + order: 0 short: One or multiple unique identifiers of the user. type: keyword name: @@ -3064,10 +3351,13 @@ user: ignore_above: 1024 level: core name: name + order: 1 short: Short name or login of the user. type: keyword group: 2 name: user + nestings: + - group prefix: user. reusable: expected: @@ -3092,6 +3382,7 @@ user_agent: ignore_above: 1024 level: extended name: device.name + order: 3 short: Name of the device. type: keyword name: @@ -3101,6 +3392,7 @@ user_agent: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the user agent. type: keyword original: @@ -3111,6 +3403,7 @@ user_agent: ignore_above: 1024 level: extended name: original + order: 0 short: Unparsed version of the user_agent. type: keyword os.family: @@ -3120,6 +3413,7 @@ user_agent: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -3130,6 +3424,7 @@ user_agent: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -3140,6 +3435,7 @@ user_agent: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -3150,6 +3446,7 @@ user_agent: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -3160,6 +3457,7 @@ user_agent: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -3170,6 +3468,7 @@ user_agent: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -3180,10 +3479,13 @@ user_agent: ignore_above: 1024 level: extended name: version + order: 2 short: Version of the user agent. type: keyword group: 2 name: user_agent + nestings: + - os prefix: user_agent. short: Fields to describe a browser user_agent string. title: User agent diff --git a/schemas/event.yml b/schemas/event.yml index 42e113017f..9bc7fee9bb 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -84,6 +84,7 @@ - name: module level: core type: keyword + short: Name of the module this data is coming from. description: > Name of the module this data is coming from. diff --git a/scripts/generator.py b/scripts/generator.py index 64d8073532..816cb8459e 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -5,6 +5,7 @@ from generators import csv_generator from generators import es_template from generators import beats +from generators import asciidoc_fields def main(): @@ -22,6 +23,7 @@ def main(): csv_generator.generate(ecs_flat, ecs_version) es_template.generate(ecs_flat, ecs_version) beats.generate(ecs_nested, ecs_version) + asciidoc_fields.generate(ecs_nested, ecs_version) def argument_parser(): diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py new file mode 100644 index 0000000000..6d0bd0c8a7 --- /dev/null +++ b/scripts/generators/asciidoc_fields.py @@ -0,0 +1,246 @@ +import sys + +from generators import ecs_helpers + + +def generate(ecs_nested, ecs_version): + save_asciidoc('docs/fields.asciidoc', page_field_index(ecs_nested)) + save_asciidoc('docs/field-details.asciidoc', page_field_details(ecs_nested)) + +# Helpers + + +def save_asciidoc(file, text): + open_mode = "wb" + if sys.version_info >= (3, 0): + open_mode = "w" + with open(file, open_mode) as outfile: + outfile.write(text) + + +# Rendering + +# Field Index + + +def page_field_index(ecs_nested): + page_text = index_header() + for fieldset in ecs_helpers.dict_sorted_by_keys(ecs_nested, ['group', 'name']): + page_text += render_field_index_row(fieldset) + page_text += table_footer() + page_text += index_footer() + return page_text + + +def render_field_index_row(fieldset): + return index_row().format( + fieldset_id='ecs-' + fieldset['name'], + fieldset_title=fieldset['title'], + fieldset_short=fieldset.get('short', fieldset['description']) + ) + + +# Field Details Page + +def page_field_details(ecs_nested): + page_text = '' + for fieldset in ecs_helpers.dict_sorted_by_keys(ecs_nested, ['group', 'name']): + page_text += render_fieldset(fieldset, ecs_nested) + return page_text + + +def render_fieldset(fieldset, ecs_nested): + text = field_details_table_header().format( + fieldset_title=fieldset['title'], + fieldset_name=fieldset['name'], + fieldset_description=fieldset['description'] + ) + + for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): + if 'original_fieldset' not in field: + text += render_field_details_row(field) + + text += table_footer() + + text += render_fieldset_reuse_section(fieldset, ecs_nested) + + return text + + +def render_field_details_row(field): + example = '' + if 'example' in field: + example = "example: `{}`".format(str(field['example'])) + text = field_details_row().format( + field_flat_name=field['flat_name'], + field_description=field['description'], + field_example=example, + field_level=field['level'], + field_type=field['type'], + ) + return text + + +def render_fieldset_reuse_section(fieldset, ecs_nested): + '''Render the section on where field set can be nested, and which field sets can be nested here''' + if not ('nestings' in fieldset or 'reusable' in fieldset): + return '' + + text = field_reuse_section().format( + reuse_of_fieldset=render_fieldset_reuses_text(fieldset) + ) + if 'nestings' in fieldset: + text += nestings_table_header().format( + fieldset_name=fieldset['name'], + fieldset_title=fieldset['title'] + ) + nestings = [] + for nested_fs_name in sorted(fieldset['nestings']): + text += render_nesting_row({ + 'flat_nesting': "{}.{}.*".format(fieldset['name'], nested_fs_name), + 'name': nested_fs_name, + 'short': ecs_nested[nested_fs_name]['short'] + }) + text += table_footer() + return text + + +def render_fieldset_reuses_text(fieldset): + '''Render where a given field set is expected to be reused''' + if 'reusable' not in fieldset: + return '' + + section_name = fieldset['name'] + sorted_fields = sorted(fieldset['reusable']['expected']) + rendered_fields = map(lambda f: "`{}.{}`".format(f, section_name), sorted_fields) + text = "The `{}` fields are expected to be nested at: {}.\n\n".format( + section_name, ', '.join(rendered_fields)) + + if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: + template = "Note also that the `{}` fields may be used directly at the top level.\n\n" + else: + template = "Note also that the `{}` fields are not expected to " + \ + "be used directly at the top level.\n\n" + text += template.format(section_name) + return text + + +def render_nesting_row(nesting): + text = nestings_row().format( + nesting_name=nesting['name'], + flat_nesting=nesting['flat_nesting'], + nesting_short=nesting['short'], + ) + return text + + +# Templates + + +def table_footer(): + return ''' +|===== +''' + + +# Field Index + + +def index_header(): + return ''' +[[ecs-fields]] +== {ecs} Fields + +[float] +[[ecs-fieldsets]] +=== Field Sets +[cols="<,<",options="header",] +|======================================================================= +| Field Set | Description +''' + + +def index_row(): + return ''' +| <<{fieldset_id},{fieldset_title}>> | {fieldset_short} +''' + + +def index_footer(): + return ''' +include::field-details.asciidoc[] +''' + + +# Field Details Page + +# Main Fields Table + + +def field_details_table_header(): + return ''' +[[ecs-{fieldset_name}]] +=== {fieldset_title} Fields + +{fieldset_description} + +==== {fieldset_title} Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== +''' + + +def field_details_row(): + return ''' +| {field_flat_name} +| {field_description} + +type: {field_type} + +{field_example} + +| {field_level} + +// =============================================================== +''' + + +# Field reuse + +def field_reuse_section(): + return ''' +==== Field Reuse + +{reuse_of_fieldset} + +''' + + +# Nestings table + +def nestings_table_header(): + return ''' +[[ecs-{fieldset_name}-nestings]] +===== Field sets that can be nested under {fieldset_title} + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +''' + + +def nestings_row(): + return ''' +| http://localhost:8000/ecs-{nesting_name}.html[{flat_nesting}] +| {nesting_short} + +// =============================================================== + +''' diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 0c4a14fcb5..2c9a59dc63 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -1,5 +1,7 @@ import yaml + from collections import OrderedDict +from generators import ecs_helpers def generate(ecs_nested, ecs_version): @@ -13,7 +15,7 @@ def generate(ecs_nested, ecs_version): continue fieldset = ecs_nested[fieldset_name] - beats_field = dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) + beats_field = ecs_helpers.dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) beats_field['fields'] = fieldset_field_array(fieldset['fields']) beats_fields.append(beats_field) @@ -33,7 +35,7 @@ def fieldset_field_array(source_fields): fields = [] for nested_field_name in source_fields: ecs_field = source_fields[nested_field_name] - beats_field = dict_copy_keys_ordered(ecs_field, allowed_keys) + beats_field = ecs_helpers.dict_copy_keys_ordered(ecs_field, allowed_keys) beats_field['name'] = nested_field_name fields.append(beats_field) return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) @@ -42,37 +44,9 @@ def fieldset_field_array(source_fields): def write_beats_yaml(beats_file, ecs_version): + warning = file_header().format(version=ecs_version) + ecs_helpers.yaml_dump('generated/beats/fields.ecs.yml', [beats_file], preamble=warning) - with open('generated/beats/fields.ecs.yml', 'w') as outfile: - outfile.write(file_header().format(version=ecs_version)) - yaml.dump([beats_file], outfile, default_flow_style=False) - - -def dict_copy_keys_ordered(dict, copied_keys): - ordered_dict = OrderedDict() - for key in copied_keys: - if key in dict: - ordered_dict[key] = dict[key] - return ordered_dict - - -def indent(text, indent, char=' '): - padding = indent * char - return ''.join(padding + line for line in text.splitlines(True)) - -# Rendering - - -def yaml_ordereddict(dumper, data): - value = [] - for item_key, item_value in data.items(): - node_key = dumper.represent_data(item_key) - node_value = dumper.represent_data(item_value) - value.append((node_key, node_value)) - return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) - - -yaml.add_representer(OrderedDict, yaml_ordereddict) # Templates diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py new file mode 100644 index 0000000000..75adb8a096 --- /dev/null +++ b/scripts/generators/ecs_helpers.py @@ -0,0 +1,59 @@ +import yaml + +from collections import OrderedDict + +# Dictionary helpers + + +def dict_copy_keys_ordered(dict, copied_keys): + ordered_dict = OrderedDict() + for key in copied_keys: + if key in dict: + ordered_dict[key] = dict[key] + return ordered_dict + + +def dict_copy_existing_keys(source, destination, keys): + for key in keys: + if key in source: + destination[key] = source[key] + + +def dict_sorted_by_keys(dict, sort_keys): + if not isinstance(sort_keys, list): + sort_keys = [sort_keys] + tuples = [] + for key in dict: + nested = dict[key] + + sort_criteria = [] + for sort_key in sort_keys: + sort_criteria.append(nested[sort_key]) + sort_criteria.append(nested) + tuples.append(sort_criteria) + + return list(map(lambda t: t[-1], sorted(tuples))) + + +def yaml_ordereddict(dumper, data): + # YAML representation of an OrderedDict will be like a dictionary, but + # respecting the order of the dictionary. + # Almost sure it's unndecessary with Python 3. + value = [] + for item_key, item_value in data.items(): + node_key = dumper.represent_data(item_key) + node_value = dumper.represent_data(item_value) + value.append((node_key, node_value)) + return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) + + +yaml.add_representer(OrderedDict, yaml_ordereddict) + +# File helpers + + +def yaml_dump(filename, data, preamble=None): + with open(filename, 'w') as outfile: + if preamble: + outfile.write(preamble) + yaml.dump(data, outfile, default_flow_style=False) diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 12502148b9..62fae3d3c3 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -1,6 +1,8 @@ import json import sys +from generators import ecs_helpers + def generate(ecs_flat, ecs_version): field_mappings = {} @@ -32,22 +34,16 @@ def dict_add_nested(dict, nestings, value): dict[current_nesting] = value -def dict_copy_existing_keys(source, destination, keys): - for key in keys: - if key in source: - destination[key] = source[key] - - def entry_for(field): dict = {'type': field['type']} try: if 'index' in field and not field['index']: - dict_copy_existing_keys(field, dict, ['index', 'doc_values']) + ecs_helpers.dict_copy_existing_keys(field, dict, ['index', 'doc_values']) if field['type'] == 'keyword': - dict_copy_existing_keys(field, dict, ['ignore_above']) + ecs_helpers.dict_copy_existing_keys(field, dict, ['ignore_above']) elif field['type'] == 'text': - dict_copy_existing_keys(field, dict, ['norms']) + ecs_helpers.dict_copy_existing_keys(field, dict, ['norms']) except KeyError as ex: print ex, field raise ex diff --git a/scripts/generators/intermediate_files.py b/scripts/generators/intermediate_files.py index cdf2c46303..b6a25a1868 100644 --- a/scripts/generators/intermediate_files.py +++ b/scripts/generators/intermediate_files.py @@ -1,9 +1,6 @@ -import yaml +from generators import ecs_helpers def generate(ecs_nested, ecs_flat): - with open('generated/ecs/fields_flat.yml', 'w') as outfile: - yaml.dump(ecs_flat, outfile, default_flow_style=False) - - with open('generated/ecs/fields_nested.yml', 'w') as outfile: - yaml.dump(ecs_nested, outfile, default_flow_style=False) + ecs_helpers.yaml_dump('generated/ecs/fields_flat.yml', ecs_flat) + ecs_helpers.yaml_dump('generated/ecs/fields_nested.yml', ecs_nested) diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 66d2a4b5b6..8c01756621 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -57,6 +57,9 @@ def schema_cleanup_values(schema): def schema_set_default_values(schema): schema['type'] = 'group' dict_set_default(schema, 'group', 2) + dict_set_default(schema, 'short', schema['description']) + if "\n" in schema['short']: + raise ValueError("Short descriptions must be single line.\nFieldset: {}\n{}".format(schema['name'], schema)) def schema_set_fieldset_prefix(schema): @@ -70,7 +73,8 @@ def schema_fields_as_dictionary(schema): """Re-nest the array of field names as a dictionary of 'fieldname' => { field definition }""" field_array = schema.pop('fields') schema['fields'] = {} - for field in field_array: + for order, field in enumerate(field_array): + field['order'] = order schema['fields'][field['name']] = field # Field definitions @@ -78,8 +82,8 @@ def schema_fields_as_dictionary(schema): def field_cleanup_values(field, prefix): dict_clean_string_values(field) - field_set_defaults(field) field_set_flat_name(field, prefix) + field_set_defaults(field) def field_set_flat_name(field, prefix): @@ -87,7 +91,6 @@ def field_set_flat_name(field, prefix): def field_set_defaults(field): - dict_set_default(field, 'short', field['description']) if field['type'] == 'keyword': dict_set_default(field, 'ignore_above', 1024) if field['type'] == 'text': @@ -95,6 +98,11 @@ def field_set_defaults(field): if field['type'] == 'object': dict_set_default(field, 'object_type', 'keyword') + dict_set_default(field, 'short', field['description']) + if "\n" in field['short']: + raise ValueError("Short descriptions must be single line.\nField: {}\n{}".format(field['flat_name'], field)) + # print(" Short descriptions must be single line. Field: {}".format(field['flat_name'])) + if 'index' in field and not field['index']: dict_set_default(field, 'doc_values', False) if 'multi_fields' in field: @@ -117,6 +125,13 @@ def duplicate_reusable_fieldsets(schema, fields_flat, fields_nested): if 'reusable' in schema: for new_nesting in schema['reusable']['expected']: + # List field set names expected under another field set. + # E.g. host.nestings = [ 'geo', 'os', 'user' ] + if 'nestings' not in fields_nested[new_nesting]: + fields_nested[new_nesting]['nestings'] = [] + fields_nested[new_nesting]['nestings'].append(schema['name']) + + # Explicitly list all leaf fields coming from field set reuse. for (name, field) in schema['fields'].items(): # Poor folks deepcopy, sorry -- A Rubyist copied_field = field.copy() diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py new file mode 100644 index 0000000000..f13f7f7db6 --- /dev/null +++ b/scripts/tests/test_ecs_helpers.py @@ -0,0 +1,61 @@ +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +from scripts.generators import ecs_helpers + + +class TestECSHelpers(unittest.TestCase): + # dict_copy_existing_keys + + def test_dict_copy_existing_keys(self): + source = {'key1': 'value1'} + destination = {} + ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'missingkey']) + self.assertEqual(destination, {'key1': 'value1'}) + + def test_dict_copy_existing_keys_overwrites(self): + source = {'key1': 'new_value'} + destination = {'key1': 'overwritten', 'untouched': 'untouched'} + ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'untouched']) + self.assertEqual(destination, {'key1': 'new_value', 'untouched': 'untouched'}) + + # dict_sorted_by_keys + + def test_sorted_by_one_key(self): + dict = { + '@timestamp': {'order': 0, 'name': '@timestamp'}, + 'message': {'order': 3, 'name': 'message'}, + 'labels': {'order': 1, 'name': 'labels'}, + 'tags': {'order': 2, 'name': 'tags'} + } + expected = [ + {'order': 0, 'name': '@timestamp'}, + {'order': 1, 'name': 'labels'}, + {'order': 2, 'name': 'tags'}, + {'order': 3, 'name': 'message'} + ] + result = ecs_helpers.dict_sorted_by_keys(dict, 'order') + self.assertEqual(result, expected) + result = ecs_helpers.dict_sorted_by_keys(dict, ['order']) + self.assertEqual(result, expected) + + def test_sorted_by_multiple_keys(self): + dict = { + 'cloud': {'group': 2, 'name': 'cloud'}, + 'agent': {'group': 2, 'name': 'agent'}, + 'base': {'group': 1, 'name': 'base'}, + } + expected = [ + {'group': 1, 'name': 'base'}, + {'group': 2, 'name': 'agent'}, + {'group': 2, 'name': 'cloud'} + ] + result = ecs_helpers.dict_sorted_by_keys(dict, ['group', 'name']) + self.assertEqual(result, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index b69e5e11fe..51dc806233 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -1,6 +1,12 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts import schema_reader + (nested, flat) = schema_reader.load_ecs() diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py index fb211fe20c..2c0dcb1062 100644 --- a/scripts/tests/test_es_template.py +++ b/scripts/tests/test_es_template.py @@ -1,4 +1,9 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts.generators import es_template @@ -25,20 +30,6 @@ def test_add_siblings(self): 'key3': 'value3' }) - # dict_copy_existing_keys - - def test_dict_copy_existing_keys(self): - source = {'key1': 'value1'} - destination = {} - es_template.dict_copy_existing_keys(source, destination, ['key1', 'missingkey']) - self.assertEqual(destination, {'key1': 'value1'}) - - def test_dict_copy_existing_keys_overwrites(self): - source = {'key1': 'new_value'} - destination = {'key1': 'overwritten', 'untouched': 'untouched'} - es_template.dict_copy_existing_keys(source, destination, ['key1', 'untouched']) - self.assertEqual(destination, {'key1': 'new_value', 'untouched': 'untouched'}) - if __name__ == '__main__': unittest.main() diff --git a/scripts/tests/test_schema_reader.py b/scripts/tests/test_schema_reader.py index bb7de86a05..1796308cea 100644 --- a/scripts/tests/test_schema_reader.py +++ b/scripts/tests/test_schema_reader.py @@ -1,4 +1,9 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts import schema_reader @@ -38,14 +43,14 @@ def test_schema_set_fieldset_prefix_not_at_root(self): {'prefix': 'myfieldset.', 'root': False, 'name': 'myfieldset'}) def test_set_default_values_defaults(self): - schema = {} + schema = {'description': '...'} schema_reader.schema_set_default_values(schema) - self.assertEqual(schema, {'group': 2, 'type': 'group'}) + self.assertEqual(schema, {'group': 2, 'type': 'group', 'description': '...', 'short': '...'}) def test_set_default_values_no_overwrite(self): - schema = {'group': 1} + schema = {'group': 1, 'description': '...'} schema_reader.schema_set_default_values(schema) - self.assertEqual(schema, {'group': 1, 'type': 'group'}) + self.assertEqual(schema, {'group': 1, 'type': 'group', 'description': '...', 'short': '...'}) # field definitions