-
Notifications
You must be signed in to change notification settings - Fork 529
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improves package generation. - Splits metrics data stream into app metrics, internal and profiles. This is following the indexing strategy proposal. - Adds the dataset property to the manifest files so the right templates are created. - Sort and dedup fields (within each yml file). Fields are now flattened. - Generates data stream pipelines out of our definitions.json file
- Loading branch information
Showing
49 changed files
with
2,541 additions
and
3,081 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| { | ||
| "description": "Default enrichment for APM events", | ||
| "processors": [ | ||
| { | ||
| "pipeline": { | ||
| "name": "apm_user_agent" | ||
| } | ||
| }, | ||
| { | ||
| "pipeline": { | ||
| "name": "apm_user_geo" | ||
| } | ||
| }, | ||
| { | ||
| "pipeline": { | ||
| "name": "apm_ingest_timestamp" | ||
| } | ||
| }, | ||
| { | ||
| "pipeline": { | ||
| "name": "apm_remove_span_metadata" | ||
| } | ||
| } | ||
| ] | ||
| } |
12 changes: 12 additions & 0 deletions
12
....1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_ingest_timestamp.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "description": "Add an ingest timestamp for APM events", | ||
| "processors": [ | ||
| { | ||
| "set": { | ||
| "field": "event.ingested", | ||
| "if": "ctx.processor?.event != 'span'", | ||
| "value": "{{_ingest.timestamp}}" | ||
| } | ||
| } | ||
| ] | ||
| } |
25 changes: 25 additions & 0 deletions
25
.../data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_remove_span_metadata.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| { | ||
| "description": "Removes metadata fields available already on the parent transaction, to save storage", | ||
| "processors": [ | ||
| { | ||
| "remove": { | ||
| "field": [ | ||
| "host", | ||
| "process", | ||
| "user", | ||
| "user_agent", | ||
| "container", | ||
| "kubernetes", | ||
| "service.node", | ||
| "service.version", | ||
| "service.language", | ||
| "service.runtime", | ||
| "service.framework" | ||
| ], | ||
| "if": "ctx.processor?.event == 'span'", | ||
| "ignore_failure": true, | ||
| "ignore_missing": true | ||
| } | ||
| } | ||
| ] | ||
| } |
13 changes: 13 additions & 0 deletions
13
.../apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_user_agent.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| { | ||
| "description": "Add user agent information for APM events", | ||
| "processors": [ | ||
| { | ||
| "user_agent": { | ||
| "field": "user_agent.original", | ||
| "ignore_failure": true, | ||
| "ignore_missing": true, | ||
| "target_field": "user_agent" | ||
| } | ||
| } | ||
| ] | ||
| } |
22 changes: 22 additions & 0 deletions
22
...ge/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_user_geo.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| { | ||
| "description": "Add user geo information for APM events", | ||
| "processors": [ | ||
| { | ||
| "geoip": { | ||
| "database_file": "GeoLite2-City.mmdb", | ||
| "field": "client.ip", | ||
| "ignore_missing": true, | ||
| "on_failure": [ | ||
| { | ||
| "remove": { | ||
| "field": "client.ip", | ||
| "ignore_failure": true, | ||
| "ignore_missing": true | ||
| } | ||
| } | ||
| ], | ||
| "target_field": "client.geo" | ||
| } | ||
| } | ||
| ] | ||
| } |
12 changes: 12 additions & 0 deletions
12
apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/base-fields.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| - name: '@timestamp' | ||
| type: date | ||
| description: Event timestamp. | ||
| - name: data_stream.type | ||
| type: constant_keyword | ||
| description: Data stream type. | ||
| - name: data_stream.dataset | ||
| type: constant_keyword | ||
| description: Data stream dataset. | ||
| - name: data_stream.namespace | ||
| type: constant_keyword | ||
| description: Data stream namespace. |
218 changes: 218 additions & 0 deletions
218
apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/ecs.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,218 @@ | ||
| - name: agent.ephemeral_id | ||
| type: keyword | ||
| description: | | ||
| The Ephemeral ID identifies a running process. | ||
| - name: agent.name | ||
| type: keyword | ||
| description: | | ||
| Name of the agent used. | ||
| - name: agent.version | ||
| type: keyword | ||
| description: | | ||
| Version of the agent used. | ||
| - name: client.ip | ||
| type: ip | ||
| description: | | ||
| IP address of the client of a recorded event. This is typically obtained from a request's X-Forwarded-For or the X-Real-IP header or falls back to a given configuration for remote address. | ||
| - name: cloud.account.id | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud account ID | ||
| ignore_above: 1024 | ||
| - name: cloud.account.name | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud account name | ||
| ignore_above: 1024 | ||
| - name: cloud.availability_zone | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud availability zone name | ||
| ignore_above: 1024 | ||
| - name: cloud.instance.id | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud instance/machine ID | ||
| ignore_above: 1024 | ||
| - name: cloud.instance.name | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud instance/machine name | ||
| ignore_above: 1024 | ||
| - name: cloud.machine.type | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud instance/machine type | ||
| ignore_above: 1024 | ||
| - name: cloud.project.id | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud project ID | ||
| ignore_above: 1024 | ||
| - name: cloud.project.name | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud project name | ||
| ignore_above: 1024 | ||
| - name: cloud.provider | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud provider name | ||
| ignore_above: 1024 | ||
| - name: cloud.region | ||
| level: extended | ||
| type: keyword | ||
| description: Cloud region name | ||
| ignore_above: 1024 | ||
| - name: container.id | ||
| type: keyword | ||
| description: | | ||
| Unique container id. | ||
| - name: destination.address | ||
| level: extended | ||
| type: keyword | ||
| description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | ||
| ignore_above: 1024 | ||
| - name: destination.ip | ||
| level: core | ||
| type: ip | ||
| description: IP addess of the destination. Can be one of multiple IPv4 or IPv6 addresses. | ||
| - name: destination.port | ||
| level: core | ||
| type: long | ||
| format: string | ||
| description: Port of the destination. | ||
| - name: event.outcome | ||
| level: core | ||
| type: keyword | ||
| description: | | ||
| `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. | ||
| ignore_above: 1024 | ||
| - name: host.architecture | ||
| type: keyword | ||
| description: | | ||
| The architecture of the host the event was recorded on. | ||
| - name: host.hostname | ||
| type: keyword | ||
| description: | | ||
| The hostname of the host the event was recorded on. | ||
| - name: host.ip | ||
| type: ip | ||
| description: | | ||
| IP of the host that records the event. | ||
| - name: host.name | ||
| type: keyword | ||
| description: | | ||
| Name of the host the event was recorded on. It can contain same information as host.hostname or a name specified by the user. | ||
| - name: host.os.platform | ||
| type: keyword | ||
| description: | | ||
| The platform of the host the event was recorded on. | ||
| - name: labels | ||
| type: object | ||
| description: | | ||
| A flat mapping of user-defined labels with string, boolean or number values. | ||
| - name: observer.hostname | ||
| type: keyword | ||
| description: | | ||
| Hostname of the APM Server. | ||
| - name: observer.type | ||
| type: keyword | ||
| description: | | ||
| The type will be set to `apm-server`. | ||
| - name: observer.version | ||
| type: keyword | ||
| description: | | ||
| APM Server version. | ||
| - name: process.args | ||
| level: extended | ||
| type: keyword | ||
| description: | | ||
| Process arguments. May be filtered to protect sensitive information. | ||
| - name: process.pid | ||
| type: long | ||
| description: | | ||
| Numeric process ID of the service process. | ||
| - name: process.ppid | ||
| type: long | ||
| description: | | ||
| Numeric ID of the service's parent process. | ||
| - name: process.title | ||
| type: keyword | ||
| description: | | ||
| Service process title. | ||
| - name: service.name | ||
| type: keyword | ||
| description: | | ||
| Immutable name of the service emitting this event. | ||
| - name: service.node.name | ||
| type: keyword | ||
| description: | | ||
| Unique meaningful name of the service node. | ||
| - name: service.version | ||
| type: keyword | ||
| description: | | ||
| Version of the service emitting this event. | ||
| - name: source.ip | ||
| type: ip | ||
| description: | | ||
| IP address of the source of a recorded event. This is typically obtained from a request's X-Forwarded-For or the X-Real-IP header or falls back to a given configuration for remote address. | ||
| - name: transaction.id | ||
| type: keyword | ||
| description: | | ||
| The transaction ID. | ||
| - name: user.email | ||
| type: keyword | ||
| description: | | ||
| Email of the logged in user. | ||
| - name: user.id | ||
| type: keyword | ||
| description: | | ||
| Identifier of the logged in user. | ||
| - name: user.name | ||
| type: keyword | ||
| description: | | ||
| The username of the logged in user. | ||
| - name: user_agent.device.name | ||
| type: keyword | ||
| description: | | ||
| Name of the device. | ||
| - name: user_agent.name | ||
| type: keyword | ||
| description: | | ||
| Name of the user agent. | ||
| - name: user_agent.original | ||
| type: keyword | ||
| description: | | ||
| Unparsed version of the user_agent. | ||
| multi_fields: | ||
| - name: text | ||
| type: text | ||
| - name: user_agent.os.family | ||
| type: keyword | ||
| description: | | ||
| OS family (such as redhat, debian, freebsd, windows). | ||
| - name: user_agent.os.full | ||
| type: keyword | ||
| description: | | ||
| Operating system name, including the version or code name. | ||
| - name: user_agent.os.kernel | ||
| type: keyword | ||
| description: | | ||
| Operating system kernel version as a raw string. | ||
| - name: user_agent.os.name | ||
| type: keyword | ||
| description: | | ||
| Operating system name, without the version. | ||
| - name: user_agent.os.platform | ||
| type: keyword | ||
| description: | | ||
| Operating system platform (such centos, ubuntu, windows). | ||
| - name: user_agent.os.version | ||
| type: keyword | ||
| description: | | ||
| Operating system version as a raw string. | ||
| - name: user_agent.version | ||
| type: keyword | ||
| description: | | ||
| Version of the user agent. |
Oops, something went wrong.