diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 44453724536..be056c60d7c 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.36.4" + changes: + - description: Add support for TSDB on kinesis data stream (metric type). + type: enhancement + link: https://github.com/elastic/integrations/pull/5866 - version: "1.36.3" changes: - description: Add dimensions to Redshift data stream. diff --git a/packages/aws/data_stream/kinesis/fields/fields.yml b/packages/aws/data_stream/kinesis/fields/fields.yml index e5910e3dc4c..3b36d827f3a 100644 --- a/packages/aws/data_stream/kinesis/fields/fields.yml +++ b/packages/aws/data_stream/kinesis/fields/fields.yml @@ -8,131 +8,158 @@ dimension: true type: keyword description: The name of the Kinesis stream. All available statistics are filtered by StreamName. + - name: kinesis.metrics type: group fields: - name: GetRecords_Bytes.avg type: double + metric_type: gauge description: > The average number of bytes retrieved from the Kinesis stream, measured over the specified time period. - name: GetRecords_IteratorAgeMilliseconds.avg type: double + metric_type: gauge description: > The age of the last record in all GetRecords calls made against a Kinesis stream, measured over the specified time period. Age is the difference between the current time and when the last record of the GetRecords call was written to the stream. - name: GetRecords_Latency.avg type: double + metric_type: gauge description: > The time taken per GetRecords operation, measured over the specified time period. - name: GetRecords_Records.sum type: long + metric_type: gauge description: > The number of records retrieved from the shard, measured over the specified time period. - name: GetRecords_Success.sum type: long + metric_type: gauge description: > The number of successful GetRecords operations per stream, measured over the specified time period. - name: IncomingBytes.avg type: double + metric_type: gauge description: > The number of bytes successfully put to the Kinesis stream over the specified time period. This metric includes bytes from PutRecord and PutRecords operations. - name: IncomingRecords.avg type: double + metric_type: gauge description: > The number of records successfully put to the Kinesis stream over the specified time period. This metric includes record counts from PutRecord and PutRecords operations. - name: PutRecord_Bytes.avg type: double + metric_type: gauge description: > The number of bytes put to the Kinesis stream using the PutRecord operation over the specified time period. - name: PutRecord_Latency.avg type: double + metric_type: gauge description: > The time taken per PutRecord operation, measured over the specified time period. - name: PutRecord_Success.avg type: double + metric_type: gauge description: > The percentage of successful writes to a Kinesis stream, measured over the specified time period. - name: PutRecords_Bytes.avg type: double + metric_type: gauge description: > The average number of bytes put to the Kinesis stream using the PutRecords operation over the specified time period. - name: PutRecords_Latency.avg type: double + metric_type: gauge description: > The average time taken per PutRecords operation, measured over the specified time period. - name: PutRecords_Success.avg type: long + metric_type: gauge description: > The total number of PutRecords operations where at least one record succeeded, per Kinesis stream, measured over the specified time period. + - name: PutRecords_TotalRecords.sum type: long + metric_type: gauge description: > The total number of records sent in a PutRecords operation per Kinesis data stream, measured over the specified time period. - name: PutRecords_SuccessfulRecords.sum type: long + metric_type: gauge description: > The number of successful records in a PutRecords operation per Kinesis data stream, measured over the specified time period. - name: PutRecords_FailedRecords.sum type: long + metric_type: gauge description: > The number of records rejected due to internal failures in a PutRecords operation per Kinesis data stream, measured over the specified time period. - name: PutRecords_ThrottledRecords.sum type: long + metric_type: gauge description: > The number of records rejected due to throttling in a PutRecords operation per Kinesis data stream, measured over the specified time period. - name: ReadProvisionedThroughputExceeded.avg type: long + metric_type: gauge description: > The number of GetRecords calls throttled for the stream over the specified time period. - name: SubscribeToShard_RateExceeded.avg type: long + metric_type: gauge description: > This metric is emitted when a new subscription attempt fails because there already is an active subscription by the same consumer or if you exceed the number of calls per second allowed for this operation. - name: SubscribeToShard_Success.avg type: long + metric_type: gauge description: > This metric records whether the SubscribeToShard subscription was successfully established. - name: SubscribeToShardEvent_Bytes.avg type: long + metric_type: gauge description: > The number of bytes received from the shard, measured over the specified time period. - name: SubscribeToShardEvent_MillisBehindLatest.avg type: long + metric_type: gauge description: > The difference between the current time and when the last record of the SubscribeToShard event was written to the stream. - name: SubscribeToShardEvent_Records.sum type: long + metric_type: gauge description: > The number of records received from the shard, measured over the specified time period. - name: SubscribeToShardEvent_Success.avg type: long + metric_type: gauge description: > This metric is emitted every time an event is published successfully. It is only emitted when there's an active subscription. - name: WriteProvisionedThroughputExceeded.avg type: long + metric_type: gauge description: > The number of records rejected due to throttling for the stream over the specified time period. This metric includes throttling from PutRecord and PutRecords operations. diff --git a/packages/aws/docs/kinesis.md b/packages/aws/docs/kinesis.md index d2fec8a10b7..f140fa022e7 100644 --- a/packages/aws/docs/kinesis.md +++ b/packages/aws/docs/kinesis.md @@ -121,77 +121,77 @@ An example event for `kinesis` looks as following: **Exported fields** -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.\* | Metric dimensions. | object | -| aws.dimensions.StreamName | The name of the Kinesis stream. All available statistics are filtered by StreamName. | keyword | -| aws.kinesis.metrics.GetRecords_Bytes.avg | The average number of bytes retrieved from the Kinesis stream, measured over the specified time period. | double | -| aws.kinesis.metrics.GetRecords_IteratorAgeMilliseconds.avg | The age of the last record in all GetRecords calls made against a Kinesis stream, measured over the specified time period. Age is the difference between the current time and when the last record of the GetRecords call was written to the stream. | double | -| aws.kinesis.metrics.GetRecords_Latency.avg | The time taken per GetRecords operation, measured over the specified time period. | double | -| aws.kinesis.metrics.GetRecords_Records.sum | The number of records retrieved from the shard, measured over the specified time period. | long | -| aws.kinesis.metrics.GetRecords_Success.sum | The number of successful GetRecords operations per stream, measured over the specified time period. | long | -| aws.kinesis.metrics.IncomingBytes.avg | The number of bytes successfully put to the Kinesis stream over the specified time period. This metric includes bytes from PutRecord and PutRecords operations. | double | -| aws.kinesis.metrics.IncomingRecords.avg | The number of records successfully put to the Kinesis stream over the specified time period. This metric includes record counts from PutRecord and PutRecords operations. | double | -| aws.kinesis.metrics.PutRecord_Bytes.avg | The number of bytes put to the Kinesis stream using the PutRecord operation over the specified time period. | double | -| aws.kinesis.metrics.PutRecord_Latency.avg | The time taken per PutRecord operation, measured over the specified time period. | double | -| aws.kinesis.metrics.PutRecord_Success.avg | The percentage of successful writes to a Kinesis stream, measured over the specified time period. | double | -| aws.kinesis.metrics.PutRecords_Bytes.avg | The average number of bytes put to the Kinesis stream using the PutRecords operation over the specified time period. | double | -| aws.kinesis.metrics.PutRecords_FailedRecords.sum | The number of records rejected due to internal failures in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | -| aws.kinesis.metrics.PutRecords_Latency.avg | The average time taken per PutRecords operation, measured over the specified time period. | double | -| aws.kinesis.metrics.PutRecords_Success.avg | The total number of PutRecords operations where at least one record succeeded, per Kinesis stream, measured over the specified time period. | long | -| aws.kinesis.metrics.PutRecords_SuccessfulRecords.sum | The number of successful records in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | -| aws.kinesis.metrics.PutRecords_ThrottledRecords.sum | The number of records rejected due to throttling in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | -| aws.kinesis.metrics.PutRecords_TotalRecords.sum | The total number of records sent in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | -| aws.kinesis.metrics.ReadProvisionedThroughputExceeded.avg | The number of GetRecords calls throttled for the stream over the specified time period. | long | -| aws.kinesis.metrics.SubscribeToShardEvent_Bytes.avg | The number of bytes received from the shard, measured over the specified time period. | long | -| aws.kinesis.metrics.SubscribeToShardEvent_MillisBehindLatest.avg | The difference between the current time and when the last record of the SubscribeToShard event was written to the stream. | long | -| aws.kinesis.metrics.SubscribeToShardEvent_Records.sum | The number of records received from the shard, measured over the specified time period. | long | -| aws.kinesis.metrics.SubscribeToShardEvent_Success.avg | This metric is emitted every time an event is published successfully. It is only emitted when there's an active subscription. | long | -| aws.kinesis.metrics.SubscribeToShard_RateExceeded.avg | This metric is emitted when a new subscription attempt fails because there already is an active subscription by the same consumer or if you exceed the number of calls per second allowed for this operation. | long | -| aws.kinesis.metrics.SubscribeToShard_Success.avg | This metric records whether the SubscribeToShard subscription was successfully established. | long | -| aws.kinesis.metrics.WriteProvisionedThroughputExceeded.avg | The number of records rejected due to throttling for the stream over the specified time period. This metric includes throttling from PutRecord and PutRecords operations. | long | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.\* | Tag key value pairs from aws resources. | object | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | -| error.message | Error message. | match_only_text | -| event.dataset | Event dataset | constant_keyword | -| event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| Field | Description | Type | Metric Type | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | +| aws.dimensions.\* | Metric dimensions. | object | | +| aws.dimensions.StreamName | The name of the Kinesis stream. All available statistics are filtered by StreamName. | keyword | | +| aws.kinesis.metrics.GetRecords_Bytes.avg | The average number of bytes retrieved from the Kinesis stream, measured over the specified time period. | double | gauge | +| aws.kinesis.metrics.GetRecords_IteratorAgeMilliseconds.avg | The age of the last record in all GetRecords calls made against a Kinesis stream, measured over the specified time period. Age is the difference between the current time and when the last record of the GetRecords call was written to the stream. | double | gauge | +| aws.kinesis.metrics.GetRecords_Latency.avg | The time taken per GetRecords operation, measured over the specified time period. | double | gauge | +| aws.kinesis.metrics.GetRecords_Records.sum | The number of records retrieved from the shard, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.GetRecords_Success.sum | The number of successful GetRecords operations per stream, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.IncomingBytes.avg | The number of bytes successfully put to the Kinesis stream over the specified time period. This metric includes bytes from PutRecord and PutRecords operations. | double | gauge | +| aws.kinesis.metrics.IncomingRecords.avg | The number of records successfully put to the Kinesis stream over the specified time period. This metric includes record counts from PutRecord and PutRecords operations. | double | gauge | +| aws.kinesis.metrics.PutRecord_Bytes.avg | The number of bytes put to the Kinesis stream using the PutRecord operation over the specified time period. | double | gauge | +| aws.kinesis.metrics.PutRecord_Latency.avg | The time taken per PutRecord operation, measured over the specified time period. | double | gauge | +| aws.kinesis.metrics.PutRecord_Success.avg | The percentage of successful writes to a Kinesis stream, measured over the specified time period. | double | gauge | +| aws.kinesis.metrics.PutRecords_Bytes.avg | The average number of bytes put to the Kinesis stream using the PutRecords operation over the specified time period. | double | gauge | +| aws.kinesis.metrics.PutRecords_FailedRecords.sum | The number of records rejected due to internal failures in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.PutRecords_Latency.avg | The average time taken per PutRecords operation, measured over the specified time period. | double | gauge | +| aws.kinesis.metrics.PutRecords_Success.avg | The total number of PutRecords operations where at least one record succeeded, per Kinesis stream, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.PutRecords_SuccessfulRecords.sum | The number of successful records in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.PutRecords_ThrottledRecords.sum | The number of records rejected due to throttling in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.PutRecords_TotalRecords.sum | The total number of records sent in a PutRecords operation per Kinesis data stream, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.ReadProvisionedThroughputExceeded.avg | The number of GetRecords calls throttled for the stream over the specified time period. | long | gauge | +| aws.kinesis.metrics.SubscribeToShardEvent_Bytes.avg | The number of bytes received from the shard, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.SubscribeToShardEvent_MillisBehindLatest.avg | The difference between the current time and when the last record of the SubscribeToShard event was written to the stream. | long | gauge | +| aws.kinesis.metrics.SubscribeToShardEvent_Records.sum | The number of records received from the shard, measured over the specified time period. | long | gauge | +| aws.kinesis.metrics.SubscribeToShardEvent_Success.avg | This metric is emitted every time an event is published successfully. It is only emitted when there's an active subscription. | long | gauge | +| aws.kinesis.metrics.SubscribeToShard_RateExceeded.avg | This metric is emitted when a new subscription attempt fails because there already is an active subscription by the same consumer or if you exceed the number of calls per second allowed for this operation. | long | gauge | +| aws.kinesis.metrics.SubscribeToShard_Success.avg | This metric records whether the SubscribeToShard subscription was successfully established. | long | gauge | +| aws.kinesis.metrics.WriteProvisionedThroughputExceeded.avg | The number of records rejected due to throttling for the stream over the specified time period. This metric includes throttling from PutRecord and PutRecords operations. | long | gauge | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | | +| aws.tags.\* | Tag key value pairs from aws resources. | object | | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | +| cloud.instance.name | Instance name of the host machine. | keyword | | +| cloud.machine.type | Machine type of the host machine. | keyword | | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | +| container.id | Unique container id. | keyword | | +| container.image.name | Name of the image the container was built on. | keyword | | +| container.labels | Image labels. | object | | +| container.name | Container name. | keyword | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | +| error.message | Error message. | match_only_text | | +| event.dataset | Event dataset | constant_keyword | | +| event.module | Event module | constant_keyword | | +| host.architecture | Operating system architecture. | keyword | | +| host.containerized | If the host is a container. | boolean | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.os.build | OS build information. | keyword | | +| host.os.codename | OS codename, if any. | keyword | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | +| host.os.name | Operating system name, without the version. | keyword | | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | +| host.os.version | Operating system version as a raw string. | keyword | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 77c57c60737..61b6a743267 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.36.3 +version: 1.36.4 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration