Skip to content

Commit

Permalink
Implement aws.ecs.* and aws.logs.* resource attributes
Browse files Browse the repository at this point in the history 
Add support for the aws.ecs.* and aws.logs.* resource
attributes in the `AwsEcsResourceDetector` detector when
the ECS Metadata v4 is available
  • Loading branch information
Michele Mancioppi committed Sep 17, 2022
1 parent 76a565e commit ad00913
Show file tree
Hide file tree
Showing 7 changed files with 556 additions and 2 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
([#1242](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1242))
- `opentelemetry-util-http` Add support for sanitizing HTTP header values.
([#1253](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1253))
- Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available
([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212))

### Fixed

Expand Down
114 changes: 113 additions & 1 deletion ...nsion/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import logging
import os
import re
import socket
from urllib.request import Request, urlopen

from opentelemetry.sdk.resources import Resource, ResourceDetector
from opentelemetry.semconv.resource import (
Expand Down Expand Up @@ -58,18 +61,127 @@ def detect(self) -> "Resource":
"Failed to get container ID on ECS: %s.", exception
)

return Resource(
base_resource = Resource(
{
ResourceAttributes.CLOUD_PROVIDER: CloudProviderValues.AWS.value,
ResourceAttributes.CLOUD_PLATFORM: CloudPlatformValues.AWS_ECS.value,
ResourceAttributes.CONTAINER_NAME: socket.gethostname(),
ResourceAttributes.CONTAINER_ID: container_id,
}
)

metadata_v4_endpoint = os.environ.get(
"ECS_CONTAINER_METADATA_URI_V4"
)

if not metadata_v4_endpoint:
return base_resource

# Returns https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html#task-metadata-endpoint-v4-response
metadata_container = json.loads(_http_get(metadata_v4_endpoint))
metadata_task = json.loads(
_http_get(f"{metadata_v4_endpoint}/task")
)

task_arn = metadata_task["TaskARN"]
base_arn = task_arn[0 : task_arn.rindex(":")] # noqa
cluster: str = metadata_task["Cluster"]
cluster_arn = (
cluster
if cluster.startswith("arn:")
else f"{base_arn}:cluster/{cluster}"
)

logs_resource = _get_logs_resource(metadata_container)

return base_resource.merge(logs_resource).merge(
Resource(
{
ResourceAttributes.AWS_ECS_CONTAINER_ARN: metadata_container[
"ContainerARN"
],
ResourceAttributes.AWS_ECS_CLUSTER_ARN: cluster_arn,
ResourceAttributes.AWS_ECS_LAUNCHTYPE: metadata_task[
"LaunchType"
].lower(),
ResourceAttributes.AWS_ECS_TASK_ARN: task_arn,
ResourceAttributes.AWS_ECS_TASK_FAMILY: metadata_task[
"Family"
],
ResourceAttributes.AWS_ECS_TASK_REVISION: metadata_task[
"Revision"
],
}
)
)
# pylint: disable=broad-except
except Exception as exception:
if self.raise_on_error:
raise exception

logger.warning("%s failed: %s", self.__class__.__name__, exception)
return Resource.get_empty()


def _get_logs_resource(metadata_container):
if metadata_container.get("LogDriver") == "awslogs":
log_options = metadata_container.get("LogOptions")
if log_options:
logs_region = log_options.get("awslogs-region")
logs_group_name = log_options.get("awslogs-group")
logs_stream_name = log_options.get("awslogs-stream")

container_arn = metadata_container["ContainerARN"]

if not logs_region:
aws_region_match = re.match(
r'arn:aws:ecs:([^:]+):.*', container_arn
)
if aws_region_match:
logs_region = aws_region_match.group(1)

else:
logger.warning("Cannot parse AWS account out of ECS ARN")

# We need to retrieve the account it from some other ARN to create the
# log-group and log-stream ARNs
aws_account = None
aws_account_match = re.match(
r'arn:aws:ecs:[^:]+:([^:]+):.*', container_arn
)
if aws_account_match:
aws_account = aws_account_match.group(1)

logs_group_arn = None
logs_stream_arn = None
if logs_region and aws_account:
if logs_group_name:
logs_group_arn = f"arn:aws:logs:{logs_region}:{aws_account}:log-group:{logs_group_name}"

if logs_stream_name:
logs_stream_arn = f"arn:aws:logs:{logs_region}:{aws_account}:log-group:{logs_group_name}:log-stream:{logs_stream_name}"

return Resource(
{
ResourceAttributes.AWS_LOG_GROUP_NAMES: [logs_group_name],
ResourceAttributes.AWS_LOG_GROUP_ARNS: [logs_group_arn],
ResourceAttributes.AWS_LOG_STREAM_NAMES: [
logs_stream_name
],
ResourceAttributes.AWS_LOG_STREAM_ARNS: [logs_stream_arn],
}
)
else:
logger.warning(
"The metadata endpoint v4 has returned 'awslogs' as 'LogDriver', but there is no 'LogOptions' data"
)

return Resource.get_empty()


def _http_get(url):
with urlopen(
Request(url, method="GET"),
timeout=5,
) as response:
return response.read().decode("utf-8")
44 changes: 44 additions & 0 deletions ...opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-ec2.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"DockerId": "ea32192c8553fbff06c9340478a2ff089b2bb5646fb718b4ee206641c9086d66",
"Name": "curl",
"DockerName": "ecs-curltest-24-curl-cca48e8dcadd97805600",
"Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
"ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
"Labels": {
"com.amazonaws.ecs.cluster": "default",
"com.amazonaws.ecs.container-name": "curl",
"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/8f03e41243824aea923aca126495f665",
"com.amazonaws.ecs.task-definition-family": "curltest",
"com.amazonaws.ecs.task-definition-version": "24"
},
"DesiredStatus": "RUNNING",
"KnownStatus": "RUNNING",
"Limits": {
"CPU": 10,
"Memory": 128
},
"CreatedAt": "2020-10-02T00:15:07.620912337Z",
"StartedAt": "2020-10-02T00:15:08.062559351Z",
"Type": "NORMAL",
"LogDriver": "awslogs",
"LogOptions": {
"awslogs-create-group": "true",
"awslogs-group": "/ecs/metadata",
"awslogs-region": "us-west-2",
"awslogs-stream": "ecs/curl/8f03e41243824aea923aca126495f665"
},
"ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9",
"Networks": [
{
"NetworkMode": "awsvpc",
"IPv4Addresses": [
"10.0.2.100"
],
"AttachmentIndex": 0,
"MACAddress": "0e:9e:32:c7:48:85",
"IPv4SubnetCIDRBlock": "10.0.2.0/24",
"PrivateDNSName": "ip-10-0-2-100.us-west-2.compute.internal",
"SubnetGatewayIpv4Address": "10.0.2.1/24"
}
]
}
50 changes: 50 additions & 0 deletions ...telemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-fargate.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
{
"DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
"Name": "curl",
"DockerName": "curl",
"Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
"ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
"Labels": {
"com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
"com.amazonaws.ecs.container-name": "curl",
"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50",
"com.amazonaws.ecs.task-definition-family": "curltest",
"com.amazonaws.ecs.task-definition-version": "2"
},
"DesiredStatus": "RUNNING",
"KnownStatus": "RUNNING",
"Limits": {
"CPU": 10,
"Memory": 128
},
"CreatedAt": "2020-10-08T20:09:11.44527186Z",
"StartedAt": "2020-10-08T20:09:11.44527186Z",
"Type": "NORMAL",
"Networks": [
{
"NetworkMode": "awsvpc",
"IPv4Addresses": [
"192.0.2.3"
],
"AttachmentIndex": 0,
"MACAddress": "0a:de:f6:10:51:e5",
"IPv4SubnetCIDRBlock": "192.0.2.0/24",
"DomainNameServers": [
"192.0.2.2"
],
"DomainNameSearchList": [
"us-west-2.compute.internal"
],
"PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal",
"SubnetGatewayIpv4Address": "192.0.2.0/24"
}
],
"ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1",
"LogOptions": {
"awslogs-create-group": "true",
"awslogs-group": "/ecs/containerlogs",
"awslogs-region": "us-west-2",
"awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50"
},
"LogDriver": "awslogs"
}
94 changes: 94 additions & 0 deletions ...sion/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-ec2.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"Cluster": "default",
"TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
"Family": "curltest",
"Revision": "26",
"DesiredStatus": "RUNNING",
"KnownStatus": "RUNNING",
"PullStartedAt": "2020-10-02T00:43:06.202617438Z",
"PullStoppedAt": "2020-10-02T00:43:06.31288465Z",
"AvailabilityZone": "us-west-2d",
"LaunchType": "EC2",
"Containers": [
{
"DockerId": "598cba581fe3f939459eaba1e071d5c93bb2c49b7d1ba7db6bb19deeb70d8e38",
"Name": "~internal~ecs~pause",
"DockerName": "ecs-curltest-26-internalecspause-e292d586b6f9dade4a00",
"Image": "amazon/amazon-ecs-pause:0.1.0",
"ImageID": "",
"Labels": {
"com.amazonaws.ecs.cluster": "default",
"com.amazonaws.ecs.container-name": "~internal~ecs~pause",
"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
"com.amazonaws.ecs.task-definition-family": "curltest",
"com.amazonaws.ecs.task-definition-version": "26"
},
"DesiredStatus": "RESOURCES_PROVISIONED",
"KnownStatus": "RESOURCES_PROVISIONED",
"Limits": {
"CPU": 0,
"Memory": 0
},
"CreatedAt": "2020-10-02T00:43:05.602352471Z",
"StartedAt": "2020-10-02T00:43:06.076707576Z",
"Type": "CNI_PAUSE",
"Networks": [
{
"NetworkMode": "awsvpc",
"IPv4Addresses": [
"10.0.2.61"
],
"AttachmentIndex": 0,
"MACAddress": "0e:10:e2:01:bd:91",
"IPv4SubnetCIDRBlock": "10.0.2.0/24",
"PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
"SubnetGatewayIpv4Address": "10.0.2.1/24"
}
]
},
{
"DockerId": "ee08638adaaf009d78c248913f629e38299471d45fe7dc944d1039077e3424ca",
"Name": "curl",
"DockerName": "ecs-curltest-26-curl-a0e7dba5aca6d8cb2e00",
"Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
"ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
"Labels": {
"com.amazonaws.ecs.cluster": "default",
"com.amazonaws.ecs.container-name": "curl",
"com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
"com.amazonaws.ecs.task-definition-family": "curltest",
"com.amazonaws.ecs.task-definition-version": "26"
},
"DesiredStatus": "RUNNING",
"KnownStatus": "RUNNING",
"Limits": {
"CPU": 10,
"Memory": 128
},
"CreatedAt": "2020-10-02T00:43:06.326590752Z",
"StartedAt": "2020-10-02T00:43:06.767535449Z",
"Type": "NORMAL",
"LogDriver": "awslogs",
"LogOptions": {
"awslogs-create-group": "true",
"awslogs-group": "/ecs/metadata",
"awslogs-region": "us-west-2",
"awslogs-stream": "ecs/curl/158d1c8083dd49d6b527399fd6414f5c"
},
"ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/abb51bdd-11b4-467f-8f6c-adcfe1fe059d",
"Networks": [
{
"NetworkMode": "awsvpc",
"IPv4Addresses": [
"10.0.2.61"
],
"AttachmentIndex": 0,
"MACAddress": "0e:10:e2:01:bd:91",
"IPv4SubnetCIDRBlock": "10.0.2.0/24",
"PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
"SubnetGatewayIpv4Address": "10.0.2.1/24"
}
]
}
]
}
Loading

0 comments on commit ad00913

Please sign in to comment.