From 3729fd71deb36fef8ea16cbf3d745df71cf34a0c Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 13 Jul 2022 09:55:39 +0100 Subject: [PATCH 1/9] [AWS Fargate] Added DesiredStatus and KnownStatus for ECS tasks, plus some refactoring --- x-pack/metricbeat/module/awsfargate/fields.go | 2 +- .../awsfargate/task_stats/_meta/fields.yml | 8 +++ .../module/awsfargate/task_stats/container.go | 11 +--- .../module/awsfargate/task_stats/data.go | 12 +++- .../awsfargate/task_stats/task_stats.go | 65 ++++++++++++------- 5 files changed, 62 insertions(+), 36 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go index 5d4cf0474c60..ead4ebc95252 100644 --- a/x-pack/metricbeat/module/awsfargate/fields.go +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwsfargate returns asset data. // This is the base64 encoded zlib format compressed contents of module/awsfargate. func AssetAwsfargate() string { - return "eJzsWsFy2zYQvfsrdjydSZvE8iXTgw6ZURVnxgc7HttpjvQKXMmoSIABwChK23/vACQlmgQlirJdOiMdRWD3vd23jxShE5jTcgi40FNUMzR0BGC4iWgIx6MvN/Ax+/b4CCAkzRRPDJdiCO+PAADu1vvuIJZhGhEwGUXEjAa7Pb8IMRnFmYapkjEY1HP7DYZoEEiEieTCDI4AFEWEmoYwIYNHAFNOUaiHLtcJCIypAtV+zDKhIcyUTJP8m/K28lYmhUEuSA0inFCkB0zGAcb4Qwpc6ICYDliUakNqtbUIP6flQqqw9P2DcpyNb2CcbXW5OqUulpw8iLADglWEzhhsa05QiS7ZXVtH15ed84Y05YLbgCdTjHm07IxiHQkqkfbA9I2U5rJ7aUqg8lDZ6ho2uzrQBo2u5SqrvJbpfekCwN06zF1BV4O5Xw+juUcDC1IEmilMKMzGszy3DriLAb/8fTa+CcafLm9H55dn18HF2e3ow+h2FHy+Pg/+fPfvqV17mq31DnfxqQ558alO7YN2ZbMVVCZjUxNq5SnPJ/x6xZOElPzNm86Vbp9cq6bXghQpeEjC8Cl/4DU75XhfuQgwXs3/OjogU1Jrh0YDirCopX4Li3vO7oG+phhpMLI0FK5Gb+DV6St4U/qahwN/e5LUy6Iq1xqH61QYHhOMrz4XshxU1vtkUc49JyUoGiTM1JYUMDTDiMJgGkn0LZpKFaMZQkKKkfCt2FJ3+7nKNuOMQE7BkeIiBwc6QUZVYjUKQqr4BfAAixMj/oNCmCydo4g0npCyG2wjmVRUa2ONreFsrhupRlLMuhGwCFzs9vXXS22od6WXBqOcTdYEW+kc6jYqfZTSBj77Sipn/ZSSyoG6FM1AUk2qb3XPq22hbRsEB7+P2qlx2FcxjumzWFCburvZ+N9rfrua0FTjbCvcXuikgrmLLApWIddzLrs9x3zgeg7np5+6PsQowuqz3ubsLcszYiyN0wgNhS6HhjBVXMxcZSI+zabrntbPeL5GNYAvE5CJb5Bg2zC1pGE/l6s+diayhjtZGuoMuBDupiAtSf1hQzhC+/FR69chO9PZuf4sVYqEyfuQWHsjJoVPwKVbNKlvnFFgrfyJkWae4O4ZRhaJ4fwTKPqakjb6rfVlgUJmuP09LIAvkJtnRl3gBJ3YOlsEVhpcwNeUUtJWgQWvnbi47f4+PTaRte1mSVekNnpg8914y80kpEQRsz43hN8H77q5ZQeJr1SiuHcCH9XBXZKfwsK7M+mrh1tGhsQLtPG8FwcfP/i4h8juPp7pqWdGvl3kK4GncYxq+TxP4yh+Jle3P4BkQgrtlp/K3d0TetGsl2nzpcYcrP5g9R4iu1u9exHUE6evGWqDygvsMcWyZvMtX/dcuL1dX/ZUj3YfJpeTv8j7Ui27EGzseWlNEGOScDHLNxy/Pu5W4Wtc5LXKD4Kz/3BYs3N10vnVgb0KXBhSU2T10VufvMcxb35puMftdewiG/fqzwpACvjCRSgXvmK3uXc2yRue+R6UjWdzoAJvQjjvAdwrwnne5lbNWMFX/BsaChZSza1wNZnBZodp4LONSwseORbIsYAm04rDFHk0YDL1vs9uaYtbwX1EHoFLQqr5LX3EN83Z05UuN0eXvhmd0s0WuIcHXN/cPHCrOoCXOPo5I0Xa/bHDqhE0/+E5oilz8J/NQFshQqszmh24XDSwKKJvPHNyxztPoZjP7tzoMTQT4/ceKOYCvxdsGo7xoP8ayQ7zNukC+j6snuIXeAUZe18ZvB5wMZFp7WdYs6a3PVo2kXg8z//Hy776A4ILJmN733Qxm6c6VDJJvD9v2p/Bt0OUZ1ojS5DNyTfsBTZSSqr9/h6wCVoW3j5TtIeUL3iGcm3GVBeyTM1ByQclvwgl/xcAAP//QI02Qw==" + return "eJzsWk1z2zYQvftX7Hg6kzaJ5UumBx0yoyrOjA92PLbTHOkVuJRRkQADgFGUtv+9A5CUaH6JomyXzohHEth9b/ftggR4AgtajQGXOkA1R0NHAIabkMZwPPlyAx/Tu8dHAD5ppnhsuBRjeH8EAHC3mXcHkfSTkIDJMCRmNNjp2UOIyCjONARKRmBQL+wd9NEgkPBjyYUZHQEoCgk1jWFGBo8AAk6hr8fO1wkIjKgE1V5mFdMY5komcXanOK04lUlhkAtSoxBnFOoRk5GHEf6QApfaI6Y9FibakFpPzc0vaLWUyi/cfxCOs+kNTNOpzlcv1/mQkwcWdkCwttAbg03NCSrRx7tL6+T6srdfnwIuuDV4EmDEw1VvFBtLULK0B6ZvpDSX/UNTAJWZSkdXsNnRnjZodMVXUeUVT+8LDwDuNmbucroazP2mGM09GliSItBMYUx+Wp7FunXAnQ345e+z6Y03/XR5Ozm/PLv2Ls5uJx8mtxPv8/W59+e7f0/t2NN0bG1x51e5yPOrXLUP0pXWlleqjLYkVMJTrE/49YrHMSn5W607F7p9fK2TXjGSu+A+CcMD/qDX7OTjfekhwHRd/xvrgExJrR0aDSj8PJb6LSzvObsH+ppgqMHIQlG4GL2BV6ev4E3hNvdHzfHySXNFvpNcoh+P1e09QWYbUtsQSOWU7EKcitaVro17C8KFkEvxJPic5f7oWJzU4imXewXNdSIMjwimV5/zsh6VxteVVdH3gpSgcBQzUxmSw9AMQ/K9IJRYNyiQKkIzhpgUI1E3YksE7XWVTsY5gQzAkeIiAwc6RkZlYhUKQqroBfAAixND/oN8mK2cUkQSzUjZCTaRTCqqpLHC1nC2KKt4QzWUYt6PgEXgbHePv15pQ4MLvTQYZmzSJNhIZ1C3URmilFr47CupjPVTSioD6lw0A0k0qaHFPYu2hbatEBz8IWqnwmFfxTimz9KCusTd1cb/HvPbdYUmGudb4Q5CJyXMfWSRs/K5XnDZ7z3mA9cLOD/91PclRhGW39ravXcMz4SxJEpCNOQ7Hxr8RHExd5EJeZBW1z1t3pHrEtUAvkhAxnWFBNuKqSMNe12u89ibyAbubGWoN+BcuG1GOpL6w5pwhPbjozbbSTvT2Tn+LFGKhMnyENv2RkyKOgEXlmhS3zgjz7byJ0aa9gS3ZhiZO4bzT6Doa0La6Le2LwsUMsVdn8Mc+BK5eWbUOU7QsY2zRWClwQV8TSih9Ast47UTFze9Pk+PTWTTdlOna1KtPbB5Nd6ymPgUK2K2z43h99G7ft2yh8TXKlG8tgIftYM7Jz9FC+/PZKg93DIyJF5gG89ycejjhz5eQ2T3Pp7qaWCNfLvI1wJPogjV6nnexlH8TF3dfgDJmBTaKT9Vd3dv6HmyXmabLyTm0OoPrb6GyO6t3m0EDaTTVxpqg8pz7BFFstLmO273XLi5fTd7ykfjD53L2V9Uu6mWPvBac14Y40UYx1zMswnHr4/7Rfgal1mssoP09B8Y2+xcnHT2dGSfAheGVICsWnqbPxeiiDdvGu6xvE6dZeO2/qwApIAvXPhyWRfsLmtnk7zhmdegtDybDeV4Y8LFAOBeES6yNHdKxhq+4t/QkLeUamGFq8mM2jtMA59tXDrwyLBAhgU0mU4cAuThiMmkdj+7Y1vcCu4j8hCcE1LNu/Qhb6uzpwtd1hyd+2Z0Sje3wD16wPXNzYNuVQXwEks/Y6RIux9jrBpB8x81RzRFDvVnM9BViNDpjGYHLhcNLHLrrWdO7njnKRTz2Z0bPYZmIvw+AMVc4PecTcMxHgxfI+lhXpsuYOjFWhP8HK8gY9eV0esRFzOZVD7DmjW97dWyicTj9fx/atmXPyC4YDKy66az2VzVvpJxXPt50/0MvhuizNMGWYxsQXXFnmMjpaTa7/eANmipeftO0R1SNuAZwtWOqSpkmZiDkg9KfhFK/i8AAP//k1OOZw==" } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml index afb9b594d4ef..b0dddd97691d 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -14,6 +14,14 @@ type: keyword description: > Container identifier across tasks and clusters, which equals to container.name + '/' + container.id. + - name: task_desired_status + type: keyword + description: > + The desired status for the task from Amazon ECS. + - name: task_known_status + type: keyword + description: > + The known status for the task from Amazon ECS. - name: cpu type: group description: Runtime CPU metrics. diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index 33c6e2f13968..3479fc580e87 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -17,16 +17,7 @@ type container struct { Labels map[string]string } -// ContainerMetadata is an struct represents container metadata -type ContainerMetadata struct { - Cluster string - TaskARN string - Family string - Revision string - Container *container -} - -func getContainerStats(c *container) *container { +func getContainerMetadata(c *container) *container { return &container{ DockerId: c.DockerId, Image: c.Image, diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 7b9f09190902..6a477be0852f 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -42,11 +42,21 @@ func createEvent(stats *Stats) mb.Event { e.MetricSetFields.Put("cluster_name", clusterName) } - taskName := stats.Container.Labels[taskLabel] + taskName := stats.taskInfo.Family if taskName != "" { e.MetricSetFields.Put("task_name", taskName) } + taskDesiredStatus := stats.taskInfo.TaskDesiredStatus + if taskDesiredStatus != "" { + e.MetricSetFields.Put("task_desired_status", taskDesiredStatus) + } + + taskKnownStatus := stats.taskInfo.TaskKnownStatus + if taskKnownStatus != "" { + e.MetricSetFields.Put("task_known_status", taskKnownStatus) + } + e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) return e } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 603867086b39..f2337b0cb12f 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -47,9 +47,20 @@ type MetricSet struct { taskEndpoint string } -// Stats is a struct represents information regarding a container +// TaskInfo is a struct that represents information about a specific ECS Fargate Task +type TaskInfo struct { + Cluster string + TaskARN string + Family string + Revision string + TaskDesiredStatus string + TaskKnownStatus string +} + +// Stats is a struct that represents information regarding a container type Stats struct { Time common.Time + taskInfo *TaskInfo Container *container cpuStats cpu.CPUStats memoryStats memoryStats @@ -57,13 +68,15 @@ type Stats struct { blkioStats blkioStats } -// TaskMetadata is an struct represents response body from ${ECS_CONTAINER_METADATA_URI_V4}/task +// TaskMetadata is a struct that represents response body from ${ECS_CONTAINER_METADATA_URI_V4}/task type TaskMetadata struct { - Cluster string `json:"Cluster"` - TaskARN string `json:"TaskARN"` - Family string `json:"Family"` - Revision string `json:"Revision"` - Containers []*container `json:"Containers"` + Cluster string `json:"Cluster"` + TaskARN string `json:"TaskARN"` + Family string `json:"Family"` + Revision string `json:"Revision"` + DesiredStatus string `json:"DesiredStatus"` + KnownStatus string `json:"KnownStatus"` + Containers []*container `json:"Containers"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -157,33 +170,37 @@ func getTask(taskResp *http.Response) (TaskMetadata, error) { } func getStatsList(taskStatsOutput map[string]types.StatsJSON, taskOutput TaskMetadata) []Stats { - containersInfo := map[string]ContainerMetadata{} + containersInfo := map[string]container{} + + taskInfo := TaskInfo{ + Family: taskOutput.Family, + TaskARN: taskOutput.TaskARN, + Cluster: taskOutput.Cluster, + Revision: taskOutput.Revision, + TaskDesiredStatus: taskOutput.DesiredStatus, + TaskKnownStatus: taskOutput.KnownStatus, + } + for _, c := range taskOutput.Containers { // Skip ~internal~ecs~pause container if c.Name == "~internal~ecs~pause" { continue } - containerMetadata := ContainerMetadata{ - Container: c, - Family: taskOutput.Family, - TaskARN: taskOutput.TaskARN, - Cluster: taskOutput.Cluster, - Revision: taskOutput.Revision, - } - containersInfo[c.DockerId] = containerMetadata + containersInfo[c.DockerId] = *c } var formattedStats []Stats - for id, taskStats := range taskStatsOutput { - if cInfo, ok := containersInfo[id]; ok { + for id, containerStats := range taskStatsOutput { + if c, ok := containersInfo[id]; ok { statsPerContainer := Stats{ - Time: common.Time(taskStats.Stats.Read), - Container: getContainerStats(cInfo.Container), - cpuStats: getCPUStats(taskStats), - memoryStats: getMemoryStats(taskStats), - networkStats: getNetworkStats(taskStats), - blkioStats: getBlkioStats(taskStats.BlkioStats), + Time: common.Time(containerStats.Stats.Read), + taskInfo: &taskInfo, + Container: getContainerMetadata(&c), + cpuStats: getCPUStats(containerStats), + memoryStats: getMemoryStats(containerStats), + networkStats: getNetworkStats(containerStats), + blkioStats: getBlkioStats(containerStats.BlkioStats), } formattedStats = append(formattedStats, statsPerContainer) From dd2d2142d8d6cdc1fc9f2ea174a8e5aef9ccc327 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:30:52 +0100 Subject: [PATCH 2/9] [AWS Fargate] Modified tests --- .../awsfargate/task_stats/_meta/testdata/task.json | 2 ++ .../module/awsfargate/task_stats/task_stats.go | 12 ++++++------ .../module/awsfargate/task_stats/task_stats_test.go | 4 ++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json index 7c3d4a3e1c93..37399e027a93 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -3,6 +3,8 @@ "TaskARN": "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", "Family": "query-metadata-1", "Revision": "7", + "DesiredStatus": "RUNNING", + "KnownStatus": "ACTIVATING", "Containers": [{ "DockerId": "1234", "Name": "query-metadata", diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index f2337b0cb12f..995a3fd539cb 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -191,16 +191,16 @@ func getStatsList(taskStatsOutput map[string]types.StatsJSON, taskOutput TaskMet } var formattedStats []Stats - for id, containerStats := range taskStatsOutput { + for id, taskStats := range taskStatsOutput { if c, ok := containersInfo[id]; ok { statsPerContainer := Stats{ - Time: common.Time(containerStats.Stats.Read), + Time: common.Time(taskStats.Stats.Read), taskInfo: &taskInfo, Container: getContainerMetadata(&c), - cpuStats: getCPUStats(containerStats), - memoryStats: getMemoryStats(containerStats), - networkStats: getNetworkStats(containerStats), - blkioStats: getBlkioStats(containerStats.BlkioStats), + cpuStats: getCPUStats(taskStats), + memoryStats: getMemoryStats(taskStats), + networkStats: getNetworkStats(taskStats), + blkioStats: getBlkioStats(taskStats.BlkioStats), } formattedStats = append(formattedStats, statsPerContainer) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go index 2ab2e5f934dd..7756a15f1e77 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -46,6 +46,8 @@ var ( "TaskARN": "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", "Family": "query-metadata-1", "Revision": "7", + "DesiredStatus": "RUNNING", + "KnownStatus": "ACTIVATING", "Containers": [{ "DockerId": "query-metadata-1", "Name": "query-metadata", @@ -82,6 +84,8 @@ func TestGetTask(t *testing.T) { assert.Equal(t, "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", taskOutput.TaskARN) assert.Equal(t, "query-metadata-1", taskOutput.Family) assert.Equal(t, "7", taskOutput.Revision) + assert.Equal(t, "RUNNING", taskOutput.DesiredStatus) + assert.Equal(t, "ACTIVATING", taskOutput.KnownStatus) assert.Equal(t, 1, len(taskOutput.Containers)) assert.Equal(t, "query-metadata-1", taskOutput.Containers[0].DockerId) From 5eddad69f71953c6f9a03835fab59e534dbf6549 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:55:02 +0100 Subject: [PATCH 3/9] [AWS Fargate] Updated data.json --- .../module/awsfargate/task_stats/_meta/data.json | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json index e37d19a398b3..002da6310183 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json @@ -128,7 +128,9 @@ } } }, - "task_name": "query-metadata" + "task_desired_status": "RUNNING", + "task_known_status": "ACTIVATING", + "task_name": "query-metadata-1" } }, "cloud": { @@ -148,15 +150,6 @@ }, "name": "query-metadata" }, - "event": { - "dataset": "awsfargate.task_stats", - "duration": 115000, - "module": "awsfargate" - }, - "metricset": { - "name": "task_stats", - "period": 10000 - }, "service": { "type": "awsfargate" } From 2c49feed5eb706e78bdeb947114d02489f659fd8 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:57:35 +0100 Subject: [PATCH 4/9] Added changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5374b3ed8d79..ce17eb046091 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -121,6 +121,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Upgrade Mongodb library in Beats to v5 {pull}31185[31185] - Azure Billing: upgrade Usage Details API to version 2019-10-01 {pull}31970[31970] * Differentiate between actual idle CPU states and an uninterruptible disk sleep. https://github.com/elastic/elastic-agent-system-metrics/pull/32[system-metrics#32] +- AWS Fargate: Added support for DesiredStatus and KnownStatus {issue}32077[32077] {pull}32342[#32342] *Packetbeat* From 918511c256b2dfce2226123531fea9ef67bc8fb5 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:15:04 +0100 Subject: [PATCH 5/9] Removed unused variable --- x-pack/metricbeat/module/awsfargate/task_stats/data.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 6a477be0852f..4e21267136ec 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -16,7 +16,6 @@ import ( var ( clusterLabel = "com_amazonaws_ecs_cluster" - taskLabel = "com_amazonaws_ecs_task-definition-family" ) func eventsMapping(r mb.ReporterV2, statsList []Stats) { From cf19a083b4f7c2da89a846e189323ad7f5dfa048 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 13 Jul 2022 17:22:39 +0100 Subject: [PATCH 6/9] Updated fields.asciidoc --- metricbeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4aa05501998b..b924edddc521 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4944,6 +4944,26 @@ type: keyword Container identifier across tasks and clusters, which equals to container.name + '/' + container.id. +type: keyword + +-- + +*`awsfargate.task_stats.task_desired_status`*:: ++ +-- +The desired status for the task from Amazon ECS. + + +type: keyword + +-- + +*`awsfargate.task_stats.task_known_status`*:: ++ +-- +The known status for the task from Amazon ECS. + + type: keyword -- From f36732e02e8447533a18565c3e6ff2cf77d5e376 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Thu, 14 Jul 2022 09:47:53 +0100 Subject: [PATCH 7/9] Fixed mage llc errors --- .../module/awsfargate/task_stats/data.go | 43 ++++++++++++++----- .../awsfargate/task_stats/task_stats.go | 19 +++++--- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 4e21267136ec..1b6288b3bd05 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -5,6 +5,7 @@ package task_stats import ( + "fmt" "strings" "time" @@ -19,8 +20,8 @@ var ( ) func eventsMapping(r mb.ReporterV2, statsList []Stats) { - for _, stats := range statsList { - r.Event(createEvent(&stats)) + for i := range statsList { + r.Event(createEvent(&statsList[i])) } } @@ -38,25 +39,40 @@ func createEvent(stats *Stats) mb.Event { regionName, clusterName := getRegionAndClusterName(stats.Container.Labels) e.RootFields = createRootFields(stats, regionName) if clusterName != "" { - e.MetricSetFields.Put("cluster_name", clusterName) + _, err := e.MetricSetFields.Put("cluster_name", clusterName) + if err != nil { + _ = fmt.Errorf("error putting metric set field 'cluster_name': %w", err) + } } taskName := stats.taskInfo.Family if taskName != "" { - e.MetricSetFields.Put("task_name", taskName) + _, err := e.MetricSetFields.Put("task_name", taskName) + if err != nil { + _ = fmt.Errorf("error putting metric set field 'task_name': %w", err) + } } taskDesiredStatus := stats.taskInfo.TaskDesiredStatus if taskDesiredStatus != "" { - e.MetricSetFields.Put("task_desired_status", taskDesiredStatus) + _, err := e.MetricSetFields.Put("task_desired_status", taskDesiredStatus) + if err != nil { + _ = fmt.Errorf("error putting metric set field 'task_desired_status': %w", err) + } } taskKnownStatus := stats.taskInfo.TaskKnownStatus if taskKnownStatus != "" { - e.MetricSetFields.Put("task_known_status", taskKnownStatus) + _, err := e.MetricSetFields.Put("task_known_status", taskKnownStatus) + if err != nil { + _ = fmt.Errorf("error putting metric set field 'task_known_status': %w", err) + } } - e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) + _, err := e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) + if err != nil { + _ = fmt.Errorf("error putting metric set field 'identifier': %w", err) + } return e } @@ -75,9 +91,8 @@ func getRegionAndClusterName(labels map[string]string) (regionName string, clust if err == nil { regionName = arnParsed.Region } - return } - return + return regionName, clusterName } func createRootFields(stats *Stats, regionName string) mapstr.M { @@ -97,7 +112,10 @@ func createRootFields(stats *Stats, regionName string) mapstr.M { cloud := mapstr.M{ "region": regionName, } - rootFields.Put("cloud", cloud) + _, err := rootFields.Put("cloud", cloud) + if err != nil { + _ = fmt.Errorf("error putting root field 'cloud': %w", err) + } } return rootFields } @@ -172,7 +190,7 @@ func createMemoryFields(stats *Stats) mapstr.M { func createNetworkFields(stats *Stats) mapstr.M { networkFields := mapstr.M{} for _, n := range stats.networkStats { - networkFields.Put(n.NameInterface, + _, err := networkFields.Put(n.NameInterface, mapstr.M{"inbound": mapstr.M{ "bytes": n.Total.RxBytes, "dropped": n.Total.RxDropped, @@ -185,6 +203,9 @@ func createNetworkFields(stats *Stats) mapstr.M { "errors": n.Total.TxErrors, "packets": n.Total.TxPackets, }}) + if err != nil { + _ = fmt.Errorf("error while putting network fields: %w", err) + } } return networkFields } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 995a3fd539cb..8adfd251f779 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -5,6 +5,7 @@ package task_stats import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -60,7 +61,7 @@ type TaskInfo struct { // Stats is a struct that represents information regarding a container type Stats struct { Time common.Time - taskInfo *TaskInfo + taskInfo TaskInfo Container *container cpuStats cpu.CPUStats memoryStats memoryStats @@ -117,8 +118,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { - // Get response from ${ECS_CONTAINER_METADATA_URI_V4}/task/stats - taskStatsResp, err := http.Get(m.taskStatsEndpoint) + // Collect information from ${ECS_CONTAINER_METADATA_URI_V4}/task/stats + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, m.taskStatsEndpoint, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequestWithContext: %w", err) + } + taskStatsResp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("http.Get failed: %w", err) } @@ -128,7 +133,11 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { } // Collect container metadata information from ${ECS_CONTAINER_METADATA_URI_V4}/task - taskResp, err := http.Get(m.taskEndpoint) + req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, m.taskStatsEndpoint, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequestWithContext: %w", err) + } + taskResp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("http.Get failed: %w", err) } @@ -195,7 +204,7 @@ func getStatsList(taskStatsOutput map[string]types.StatsJSON, taskOutput TaskMet if c, ok := containersInfo[id]; ok { statsPerContainer := Stats{ Time: common.Time(taskStats.Stats.Read), - taskInfo: &taskInfo, + taskInfo: taskInfo, Container: getContainerMetadata(&c), cpuStats: getCPUStats(taskStats), memoryStats: getMemoryStats(taskStats), From 2ff2c515ae7371813274395a677b57baf0a41060 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Thu, 14 Jul 2022 10:34:49 +0100 Subject: [PATCH 8/9] Minor fixes --- x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 8adfd251f779..5b020e50b360 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -119,7 +119,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { // Collect information from ${ECS_CONTAINER_METADATA_URI_V4}/task/stats - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, m.taskStatsEndpoint, nil) + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, m.taskStatsEndpoint, nil) if err != nil { return nil, fmt.Errorf("http.NewRequestWithContext: %w", err) } @@ -133,7 +133,7 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { } // Collect container metadata information from ${ECS_CONTAINER_METADATA_URI_V4}/task - req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, m.taskStatsEndpoint, nil) + req, err = http.NewRequestWithContext(context.TODO(), http.MethodGet, m.taskEndpoint, nil) if err != nil { return nil, fmt.Errorf("http.NewRequestWithContext: %w", err) } From d1ed939cbd80ce455b154968758430d6693e4797 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Tue, 19 Jul 2022 16:27:32 +0100 Subject: [PATCH 9/9] Added timeout and removed explicit error logging --- .../module/awsfargate/task_stats/data.go | 25 ++++--------------- .../awsfargate/task_stats/task_stats.go | 14 +++++++---- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 1b6288b3bd05..d1a4eb3277e9 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -39,40 +39,25 @@ func createEvent(stats *Stats) mb.Event { regionName, clusterName := getRegionAndClusterName(stats.Container.Labels) e.RootFields = createRootFields(stats, regionName) if clusterName != "" { - _, err := e.MetricSetFields.Put("cluster_name", clusterName) - if err != nil { - _ = fmt.Errorf("error putting metric set field 'cluster_name': %w", err) - } + _, _ = e.MetricSetFields.Put("cluster_name", clusterName) } taskName := stats.taskInfo.Family if taskName != "" { - _, err := e.MetricSetFields.Put("task_name", taskName) - if err != nil { - _ = fmt.Errorf("error putting metric set field 'task_name': %w", err) - } + _, _ = e.MetricSetFields.Put("task_name", taskName) } taskDesiredStatus := stats.taskInfo.TaskDesiredStatus if taskDesiredStatus != "" { - _, err := e.MetricSetFields.Put("task_desired_status", taskDesiredStatus) - if err != nil { - _ = fmt.Errorf("error putting metric set field 'task_desired_status': %w", err) - } + _, _ = e.MetricSetFields.Put("task_desired_status", taskDesiredStatus) } taskKnownStatus := stats.taskInfo.TaskKnownStatus if taskKnownStatus != "" { - _, err := e.MetricSetFields.Put("task_known_status", taskKnownStatus) - if err != nil { - _ = fmt.Errorf("error putting metric set field 'task_known_status': %w", err) - } + _, _ = e.MetricSetFields.Put("task_known_status", taskKnownStatus) } - _, err := e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) - if err != nil { - _ = fmt.Errorf("error putting metric set field 'identifier': %w", err) - } + _, _ = e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) return e } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 5b020e50b360..21eab0f0bc74 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "net/http" "os" + "time" "github.com/docker/docker/api/types" @@ -22,9 +23,10 @@ import ( ) var ( - metricsetName = "task_stats" - taskStatsPath = "task/stats" - taskPath = "task" + metricsetName = "task_stats" + taskStatsPath = "task/stats" + taskPath = "task" + queryTaskMetadataEndpointTimeout = 60 * time.Second ) // init registers the MetricSet with the central registry as soon as the program @@ -118,8 +120,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { + context, cancel := context.WithTimeout(context.Background(), queryTaskMetadataEndpointTimeout) + defer cancel() // Collect information from ${ECS_CONTAINER_METADATA_URI_V4}/task/stats - req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, m.taskStatsEndpoint, nil) + req, err := http.NewRequestWithContext(context, http.MethodGet, m.taskStatsEndpoint, nil) if err != nil { return nil, fmt.Errorf("http.NewRequestWithContext: %w", err) } @@ -133,7 +137,7 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { } // Collect container metadata information from ${ECS_CONTAINER_METADATA_URI_V4}/task - req, err = http.NewRequestWithContext(context.TODO(), http.MethodGet, m.taskEndpoint, nil) + req, err = http.NewRequestWithContext(context, http.MethodGet, m.taskEndpoint, nil) if err != nil { return nil, fmt.Errorf("http.NewRequestWithContext: %w", err) }