From e73f7c7706493a4f612de002d725a9d357607fbf Mon Sep 17 00:00:00 2001 From: samidbb Date: Thu, 1 Aug 2024 14:59:27 +0200 Subject: [PATCH 1/5] scrape worker nodes logs --- .../helm-grafana-agent/values/values.yaml | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/_sub/monitoring/helm-grafana-agent/values/values.yaml b/_sub/monitoring/helm-grafana-agent/values/values.yaml index 6b8c376bb..b0c7eb459 100644 --- a/_sub/monitoring/helm-grafana-agent/values/values.yaml +++ b/_sub/monitoring/helm-grafana-agent/values/values.yaml @@ -101,6 +101,27 @@ logs: enabled: true cluster_events: enabled: true + extraConfig: | + local.file_match "worker_nodes_logs" { + path_targets = [{ + __address__ = "localhost", + __path__ = "/var/log/**", + __path_exclude__ = "/var/log/{pods,containers,journal}/**", + instance = constants.hostname, + job = "integrations/worker-node", + }] + } + + loki.source.file "worker_nodes_logs" { + targets = local.file_match.worker_nodes_logs.targets + forward_to = [loki.process.worker_nodes_logs.receiver] + } + + loki.process "worker_nodes_logs" { + forward_to = [ + loki.process.logs_service.receiver, + ] + } alloy-events: {} alloy-logs: {} @@ -108,7 +129,7 @@ alloy-logs: {} # Metrics configurations and infrastructure metrics: - enabled: true + enabled: false autoDiscover: enabled: false cost: @@ -357,4 +378,4 @@ extraObjects: matchLabels: component: apiserver provider: kubernetes - #### \ No newline at end of file + #### From 68649eb32a36c03699ed8ff5b613f0b3d59066ef Mon Sep 17 00:00:00 2001 From: samidbb Date: Thu, 1 Aug 2024 15:03:45 +0200 Subject: [PATCH 2/5] re-enable metrics --- _sub/monitoring/helm-grafana-agent/values/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_sub/monitoring/helm-grafana-agent/values/values.yaml b/_sub/monitoring/helm-grafana-agent/values/values.yaml index b0c7eb459..e738e7a54 100644 --- a/_sub/monitoring/helm-grafana-agent/values/values.yaml +++ b/_sub/monitoring/helm-grafana-agent/values/values.yaml @@ -129,7 +129,7 @@ alloy-logs: {} # Metrics configurations and infrastructure metrics: - enabled: false + enabled: true autoDiscover: enabled: false cost: From 96c912f034267e8981da31dc63d23755f6bc889d Mon Sep 17 00:00:00 2001 From: samidbb Date: Fri, 2 Aug 2024 09:53:24 +0200 Subject: [PATCH 3/5] enabled journal logs on nodes --- _sub/monitoring/helm-grafana-agent/values/values.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_sub/monitoring/helm-grafana-agent/values/values.yaml b/_sub/monitoring/helm-grafana-agent/values/values.yaml index e738e7a54..fe99baffe 100644 --- a/_sub/monitoring/helm-grafana-agent/values/values.yaml +++ b/_sub/monitoring/helm-grafana-agent/values/values.yaml @@ -101,6 +101,9 @@ logs: enabled: true cluster_events: enabled: true + journal: + enabled: true + units: [] extraConfig: | local.file_match "worker_nodes_logs" { path_targets = [{ @@ -108,7 +111,7 @@ logs: __path__ = "/var/log/**", __path_exclude__ = "/var/log/{pods,containers,journal}/**", instance = constants.hostname, - job = "integrations/worker-node", + job = "integrations/kubernetes/worker-node", }] } From 3de64f517b2b38263cbbf2dccb46840251f25db7 Mon Sep 17 00:00:00 2001 From: samidbb Date: Fri, 2 Aug 2024 10:49:15 +0200 Subject: [PATCH 4/5] update grafana in QA --- test/integration/eu-west-1/k8s-qa/services/terragrunt.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/eu-west-1/k8s-qa/services/terragrunt.hcl b/test/integration/eu-west-1/k8s-qa/services/terragrunt.hcl index 7554b7a87..3ec700190 100644 --- a/test/integration/eu-west-1/k8s-qa/services/terragrunt.hcl +++ b/test/integration/eu-west-1/k8s-qa/services/terragrunt.hcl @@ -253,7 +253,7 @@ inputs = { grafana_agent_deploy = true grafana_deploy = false - grafana_agent_chart_version = "1.0.12" + grafana_agent_chart_version = "1.4.4" grafana_agent_resource_memory_request = "4Gi" grafana_agent_resource_memory_limit = "4Gi" grafana_agent_storage_enabled = true From f2f3de6a76998b927c5a2191beae3780b2170f43 Mon Sep 17 00:00:00 2001 From: samidbb Date: Fri, 2 Aug 2024 12:02:02 +0200 Subject: [PATCH 5/5] expose node real instance name --- _sub/monitoring/helm-grafana-agent/values/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_sub/monitoring/helm-grafana-agent/values/values.yaml b/_sub/monitoring/helm-grafana-agent/values/values.yaml index fe99baffe..b0e0e361b 100644 --- a/_sub/monitoring/helm-grafana-agent/values/values.yaml +++ b/_sub/monitoring/helm-grafana-agent/values/values.yaml @@ -110,7 +110,7 @@ logs: __address__ = "localhost", __path__ = "/var/log/**", __path_exclude__ = "/var/log/{pods,containers,journal}/**", - instance = constants.hostname, + instance = env("HOSTNAME"), job = "integrations/kubernetes/worker-node", }] }