From 77375cdfef6559aee00324d047d1c2d7d089331e Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 21 Feb 2020 12:15:58 +0200 Subject: [PATCH 1/3] Fix k8s pod labels tier in metadata Signed-off-by: chrismark --- libbeat/common/kubernetes/metadata/pod.go | 13 ++++++++ .../common/kubernetes/metadata/pod_test.go | 30 +++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index b42fa005cc0..7611749a9c5 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -49,6 +49,8 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map } out := p.resource.Generate("pod", obj, opts...) + // TODO: remove this call when moving to 8.0 + out = p.fixLabels(out) if p.node != nil { meta := p.node.GenerateFromName(po.Spec.NodeName) @@ -89,3 +91,14 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } + +func (p *pod) fixLabels(in common.MapStr) common.MapStr { + labels, err := in.GetValue("pod.labels") + if err != nil { + return in + } + in.Put("labels", labels) + in.Delete("pod.labels") + + return in +} diff --git a/libbeat/common/kubernetes/metadata/pod_test.go b/libbeat/common/kubernetes/metadata/pod_test.go index 2c42958a759..c3838ad2c32 100644 --- a/libbeat/common/kubernetes/metadata/pod_test.go +++ b/libbeat/common/kubernetes/metadata/pod_test.go @@ -67,9 +67,9 @@ func TestPod_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "node": common.MapStr{ @@ -110,9 +110,6 @@ func TestPod_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "deployment": common.MapStr{ @@ -121,6 +118,9 @@ func TestPod_Generate(t *testing.T) { "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, } @@ -168,14 +168,14 @@ func TestPod_GenerateFromName(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, { @@ -211,9 +211,6 @@ func TestPod_GenerateFromName(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "deployment": common.MapStr{ @@ -222,6 +219,9 @@ func TestPod_GenerateFromName(t *testing.T) { "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, } @@ -304,9 +304,6 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "namespace_uid": uid, @@ -320,6 +317,9 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { "nodekey": "nodevalue", }, }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, } From 1f4d866ebfe844e2878aee1680fc1a4d75a076ab Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 21 Feb 2020 14:30:18 +0200 Subject: [PATCH 2/3] change method name Signed-off-by: chrismark --- libbeat/common/kubernetes/metadata/pod.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index 7611749a9c5..7385d2abdf5 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -50,7 +50,7 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map out := p.resource.Generate("pod", obj, opts...) // TODO: remove this call when moving to 8.0 - out = p.fixLabels(out) + out = p.exportPodLabels(out) if p.node != nil { meta := p.node.GenerateFromName(po.Spec.NodeName) @@ -92,7 +92,7 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } -func (p *pod) fixLabels(in common.MapStr) common.MapStr { +func (p *pod) exportPodLabels(in common.MapStr) common.MapStr { labels, err := in.GetValue("pod.labels") if err != nil { return in From b71eabcc1f099978175c37b304680b77109d8e15 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 25 Feb 2020 11:05:29 +0200 Subject: [PATCH 3/3] fix tests Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 3 +- .../add_kubernetes_metadata/indexers_test.go | 30 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 960154d5897..0eede63e580 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -57,6 +57,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] - Fix loading processors from annotation hints. {pull}16348[16348] +- Fix k8s pods labels broken schema. {pull}16480[16480] *Auditbeat* @@ -125,7 +126,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add MQTT input. {issue}15602[15602] {pull}16204[16204] - Add ECS categorization fields to activemq module. {issue}16151[16151] {pull}16201[16201] - Add a TLS test and more debug output to httpjson input {pull}16315[16315] -- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] +- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] - Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280] *Heartbeat* diff --git a/libbeat/processors/add_kubernetes_metadata/indexers_test.go b/libbeat/processors/add_kubernetes_metadata/indexers_test.go index 0d74cece012..acbe1bf51eb 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers_test.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers_test.go @@ -66,9 +66,9 @@ func TestPodIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, + }, + "labels": common.MapStr{ + "labelkey": "labelvalue", }, "namespace": "testns", "node": common.MapStr{ @@ -117,14 +117,14 @@ func TestPodUIDIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, }, "namespace": "testns", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "labelkey": "labelvalue", + }, } assert.Equal(t, expected.String(), indexers[0].Data.String()) @@ -173,14 +173,14 @@ func TestContainerIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, }, "namespace": "testns", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "labelkey": "labelvalue", + }, } container1 := "docker://abcde" pod.Spec.NodeName = nodeName @@ -250,7 +250,7 @@ func TestFilteredGenMeta(t *testing.T) { indexers := podIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) - rawLabels, _ := indexers[0].Data.GetValue("pod.labels") + rawLabels, _ := indexers[0].Data.GetValue("labels") assert.NotNil(t, rawLabels) labelMap, ok := rawLabels.(common.MapStr) @@ -274,7 +274,7 @@ func TestFilteredGenMeta(t *testing.T) { indexers = podIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) - rawLabels, _ = indexers[0].Data.GetValue("pod.labels") + rawLabels, _ = indexers[0].Data.GetValue("labels") assert.NotNil(t, rawLabels) labelMap, ok = rawLabels.(common.MapStr) @@ -331,7 +331,7 @@ func TestFilteredGenMetaExclusion(t *testing.T) { indexers := podIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) - rawLabels, _ := indexers[0].Data.GetValue("pod.labels") + rawLabels, _ := indexers[0].Data.GetValue("labels") assert.NotNil(t, rawLabels) labelMap, ok := rawLabels.(common.MapStr) @@ -392,14 +392,14 @@ func TestIpPortIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, }, "namespace": "testns", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "labelkey": "labelvalue", + }, } pod.Spec.Containers = []v1.Container{