From 402eec8195aa3da26f2be04c7e268eba5bc44b78 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 12 Mar 2020 14:53:53 +0200 Subject: [PATCH] Cherry-pick #16834 to 7.6: Fix k8s metadata issue (#16977) * Fix k8s metadata issue (#16834) (cherry picked from commit 034ee554ef6849fafea21b72db228f3c7d0fc159) * Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + .../common/kubernetes/metadata/metadata.go | 10 +++++ .../common/kubernetes/metadata/namespace.go | 14 +++++-- .../kubernetes/metadata/namespace_test.go | 23 +++++------ .../common/kubernetes/metadata/node_test.go | 12 +++--- libbeat/common/kubernetes/metadata/pod.go | 22 +---------- .../common/kubernetes/metadata/resource.go | 4 +- .../kubernetes/metadata/resource_test.go | 12 +++--- .../kubernetes/metadata/service_test.go | 39 ++++++++++++------- 9 files changed, 73 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9d3d600f254..82879c3b8c0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -40,6 +40,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* +- Fix k8s metadata issue regarding node labels not shown up on root level of metadata. {pull}16834[16834] *Auditbeat* diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go index b2205c91dd9..6982de34358 100644 --- a/libbeat/common/kubernetes/metadata/metadata.go +++ b/libbeat/common/kubernetes/metadata/metadata.go @@ -18,6 +18,8 @@ package metadata import ( + "strings" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/kubernetes" "github.com/elastic/beats/libbeat/common/safemapstr" @@ -40,3 +42,11 @@ func WithFields(key string, value interface{}) FieldOptions { safemapstr.Put(meta, key, value) } } + +// WithLabels FieldOption allows adding labels under sub-resource(kind) +// example if kind=namespace namespace.labels key will be added +func WithLabels(kind string) FieldOptions { + return func(meta common.MapStr) { + safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) + } +} diff --git a/libbeat/common/kubernetes/metadata/namespace.go b/libbeat/common/kubernetes/metadata/namespace.go index 7eb2d12eb5a..88a3e537b74 100644 --- a/libbeat/common/kubernetes/metadata/namespace.go +++ b/libbeat/common/kubernetes/metadata/namespace.go @@ -67,9 +67,8 @@ func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) common.M } return n.Generate(no, opts...) - } else { - return nil } + return nil } func flattenMetadata(in common.MapStr) common.MapStr { @@ -83,7 +82,6 @@ func flattenMetadata(in common.MapStr) common.MapStr { if !ok { return nil } - for k, v := range fields { if k == "name" { out[resource] = v @@ -92,5 +90,15 @@ func flattenMetadata(in common.MapStr) common.MapStr { } } + rawLabels, err := in.GetValue("labels") + if err != nil { + return out + } + labels, ok := rawLabels.(common.MapStr) + if !ok { + return out + } + out[resource+"_labels"] = labels + return out } diff --git a/libbeat/common/kubernetes/metadata/namespace_test.go b/libbeat/common/kubernetes/metadata/namespace_test.go index 4011a2f0053..0ecf7cf464a 100644 --- a/libbeat/common/kubernetes/metadata/namespace_test.go +++ b/libbeat/common/kubernetes/metadata/namespace_test.go @@ -58,17 +58,18 @@ func TestNamespace_Generate(t *testing.T) { }, }, // Use this for 8.0 - /*output: common.MapStr{ - "namespace": common.MapStr{ - "name": "obj", - "uid": uid, - "labels": common.MapStr{ - "foo": "bar", + /* + output: common.MapStr{ + "namespace": common.MapStr{ + "name": name, + "uid": uid, + "labels": common.MapStr{ + "foo": "bar", + }, }, - }, - },*/ + },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", @@ -114,7 +115,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { /* output: common.MapStr{ "namespace": common.MapStr{ - "name": "obj", + "name": name, "uid": uid, "labels": common.MapStr{ "foo": "bar", @@ -122,7 +123,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { }, },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go index 9d198361ab9..40eb54ef3de 100644 --- a/libbeat/common/kubernetes/metadata/node_test.go +++ b/libbeat/common/kubernetes/metadata/node_test.go @@ -61,9 +61,9 @@ func TestNode_Generate(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, @@ -106,9 +106,9 @@ func TestNode_GenerateFromName(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index 018b710e4b4..9b7f3670625 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -49,11 +49,9 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map } out := p.resource.Generate("pod", obj, opts...) - // TODO: remove this call when moving to 8.0 - out = p.exportPodLabelsAndAnnotations(out) if p.node != nil { - meta := p.node.GenerateFromName(po.Spec.NodeName) + meta := p.node.GenerateFromName(po.Spec.NodeName, WithLabels("node")) if meta != nil { out.Put("node", meta["node"]) } else { @@ -91,21 +89,3 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } - -func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr { - labels, err := in.GetValue("pod.labels") - if err != nil { - return in - } - in.Put("labels", labels) - in.Delete("pod.labels") - - annotations, err := in.GetValue("pod.annotations") - if err != nil { - return in - } - in.Put("annotations", annotations) - in.Delete("pod.annotations") - - return in -} diff --git a/libbeat/common/kubernetes/metadata/resource.go b/libbeat/common/kubernetes/metadata/resource.go index eb00455ffab..e4610ce6b50 100644 --- a/libbeat/common/kubernetes/metadata/resource.go +++ b/libbeat/common/kubernetes/metadata/resource.go @@ -98,11 +98,11 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie } if len(labelMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".labels", labelMap) + safemapstr.Put(meta, "labels", labelMap) } if len(annotationsMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".annotations", annotationsMap) + safemapstr.Put(meta, "annotations", annotationsMap) } for _, option := range options { diff --git a/libbeat/common/kubernetes/metadata/resource_test.go b/libbeat/common/kubernetes/metadata/resource_test.go index dbf168644af..df79241c83a 100644 --- a/libbeat/common/kubernetes/metadata/resource_test.go +++ b/libbeat/common/kubernetes/metadata/resource_test.go @@ -60,9 +60,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -97,9 +97,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go index dfda4e9db27..50c13cae397 100644 --- a/libbeat/common/kubernetes/metadata/service_test.go +++ b/libbeat/common/kubernetes/metadata/service_test.go @@ -64,9 +64,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -101,9 +101,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -153,9 +153,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -190,9 +190,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -262,10 +262,19 @@ func TestService_GenerateWithNamespace(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, + "labels": common.MapStr{ + "foo": "bar", + }, + // Use this for 8.0 + /* + "namespace": common.MapStr{ + "name": "default", + "uid": uid, + "labels": common.MapStr{ + "nskey": "nsvalue", + }, + },*/ "namespace": "default", "namespace_uid": uid, "namespace_labels": common.MapStr{