Fix k8s metadata issue

libbeat/common/kubernetes/metadata/metadata.go

@@ -21,6 +21,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/kubernetes"
 	"github.com/elastic/beats/v7/libbeat/common/safemapstr"
+	"strings"
 )
 
 // MetaGen allows creation of metadata from either Kubernetes resources or their Resource names.
@@ -40,3 +41,11 @@ func WithFields(key string, value interface{}) FieldOptions {
 		safemapstr.Put(meta, key, value)
 	}
 }
+
+// WithLabels FieldOption allows adding labels under sub-resource(kind)
+// example if kind=namespace namespace.labels key will be added
+func WithLabels(kind string) FieldOptions {
+	return func(meta common.MapStr) {
+		safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"])
+	}
+}

libbeat/common/kubernetes/metadata/namespace.go

@@ -47,8 +47,6 @@ func (n *namespace) Generate(obj kubernetes.Resource, opts ...FieldOptions) comm
 	}
 
 	meta := n.resource.Generate(resource, obj, opts...)
-	// TODO: remove this call when moving to 8.0
-	meta = flattenMetadata(meta)
 
 	// TODO: Add extra fields in here if need be
 	return meta
@@ -60,38 +58,14 @@ func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) common.M
 		return nil
 	}
 
-	obj, ok, _ := n.store.GetByKey(name)
-	if !ok {
-		return nil
-	}
-
-	no, ok := obj.(*kubernetes.Namespace)
-	if !ok {
-		return nil
-	}
-
-	return n.Generate(no, opts...)
-}
-
-func flattenMetadata(in common.MapStr) common.MapStr {
-	out := common.MapStr{}
-	rawFields, err := in.GetValue(resource)
-	if err != nil {
-		return nil
-	}
-
-	fields, ok := rawFields.(common.MapStr)
-	if !ok {
-		return nil
-	}
-
-	for k, v := range fields {
-		if k == "name" {
-			out[resource] = v
-		} else {
-			out[resource+"_"+k] = v
+	if obj, ok, _ := n.store.GetByKey(name); ok {
+		no, ok := obj.(*kubernetes.Namespace)
+		if !ok {
+			return nil
 		}
+
+		return n.Generate(no, opts...)
 	}
 
-	return out
+	return nil
 }

libbeat/common/kubernetes/metadata/namespace_test.go

@@ -57,20 +57,12 @@ func TestNamespace_Generate(t *testing.T) {
 					APIVersion: "v1",
 				},
 			},
-			// Use this for 8.0
-			/*output: common.MapStr{
+			output: common.MapStr{
 				"namespace": common.MapStr{
 					"name": "obj",
-					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+					"uid": uid,
 				},
-			},*/
-			output: common.MapStr{
-				"namespace":     "obj",
-				"namespace_uid": uid,
-				"namespace_labels": common.MapStr{
+				"labels": common.MapStr{
 					"foo": "bar",
 				},
 			},
@@ -122,9 +114,11 @@ func TestNamespace_GenerateFromName(t *testing.T) {
 					},
 				},*/
 			output: common.MapStr{
-				"namespace":     "obj",
-				"namespace_uid": uid,
-				"namespace_labels": common.MapStr{
+				"namespace": common.MapStr{
+					"name": "obj",
+					"uid":  uid,
+				},
+				"labels": common.MapStr{
 					"foo": "bar",
 				},
 			},

libbeat/common/kubernetes/metadata/node_test.go

@@ -61,9 +61,9 @@ func TestNode_Generate(t *testing.T) {
 				"node": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 			},
 		},
@@ -106,9 +106,9 @@ func TestNode_GenerateFromName(t *testing.T) {
 				"node": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 			},
 		},

libbeat/common/kubernetes/metadata/pod.go

@@ -49,11 +49,9 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map
 	}
 
 	out := p.resource.Generate("pod", obj, opts...)
-	// TODO: remove this call when moving to 8.0
-	out = p.exportPodLabelsAndAnnotations(out)
 
 	if p.node != nil {
-		meta := p.node.GenerateFromName(po.Spec.NodeName)
+		meta := p.node.GenerateFromName(po.Spec.NodeName, WithLabels("node"))
 		if meta != nil {
 			out.Put("node", meta["node"])
 		} else {
@@ -64,11 +62,9 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map
 	}
 
 	if p.namespace != nil {
-		meta := p.namespace.GenerateFromName(po.GetNamespace())
+		meta := p.namespace.GenerateFromName(po.GetNamespace(), WithLabels("namespace"))
 		if meta != nil {
-			// Use this in 8.0
-			//out.Put("namespace", meta["namespace"])
-			out.DeepUpdate(meta)
+			out.Put("namespace", meta["namespace"])
 		}
 	}
 	return out
@@ -91,21 +87,3 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr
 
 	return nil
 }
-
-func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr {
-	labels, err := in.GetValue("pod.labels")
-	if err != nil {
-		return in
-	}
-	in.Put("labels", labels)
-	in.Delete("pod.labels")
-
-	annotations, err := in.GetValue("pod.annotations")
-	if err != nil {
-		return in
-	}
-	in.Put("annotations", annotations)
-	in.Delete("pod.annotations")
-
-	return in
-}

libbeat/common/kubernetes/metadata/pod_test.go

@@ -334,10 +334,12 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
 					"name": "obj",
 					"uid":  uid,
 				},
-				"namespace":     "default",
-				"namespace_uid": uid,
-				"namespace_labels": common.MapStr{
-					"nskey": "nsvalue",
+				"namespace": common.MapStr{
+					"name": "default",
+					"uid":  uid,
+					"labels": common.MapStr{
+						"nskey": "nsvalue",
+					},
 				},
 				"node": common.MapStr{
 					"name": "testnode",

libbeat/common/kubernetes/metadata/resource.go

@@ -98,11 +98,11 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie
 	}
 
 	if len(labelMap) != 0 {
-		safemapstr.Put(meta, strings.ToLower(kind)+".labels", labelMap)
+		safemapstr.Put(meta, "labels", labelMap)
 	}
 
 	if len(annotationsMap) != 0 {
-		safemapstr.Put(meta, strings.ToLower(kind)+".annotations", annotationsMap)
+		safemapstr.Put(meta, "annotations", annotationsMap)
 	}
 
 	for _, option := range options {

libbeat/common/kubernetes/metadata/resource_test.go

@@ -60,9 +60,9 @@ func TestResource_Generate(t *testing.T) {
 				"pod": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 				"namespace": "default",
 			},
@@ -97,9 +97,9 @@ func TestResource_Generate(t *testing.T) {
 				"pod": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 				"namespace": "default",
 				"deployment": common.MapStr{

libbeat/common/kubernetes/metadata/service.go

@@ -49,11 +49,9 @@ func (s *service) Generate(obj kubernetes.Resource, opts ...FieldOptions) common
 	out := s.resource.Generate("service", obj, opts...)
 
 	if s.namespace != nil {
-		meta := s.namespace.GenerateFromName(svc.GetNamespace())
+		meta := s.namespace.GenerateFromName(svc.GetNamespace(), WithLabels("namespace"))
 		if meta != nil {
-			// Use this in 8.0
-			//out.Put("namespace", meta["namespace"])
-			out.DeepUpdate(meta)
+			out.Put("namespace", meta["namespace"])
 		}
 	}
 

libbeat/common/kubernetes/metadata/service_test.go

@@ -64,9 +64,9 @@ func TestService_Generate(t *testing.T) {
 				"service": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 				"namespace": "default",
 			},
@@ -101,9 +101,9 @@ func TestService_Generate(t *testing.T) {
 				"service": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 				"namespace": "default",
 				"deployment": common.MapStr{
@@ -153,9 +153,9 @@ func TestService_GenerateFromName(t *testing.T) {
 				"service": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 				"namespace": "default",
 			},
@@ -190,9 +190,9 @@ func TestService_GenerateFromName(t *testing.T) {
 				"service": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
-					"labels": common.MapStr{
-						"foo": "bar",
-					},
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
 				},
 				"namespace": "default",
 				"deployment": common.MapStr{
@@ -262,15 +262,17 @@ func TestService_GenerateWithNamespace(t *testing.T) {
 				"service": common.MapStr{
 					"name": "obj",
 					"uid":  uid,
+				},
+				"labels": common.MapStr{
+					"foo": "bar",
+				},
+				"namespace":   common.MapStr{
+					"name": "default",
+					"uid": uid,
 					"labels": common.MapStr{
-						"foo": "bar",
+						"nskey": "nsvalue",
 					},
 				},
-				"namespace":     "default",
-				"namespace_uid": uid,
-				"namespace_labels": common.MapStr{
-					"nskey": "nsvalue",
-				},
 			},
 		},
 	}