Cherry-pick #19398 to 7.x: Add support for named ports in autodiscover (hints & templates)

CHANGELOG.next.asciidoc

@@ -602,6 +602,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Add dashboards for googlecloud load balancing metricset. {pull}18369[18369]
 - Update Couchbase to version 6.5 {issue}18595[18595] {pull}19055[19055]
 - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268]
+- Add support for named ports in autodiscover. {pull}19398[19398]
 - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423]
 
 *Packetbeat*

libbeat/autodiscover/providers/kubernetes/pod.go

@@ -191,6 +191,9 @@ func (p *pod) GenerateHints(event bus.Event) bus.Event {
 	if port, ok := event["port"]; ok {
 		e["port"] = port
 	}
+	if ports, ok := event["ports"]; ok {
+		e["ports"] = ports
+	}
 
 	if rawCont, ok := kubeMeta["container"]; ok {
 		container = rawCont.(common.MapStr)
@@ -300,6 +303,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 		}
 	}
 
+	podPorts := common.MapStr{}
 	// Emit container and port information
 	for _, c := range containers {
 		// If it doesn't have an ID, container doesn't exist in
@@ -349,6 +353,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 		}
 
 		for _, port := range c.Ports {
+			podPorts[port.Name] = port.ContainerPort
 			event := bus.Event{
 				"provider":   p.uuid,
 				"id":         eventID,
@@ -385,6 +390,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet
 			"id":         fmt.Sprint(pod.GetObjectMeta().GetUID()),
 			flag:         true,
 			"host":       host,
+			"ports":      podPorts,
 			"kubernetes": kubemeta,
 			"meta": common.MapStr{
 				"kubernetes": meta,

libbeat/autodiscover/providers/kubernetes/pod_test.go

@@ -395,6 +395,7 @@ func TestEmitEvent(t *testing.T) {
 					"host":     "127.0.0.1",
 					"id":       uid,
 					"provider": UUID,
+					"ports":    common.MapStr{},
 					"kubernetes": common.MapStr{
 						"pod": common.MapStr{
 							"name": "filebeat",
@@ -493,9 +494,11 @@ func TestEmitEvent(t *testing.T) {
 							Ports: []v1.ContainerPort{
 								{
 									ContainerPort: 8080,
+									Name:          "port1",
 								},
 								{
 									ContainerPort: 9090,
+									Name:          "port2",
 								},
 							},
 						},
@@ -508,6 +511,10 @@ func TestEmitEvent(t *testing.T) {
 					"host":     "127.0.0.1",
 					"id":       uid,
 					"provider": UUID,
+					"ports": common.MapStr{
+						"port1": int32(8080),
+						"port2": int32(9090),
+					},
 					"kubernetes": common.MapStr{
 						"pod": common.MapStr{
 							"name": "filebeat",
@@ -711,6 +718,7 @@ func TestEmitEvent(t *testing.T) {
 					"host":     "",
 					"id":       uid,
 					"provider": UUID,
+					"ports":    common.MapStr{},
 					"kubernetes": common.MapStr{
 						"pod": common.MapStr{
 							"name": "filebeat",
@@ -812,6 +820,7 @@ func TestEmitEvent(t *testing.T) {
 					"host":     "127.0.0.1",
 					"id":       uid,
 					"provider": UUID,
+					"ports":    common.MapStr{},
 					"kubernetes": common.MapStr{
 						"pod": common.MapStr{
 							"name": "filebeat",
@@ -913,6 +922,7 @@ func TestEmitEvent(t *testing.T) {
 					"host":     "127.0.0.1",
 					"id":       uid,
 					"provider": UUID,
+					"ports":    common.MapStr{},
 					"kubernetes": common.MapStr{
 						"pod": common.MapStr{
 							"name": "filebeat",

libbeat/autodiscover/template/config_test.go

@@ -35,6 +35,11 @@ func TestConfigsMapping(t *testing.T) {
 		"correct": "config",
 	})
 
+	configPorts, _ := common.NewConfigFrom(map[string]interface{}{
+		"correct": "config",
+		"hosts":   [1]string{"1.2.3.4:8080"},
+	})
+
 	tests := []struct {
 		mapping  string
 		event    bus.Event
@@ -74,6 +79,38 @@ func TestConfigsMapping(t *testing.T) {
 			},
 			expected: []*common.Config{config},
 		},
+		// Match config and replace data.host and data.ports.<name> properly
+		{
+			mapping: `
+- condition.equals:
+    foo: 3
+  config:
+  - correct: config
+    hosts: ["${data.host}:${data.ports.web}"]`,
+			event: bus.Event{
+				"foo":  3,
+				"host": "1.2.3.4",
+				"ports": common.MapStr{
+					"web": 8080,
+				},
+			},
+			expected: []*common.Config{configPorts},
+		},
+		// Match config and replace data.host and data.port properly
+		{
+			mapping: `
+- condition.equals:
+    foo: 3
+  config:
+  - correct: config
+    hosts: ["${data.host}:${data.port}"]`,
+			event: bus.Event{
+				"foo":  3,
+				"host": "1.2.3.4",
+				"port": 8080,
+			},
+			expected: []*common.Config{configPorts},
+		},
 	}
 
 	for _, test := range tests {

libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go

@@ -101,6 +101,9 @@ func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interf
 func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString, error) {
 	// key = "kubernetes.somenamespace.somesecret.value"
 	tokens := strings.Split(key, ".")
+	if len(tokens) > 0 && tokens[0] != "kubernetes" {
+		return nil, keystore.ErrKeyDoesntExists
+	}
 	if len(tokens) != 4 {
 		k.logger.Debugf(
 			"not valid secret key: %v. Secrets should be of the following format %v",

metricbeat/autodiscover/builder/hints/metrics.go

@@ -219,9 +219,16 @@ func (m *metricHints) getHostsWithPort(hints common.MapStr, port int, noPort boo
 	var result []string
 	thosts := builder.GetHintAsList(hints, m.Key, hosts)
 
-	// Only pick hosts that have ${data.port} or the port on current event. This will make
-	// sure that incorrect meta mapping doesn't happen
+	// Only pick hosts that:
+	// 1. have noPort (pod level event) and data.ports.<port_name> defined
+	// 2. have ${data.port} or the port on current event.
+	// This will make sure that incorrect meta mapping doesn't happen
 	for _, h := range thosts {
+		if strings.Contains(h, "data.ports.") && noPort {
+			result = append(result, fmt.Sprintf("'%v'", h))
+			// move on to the next host
+			continue
+		}
 		if strings.Contains(h, "data.port") && port != 0 && !noPort || m.checkHostPort(h, port) ||
 			// Use the event that has no port config if there is a ${data.host}:9090 like input
 			(noPort && strings.Contains(h, "data.host")) {

metricbeat/autodiscover/builder/hints/metrics_test.go

@@ -368,6 +368,85 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		{
+			message: "Named port in the hints should return the corresponding container port",
+			event: bus.Event{
+				"host":  "1.2.3.4",
+				"ports": common.MapStr{"some": 3306},
+				"hints": common.MapStr{
+					"metrics": common.MapStr{
+						"module":    "mockmoduledefaults",
+						"namespace": "test",
+						"hosts":     "${data.host}:${data.ports.some}",
+					},
+				},
+			},
+			len: 1,
+			result: []common.MapStr{
+				{
+					"module":     "mockmoduledefaults",
+					"namespace":  "test",
+					"metricsets": []string{"default"},
+					"hosts":      []interface{}{"1.2.3.4:3306"},
+					"timeout":    "3s",
+					"period":     "1m",
+					"enabled":    true,
+				},
+			},
+		},
+		{
+			message: "Named port in the hints should return the corresponding container port for complex hosts",
+			event: bus.Event{
+				"host":  "1.2.3.4",
+				"ports": common.MapStr{"prometheus": 3306},
+				"hints": common.MapStr{
+					"metrics": common.MapStr{
+						"module":    "mockmoduledefaults",
+						"namespace": "test",
+						"hosts":     "http://${data.host}:${data.ports.prometheus}/metrics",
+					},
+				},
+			},
+			len: 1,
+			result: []common.MapStr{
+				{
+					"module":     "mockmoduledefaults",
+					"namespace":  "test",
+					"metricsets": []string{"default"},
+					"hosts":      []interface{}{"http://1.2.3.4:3306/metrics"},
+					"timeout":    "3s",
+					"period":     "1m",
+					"enabled":    true,
+				},
+			},
+		},
+		{
+			message: "data.port in the hints should return the corresponding container port",
+			event: bus.Event{
+				"host":  "1.2.3.4",
+				"port":  3306,
+				"ports": common.MapStr{"prometheus": 3306},
+				"hints": common.MapStr{
+					"metrics": common.MapStr{
+						"module":    "mockmoduledefaults",
+						"namespace": "test",
+						"hosts":     "${data.host}:${data.port}",
+					},
+				},
+			},
+			len: 1,
+			result: []common.MapStr{
+				{
+					"module":     "mockmoduledefaults",
+					"namespace":  "test",
+					"metricsets": []string{"default"},
+					"hosts":      []interface{}{"1.2.3.4:3306"},
+					"timeout":    "3s",
+					"period":     "1m",
+					"enabled":    true,
+				},
+			},
+		},
 		{
 			message: "Module with mutliple sets of hints must return the right configs",
 			event: bus.Event{

metricbeat/docs/autodiscover-hints.asciidoc

@@ -11,8 +11,11 @@ it. Hints tell {beatname_uc} how to get metrics for the given container. This is
 [float]
 ===== `co.elastic.metrics/hosts`
 
-Hosts setting to use with the given module. Hosts can include `${data.host}` and `${data.port}`
-values from the autodiscover event, ie: `${data.host}:80`.
+Hosts setting to use with the given module. Hosts can include `${data.host}`, `${data.port}`,
+`${data.ports.<port_name>}` values from the autodiscover event, ie: `${data.host}:80`.
+For Kuberentes autodiscover events users can leverage port names as well,
+like `http://${data.host}:${data.ports.prometheus}/metrics`.
+In the last one we refer to the container port with name `prometheus`.
 
 [float]
 ===== `co.elastic.metrics/metricsets`

metricbeat/docs/autodiscover-kubernetes-config.asciidoc

@@ -34,6 +34,9 @@ Pods share an identical host. If only the `{data.host}` variable is interpolated
 then one config will be generated per host. The configs will be identical.
 After they are de-duplicated, only one will be used.
 
+In order to target one specific exposed port `{data.host}:{data.ports.web}` can be used
+in template config, where `web` is the name of the exposed container port.
+
 [float]
 [[kubernetes-secrets]]
 ===== Metricbeat Autodiscover Secret Management