[Elastic Agent] Add support for synthetics inputs that run heartbeat (e…

…lastic#20387)

* Add support for synthetics inputs that run heartbeat.

* Run mage fmt.

* Add changelog.

* Update test configs to use host instead of hosts.

* Enable support for top-level inputs from Elastic Agent.

dev-tools/packaging/packages.yml

@@ -64,6 +64,16 @@ shared:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      /var/lib/{{.BeatName}}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+      /var/lib/{{.BeatName}}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+      /var/lib/{{.BeatName}}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
       /var/lib/{{.BeatName}}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
@@ -135,6 +145,17 @@ shared:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      /etc/{{.BeatName}}/data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+      /etc/{{.BeatName}}/data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      /etc/{{.BeatName}}/data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
       /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
@@ -195,6 +216,16 @@ shared:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
       'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
@@ -239,6 +270,16 @@ shared:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
+      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
+        mode: 0644
+      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
+        mode: 0644
+      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+        source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
+        mode: 0644
+        skip_on_missing: true
       'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644

heartbeat/beater/heartbeat.go

@@ -139,6 +139,8 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) error {
 func (bt *Heartbeat) RunCentralMgmtMonitors(b *beat.Beat) {
 	monitors := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher)
 	reload.Register.MustRegisterList(b.Info.Beat+".monitors", monitors)
+	inputs := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher)
+	reload.Register.MustRegisterList("inputs", inputs)
 }
 
 // RunReloadableMonitors runs the `heartbeat.config.monitors` portion of the yaml config if present.

heartbeat/monitors/active/http/http.go

@@ -33,6 +33,7 @@ import (
 
 func init() {
 	monitors.RegisterActive("http", create)
+	monitors.RegisterActive("synthetics/http", create)
 }
 
 var debugf = logp.MakeDebug("http")

heartbeat/monitors/active/icmp/icmp.go

@@ -36,6 +36,7 @@ var debugf = logp.MakeDebug("icmp")
 
 func init() {
 	monitors.RegisterActive("icmp", create)
+	monitors.RegisterActive("synthetics/icmp", create)
 }
 
 func create(

heartbeat/monitors/active/tcp/tcp.go

@@ -40,6 +40,7 @@ import (
 
 func init() {
 	monitors.RegisterActive("tcp", create)
+	monitors.RegisterActive("synthetics/tcp", create)
 }
 
 var debugf = logp.MakeDebug("tcp")

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -102,3 +102,4 @@
 - Improved version CLI {pull}20359[20359]
 - Enroll CLI now restarts running daemon {pull}20359[20359]
 - Add restart CLI cmd {pull}20359[20359]
+- Add new `synthetics/*` inputs to run Heartbeat {pull}20387[20387]

x-pack/elastic-agent/magefile.go

@@ -523,7 +523,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 		defer os.RemoveAll(dropPath)
 		defer os.Unsetenv(agentDropPath)
 
-		packedBeats := []string{"filebeat", "metricbeat"}
+		packedBeats := []string{"filebeat", "heartbeat", "metricbeat"}
 
 		for _, b := range packedBeats {
 			pwd, err := filepath.Abs(filepath.Join("..", b))

x-pack/elastic-agent/pkg/agent/program/program_test.go

@@ -385,8 +385,8 @@ func TestConfiguration(t *testing.T) {
 		err      bool
 	}{
 		"single_config": {
-			programs: []string{"filebeat", "metricbeat", "endpoint"},
-			expected: 3,
+			programs: []string{"filebeat", "heartbeat", "metricbeat", "endpoint"},
+			expected: 4,
 		},
 		"constraints_config": {
 			programs: []string{"filebeat"},
@@ -400,10 +400,10 @@ func TestConfiguration(t *testing.T) {
 		// 	programs: []string{"journalbeat"},
 		// 	expected: 1,
 		// },
-		// "monitor_config": {
-		// 	programs: []string{"heartbeat"},
-		// 	expected: 1,
-		// },
+		"synthetics_config": {
+			programs: []string{"heartbeat"},
+			expected: 1,
+		},
 		"enabled_true": {
 			programs: []string{"filebeat"},
 			expected: 1,

x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml

@@ -21,6 +21,7 @@ output:
 
 inputs:
 - id: endpoint-id
+  type: endpoint
   name: endpoint-1
   enabled: true
   package:

x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml

@@ -20,6 +20,7 @@ outputs:
 
 inputs:
 - id: endpoint-id
+  type: endpoint
   name: endpoint-1
   enabled: true
   package:

x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_no_fleet.yml

@@ -11,6 +11,7 @@ outputs:
 
 inputs:
 - id: endpoint-id
+  type: endpoint
   name: endpoint-1
   enabled: true
   package:

x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_unknown_output.yml

@@ -16,6 +16,7 @@ outputs:
 
 inputs:
 - id: endpoint-id
+  type: endpoint
   name: endpoint-1
   enabled: true
   package:

x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml

@@ -21,6 +21,7 @@ output:
 
 inputs:
 - id: endpoint-id
+  type: endpoint
   name: endpoint-1
   enabled: true
   package:

x-pack/elastic-agent/pkg/agent/program/testdata/single_config-heartbeat.yml

@@ -0,0 +1,18 @@
+inputs:
+- type: synthetics/http
+  id: unique-http-id
+  name: my-http
+  schedule: '*/5 * * * * * *'
+  host: "http://localhost:80/service/status"
+  timeout: 16s
+  wait: 1s
+  dataset.namespace: default
+output:
+  elasticsearch:
+    hosts:
+      - 127.0.0.1:9200
+      - 127.0.0.1:9300
+    username: elastic
+    password: changeme
+    api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA
+    ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y=

x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml

@@ -67,7 +67,15 @@ inputs:
       metricset: info
       hosts: ["http://apache.remote"]
   hosts: ["http://apache.local"]
+- type: synthetics/http
+  id: unique-http-id
+  name: my-http
+  schedule: '*/5 * * * * * *'
+  host: "http://localhost:80/service/status"
+  timeout: 16s
+  wait: 1s
 - id: endpoint-id
+  type: endpoint
   name: endpoint-1
   enabled: true
   package:

x-pack/elastic-agent/pkg/agent/program/testdata/synthetics_config-heartbeat.yml

@@ -0,0 +1,33 @@
+inputs:
+- type: synthetics/http
+  id: unique-http-id
+  name: my-http
+  schedule: '*/5 * * * * * *'
+  host: "http://localhost:80/service/status"
+  timeout: 16s
+  wait: 1s
+  dataset.namespace: default
+- type: synthetics/tcp
+  id: unique-tcp-id
+  name: my-tcp
+  schedule: '*/5 * * * * * *'
+  host: "localhost:777"
+  timeout: 16s
+  wait: 1s
+  dataset.namespace: default
+- type: synthetics/icmp
+  id: unique-icmp-id
+  name: my-icmp
+  schedule: '@every 5s'
+  host: "localhost"
+  ipv4: true
+  ipv6: true
+  mode: any
+  timeout: 16s
+  wait: 1s
+  dataset.namespace: default
+output:
+  elasticsearch:
+    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    username: elastic
+    password: changeme

x-pack/elastic-agent/pkg/agent/program/testdata/synthetics_config.yml

@@ -0,0 +1,31 @@
+inputs:
+- type: synthetics/http
+  id: unique-http-id
+  name: my-http
+  schedule: '*/5 * * * * * *'
+  host: "http://localhost:80/service/status"
+  timeout: 16s
+  wait: 1s
+- type: synthetics/tcp
+  id: unique-tcp-id
+  name: my-tcp
+  schedule: '*/5 * * * * * *'
+  host: "localhost:777"
+  timeout: 16s
+  wait: 1s
+- type: synthetics/icmp
+  id: unique-icmp-id
+  name: my-icmp
+  schedule: '@every 5s'
+  host: "localhost"
+  ipv4: true
+  ipv6: true
+  mode: any
+  timeout: 16s
+  wait: 1s
+outputs:
+  default:
+    type: elasticsearch
+    hosts: [127.0.0.1:9200, 127.0.0.1:9300]
+    username: elastic
+    password: changeme

x-pack/elastic-agent/pkg/agent/transpiler/rules.go

@@ -1199,7 +1199,7 @@ func (r *FilterValuesWithRegexpRule) Apply(ast *AST) error {
 
 		newAST, ok := Lookup(newRoot, r.Key)
 		if !ok {
-			newNodes = append(newNodes, item)
+			// doesn't have key so its filtered out
 			continue
 		}
 
@@ -1210,7 +1210,7 @@ func (r *FilterValuesWithRegexpRule) Apply(ast *AST) error {
 		}
 
 		if n.name != r.Key {
-			newNodes = append(newNodes, item)
+			// doesn't match so its filtered out
 			continue
 		}
 

x-pack/elastic-agent/spec/heartbeat.yml

@@ -0,0 +1,22 @@
+name: Heartbeat
+cmd: heartbeat
+args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"]
+artifact: beats/heartbeat
+rules:
+  - fix_stream: {}
+  - filter_values_with_regexp:
+      key: type
+      re: ^synthetics/.+
+      selector: inputs
+  - filter_values:
+      selector: inputs
+      key: enabled
+      values:
+        - true
+  - filter:
+      selectors:
+        - inputs
+        - output
+        - keystore
+when: HasItems(%{[inputs]}) && HasNamespace('output', 'elasticsearch', 'redis',
+  'kafka', 'logstash')