feat: implement system integration as chart built-in

deploy/helm/elastic-agent/README.md

@@ -123,12 +123,21 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations
 | kubernetes.system.metrics.enabled | bool | `true` | enable system metric stream (kubelet) [ref](https://www.elastic.co/docs/current/integrations/kubernetes/kubelet#system) |
 | kubernetes.system.metrics.vars | object | `{}` | system metric stream vars |
 
-### 3 - User Extra Integrations
+### 4 - System integration
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| system.enabled | bool | `false` | enable System integration. |
+| system.output | string | `"default"` | name of the output used in kubernetes integration. Note that this output needs to be defined in [outputs](#1-outputs) |
+| system.namespace | string | `"default"` | output namespace |
+| system.logs.enabled | bool | `true` | enable logs |
+| system.metrics.enabled | bool | `true` | enable metrics |
+
+### 5 - User Extra Integrations
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | extraIntegrations | object | `{}` | extra [user-defined integrations](https://www.elastic.co/guide/en/fleet/current/elastic-agent-input-configuration.html) to be added to the Elastic Agent An example can be found [here](./examples/nginx-custom-integration/README.md) |
 
-### 3 - Elastic-Agent Configuration
+### 6 - Elastic-Agent Configuration
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | agent.version | string | `"9.0.0"` | elastic-agent version |
@@ -137,7 +146,7 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations
 | agent.unprivileged | bool | `false` | enable unprivileged mode |
 | agent.presets | map[string]{} | `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }` | Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets required by the built-in Kubernetes integration [here](./values.yaml) |
 
-### 3.1 - Elastic-Agent Managed Configuration
+### 6.1 - Elastic-Agent Managed Configuration
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | agent.fleet.enabled | bool | `false` | enable elastic-agent managed |

deploy/helm/elastic-agent/templates/NOTES.txt

@@ -22,6 +22,9 @@ Installed integrations:
 {{- if eq $.Values.kubernetes.enabled true }}
   - kubernetes [built-in chart integration]
 {{- end }}
+{{- if eq $.Values.system.enabled true }}
+  - system [built-in chart integration]
+{{- end }}
 {{- range $customInputName, $customInputVal := $.Values.extraIntegrations }}
   - {{$customInputName}} [user-defined integration]
 {{- end }}

deploy/helm/elastic-agent/templates/agent/_helpers.tpl

@@ -65,6 +65,7 @@ Initialise input templates if we are not deploying as managed
 {{- if eq $.Values.agent.fleet.enabled false -}}
 {{/* standalone agent so initialise inputs */}}
 {{- include "elasticagent.kubernetes.init" $ -}}
+{{- include "elasticagent.system.init" $ -}}
 {{- range $customInputName, $customInputVal := $.Values.extraIntegrations -}}
 {{- $customInputPresetName := ($customInputVal).preset -}}
 {{- $presetVal := get $.Values.agent.presets $customInputPresetName -}}

deploy/helm/elastic-agent/templates/integrations/_system/_system.tpl

@@ -0,0 +1,6 @@
+{{- define "elasticagent.system.init" -}}
+{{- if eq $.Values.system.enabled true -}}
+{{- include "elasticagent.system.config.logs.init" $ -}}
+{{- include "elasticagent.system.config.metrics.init" $ -}}
+{{- end -}}
+{{- end -}}

deploy/helm/elastic-agent/templates/integrations/_system/_system_logs.tpl

@@ -0,0 +1,45 @@
+{{- define "elasticagent.system.config.logs.init" -}}
+{{- if $.Values.system.logs.enabled}}
+{{- $preset := $.Values.agent.presets.perNode -}}
+{{- $inputVal := (include "elasticagent.system.config.logs.input" $ | fromYamlArray) -}}
+{{- include "elasticagent.preset.mutate.inputs" (list $ $preset $inputVal) -}}
+{{- include "elasticagent.preset.applyOnce" (list $ $preset "elasticagent.kubernetes.pernode.preset") -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "elasticagent.system.config.logs.input" -}}
+- id: system-logs
+  type: logfile
+  use_output: {{ $.Values.system.output }}
+  data_stream:
+    namespace: {{ $.Values.system.namespace }}
+  streams:
+    - data_stream:
+        dataset: system.auth
+        type: logs
+      paths:
+        - /var/log/auth.log*
+        - /var/log/secure*
+      exclude_files:
+        - .gz$
+      multiline:
+        pattern: ^\s
+        match: after
+      processors:
+        - add_locale: null
+      ignore_older: 72h
+    - data_stream:
+        dataset: system.syslog
+        type: logs
+      paths:
+        - /var/log/messages*
+        - /var/log/syslog*
+      exclude_files:
+        - .gz$
+      multiline:
+        pattern: ^\s
+        match: after
+      processors:
+        - add_locale: null
+      ignore_older: 72h
+{{- end -}}

deploy/helm/elastic-agent/templates/integrations/_system/_system_metrics.tpl

@@ -0,0 +1,103 @@
+{{- define "elasticagent.system.config.metrics.init" -}}
+{{- if $.Values.system.metrics.enabled}}
+{{- $preset := $.Values.agent.presets.perNode -}}
+{{- $inputVal := (include "elasticagent.system.config.metrics.input" $ | fromYamlArray) -}}
+{{- include "elasticagent.preset.mutate.inputs" (list $ $preset $inputVal) -}}
+{{- include "elasticagent.preset.applyOnce" (list $ $preset "elasticagent.kubernetes.pernode.preset") -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "elasticagent.system.config.metrics.input" -}}
+- id: system-metrics
+  type: system/metrics
+  use_output: {{ $.Values.system.output }}
+  data_stream:
+    namespace: {{ $.Values.system.namespace }}
+  streams:
+    - data_stream:
+        dataset: system.cpu
+        type: metrics
+      period: 10s
+      cpu.metrics:
+        - percentages
+        - normalized_percentages
+      metricsets:
+        - cpu
+    - data_stream:
+        dataset: system.diskio
+        type: metrics
+      period: 10s
+      diskio.include_devices: null
+      metricsets:
+        - diskio
+    - data_stream:
+        dataset: system.filesystem
+        type: metrics
+      period: 1m
+      metricsets:
+        - filesystem
+      processors:
+        - drop_event.when.regexp:
+            system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
+    - data_stream:
+        dataset: system.fsstat
+        type: metrics
+      period: 1m
+      metricsets:
+        - fsstat
+      processors:
+        - drop_event.when.regexp:
+            system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
+    - data_stream:
+        dataset: system.load
+        type: metrics
+      condition: '${host.platform} != ''windows'''
+      period: 10s
+      metricsets:
+        - load
+    - data_stream:
+        dataset: system.memory
+        type: metrics
+      period: 10s
+      metricsets:
+        - memory
+    - data_stream:
+        dataset: system.network
+        type: metrics
+      period: 10s
+      network.interfaces: null
+      metricsets:
+        - network
+    - data_stream:
+        dataset: system.process
+        type: metrics
+      period: 10s
+      processes:
+        - .*
+      process.include_top_n.by_cpu: 5
+      process.include_top_n.by_memory: 5
+      process.cmdline.cache.enabled: true
+      process.cgroups.enabled: false
+      process.include_cpu_ticks: false
+      metricsets:
+        - process
+      process.include_cpu_ticks: false
+    - data_stream:
+        dataset: system.process_summary
+        type: metrics
+      period: 10s
+      metricsets:
+        - process_summary
+    - data_stream:
+        dataset: system.socket_summary
+        type: metrics
+      period: 10s
+      metricsets:
+        - socket_summary
+    - data_stream:
+        type: metrics
+        dataset: system.uptime
+      metricsets:
+        - uptime
+      period: 10s
+{{- end -}}

deploy/helm/elastic-agent/values.schema.json

@@ -18,6 +18,53 @@
                 }
             ]
         },
+        "system": {
+            "type": "object",
+            "description": "Configuration for system integration.",
+            "properties": {
+                "enabled": {
+                    "type": "boolean",
+                    "description": "Enable system integration."
+                },
+                "output": {
+                    "type": "string",
+                    "description": "Name of the output used in system integration. Must be defined in outputs."
+                },
+                "namespace": {
+                    "type": "string",
+                    "description": "output namespace.",
+                    "default": "default"
+                },
+                "logs": {
+                    "type": "object",
+                    "properties": {
+                        "enabled": {
+                            "type": "boolean",
+                            "description": "Enable system logs stream."
+                        }
+                    },
+                    "required": [
+                        "enabled"
+                    ]
+                },
+                "metrics": {
+                    "type": "object",
+                    "properties": {
+                        "enabled": {
+                            "type": "boolean",
+                            "description": "Enable system metrics stream."
+                        }
+                    },
+                    "required": [
+                        "enabled"
+                    ]
+                }
+            },
+            "required": [
+                "enabled",
+                "output"
+            ]
+        },
         "kubernetes": {
             "type": "object",
             "description": "Configuration for Kubernetes integration.",
@@ -528,7 +575,8 @@
                             "password"
                         ]
                     }
-                },{
+                },
+                {
                     "if": {
                         "properties": {
                             "type": {

deploy/helm/elastic-agent/values.yaml

@@ -64,7 +64,7 @@ kubernetes:
     # results in overriding and *disabling all* the respective state streams
     # @section -- 2 - Kubernetes integration
     enabled: true
-    # -- deploy kube-state-metrics service as a sidecar container to the elastic agent of `ksmShared` preset.
+    # -- deploy kube-state-metrics service as a sidecar container to the elastic agent of `ksmSharded` preset.
     # If set to `false`, kube-state-metrics will *not* get deployed and `clusterWide` agent preset will be used for
     # collecting kube-state-metrics.
     # @section -- 2 - Kubernetes integration
@@ -287,55 +287,74 @@ kubernetes:
       # -- system metric stream vars
       # @section -- 2 - Kubernetes integration
       vars: {}
+system:
+  # -- enable System integration.
+  # @section -- 4 - System integration
+  enabled: false
+  # -- name of the output used in kubernetes integration. Note that this output needs to be defined in
+  # [outputs](#1-outputs)
+  # @section -- 4 - System integration
+  output: default
+  # -- output namespace
+  # @section -- 4 - System integration
+  namespace: default
+  logs:
+    # -- enable logs
+    # @section -- 4 - System integration
+    enabled: true
+  metrics:
+    # -- enable metrics
+    # @section -- 4 - System integration
+    enabled: true
 # -- extra [user-defined integrations](https://www.elastic.co/guide/en/fleet/current/elastic-agent-input-configuration.html) to be added to the Elastic Agent
 # An example can be found [here](./examples/nginx-custom-integration/README.md)
-# @section -- 3 - User Extra Integrations
+# @section -- 5 - User Extra Integrations
 extraIntegrations: {}
 agent:
   # -- elastic-agent version
-  # @section -- 3 - Elastic-Agent Configuration
+  # @section -- 6 - Elastic-Agent Configuration
   version: 9.0.0
   # -- image configuration
-  # @section -- 3 - Elastic-Agent Configuration
+  # @section -- 6 - Elastic-Agent Configuration
   image:
     repository: docker.elastic.co/beats/elastic-agent
     pullPolicy: IfNotPresent
     tag: "9.0.0-SNAPSHOT"
   # -- generate kubernetes manifests or [ECK](https://github.com/elastic/cloud-on-k8s) CRDs
-  # @section -- 3 - Elastic-Agent Configuration
+  # @section -- 6 - Elastic-Agent Configuration
   engine: k8s # k8s or eck
   # -- enable unprivileged mode
-  # @section -- 3 - Elastic-Agent Configuration
+  # @section -- 6 - Elastic-Agent Configuration
   unprivileged: false
   fleet:
     # -- enable elastic-agent managed
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     enabled: false
     # -- Fleet server URL
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     url: ""
     # -- Fleet enrollment token
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     token: ""
     # -- Fleet insecure url
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     insecure: false
     # -- Kibana host to fallback if enrollment token is not supplied
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     kibanaHost: ""
     # -- Kibana username to fallback if enrollment token is not supplied
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     kibanaUser: ""
     # -- Kibana password to fallback if enrollment token is not supplied
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     kibanaPassword: ""
     # -- Agent preset to deploy
-    # @section -- 3.1 - Elastic-Agent Managed Configuration
+    # @section -- 6.1 - Elastic-Agent Managed Configuration
     preset: perNode
   # -- Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets
   # required by the built-in Kubernetes integration [here](./values.yaml)
   # @notationType -- map[string]{}
-  # @section -- 3 - Elastic-Agent Configuration
+  # @section -- 6 - Elastic-Agent Configuration
   # @default -- `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }`
   presets:
     #  examplePreset: