From 64e45129a3c8e60355beb9e5338caf7f0b8db01f Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Fri, 25 Oct 2024 15:31:51 +0300 Subject: [PATCH] feat: implement system integration as chart built-in --- deploy/helm/elastic-agent/README.md | 15 ++- deploy/helm/elastic-agent/templates/NOTES.txt | 3 + .../templates/agent/_helpers.tpl | 1 + .../integrations/_system/_system.tpl | 6 + .../integrations/_system/_system_logs.tpl | 45 ++++++++ .../integrations/_system/_system_metrics.tpl | 103 ++++++++++++++++++ deploy/helm/elastic-agent/values.schema.json | 50 ++++++++- deploy/helm/elastic-agent/values.yaml | 49 ++++++--- 8 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 deploy/helm/elastic-agent/templates/integrations/_system/_system.tpl create mode 100644 deploy/helm/elastic-agent/templates/integrations/_system/_system_logs.tpl create mode 100644 deploy/helm/elastic-agent/templates/integrations/_system/_system_metrics.tpl diff --git a/deploy/helm/elastic-agent/README.md b/deploy/helm/elastic-agent/README.md index 876d432ea4f..c5939ef323a 100644 --- a/deploy/helm/elastic-agent/README.md +++ b/deploy/helm/elastic-agent/README.md @@ -123,12 +123,21 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations | kubernetes.system.metrics.enabled | bool | `true` | enable system metric stream (kubelet) [ref](https://www.elastic.co/docs/current/integrations/kubernetes/kubelet#system) | | kubernetes.system.metrics.vars | object | `{}` | system metric stream vars | -### 3 - User Extra Integrations +### 4 - System integration +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| system.enabled | bool | `false` | enable System integration. | +| system.output | string | `"default"` | name of the output used in kubernetes integration. Note that this output needs to be defined in [outputs](#1-outputs) | +| system.namespace | string | `"default"` | output namespace | +| system.logs.enabled | bool | `true` | enable logs | +| system.metrics.enabled | bool | `true` | enable metrics | + +### 5 - User Extra Integrations | Key | Type | Default | Description | |-----|------|---------|-------------| | extraIntegrations | object | `{}` | extra [user-defined integrations](https://www.elastic.co/guide/en/fleet/current/elastic-agent-input-configuration.html) to be added to the Elastic Agent An example can be found [here](./examples/nginx-custom-integration/README.md) | -### 3 - Elastic-Agent Configuration +### 6 - Elastic-Agent Configuration | Key | Type | Default | Description | |-----|------|---------|-------------| | agent.version | string | `"9.0.0"` | elastic-agent version | @@ -137,7 +146,7 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations | agent.unprivileged | bool | `false` | enable unprivileged mode | | agent.presets | map[string]{} | `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }` | Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets required by the built-in Kubernetes integration [here](./values.yaml) | -### 3.1 - Elastic-Agent Managed Configuration +### 6.1 - Elastic-Agent Managed Configuration | Key | Type | Default | Description | |-----|------|---------|-------------| | agent.fleet.enabled | bool | `false` | enable elastic-agent managed | diff --git a/deploy/helm/elastic-agent/templates/NOTES.txt b/deploy/helm/elastic-agent/templates/NOTES.txt index e7f9d2caede..0b8587db0fb 100644 --- a/deploy/helm/elastic-agent/templates/NOTES.txt +++ b/deploy/helm/elastic-agent/templates/NOTES.txt @@ -22,6 +22,9 @@ Installed integrations: {{- if eq $.Values.kubernetes.enabled true }} - kubernetes [built-in chart integration] {{- end }} +{{- if eq $.Values.system.enabled true }} + - system [built-in chart integration] +{{- end }} {{- range $customInputName, $customInputVal := $.Values.extraIntegrations }} - {{$customInputName}} [user-defined integration] {{- end }} diff --git a/deploy/helm/elastic-agent/templates/agent/_helpers.tpl b/deploy/helm/elastic-agent/templates/agent/_helpers.tpl index 0de739754ee..d3021d69bd4 100644 --- a/deploy/helm/elastic-agent/templates/agent/_helpers.tpl +++ b/deploy/helm/elastic-agent/templates/agent/_helpers.tpl @@ -65,6 +65,7 @@ Initialise input templates if we are not deploying as managed {{- if eq $.Values.agent.fleet.enabled false -}} {{/* standalone agent so initialise inputs */}} {{- include "elasticagent.kubernetes.init" $ -}} +{{- include "elasticagent.system.init" $ -}} {{- range $customInputName, $customInputVal := $.Values.extraIntegrations -}} {{- $customInputPresetName := ($customInputVal).preset -}} {{- $presetVal := get $.Values.agent.presets $customInputPresetName -}} diff --git a/deploy/helm/elastic-agent/templates/integrations/_system/_system.tpl b/deploy/helm/elastic-agent/templates/integrations/_system/_system.tpl new file mode 100644 index 00000000000..fee22b990e9 --- /dev/null +++ b/deploy/helm/elastic-agent/templates/integrations/_system/_system.tpl @@ -0,0 +1,6 @@ +{{- define "elasticagent.system.init" -}} +{{- if eq $.Values.system.enabled true -}} +{{- include "elasticagent.system.config.logs.init" $ -}} +{{- include "elasticagent.system.config.metrics.init" $ -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/elastic-agent/templates/integrations/_system/_system_logs.tpl b/deploy/helm/elastic-agent/templates/integrations/_system/_system_logs.tpl new file mode 100644 index 00000000000..39788e672db --- /dev/null +++ b/deploy/helm/elastic-agent/templates/integrations/_system/_system_logs.tpl @@ -0,0 +1,45 @@ +{{- define "elasticagent.system.config.logs.init" -}} +{{- if $.Values.system.logs.enabled}} +{{- $preset := $.Values.agent.presets.perNode -}} +{{- $inputVal := (include "elasticagent.system.config.logs.input" $ | fromYamlArray) -}} +{{- include "elasticagent.preset.mutate.inputs" (list $ $preset $inputVal) -}} +{{- include "elasticagent.preset.applyOnce" (list $ $preset "elasticagent.kubernetes.pernode.preset") -}} +{{- end -}} +{{- end -}} + +{{- define "elasticagent.system.config.logs.input" -}} +- id: system-logs + type: logfile + use_output: {{ $.Values.system.output }} + data_stream: + namespace: {{ $.Values.system.namespace }} + streams: + - data_stream: + dataset: system.auth + type: logs + paths: + - /var/log/auth.log* + - /var/log/secure* + exclude_files: + - .gz$ + multiline: + pattern: ^\s + match: after + processors: + - add_locale: null + ignore_older: 72h + - data_stream: + dataset: system.syslog + type: logs + paths: + - /var/log/messages* + - /var/log/syslog* + exclude_files: + - .gz$ + multiline: + pattern: ^\s + match: after + processors: + - add_locale: null + ignore_older: 72h +{{- end -}} diff --git a/deploy/helm/elastic-agent/templates/integrations/_system/_system_metrics.tpl b/deploy/helm/elastic-agent/templates/integrations/_system/_system_metrics.tpl new file mode 100644 index 00000000000..0661020385e --- /dev/null +++ b/deploy/helm/elastic-agent/templates/integrations/_system/_system_metrics.tpl @@ -0,0 +1,103 @@ +{{- define "elasticagent.system.config.metrics.init" -}} +{{- if $.Values.system.metrics.enabled}} +{{- $preset := $.Values.agent.presets.perNode -}} +{{- $inputVal := (include "elasticagent.system.config.metrics.input" $ | fromYamlArray) -}} +{{- include "elasticagent.preset.mutate.inputs" (list $ $preset $inputVal) -}} +{{- include "elasticagent.preset.applyOnce" (list $ $preset "elasticagent.kubernetes.pernode.preset") -}} +{{- end -}} +{{- end -}} + +{{- define "elasticagent.system.config.metrics.input" -}} +- id: system-metrics + type: system/metrics + use_output: {{ $.Values.system.output }} + data_stream: + namespace: {{ $.Values.system.namespace }} + streams: + - data_stream: + dataset: system.cpu + type: metrics + period: 10s + cpu.metrics: + - percentages + - normalized_percentages + metricsets: + - cpu + - data_stream: + dataset: system.diskio + type: metrics + period: 10s + diskio.include_devices: null + metricsets: + - diskio + - data_stream: + dataset: system.filesystem + type: metrics + period: 1m + metricsets: + - filesystem + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - data_stream: + dataset: system.fsstat + type: metrics + period: 1m + metricsets: + - fsstat + processors: + - drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - data_stream: + dataset: system.load + type: metrics + condition: '${host.platform} != ''windows''' + period: 10s + metricsets: + - load + - data_stream: + dataset: system.memory + type: metrics + period: 10s + metricsets: + - memory + - data_stream: + dataset: system.network + type: metrics + period: 10s + network.interfaces: null + metricsets: + - network + - data_stream: + dataset: system.process + type: metrics + period: 10s + processes: + - .* + process.include_top_n.by_cpu: 5 + process.include_top_n.by_memory: 5 + process.cmdline.cache.enabled: true + process.cgroups.enabled: false + process.include_cpu_ticks: false + metricsets: + - process + process.include_cpu_ticks: false + - data_stream: + dataset: system.process_summary + type: metrics + period: 10s + metricsets: + - process_summary + - data_stream: + dataset: system.socket_summary + type: metrics + period: 10s + metricsets: + - socket_summary + - data_stream: + type: metrics + dataset: system.uptime + metricsets: + - uptime + period: 10s +{{- end -}} diff --git a/deploy/helm/elastic-agent/values.schema.json b/deploy/helm/elastic-agent/values.schema.json index 443c5fc8653..466efafd058 100644 --- a/deploy/helm/elastic-agent/values.schema.json +++ b/deploy/helm/elastic-agent/values.schema.json @@ -18,6 +18,53 @@ } ] }, + "system": { + "type": "object", + "description": "Configuration for system integration.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable system integration." + }, + "output": { + "type": "string", + "description": "Name of the output used in system integration. Must be defined in outputs." + }, + "namespace": { + "type": "string", + "description": "output namespace.", + "default": "default" + }, + "logs": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable system logs stream." + } + }, + "required": [ + "enabled" + ] + }, + "metrics": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable system metrics stream." + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "enabled", + "output" + ] + }, "kubernetes": { "type": "object", "description": "Configuration for Kubernetes integration.", @@ -528,7 +575,8 @@ "password" ] } - },{ + }, + { "if": { "properties": { "type": { diff --git a/deploy/helm/elastic-agent/values.yaml b/deploy/helm/elastic-agent/values.yaml index f72e78cf949..38a311088f0 100644 --- a/deploy/helm/elastic-agent/values.yaml +++ b/deploy/helm/elastic-agent/values.yaml @@ -64,7 +64,7 @@ kubernetes: # results in overriding and *disabling all* the respective state streams # @section -- 2 - Kubernetes integration enabled: true - # -- deploy kube-state-metrics service as a sidecar container to the elastic agent of `ksmShared` preset. + # -- deploy kube-state-metrics service as a sidecar container to the elastic agent of `ksmSharded` preset. # If set to `false`, kube-state-metrics will *not* get deployed and `clusterWide` agent preset will be used for # collecting kube-state-metrics. # @section -- 2 - Kubernetes integration @@ -287,55 +287,74 @@ kubernetes: # -- system metric stream vars # @section -- 2 - Kubernetes integration vars: {} +system: + # -- enable System integration. + # @section -- 4 - System integration + enabled: false + # -- name of the output used in kubernetes integration. Note that this output needs to be defined in + # [outputs](#1-outputs) + # @section -- 4 - System integration + output: default + # -- output namespace + # @section -- 4 - System integration + namespace: default + logs: + # -- enable logs + # @section -- 4 - System integration + enabled: true + metrics: + # -- enable metrics + # @section -- 4 - System integration + enabled: true # -- extra [user-defined integrations](https://www.elastic.co/guide/en/fleet/current/elastic-agent-input-configuration.html) to be added to the Elastic Agent # An example can be found [here](./examples/nginx-custom-integration/README.md) -# @section -- 3 - User Extra Integrations +# @section -- 5 - User Extra Integrations extraIntegrations: {} agent: # -- elastic-agent version - # @section -- 3 - Elastic-Agent Configuration + # @section -- 6 - Elastic-Agent Configuration version: 9.0.0 # -- image configuration - # @section -- 3 - Elastic-Agent Configuration + # @section -- 6 - Elastic-Agent Configuration image: repository: docker.elastic.co/beats/elastic-agent pullPolicy: IfNotPresent tag: "9.0.0-SNAPSHOT" # -- generate kubernetes manifests or [ECK](https://github.com/elastic/cloud-on-k8s) CRDs - # @section -- 3 - Elastic-Agent Configuration + # @section -- 6 - Elastic-Agent Configuration engine: k8s # k8s or eck # -- enable unprivileged mode - # @section -- 3 - Elastic-Agent Configuration + # @section -- 6 - Elastic-Agent Configuration unprivileged: false fleet: # -- enable elastic-agent managed - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration enabled: false # -- Fleet server URL - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration url: "" # -- Fleet enrollment token - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration token: "" # -- Fleet insecure url - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration insecure: false # -- Kibana host to fallback if enrollment token is not supplied - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration kibanaHost: "" # -- Kibana username to fallback if enrollment token is not supplied - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration kibanaUser: "" # -- Kibana password to fallback if enrollment token is not supplied - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration kibanaPassword: "" # -- Agent preset to deploy - # @section -- 3.1 - Elastic-Agent Managed Configuration + # @section -- 6.1 - Elastic-Agent Managed Configuration preset: perNode # -- Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets # required by the built-in Kubernetes integration [here](./values.yaml) # @notationType -- map[string]{} - # @section -- 3 - Elastic-Agent Configuration + # @section -- 6 - Elastic-Agent Configuration # @default -- `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }` presets: # examplePreset: