feat: implement support for creating services for agent presets (#6002)

elastic · Jan 1, 2025 · ed4f923 · ed4f923
1 parent 68d5833
commit ed4f923
Show file tree

Hide file tree

Showing 12 changed files with 492 additions and 7 deletions.
diff --git a/deploy/helm/elastic-agent/examples/netflow-service/README.md b/deploy/helm/elastic-agent/examples/netflow-service/README.md
@@ -0,0 +1,28 @@
+# Example: Netflow Custom Integration
+
+In this example we define a `netflow` custom integration alongside a custom agent preset defined in [agent-netflow-values.yaml](agent-netflow-values.yaml). Also, we disable all `kubernetes` related providers and creation of cluster role and service account, as they are not required for this example.
+
+## Prerequisites:
+1. A k8s secret that contains the connection details to an Elasticsearch cluster such as the URL and the API key ([Kibana - Creating API Keys](https://www.elastic.co/guide/en/kibana/current/api-keys.html)):
+    ```console
+    kubectl create secret generic es-api-secret \
+       --from-literal=api_key=... \
+       --from-literal=url=...
+    ```
+
+2. `NetFlow Records` integration assets are installed through Kibana
+
+## Run:
+1. Install Helm chart
+    ```console
+    helm install elastic-agent ../../ -f ./agent-netflow-values.yaml
+    ```
+
+2. Run the netflow data generator deployment
+    ```console
+   kubectl run -it --rm netflow-generator --image=networkstatic/nflow-generator --restart=Never -- -t agent-netflow-elastic-agent.default.svc.cluster.local -p 2055
+    ```
+
+## Validate:
+
+1. The Kibana `netflow`-related dashboards should start showing netflow related data.
diff --git a/deploy/helm/elastic-agent/examples/netflow-service/agent-netflow-values.yaml b/deploy/helm/elastic-agent/examples/netflow-service/agent-netflow-values.yaml
@@ -0,0 +1,62 @@
+outputs:
+  default:
+    type: ESSecretAuthAPI
+    secretName: es-api-secret
+
+extraIntegrations:
+  netflow:
+    id: netflow-netflow-60a9d5b2-c611-4749-90bf-5e2443936c1d
+    name: netflow-1
+    preset: netflow
+    revision: 1
+    type: netflow
+    use_output: default
+    meta:
+      package:
+        name: netflow
+        version: 2.19.1
+    data_stream:
+      namespace: default
+    package_policy_id: 60a9d5b2-c611-4749-90bf-5e2443936c1d
+    streams:
+      - id: netflow-netflow.log-60a9d5b2-c611-4749-90bf-5e2443936c1d
+        data_stream:
+          dataset: netflow.log
+          type: logs
+        protocols:
+          - v1
+          - v5
+          - v6
+          - v7
+          - v8
+          - v9
+          - ipfix
+        host: '0.0.0.0:2055'
+        max_message_size: 10KiB
+        expiration_timeout: 30m
+        queue_size: 8192
+        detect_sequence_reset: true
+        tags:
+          - netflow
+          - forwarded
+        publisher_pipeline.disable_host: true
+
+kubernetes:
+  enabled: false
+
+agent:
+  unprivileged: true
+  presets:
+    netflow:
+      automountServiceAccountToken: false
+      mode: deployment
+      service:
+        type: ClusterIP
+      ports:
+        - containerPort: 2055
+          servicePort: 2055
+          protocol: UDP
+      serviceAccount:
+        create: false
+      clusterRole:
+        create: false
diff --git a/deploy/helm/elastic-agent/examples/netflow-service/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/netflow-service/rendered/manifest.yaml
@@ -0,0 +1,168 @@
+---
+# Source: elastic-agent/templates/agent/k8s/secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: agent-netflow-example
+  namespace: "default"
+  labels:
+    helm.sh/chart: elastic-agent-9.0.0-beta
+    app.kubernetes.io/name: elastic-agent
+    app.kubernetes.io/instance: example
+    app.kubernetes.io/version: 9.0.0
+stringData:
+
+  agent.yml: |-
+    id: agent-netflow-example
+    outputs:
+      default:
+        api_key: ${OUTPUT_DEFAULT_API_KEY}
+        hosts:
+        - ${OUTPUT_DEFAULT_URL}
+        type: elasticsearch
+    secret_references: []
+    inputs:
+      - data_stream:
+          namespace: default
+        id: netflow-netflow-60a9d5b2-c611-4749-90bf-5e2443936c1d
+        meta:
+          package:
+            name: netflow
+            version: 2.19.1
+        name: netflow-1
+        package_policy_id: 60a9d5b2-c611-4749-90bf-5e2443936c1d
+        preset: netflow
+        revision: 1
+        streams:
+        - data_stream:
+            dataset: netflow.log
+            type: logs
+          detect_sequence_reset: true
+          expiration_timeout: 30m
+          host: 0.0.0.0:2055
+          id: netflow-netflow.log-60a9d5b2-c611-4749-90bf-5e2443936c1d
+          max_message_size: 10KiB
+          protocols:
+          - v1
+          - v5
+          - v6
+          - v7
+          - v8
+          - v9
+          - ipfix
+          publisher_pipeline.disable_host: true
+          queue_size: 8192
+          tags:
+          - netflow
+          - forwarded
+        type: netflow
+        use_output: default
+    providers:
+      kubernetes_leaderelection:
+        enabled: false
+        leader_lease: example-netflow
+---
+# Source: elastic-agent/templates/agent/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent-netflow-example
+  namespace: "default"
+  labels:
+    helm.sh/chart: elastic-agent-9.0.0-beta
+    app.kubernetes.io/name: elastic-agent
+    app.kubernetes.io/instance: example
+    app.kubernetes.io/version: 9.0.0
+spec:
+  type: ClusterIP
+  selector:
+    name: agent-netflow-example
+  ports:
+    - port: 2055
+      targetPort: 2055
+      protocol: UDP
+---
+# Source: elastic-agent/templates/agent/k8s/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: agent-netflow-example
+  namespace: "default"
+  labels:
+    helm.sh/chart: elastic-agent-9.0.0-beta
+    app.kubernetes.io/name: elastic-agent
+    app.kubernetes.io/instance: example
+    app.kubernetes.io/version: 9.0.0
+spec:
+  selector:
+    matchLabels:
+      name: agent-netflow-example
+  template:
+    metadata:
+      labels:
+        name: agent-netflow-example
+      annotations:
+        checksum/config: 4e9f48f0d6ae172f2f6aa5d526b0ca3af7dd28250e7c06c9d4e67ec0a2fc4573
+    spec:
+      automountServiceAccountToken: false
+      containers:
+      - args:
+        - -c
+        - /etc/elastic-agent/agent.yml
+        - -e
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: STATE_PATH
+          value: /usr/share/elastic-agent/state
+        - name: OUTPUT_DEFAULT_URL
+          valueFrom:
+            secretKeyRef:
+              key: url
+              name: es-api-secret
+        - name: OUTPUT_DEFAULT_API_KEY
+          valueFrom:
+            secretKeyRef:
+              key: api_key
+              name: es-api-secret
+        image: docker.elastic.co/beats/elastic-agent:9.0.0-SNAPSHOT
+        imagePullPolicy: IfNotPresent
+        name: agent
+        ports:
+        - containerPort: 2055
+          protocol: UDP
+        securityContext:
+          capabilities:
+            add:
+            - CHOWN
+            - SETPCAP
+            - DAC_READ_SEARCH
+            - SYS_PTRACE
+            drop:
+            - ALL
+          privileged: false
+          runAsGroup: 1000
+          runAsUser: 1000
+        volumeMounts:
+        - mountPath: /usr/share/elastic-agent/state
+          name: agent-data
+        - mountPath: /etc/elastic-agent/agent.yml
+          name: config
+          readOnly: true
+          subPath: agent.yml
+      dnsPolicy: ClusterFirstWithHostNet
+      volumes:
+      - hostPath:
+          path: /etc/elastic-agent/default/agent-netflow-example/state
+          type: DirectoryOrCreate
+        name: agent-data
+      - name: config
+        secret:
+          defaultMode: 292
+          secretName: agent-netflow-example
diff --git a/deploy/helm/elastic-agent/templates/agent/eck/_pod_template.yaml b/deploy/helm/elastic-agent/templates/agent/eck/_pod_template.yaml
@@ -72,6 +72,24 @@ template:
         resources:
           {{- . | toYaml | nindent 10 }}
         {{- end }}
+        {{- with ($presetVal).ports }}
+        ports:
+        {{- range $idx, $port := . }}
+          - containerPort: {{ $port.containerPort }}
+          {{- with $port.protocol | default "TCP" }}
+            protocol: {{ . }}
+          {{- end }}
+          {{- with $port.name }}
+            name: {{ . }}
+          {{- end }}
+          {{- with $port.hostPort }}
+            hostPort: {{ . }}
+          {{- end }}
+          {{- with $port.hostIP }}
+            hostIP: {{ . }}
+          {{- end }}
+        {{- end }}
+        {{- end }}
         volumeMounts:
           {{- with ($presetVal).extraVolumeMounts }}
           {{- . | toYaml | nindent 10 }}
@@ -93,7 +111,7 @@ template:
           {{- if eq $.Values.agent.fleet.enabled false }}
           {{- with ($presetVal).outputs }}
           {{- range $outputName, $outputVal := . -}}
-          {{- (include (printf "elasticagent.output.%s.preset.envvars" ($outputVal).type) (list $ $outputName $outputVal)) | nindent 14 }}
+          {{- (include (printf "elasticagent.output.%s.preset.envvars" ($outputVal).type) (list $ $outputName $outputVal)) | nindent 10 }}
           {{- end }}
           {{- end }}
           {{- end }}

diff --git a/deploy/helm/elastic-agent/templates/agent/eck/deployment.yaml b/deploy/helm/elastic-agent/templates/agent/eck/deployment.yaml
@@ -2,7 +2,7 @@
 {{- range $presetName, $presetVal := $.Values.agent.presets -}}
 {{- if and (eq ($presetVal).mode "deployment") (eq $.Values.agent.engine "eck") -}}
 {{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName)  -}}
-{{- $podTemplateResource := include "elasticagent.engine.eck.podTemplate" (list $ $presetVal $agentName) | fromYaml -}}
+{{- $podTemplateResource := include "elasticagent.engine.eck.podTemplate" (list $ $presetVal $agentName) | fromYaml }}
 apiVersion: agent.k8s.elastic.co/v1alpha1
 kind: Agent
 metadata:

diff --git a/deploy/helm/elastic-agent/templates/agent/eck/statefulset.yaml b/deploy/helm/elastic-agent/templates/agent/eck/statefulset.yaml
@@ -2,7 +2,7 @@
 {{- range $presetName, $presetVal := $.Values.agent.presets -}}
 {{- if and (eq ($presetVal).mode "statefulset") (eq $.Values.agent.engine "eck") -}}
 {{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName)  -}}
-{{- $podTemplateResource := include "elasticagent.engine.eck.podTemplate" (list $ $presetVal $agentName) | fromYaml -}}
+{{- $podTemplateResource := include "elasticagent.engine.eck.podTemplate" (list $ $presetVal $agentName) | fromYaml }}
 apiVersion: agent.k8s.elastic.co/v1alpha1
 kind: Agent
 metadata:

diff --git a/deploy/helm/elastic-agent/templates/agent/k8s/_pod_template.yaml b/deploy/helm/elastic-agent/templates/agent/k8s/_pod_template.yaml
@@ -98,6 +98,24 @@ template:
         resources:
           {{- . | toYaml | nindent 10 }}
         {{- end }}
+        {{- with ($presetVal).ports }}
+        ports:
+        {{- range $idx, $port := . }}
+          - containerPort: {{ $port.containerPort }}
+          {{- with $port.protocol | default "TCP" }}
+            protocol: {{ . }}
+          {{- end }}
+          {{- with $port.name }}
+            name: {{ . }}
+          {{- end }}
+          {{- with $port.hostPort }}
+            hostPort: {{ . }}
+          {{- end }}
+          {{- with $port.hostIP }}
+            hostIP: {{ . }}
+          {{- end }}
+        {{- end }}
+        {{- end }}
         volumeMounts:
           {{- $definedAgentStateVolumeMount := false -}}
           {{- with ($presetVal).extraVolumeMounts }}
@@ -135,7 +153,7 @@ template:
           {{- if eq $.Values.agent.fleet.enabled false }}
           {{- with ($presetVal).outputs }}
           {{- range $outputName, $outputVal := . -}}
-          {{- (include (printf "elasticagent.output.%s.preset.envvars" ($outputVal).type) (list $ $outputName $outputVal)) | nindent 12 }}
+          {{- (include (printf "elasticagent.output.%s.preset.envvars" ($outputVal).type) (list $ $outputName $outputVal)) | nindent 10 }}
           {{- end }}
           {{- end }}
           {{- end }}

diff --git a/deploy/helm/elastic-agent/templates/agent/k8s/daemonset.yaml b/deploy/helm/elastic-agent/templates/agent/k8s/daemonset.yaml
@@ -2,7 +2,7 @@
 {{- range $presetName, $presetVal := $.Values.agent.presets -}}
 {{- if and (eq ($presetVal).mode "daemonset") (eq $.Values.agent.engine "k8s") -}}
 {{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName)  -}}
-{{- $podTemplateResource := include "elasticagent.engine.k8s.podTemplate" (list $ $presetVal $agentName) | fromYaml -}}
+{{- $podTemplateResource := include "elasticagent.engine.k8s.podTemplate" (list $ $presetVal $agentName) | fromYaml }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:

diff --git a/deploy/helm/elastic-agent/templates/agent/k8s/deployment.yaml b/deploy/helm/elastic-agent/templates/agent/k8s/deployment.yaml
@@ -2,7 +2,7 @@
 {{- range $presetName, $presetVal := $.Values.agent.presets -}}
 {{- if and (eq ($presetVal).mode "deployment") (eq $.Values.agent.engine "k8s") -}}
 {{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName)  -}}
-{{- $podTemplateResource := include "elasticagent.engine.k8s.podTemplate" (list $ $presetVal $agentName) | fromYaml  -}}
+{{- $podTemplateResource := include "elasticagent.engine.k8s.podTemplate" (list $ $presetVal $agentName) | fromYaml }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:

diff --git a/deploy/helm/elastic-agent/templates/agent/k8s/statefulset.yaml b/deploy/helm/elastic-agent/templates/agent/k8s/statefulset.yaml
@@ -2,7 +2,7 @@
 {{- range $presetName, $presetVal := $.Values.agent.presets -}}
 {{- if and (eq ($presetVal).mode "statefulset") (eq $.Values.agent.engine "k8s") -}}
 {{- $agentName := include "elasticagent.preset.fullname" (list $ $presetName)  -}}
-{{- $podTemplateResource := include "elasticagent.engine.k8s.podTemplate" (list $ $presetVal $agentName) | fromYaml  -}}
+{{- $podTemplateResource := include "elasticagent.engine.k8s.podTemplate" (list $ $presetVal $agentName) | fromYaml }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata: