OTAGENT-286 Add support for filelog receiver

charts/datadog/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Datadog changelog
 
+## 3.100.0
+
+* Mount  `datadog.otelCollector.logs.enabled` to support additional RBAC permissions required by OTel components that are not included by default with `otel-agent`.
+* Add support for additional volume mounts in `otel-agent` via `agents.containers.otelAgent.volumeMounts`.
+
 ## 3.99.0
 
 * Add `datadog.otelCollector.rbac.create` to control creation additional ClusterRole for `otel-agent` required by Kubernetes Attributes processor.

charts/datadog/Chart.yaml

@@ -1,7 +1,7 @@
 ---
 apiVersion: v1
 name: datadog
-version: 3.99.0
+version: 3.100.0
 appVersion: "7"
 description: Datadog Agent
 keywords:

charts/datadog/README.md

@@ -1,6 +1,6 @@
 # Datadog
 
-![Version: 3.99.0](https://img.shields.io/badge/Version-3.99.0-informational?style=flat-square) ![AppVersion: 7](https://img.shields.io/badge/AppVersion-7-informational?style=flat-square)
+![Version: 3.100.0](https://img.shields.io/badge/Version-3.100.0-informational?style=flat-square) ![AppVersion: 7](https://img.shields.io/badge/AppVersion-7-informational?style=flat-square)
 
 [Datadog](https://www.datadoghq.com/) is a hosted infrastructure monitoring platform. This chart adds the Datadog Agent to all nodes in your cluster via a DaemonSet. It also optionally depends on the [kube-state-metrics chart](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics). For more information about monitoring Kubernetes with Datadog, please refer to the [Datadog documentation website](https://docs.datadoghq.com/agent/basic_agent_usage/kubernetes/).
 
@@ -487,6 +487,7 @@ helm install <RELEASE_NAME> \
 | agents.containers.otelAgent.ports | list | `[]` | Allows to specify extra ports (hostPorts for instance) for this container |
 | agents.containers.otelAgent.resources | object | `{}` | Resource requests and limits for the otel-agent container |
 | agents.containers.otelAgent.securityContext | object | `{}` | Allows you to overwrite the default container SecurityContext for the otel-agent container. |
+| agents.containers.otelAgent.volumeMounts | list | `[]` | Specify additional volumes to mount in the otel-agent container |
 | agents.containers.processAgent.env | list | `[]` | Additional environment variables for the process-agent container |
 | agents.containers.processAgent.envDict | object | `{}` | Set environment variables specific to process-agent defined in a dict |
 | agents.containers.processAgent.envFrom | list | `[]` | Set environment variables specific to process-agent from configMaps and/or secrets |
@@ -808,6 +809,7 @@ helm install <RELEASE_NAME> \
 | datadog.osReleasePath | string | `"/etc/os-release"` | Specify the path to your os-release file |
 | datadog.otelCollector.config | string | `nil` | OTel collector configuration |
 | datadog.otelCollector.enabled | bool | `false` | Enable the OTel Collector |
+| datadog.otelCollector.logs.enabled | bool | `false` | Enable logs support in the OTel Collector. If true, checks OTel Collector config for filelog receiver and mounts additional volumes to collect containers and pods logs. |
 | datadog.otelCollector.ports | list | `[{"containerPort":"4317","name":"otel-grpc"},{"containerPort":"4318","name":"otel-http"}]` | Ports that OTel Collector is listening |
 | datadog.otelCollector.rbac.create | bool | `true` | If true, check OTel Collector config for k8sattributes processor and create required ClusterRole to access Kubernetes API |
 | datadog.otelCollector.rbac.rules | list | `[]` | A set of additional RBAC rules to apply to OTel Collector's ClusterRole |

charts/datadog/ci/agent-otel-collector-logs-values.yaml

@@ -0,0 +1,38 @@
+targetSystem: "linux"
+agents:
+  image:
+    repository: datadog/agent-dev
+    tag: nightly-ot-beta-main
+    doNotCheckTag: true
+  containers:
+    agent:
+      env:
+        - name: DD_HOSTNAME
+          value: "datadog"
+datadog:
+  apiKey: "00000000000000000000000000000000"
+  appKey: "0000000000000000000000000000000000000000"
+  otelCollector:
+    enabled: true
+    logs:
+      enabled: true
+    config: |
+      receivers:
+        otlp:
+        filelog:
+        filelog/datadog:
+      exporters:
+        datadog:
+          api:
+            key: "00000000000000000000000000000000"
+      service:
+        pipelines:
+          traces:
+            receivers: [otlp]
+            exporters: [datadog]
+          metrics:
+            receivers: [otlp]
+            exporters: [datadog]
+          logs:
+            receivers: [filelog]
+            exporters: [datadog]

charts/datadog/ci/agent-otel-collector-volume-mounts-values.yaml

@@ -0,0 +1,43 @@
+targetSystem: "linux"
+agents:
+  image:
+    repository: datadog/agent-dev
+    tag: nightly-ot-beta-main
+    doNotCheckTag: true
+  containers:
+    agent:
+      env:
+        - name: DD_HOSTNAME
+          value: "datadog"
+    otelAgent:
+      volumeMounts:
+        - name: logscustompath
+          mountPath: /var/log/custom
+          readOnly: true
+  volumes:
+    - hostPath:
+        path: /var/log/custom
+      name: logscustompath
+datadog:
+  apiKey: "00000000000000000000000000000000"
+  appKey: "0000000000000000000000000000000000000000"
+  otelCollector:
+    enabled: true
+    config: |
+      receivers:
+        otlp:
+      exporters:
+        datadog:
+          api:
+            key: "00000000000000000000000000000000"
+      service:
+        pipelines:
+          traces:
+            receivers: [otlp]
+            exporters: [datadog]
+          metrics:
+            receivers: [otlp]
+            exporters: [datadog]
+          logs:
+            receivers: [otlp]
+            exporters: [datadog]

charts/datadog/templates/_container-otel-agent.yaml

@@ -73,6 +73,22 @@
     - name: dsdsocket
       mountPath: {{ (dir .Values.datadog.dogstatsd.socketPath) }}
       readOnly: true
+    {{- if and .Values.datadog.otelCollector.logs.enabled (eq (include "should-mount-logs-for-otel-agent" .) "true") }}
+    - name: logpodpath
+      mountPath: /var/log/pods
+      mountPropagation: {{ .Values.datadog.hostVolumeMountPropagation }}
+      readOnly: true
+    - name: logscontainerspath
+      mountPath: /var/log/containers
+      mountPropagation: {{ .Values.datadog.hostVolumeMountPropagation }}
+      readOnly: true
+    {{- if and (not .Values.datadog.criSocketPath) (not .Values.providers.gke.gdc) }}
+    - name: logdockercontainerpath
+      mountPath: /var/lib/docker/containers
+      mountPropagation: {{ .Values.datadog.hostVolumeMountPropagation }}
+      readOnly: true
+    {{- end }}
+    {{- end }}
     {{- end }}
     {{- include "container-crisocket-volumemounts" . | nindent 4 }}
     {{- include "container-cloudinit-volumemounts" . | nindent 4 }}
@@ -82,4 +98,7 @@
 {{- if .Values.agents.volumeMounts }}
 {{ toYaml .Values.agents.volumeMounts | indent 4 }}
 {{- end }}
+{{- if .Values.agents.containers.otelAgent.volumeMounts }}
+{{ toYaml .Values.agents.containers.otelAgent.volumeMounts | indent 4 }}
+{{- end }}
 {{- end -}}

charts/datadog/templates/_daemonset-volumes-linux.yaml

@@ -198,7 +198,7 @@
   name: runtimesocketdir
 {{- end }}
 {{- end }}
-{{- if or .Values.datadog.logs.enabled .Values.datadog.logsEnabled }}
+{{- if or .Values.datadog.logs.enabled .Values.datadog.logsEnabled .Values.datadog.otelCollector.logs.enabled }}
 - hostPath:
     path: {{ template "datadog.hostMountRoot" . }}/logs
   name: pointerdir

charts/datadog/templates/_helpers.tpl

@@ -167,6 +167,20 @@ Return true if k8sattributes RBAC rules should be added to the OTel Agent Cluste
 {{- $return }}
 {{- end -}}
 
+{{/*
+Return true if conatiner and pod logs volumes should be mounted in the OTel Agent container
+*/}}
+{{- define "should-mount-logs-for-otel-agent" -}}
+{{- $return := false }}
+{{- $config := .Values.datadog.otelCollector.config | default "" | fromYaml }}
+{{- range $key, $val := $config.receivers }}
+  {{- if hasPrefix "filelog" $key }}
+    {{- $return = true }}
+  {{- end }}
+{{- end }}
+{{- $return }}
+{{- end -}}
+
 {{/*
 Return secret name to be used based on provided values.
 */}}

charts/datadog/values.yaml

@@ -611,6 +611,13 @@ datadog:
       #     resources: ["pods", "nodes"]
       #     verbs: ["get", "list", "watch"]
 
+    ## Provide OTel Collector logs configuration
+    logs:
+      # datadog.otelCollector.logs.enabled -- Enable logs support in the OTel Collector.
+      # If true, checks OTel Collector config for filelog receiver and mounts additional volumes to collect containers
+      # and pods logs.
+      enabled: false
+
   ## Continuous Profiler configuration
   ##
   ## Continuous Profiler is disabled by default and can be enabled by setting the `enabled` field to
@@ -1824,6 +1831,13 @@ agents:
 
       # agents.containers.otelAgent.ports -- Allows to specify extra ports (hostPorts for instance) for this container
       ports: []
+
+      # agents.containers.otelAgent.volumeMounts -- Specify additional volumes to mount in the otel-agent container
+      volumeMounts: []
+      #   - name: <VOLUME_NAME>
+      #     mountPath: <CONTAINER_PATH>
+      #     readOnly: true
+
     traceAgent:
       # agents.containers.traceAgent.env -- Additional environment variables for the trace-agent container
       env: []