kubelet: fix missing metrics for static pods

DataDog · May 27, 2020 · 705cd7d · 705cd7d
1 parent 6808e92
commit 705cd7d
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 8 deletions.
diff --git a/kubelet/datadog_checks/kubelet/prometheus.py b/kubelet/datadog_checks/kubelet/prometheus.py
@@ -302,21 +302,21 @@ def _process_container_metric(self, type, metric_name, metric, scraper_config, l
             if self.pod_list_utils.is_excluded(c_id, pod_uid):
                 continue
 
-            tags = tagger.tag(replace_container_rt_prefix(c_id), tagger.HIGH)
-            if not tags:
-                continue
-            tags += scraper_config['custom_tags']
-
             # FIXME we are forced to do that because the Kubelet PodList isn't updated
             # for static pods, see https://github.com/kubernetes/kubernetes/pull/59948
             pod = self._get_pod_by_metric_label(sample[self.SAMPLE_LABELS])
             if pod is not None and is_static_pending_pod(pod):
                 pod_tags = tagger.tag('kubernetes_pod_uid://%s' % pod["metadata"]["uid"], tagger.HIGH)
                 if not pod_tags:
                     continue
-                tags += pod_tags
-                tags += self._get_kube_container_name(sample[self.SAMPLE_LABELS])
-                tags = list(set(tags))
+                pod_tags += self._get_kube_container_name(sample[self.SAMPLE_LABELS])
+                tags = list(set(pod_tags))
+            else:
+                tags = tagger.tag(replace_container_rt_prefix(c_id), tagger.HIGH)
+
+            if not tags:
+                continue
+            tags += scraper_config['custom_tags']
 
             for label in labels:
                 value = sample[self.SAMPLE_LABELS].get(label)

diff --git a/kubelet/tests/test_kubelet.py b/kubelet/tests/test_kubelet.py
@@ -718,6 +718,27 @@ def test_no_tags_no_metrics(monkeypatch, aggregator, tagger):
     aggregator.assert_all_metrics_covered()
 
 
+def test_static_pods(monkeypatch, aggregator, tagger):
+    tagger.reset()
+    tagger.set_tags(
+        {
+            "kubernetes_pod_uid://260c2b1d43b094af6d6b4ccba082c2db": [
+                'pod_name:kube-proxy-gke-haissam-default-pool-be5066f1-wnvn'
+            ]
+        }
+    )
+
+    check = mock_kubelet_check(monkeypatch, [{}])
+    check.check({"cadvisor_metrics_endpoint": "http://dummy/metrics/cadvisor", "kubelet_metrics_endpoint": ""})
+
+    # Test that we get metrics for this static pod
+    aggregator.assert_metric(
+        'kubernetes.cpu.user.total',
+        109.76,
+        ['kube_container_name:kube-proxy', 'pod_name:kube-proxy-gke-haissam-default-pool-be5066f1-wnvn'],
+    )
+
+
 def test_pod_expiration(monkeypatch, aggregator, tagger):
     check = KubeletCheck('kubelet', {}, [{}])
     check.pod_list_url = "dummyurl"