Add phase tag to pod metrics (#2624)

* add phase tag label join

* Allow matching two metrics on the same label

* fix _store_labels to support updates, plus test coverage.

datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py

@@ -295,23 +295,31 @@ def process(self, scraper_config, metric_transformers=None):
             self.process_metric(metric, scraper_config, metric_transformers=metric_transformers)
 
     def _store_labels(self, metric, scraper_config):
-        scraper_config['label_joins']
         # If targeted metric, store labels
         if metric.name in scraper_config['label_joins']:
             matching_label = scraper_config['label_joins'][metric.name]['label_to_match']
             for sample in metric.samples:
-                labels_list = []
+                # metadata-only metrics that are used for label joins are always equal to 1
+                # this is required for metrics where all combinations of a state are sent
+                # but only the active one is set to 1 (others are set to 0)
+                # example: kube_pod_status_phase in kube-state-metrics
+                if sample[self.SAMPLE_VALUE] != 1:
+                    continue
+                label_dict = dict()
                 matching_value = None
                 for label_name, label_value in iteritems(sample[self.SAMPLE_LABELS]):
                     if label_name == matching_label:
                         matching_value = label_value
                     elif label_name in scraper_config['label_joins'][metric.name]['labels_to_get']:
-                        labels_list.append((label_name, label_value))
+                        label_dict[label_name] = label_value
                 try:
-                    scraper_config['_label_mapping'][matching_label][matching_value] = labels_list
+                    if scraper_config['_label_mapping'][matching_label].get(matching_value):
+                        scraper_config['_label_mapping'][matching_label][matching_value].update(label_dict)
+                    else:
+                        scraper_config['_label_mapping'][matching_label][matching_value] = label_dict
                 except KeyError:
                     if matching_value is not None:
-                        scraper_config['_label_mapping'][matching_label] = {matching_value: labels_list}
+                        scraper_config['_label_mapping'][matching_label] = {matching_value: label_dict}
 
     def _join_labels(self, metric, scraper_config):
         # Filter metric to see if we can enrich with joined labels
@@ -325,10 +333,11 @@ def _join_labels(self, metric, scraper_config):
                     scraper_config['_active_label_mapping'][label_name][sample[self.SAMPLE_LABELS][label_name]] = True
                     # If mapping found add corresponding labels
                     try:
-                        for label_tuple in (
-                            scraper_config['_label_mapping'][label_name][sample[self.SAMPLE_LABELS][label_name]]
+                        for name, val in (
+                            iteritems(scraper_config['_label_mapping'][label_name][sample[self.SAMPLE_LABELS]
+                                                                                   [label_name]])
                         ):
-                            sample[self.SAMPLE_LABELS][label_tuple[0]] = label_tuple[1]
+                            sample[self.SAMPLE_LABELS][name] = val
                     except KeyError:
                         pass
 

datadog_checks_base/tests/test_openmetrics.py

@@ -10,6 +10,7 @@
 import requests
 
 from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, SummaryMetricFamily, HistogramMetricFamily
+from six import iteritems
 
 from datadog_checks.checks.openmetrics import OpenMetricsBaseCheck
 
@@ -1440,6 +1441,48 @@ def test_label_join_with_hostname(aggregator, mocked_prometheus_check, mocked_pr
     )
 
 
+def test_label_join_state_change(aggregator, mocked_prometheus_check, mocked_prometheus_scraper_config, mock_get):
+    """
+    This test checks that the label join picks up changes for already watched labels.
+    If a phase changes for example, the tag should change as well.
+    """
+    check = mocked_prometheus_check
+    mocked_prometheus_scraper_config['namespace'] = 'ksm'
+    mocked_prometheus_scraper_config['label_joins'] = {
+        'kube_pod_info': {
+            'label_to_match': 'pod',
+            'labels_to_get': ['node']
+        },
+        'kube_pod_status_phase': {
+            'label_to_match': 'pod',
+            'labels_to_get': ['phase']
+        }
+    }
+    mocked_prometheus_scraper_config['metrics_mapper'] = {'kube_pod_status_ready': 'pod.ready'}
+    # dry run to build mapping
+    check.process(mocked_prometheus_scraper_config)
+    # run with submit
+    check.process(mocked_prometheus_scraper_config)
+
+    # check that 15 pods are in phase:Running
+    assert 15 == len(mocked_prometheus_scraper_config['_label_mapping']['pod'])
+    for _, tags in iteritems(mocked_prometheus_scraper_config['_label_mapping']['pod']):
+        assert tags.get('phase') == 'Running'
+
+    text_data = mock_get.replace(
+        'kube_pod_status_phase{namespace="default",phase="Running",pod="dd-agent-62bgh"} 1',
+        'kube_pod_status_phase{namespace="default",phase="Test",pod="dd-agent-62bgh"} 1'
+    )
+
+    mock_response = mock.MagicMock(
+        status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={'Content-Type': text_content_type}
+    )
+    with mock.patch('requests.get', return_value=mock_response, __name__="get"):
+        check.process(mocked_prometheus_scraper_config)
+        assert 15 == len(mocked_prometheus_scraper_config['_label_mapping']['pod'])
+        assert mocked_prometheus_scraper_config['_label_mapping']['pod']['dd-agent-62bgh']['phase'] == 'Test'
+
+
 def test_health_service_check_ok(mock_get, aggregator, mocked_prometheus_check, mocked_prometheus_scraper_config):
     """ Tests endpoint health service check OK """
     check = mocked_prometheus_check

kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py

@@ -235,6 +235,10 @@ def _create_kubernetes_state_prometheus_instance(self, instance):
                     'label_to_match': 'pod',
                     'labels_to_get': ['node']
                 },
+                'kube_pod_status_phase': {
+                    'label_to_match': 'pod',
+                    'labels_to_get': ['phase']
+                },
                 'kube_persistentvolume_info': {
                     'label_to_match': 'persistentvolume',
                     'labels_to_get': ['storageclass']