fix _store_labels to support updates, plus test coverage.

datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py

@@ -305,23 +305,21 @@ def _store_labels(self, metric, scraper_config):
                 # example: kube_pod_status_phase in kube-state-metrics
                 if sample[self.SAMPLE_VALUE] != 1:
                     continue
-                label_set = set()
+                label_dict = dict()
                 matching_value = None
                 for label_name, label_value in iteritems(sample[self.SAMPLE_LABELS]):
                     if label_name == matching_label:
                         matching_value = label_value
                     elif label_name in scraper_config['label_joins'][metric.name]['labels_to_get']:
-                        label_set.add((label_name, label_value))
+                        label_dict[label_name] = label_value
                 try:
                     if scraper_config['_label_mapping'][matching_label].get(matching_value):
-                        scraper_config['_label_mapping'][matching_label][matching_value] = label_set.union(
-                            scraper_config['_label_mapping'][matching_label][matching_value]
-                        )
+                        scraper_config['_label_mapping'][matching_label][matching_value].update(label_dict)
                     else:
-                        scraper_config['_label_mapping'][matching_label][matching_value] = label_set
+                        scraper_config['_label_mapping'][matching_label][matching_value] = label_dict
                 except KeyError:
                     if matching_value is not None:
-                        scraper_config['_label_mapping'][matching_label] = {matching_value: label_set}
+                        scraper_config['_label_mapping'][matching_label] = {matching_value: label_dict}
 
     def _join_labels(self, metric, scraper_config):
         # Filter metric to see if we can enrich with joined labels
@@ -335,10 +333,11 @@ def _join_labels(self, metric, scraper_config):
                     scraper_config['_active_label_mapping'][label_name][sample[self.SAMPLE_LABELS][label_name]] = True
                     # If mapping found add corresponding labels
                     try:
-                        for label_tuple in (
-                            scraper_config['_label_mapping'][label_name][sample[self.SAMPLE_LABELS][label_name]]
+                        for name, val in (
+                            scraper_config['_label_mapping'][label_name][sample[self.SAMPLE_LABELS]
+                                                                               [label_name]].iteritems()
                         ):
-                            sample[self.SAMPLE_LABELS][label_tuple[0]] = label_tuple[1]
+                            sample[self.SAMPLE_LABELS][name] = val
                     except KeyError:
                         pass
 

datadog_checks_base/tests/test_openmetrics.py

@@ -1440,6 +1440,48 @@ def test_label_join_with_hostname(aggregator, mocked_prometheus_check, mocked_pr
     )
 
 
+def test_label_join_state_change(aggregator, mocked_prometheus_check, mocked_prometheus_scraper_config, mock_get):
+    """
+    This test checks that the label join picks up changes for already watched labels.
+    If a phase changes for example, the tag should change as well.
+    """
+    check = mocked_prometheus_check
+    mocked_prometheus_scraper_config['namespace'] = 'ksm'
+    mocked_prometheus_scraper_config['label_joins'] = {
+        'kube_pod_info': {
+            'label_to_match': 'pod',
+            'labels_to_get': ['node']
+        },
+        'kube_pod_status_phase': {
+            'label_to_match': 'pod',
+            'labels_to_get': ['phase']
+        }
+    }
+    mocked_prometheus_scraper_config['metrics_mapper'] = {'kube_pod_status_ready': 'pod.ready'}
+    # dry run to build mapping
+    check.process(mocked_prometheus_scraper_config)
+    # run with submit
+    check.process(mocked_prometheus_scraper_config)
+
+    # check that 15 pods are in phase:Running
+    assert 15 == len(mocked_prometheus_scraper_config['_label_mapping']['pod'])
+    for _, tags in mocked_prometheus_scraper_config['_label_mapping']['pod'].iteritems():
+        assert tags.get('phase') == 'Running'
+
+    text_data = mock_get.replace(
+        'kube_pod_status_phase{namespace="default",phase="Running",pod="dd-agent-62bgh"} 1',
+        'kube_pod_status_phase{namespace="default",phase="Test",pod="dd-agent-62bgh"} 1'
+    )
+
+    mock_response = mock.MagicMock(
+        status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={'Content-Type': text_content_type}
+    )
+    with mock.patch('requests.get', return_value=mock_response, __name__="get"):
+        check.process(mocked_prometheus_scraper_config)
+        assert 15 == len(mocked_prometheus_scraper_config['_label_mapping']['pod'])
+        assert mocked_prometheus_scraper_config['_label_mapping']['pod']['dd-agent-62bgh']['phase'] == 'Test'
+
+
 def test_health_service_check_ok(mock_get, aggregator, mocked_prometheus_check, mocked_prometheus_scraper_config):
     """ Tests endpoint health service check OK """
     check = mocked_prometheus_check