From 846480f3cfe5e74be8685d6c5c175f7d89c75a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=C3=AFssam=20Kaj?= Date: Sun, 2 Dec 2018 16:15:42 -0800 Subject: [PATCH] Add phase tag to pod metrics (#2624) * add phase tag label join * Allow matching two metrics on the same label * fix _store_labels to support updates, plus test coverage. --- .../base/checks/openmetrics/mixins.py | 25 +++++++---- datadog_checks_base/tests/test_openmetrics.py | 43 +++++++++++++++++++ .../kubernetes_state/kubernetes_state.py | 4 ++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py index 939eb0ad830c1..5e183bb1906b3 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py @@ -295,23 +295,31 @@ def process(self, scraper_config, metric_transformers=None): self.process_metric(metric, scraper_config, metric_transformers=metric_transformers) def _store_labels(self, metric, scraper_config): - scraper_config['label_joins'] # If targeted metric, store labels if metric.name in scraper_config['label_joins']: matching_label = scraper_config['label_joins'][metric.name]['label_to_match'] for sample in metric.samples: - labels_list = [] + # metadata-only metrics that are used for label joins are always equal to 1 + # this is required for metrics where all combinations of a state are sent + # but only the active one is set to 1 (others are set to 0) + # example: kube_pod_status_phase in kube-state-metrics + if sample[self.SAMPLE_VALUE] != 1: + continue + label_dict = dict() matching_value = None for label_name, label_value in iteritems(sample[self.SAMPLE_LABELS]): if label_name == matching_label: matching_value = label_value elif label_name in scraper_config['label_joins'][metric.name]['labels_to_get']: - labels_list.append((label_name, label_value)) + label_dict[label_name] = label_value try: - scraper_config['_label_mapping'][matching_label][matching_value] = labels_list + if scraper_config['_label_mapping'][matching_label].get(matching_value): + scraper_config['_label_mapping'][matching_label][matching_value].update(label_dict) + else: + scraper_config['_label_mapping'][matching_label][matching_value] = label_dict except KeyError: if matching_value is not None: - scraper_config['_label_mapping'][matching_label] = {matching_value: labels_list} + scraper_config['_label_mapping'][matching_label] = {matching_value: label_dict} def _join_labels(self, metric, scraper_config): # Filter metric to see if we can enrich with joined labels @@ -325,10 +333,11 @@ def _join_labels(self, metric, scraper_config): scraper_config['_active_label_mapping'][label_name][sample[self.SAMPLE_LABELS][label_name]] = True # If mapping found add corresponding labels try: - for label_tuple in ( - scraper_config['_label_mapping'][label_name][sample[self.SAMPLE_LABELS][label_name]] + for name, val in ( + iteritems(scraper_config['_label_mapping'][label_name][sample[self.SAMPLE_LABELS] + [label_name]]) ): - sample[self.SAMPLE_LABELS][label_tuple[0]] = label_tuple[1] + sample[self.SAMPLE_LABELS][name] = val except KeyError: pass diff --git a/datadog_checks_base/tests/test_openmetrics.py b/datadog_checks_base/tests/test_openmetrics.py index 6e8d559b764ff..0d18f81f82a10 100644 --- a/datadog_checks_base/tests/test_openmetrics.py +++ b/datadog_checks_base/tests/test_openmetrics.py @@ -10,6 +10,7 @@ import requests from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, SummaryMetricFamily, HistogramMetricFamily +from six import iteritems from datadog_checks.checks.openmetrics import OpenMetricsBaseCheck @@ -1440,6 +1441,48 @@ def test_label_join_with_hostname(aggregator, mocked_prometheus_check, mocked_pr ) +def test_label_join_state_change(aggregator, mocked_prometheus_check, mocked_prometheus_scraper_config, mock_get): + """ + This test checks that the label join picks up changes for already watched labels. + If a phase changes for example, the tag should change as well. + """ + check = mocked_prometheus_check + mocked_prometheus_scraper_config['namespace'] = 'ksm' + mocked_prometheus_scraper_config['label_joins'] = { + 'kube_pod_info': { + 'label_to_match': 'pod', + 'labels_to_get': ['node'] + }, + 'kube_pod_status_phase': { + 'label_to_match': 'pod', + 'labels_to_get': ['phase'] + } + } + mocked_prometheus_scraper_config['metrics_mapper'] = {'kube_pod_status_ready': 'pod.ready'} + # dry run to build mapping + check.process(mocked_prometheus_scraper_config) + # run with submit + check.process(mocked_prometheus_scraper_config) + + # check that 15 pods are in phase:Running + assert 15 == len(mocked_prometheus_scraper_config['_label_mapping']['pod']) + for _, tags in iteritems(mocked_prometheus_scraper_config['_label_mapping']['pod']): + assert tags.get('phase') == 'Running' + + text_data = mock_get.replace( + 'kube_pod_status_phase{namespace="default",phase="Running",pod="dd-agent-62bgh"} 1', + 'kube_pod_status_phase{namespace="default",phase="Test",pod="dd-agent-62bgh"} 1' + ) + + mock_response = mock.MagicMock( + status_code=200, iter_lines=lambda **kwargs: text_data.split("\n"), headers={'Content-Type': text_content_type} + ) + with mock.patch('requests.get', return_value=mock_response, __name__="get"): + check.process(mocked_prometheus_scraper_config) + assert 15 == len(mocked_prometheus_scraper_config['_label_mapping']['pod']) + assert mocked_prometheus_scraper_config['_label_mapping']['pod']['dd-agent-62bgh']['phase'] == 'Test' + + def test_health_service_check_ok(mock_get, aggregator, mocked_prometheus_check, mocked_prometheus_scraper_config): """ Tests endpoint health service check OK """ check = mocked_prometheus_check diff --git a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py index b01a058fe0ec9..1143f39f1b42b 100644 --- a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py +++ b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py @@ -235,6 +235,10 @@ def _create_kubernetes_state_prometheus_instance(self, instance): 'label_to_match': 'pod', 'labels_to_get': ['node'] }, + 'kube_pod_status_phase': { + 'label_to_match': 'pod', + 'labels_to_get': ['phase'] + }, 'kube_persistentvolume_info': { 'label_to_match': 'persistentvolume', 'labels_to_get': ['storageclass']