From 7ea3f2c20c2eafe0852a8839d28706fd231bfaa3 Mon Sep 17 00:00:00 2001 From: Ahmed Mezghani Date: Thu, 20 May 2021 15:04:39 +0200 Subject: [PATCH] Support ignoring instance tags --- .../amazon_msk/data/conf.yaml.example | 9 ++++++++ .../azure_iot_edge/data/conf.yaml.example | 9 ++++++++ .../cilium/data/conf.yaml.example | 9 ++++++++ .../cockroachdb/data/conf.yaml.example | 9 ++++++++ .../coredns/data/conf.yaml.example | 9 ++++++++ .../crio/data/conf.yaml.example | 9 ++++++++ .../base/checks/openmetrics/mixins.py | 15 +++++++++++++ .../base/checks/openmetrics/v2/scraper.py | 15 +++++++++++++ .../tests/openmetrics/test_options.py | 22 +++++++++++++++++++ datadog_checks_base/tests/test_openmetrics.py | 17 ++++++++++++++ .../configuration/instances/openmetrics.yaml | 11 ++++++++++ .../instances/openmetrics_legacy.yaml | 11 ++++++++++ .../etcd/data/conf.yaml.example | 9 ++++++++ .../external_dns/data/conf.yaml.example | 9 ++++++++ .../gitlab/data/conf.yaml.example | 9 ++++++++ .../gitlab_runner/data/conf.yaml.example | 9 ++++++++ .../haproxy/data/conf.yaml.example | 9 ++++++++ .../istio/data/conf.yaml.example | 9 ++++++++ .../kong/data/conf.yaml.example | 9 ++++++++ .../data/conf.yaml.example | 9 ++++++++ .../kube_scheduler/data/conf.yaml.example | 9 ++++++++ .../linkerd/data/conf.yaml.example | 9 ++++++++ .../data/conf.yaml.example | 9 ++++++++ .../openmetrics/data/conf.yaml.example | 9 ++++++++ .../scylla/data/conf.yaml.example | 9 ++++++++ 25 files changed, 262 insertions(+) diff --git a/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example b/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example index 8e3909766e70e4..2801cf20340372 100644 --- a/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example +++ b/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example @@ -194,6 +194,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example b/azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example index 3a972546316131..2982c50c9c320b 100644 --- a/azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example +++ b/azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example @@ -162,6 +162,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/cilium/datadog_checks/cilium/data/conf.yaml.example b/cilium/datadog_checks/cilium/data/conf.yaml.example index 0423769d0d984c..a4a116777a10c4 100644 --- a/cilium/datadog_checks/cilium/data/conf.yaml.example +++ b/cilium/datadog_checks/cilium/data/conf.yaml.example @@ -165,6 +165,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example b/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example index 147a616e230c4f..ab6b21e0ee0963 100644 --- a/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example +++ b/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example @@ -157,6 +157,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/coredns/datadog_checks/coredns/data/conf.yaml.example b/coredns/datadog_checks/coredns/data/conf.yaml.example index a4cc9467cd71a3..39914ca02ceb82 100644 --- a/coredns/datadog_checks/coredns/data/conf.yaml.example +++ b/coredns/datadog_checks/coredns/data/conf.yaml.example @@ -176,6 +176,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/crio/datadog_checks/crio/data/conf.yaml.example b/crio/datadog_checks/crio/data/conf.yaml.example index 3dbd11148a8261..c7a49f0abf48c9 100644 --- a/crio/datadog_checks/crio/data/conf.yaml.example +++ b/crio/datadog_checks/crio/data/conf.yaml.example @@ -157,6 +157,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py index 790d6edd3e6ec8..11ae93308fe1d2 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py @@ -318,6 +318,21 @@ def create_scraper_configuration(self, instance=None): # Custom tags that will be sent with each metric config['custom_tags'] = instance.get('tags', []) + # Some tags can be ignored to reduce the cardinality. + # This can be useful for cost optimization in containerized environments + # when the openmetrics check is configured to collect custom metrics. + # Even when the Agent's Tagger is configured to add low-cardinality tags only, + # some tags can still generate unwanted metric contexts (e.g pod annotations as tags). + ignore_tags = instance.get('ignore_tags', default_instance.get('ignore_tags', [])) + if ignore_tags: + ignored_tag_patterns = set() + for ignored_tag in ignore_tags: + ignored_tag_patterns.add(translate(ignored_tag)) + + if ignored_tag_patterns: + ignored_tags_re = compile('|'.join(ignored_tag_patterns)) + config['custom_tags'] = [tag for tag in config['custom_tags'] if not ignored_tags_re.search(tag)] + # Additional tags to be sent with each metric config['_metric_tags'] = [] diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py index 5fd50e10f2c6fa..bc64bbdd0d0818 100644 --- a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py +++ b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py @@ -144,6 +144,21 @@ def __init__(self, check, config): if not isinstance(entry, str): raise ConfigurationError(f'Entry #{i} of setting `tags` must be a string') + # Some tags can be ignored to reduce the cardinality. + # This can be useful for cost optimization in containerized environments + # when the openmetrics check is configured to collect custom metrics. + # Even when the Agent's Tagger is configured to add low-cardinality tags only, + # some tags can still generate unwanted metric contexts (e.g pod annotations as tags). + ignore_tags = config.get('ignore_tags', []) + if ignore_tags: + ignored_tag_patterns = set() + for ignored_tag in ignore_tags: + ignored_tag_patterns.add(fnmatch.translate(ignored_tag)) + + if ignored_tag_patterns: + ignored_tags_re = re.compile('|'.join(ignored_tag_patterns)) + custom_tags = [tag for tag in custom_tags if not ignored_tags_re.search(tag)] + # 16 KiB seems optimal, and is also the standard chunk size of the Bittorrent protocol: # https://www.bittorrent.org/beps/bep_0003.html self.request_size = int(float(config.get('request_size') or 16) * KIBIBYTE) diff --git a/datadog_checks_base/tests/openmetrics/test_options.py b/datadog_checks_base/tests/openmetrics/test_options.py index 8747f9accaa78b..edd47982a42482 100644 --- a/datadog_checks_base/tests/openmetrics/test_options.py +++ b/datadog_checks_base/tests/openmetrics/test_options.py @@ -550,3 +550,25 @@ def test_excluded_metric(self, aggregator, dd_run_check, mock_http_response): ) aggregator.assert_all_metrics_covered() + + +class TestIgnoreTags: + def test(self, aggregator, dd_run_check, mock_http_response): + mock_http_response( + """ + # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. + # TYPE go_memstats_alloc_bytes gauge + go_memstats_alloc_bytes 6.396288e+06 + """ + ) + + all_tags = ['foo', 'foofoo', 'bar', 'bar:baz'] + ignored_tags = ['foo*', 'bar:baz'] + check = get_check({'metrics': ['go_memstats_alloc_bytes'], 'tags': all_tags, 'ignore_tags': ignored_tags}) + dd_run_check(check) + + aggregator.assert_metric( + 'test.go_memstats_alloc_bytes', 6396288, metric_type=aggregator.GAUGE, tags=['endpoint:test', 'bar'] + ) + + aggregator.assert_all_metrics_covered() diff --git a/datadog_checks_base/tests/test_openmetrics.py b/datadog_checks_base/tests/test_openmetrics.py index ec8b9692c85d90..6503623047e9ca 100644 --- a/datadog_checks_base/tests/test_openmetrics.py +++ b/datadog_checks_base/tests/test_openmetrics.py @@ -2781,3 +2781,20 @@ def test_empty_namespace(aggregator, mocked_prometheus_check, text_data, ref_gau check.process_metric(ref_gauge, config) aggregator.assert_metric('process.vm.bytes', count=1) + + +def test_ignore_tags(aggregator, mocked_prometheus_check, ref_gauge): + """ + Test that tags matching ignored tag patterns are properly discarded. + """ + check = mocked_prometheus_check + instance = copy.deepcopy(PROMETHEUS_CHECK_INSTANCE) + instance['tags'] = ['foo', 'foofoo', 'bar', 'bar:baz'] + instance['ignore_tags'] = ['foo*', 'bar:baz'] + + config = check.get_scraper_config(instance) + config['_dry_run'] = False + + check.process_metric(ref_gauge, config) + + aggregator.assert_metric('prometheus.process.vm.bytes', tags=['bar'], count=1) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics.yaml b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics.yaml index b76a167a9e4010..f3a83efd3824ea 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics.yaml +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics.yaml @@ -300,5 +300,16 @@ value: example: false type: boolean +- name: ignore_tags + description: A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + value: + type: array + items: + type: string + example: + - + - + - <*TAG_SUFFIX> + - <*SUBSTRING*> - template: instances/http - template: instances/default diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics_legacy.yaml b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics_legacy.yaml index 1e9e053430a620..d76e53154294b3 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics_legacy.yaml +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics_legacy.yaml @@ -167,5 +167,16 @@ example: : [, ] : ['*'] +- name: ignore_tags + description: A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + value: + type: array + items: + type: string + example: + - + - + - <*TAG_SUFFIX> + - <*SUBSTRING*> - template: instances/http - template: instances/default diff --git a/etcd/datadog_checks/etcd/data/conf.yaml.example b/etcd/datadog_checks/etcd/data/conf.yaml.example index 7251d336ce81d6..70bb15a18a4c6c 100644 --- a/etcd/datadog_checks/etcd/data/conf.yaml.example +++ b/etcd/datadog_checks/etcd/data/conf.yaml.example @@ -163,6 +163,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/external_dns/datadog_checks/external_dns/data/conf.yaml.example b/external_dns/datadog_checks/external_dns/data/conf.yaml.example index 8f76a8a5daee3d..12169a9e3b7fb9 100644 --- a/external_dns/datadog_checks/external_dns/data/conf.yaml.example +++ b/external_dns/datadog_checks/external_dns/data/conf.yaml.example @@ -157,6 +157,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/gitlab/datadog_checks/gitlab/data/conf.yaml.example b/gitlab/datadog_checks/gitlab/data/conf.yaml.example index dae03cad5ced36..6bf781b59ce5ec 100644 --- a/gitlab/datadog_checks/gitlab/data/conf.yaml.example +++ b/gitlab/datadog_checks/gitlab/data/conf.yaml.example @@ -127,6 +127,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example b/gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example index 935754d39fcab2..17cc86bebe4bf9 100644 --- a/gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example +++ b/gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example @@ -211,6 +211,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/haproxy/datadog_checks/haproxy/data/conf.yaml.example b/haproxy/datadog_checks/haproxy/data/conf.yaml.example index 437c5665451b2a..b75f574f472fda 100644 --- a/haproxy/datadog_checks/haproxy/data/conf.yaml.example +++ b/haproxy/datadog_checks/haproxy/data/conf.yaml.example @@ -304,6 +304,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/istio/datadog_checks/istio/data/conf.yaml.example b/istio/datadog_checks/istio/data/conf.yaml.example index 9785f5ba32380c..c6f30a9ae8e597 100644 --- a/istio/datadog_checks/istio/data/conf.yaml.example +++ b/istio/datadog_checks/istio/data/conf.yaml.example @@ -196,6 +196,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/kong/datadog_checks/kong/data/conf.yaml.example b/kong/datadog_checks/kong/data/conf.yaml.example index 76fcbea1396d58..65bbd9e68d658c 100644 --- a/kong/datadog_checks/kong/data/conf.yaml.example +++ b/kong/datadog_checks/kong/data/conf.yaml.example @@ -237,6 +237,15 @@ instances: # # telemetry: false + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/data/conf.yaml.example b/kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/data/conf.yaml.example index b9365be35938e2..6282ef6f238994 100644 --- a/kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/data/conf.yaml.example +++ b/kube_apiserver_metrics/datadog_checks/kube_apiserver_metrics/data/conf.yaml.example @@ -164,6 +164,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/kube_scheduler/datadog_checks/kube_scheduler/data/conf.yaml.example b/kube_scheduler/datadog_checks/kube_scheduler/data/conf.yaml.example index 14d7cceec7af91..4732c8e66f5e62 100644 --- a/kube_scheduler/datadog_checks/kube_scheduler/data/conf.yaml.example +++ b/kube_scheduler/datadog_checks/kube_scheduler/data/conf.yaml.example @@ -173,6 +173,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/linkerd/datadog_checks/linkerd/data/conf.yaml.example b/linkerd/datadog_checks/linkerd/data/conf.yaml.example index 16bec28d609227..a648e57b6f299e 100644 --- a/linkerd/datadog_checks/linkerd/data/conf.yaml.example +++ b/linkerd/datadog_checks/linkerd/data/conf.yaml.example @@ -157,6 +157,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/nginx_ingress_controller/datadog_checks/nginx_ingress_controller/data/conf.yaml.example b/nginx_ingress_controller/datadog_checks/nginx_ingress_controller/data/conf.yaml.example index 12de86aa05fa5d..2c451d92a715c9 100644 --- a/nginx_ingress_controller/datadog_checks/nginx_ingress_controller/data/conf.yaml.example +++ b/nginx_ingress_controller/datadog_checks/nginx_ingress_controller/data/conf.yaml.example @@ -165,6 +165,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/openmetrics/datadog_checks/openmetrics/data/conf.yaml.example b/openmetrics/datadog_checks/openmetrics/data/conf.yaml.example index 04ab63af517dfa..5ff6989c2dd0b7 100644 --- a/openmetrics/datadog_checks/openmetrics/data/conf.yaml.example +++ b/openmetrics/datadog_checks/openmetrics/data/conf.yaml.example @@ -171,6 +171,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ## diff --git a/scylla/datadog_checks/scylla/data/conf.yaml.example b/scylla/datadog_checks/scylla/data/conf.yaml.example index 0f7c6ec67fc927..122a302480d3a5 100644 --- a/scylla/datadog_checks/scylla/data/conf.yaml.example +++ b/scylla/datadog_checks/scylla/data/conf.yaml.example @@ -174,6 +174,15 @@ instances: # : # - '*' + ## @param ignore_tags - list of strings - optional + ## A list of tags to ignore, the "*" wildcard can be used to match multiple tag strings. + # + # ignore_tags: + # - + # - + # - <*TAG_SUFFIX> + # - <*SUBSTRING*> + ## @param proxy - mapping - optional ## This overrides the `proxy` setting in `init_config`. ##