Support "ignore_tags" configuration (#9392)

amazon_msk/datadog_checks/amazon_msk/config_models/defaults.py

@@ -84,6 +84,10 @@ def instance_ignore_metrics_by_labels(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_ignore_tags(field, value):
+    return get_default_field_value(field, value)
+
+
 def instance_jmx_exporter_port(field, value):
     return 11001
 

amazon_msk/datadog_checks/amazon_msk/config_models/instance.py

@@ -74,6 +74,7 @@ class Config:
     health_service_check: Optional[bool]
     ignore_metrics: Optional[Sequence[str]]
     ignore_metrics_by_labels: Optional[IgnoreMetricsByLabels]
+    ignore_tags: Optional[Sequence[str]]
     jmx_exporter_port: Optional[int]
     kerberos_auth: Optional[str]
     kerberos_cache: Optional[str]

amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example

@@ -194,6 +194,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example

@@ -162,6 +162,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

cilium/datadog_checks/cilium/data/conf.yaml.example

@@ -165,6 +165,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example

@@ -157,6 +157,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

coredns/datadog_checks/coredns/data/conf.yaml.example

@@ -176,6 +176,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

crio/datadog_checks/crio/data/conf.yaml.example

@@ -157,6 +157,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py

@@ -318,6 +318,16 @@ def create_scraper_configuration(self, instance=None):
         # Custom tags that will be sent with each metric
         config['custom_tags'] = instance.get('tags', [])
 
+        # Some tags can be ignored to reduce the cardinality.
+        # This can be useful for cost optimization in containerized environments
+        # when the openmetrics check is configured to collect custom metrics.
+        # Even when the Agent's Tagger is configured to add low-cardinality tags only,
+        # some tags can still generate unwanted metric contexts (e.g pod annotations as tags).
+        ignore_tags = instance.get('ignore_tags', default_instance.get('ignore_tags', []))
+        if ignore_tags:
+            ignored_tags_re = compile('|'.join(set(ignore_tags)))
+            config['custom_tags'] = [tag for tag in config['custom_tags'] if not ignored_tags_re.search(tag)]
+
         # Additional tags to be sent with each metric
         config['_metric_tags'] = []
 

datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py

@@ -144,6 +144,16 @@ def __init__(self, check, config):
             if not isinstance(entry, str):
                 raise ConfigurationError(f'Entry #{i} of setting `tags` must be a string')
 
+        # Some tags can be ignored to reduce the cardinality.
+        # This can be useful for cost optimization in containerized environments
+        # when the openmetrics check is configured to collect custom metrics.
+        # Even when the Agent's Tagger is configured to add low-cardinality tags only,
+        # some tags can still generate unwanted metric contexts (e.g pod annotations as tags).
+        ignore_tags = config.get('ignore_tags', [])
+        if ignore_tags:
+            ignored_tags_re = re.compile('|'.join(set(ignore_tags)))
+            custom_tags = [tag for tag in custom_tags if not ignored_tags_re.search(tag)]
+
         # 16 KiB seems optimal, and is also the standard chunk size of the Bittorrent protocol:
         # https://www.bittorrent.org/beps/bep_0003.html
         self.request_size = int(float(config.get('request_size') or 16) * KIBIBYTE)

datadog_checks_base/tests/openmetrics/test_options.py

@@ -550,3 +550,65 @@ def test_excluded_metric(self, aggregator, dd_run_check, mock_http_response):
         )
 
         aggregator.assert_all_metrics_covered()
+
+
+class TestIgnoreTags:
+    def test_simple_match(self, aggregator, dd_run_check, mock_http_response):
+        mock_http_response(
+            """
+            # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+            # TYPE go_memstats_alloc_bytes gauge
+            go_memstats_alloc_bytes 6.396288e+06
+            """
+        )
+        all_tags = ['foo', 'foobar', 'bar', 'bar:baz', 'bar:bar']
+        ignored_tags = ['foo', 'bar:baz']
+        wanted_tags = ['bar', 'bar:bar']
+        check = get_check({'metrics': ['go_memstats_alloc_bytes'], 'tags': all_tags, 'ignore_tags': ignored_tags})
+        dd_run_check(check)
+
+        aggregator.assert_metric(
+            'test.go_memstats_alloc_bytes', 6396288, metric_type=aggregator.GAUGE, tags=['endpoint:test'] + wanted_tags
+        )
+
+        aggregator.assert_all_metrics_covered()
+
+    def test_simple_wildcard(self, aggregator, dd_run_check, mock_http_response):
+        mock_http_response(
+            """
+            # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+            # TYPE go_memstats_alloc_bytes gauge
+            go_memstats_alloc_bytes 6.396288e+06
+            """
+        )
+        all_tags = ['foo', 'foobar', 'bar', 'bar:baz', 'bar:bar']
+        ignored_tags = ['foo*', '.*baz']
+        wanted_tags = ['bar', 'bar:bar']
+        check = get_check({'metrics': ['go_memstats_alloc_bytes'], 'tags': all_tags, 'ignore_tags': ignored_tags})
+        dd_run_check(check)
+
+        aggregator.assert_metric(
+            'test.go_memstats_alloc_bytes', 6396288, metric_type=aggregator.GAUGE, tags=['endpoint:test'] + wanted_tags
+        )
+
+        aggregator.assert_all_metrics_covered()
+
+    def test_regex(self, aggregator, dd_run_check, mock_http_response):
+        mock_http_response(
+            """
+            # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+            # TYPE go_memstats_alloc_bytes gauge
+            go_memstats_alloc_bytes 6.396288e+06
+            """
+        )
+        all_tags = ['foo', 'foobar', 'bar:baz', 'bar:bar']
+        ignored_tags = ['^foo', '.+:bar$']
+        wanted_tags = ['bar:baz']
+        check = get_check({'metrics': ['go_memstats_alloc_bytes'], 'tags': all_tags, 'ignore_tags': ignored_tags})
+        dd_run_check(check)
+
+        aggregator.assert_metric(
+            'test.go_memstats_alloc_bytes', 6396288, metric_type=aggregator.GAUGE, tags=['endpoint:test'] + wanted_tags
+        )
+
+        aggregator.assert_all_metrics_covered()

datadog_checks_base/tests/test_openmetrics.py

@@ -2781,3 +2781,63 @@ def test_empty_namespace(aggregator, mocked_prometheus_check, text_data, ref_gau
     check.process_metric(ref_gauge, config)
 
     aggregator.assert_metric('process.vm.bytes', count=1)
+
+
+def test_ignore_tags_match(aggregator, mocked_prometheus_check, ref_gauge):
+    """
+    Test that tags matching ignored tag patterns are properly discarded.
+    """
+    check = mocked_prometheus_check
+    instance = copy.deepcopy(PROMETHEUS_CHECK_INSTANCE)
+    all_tags = ['foo', 'foobar', 'bar', 'bar:baz', 'bar:bar']
+    ignored_tags = ['foo', 'bar:baz']
+    wanted_tags = ['bar', 'bar:bar']
+    instance['tags'] = all_tags
+    instance['ignore_tags'] = ignored_tags
+
+    config = check.get_scraper_config(instance)
+    config['_dry_run'] = False
+
+    check.process_metric(ref_gauge, config)
+
+    aggregator.assert_metric('prometheus.process.vm.bytes', tags=wanted_tags, count=1)
+
+
+def test_ignore_tags_wildcard(aggregator, mocked_prometheus_check, ref_gauge):
+    """
+    Test that tags matching ignored tag patters with wildcards are properly discarded.
+    """
+    check = mocked_prometheus_check
+    instance = copy.deepcopy(PROMETHEUS_CHECK_INSTANCE)
+    all_tags = ['foo', 'foobar', 'bar', 'bar:baz', 'bar:bar']
+    ignored_tags = ['foo*', '.*baz']
+    wanted_tags = ['bar', 'bar:bar']
+    instance['tags'] = all_tags
+    instance['ignore_tags'] = ignored_tags
+
+    config = check.get_scraper_config(instance)
+    config['_dry_run'] = False
+
+    check.process_metric(ref_gauge, config)
+
+    aggregator.assert_metric('prometheus.process.vm.bytes', tags=wanted_tags, count=1)
+
+
+def test_ignore_tags_regex(aggregator, mocked_prometheus_check, ref_gauge):
+    """
+    Test that tags matching ignored tag patters with regex are properly discarded.
+    """
+    check = mocked_prometheus_check
+    instance = copy.deepcopy(PROMETHEUS_CHECK_INSTANCE)
+    all_tags = ['foo', 'foobar', 'bar:baz', 'bar:bar']
+    ignored_tags = ['^foo', '.+:bar$']
+    wanted_tags = ['bar:baz']
+    instance['tags'] = all_tags
+    instance['ignore_tags'] = ignored_tags
+
+    config = check.get_scraper_config(instance)
+    config['_dry_run'] = False
+
+    check.process_metric(ref_gauge, config)
+
+    aggregator.assert_metric('prometheus.process.vm.bytes', tags=wanted_tags, count=1)

datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics.yaml

@@ -300,5 +300,15 @@
   value:
     example: false
     type: boolean
+- name: ignore_tags
+  description: A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+  value:
+    type: array
+    items:
+      type: string
+    example:
+      - <FULL:TAG>
+      - <TAG_PREFIX:.*>
+      - <TAG_SUFFIX$>
 - template: instances/http
 - template: instances/default

datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics_legacy.yaml

@@ -167,5 +167,15 @@
     example:
       <KEY_1>: [<LABEL_1>, <LABEL_2>]
       <KEY_2>: ['*']
+- name: ignore_tags
+  description: A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+  value:
+    type: array
+    items:
+      type: string
+    example:
+      - <FULL:TAG>
+      - <TAG_PREFIX:.*>
+      - <TAG_SUFFIX$>
 - template: instances/http
 - template: instances/default

etcd/datadog_checks/etcd/data/conf.yaml.example

@@ -163,6 +163,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

external_dns/datadog_checks/external_dns/data/conf.yaml.example

@@ -157,6 +157,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

gitlab/datadog_checks/gitlab/data/conf.yaml.example

@@ -127,6 +127,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

gitlab_runner/datadog_checks/gitlab_runner/config_models/defaults.py

@@ -80,6 +80,10 @@ def instance_ignore_metrics_by_labels(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_ignore_tags(field, value):
+    return get_default_field_value(field, value)
+
+
 def instance_kerberos_auth(field, value):
     return 'disabled'
 

gitlab_runner/datadog_checks/gitlab_runner/config_models/instance.py

@@ -73,6 +73,7 @@ class Config:
     health_service_check: Optional[bool]
     ignore_metrics: Optional[Sequence[str]]
     ignore_metrics_by_labels: Optional[IgnoreMetricsByLabels]
+    ignore_tags: Optional[Sequence[str]]
     kerberos_auth: Optional[str]
     kerberos_cache: Optional[str]
     kerberos_delegate: Optional[bool]

gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example

@@ -211,6 +211,14 @@ instances:
     #   <KEY_2>:
     #   - '*'
 
+    ## @param ignore_tags - list of strings - optional
+    ## A list of tags to ignore, regular expressions can be used to match multiple tag strings.
+    #
+    # ignore_tags:
+    #   - <FULL:TAG>
+    #   - <TAG_PREFIX:.*>
+    #   - <TAG_SUFFIX$>
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

haproxy/datadog_checks/haproxy/config_models/defaults.py

@@ -112,6 +112,10 @@ def instance_ignore_metrics_by_labels(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_ignore_tags(field, value):
+    return get_default_field_value(field, value)
+
+
 def instance_kerberos_auth(field, value):
     return 'disabled'