Support regular expressions in dynamic tags (#6096)

Tags coming from SNMP values can be complex, we can either want a
substring or divide it into multiple tags. This adds support for it
using regular expressions in the configuration.

Co-authored-by: ruthnaebeck <19349244+ruthnaebeck@users.noreply.github.com>
Co-authored-by: Ofek Lev <ofekmeister@gmail.com>

snmp/datadog_checks/snmp/config.py

@@ -2,8 +2,9 @@
 # All rights reserved
 # Licensed under Simplified BSD License (see LICENSE)
 import ipaddress
+import re
 from collections import defaultdict
-from typing import Any, Callable, DefaultDict, Dict, Iterator, List, Optional, Set, Tuple, Union
+from typing import Any, Callable, DefaultDict, Dict, Iterator, List, Optional, Pattern, Set, Tuple, Union
 
 from datadog_checks.base import ConfigurationError, is_affirmative
 
@@ -73,6 +74,28 @@ def __init__(self, name, symbol):
         self.name = name
         self.symbol = symbol
 
+    def matched_tags(self, value):
+        # type: (Any) -> Iterator[str]
+        yield '{}:{}'.format(self.name, value)
+
+
+class ParsedMatchMetricTags(object):
+
+    __slots__ = ('names', 'symbol', 'match')
+
+    def __init__(self, names, symbol, match):
+        # type: (dict, str, Pattern) -> None
+        self.names = names
+        self.symbol = symbol
+        self.match = match
+
+    def matched_tags(self, value):
+        # type: (Any) -> Iterator[str]
+        matched = self.match.match(str(value))
+        if matched is not None:
+            for name, match in self.names.items():
+                yield '{}:{}'.format(name, matched.expand(match))
+
 
 def _no_op(*args, **kwargs):
     # type: (*Any, **Any) -> None
@@ -478,27 +501,54 @@ def get_table_symbols(mib, table):  # type: ignore
         return all_oids, bulk_oids, parsed_metrics
 
     def parse_metric_tags(self, metric_tags):
-        # type: (List[Dict[str, Any]]) -> Tuple[List[Any], List[ParsedMetricTag]]
+        # type: (List[Dict[str, Any]]) -> Tuple[List[Any], List[Any]]
         """Parse configuration for global metric_tags."""
         oids = []
         parsed_metric_tags = []
+
         for tag in metric_tags:
-            if not ('symbol' in tag and 'tag' in tag):
-                raise ConfigurationError("A metric tag needs to specify a symbol and a tag: {}".format(tag))
-            if not ('OID' in tag or 'MIB' in tag):
-                raise ConfigurationError("A metric tag needs to specify an OID or a MIB: {}".format(tag))
+            if 'symbol' not in tag:
+                raise ConfigurationError('A metric tag needs to specify a symbol: {}'.format(tag))
             symbol = tag['symbol']
-            tag_name = tag['tag']
+
+            if not ('OID' in tag or 'MIB' in tag):
+                raise ConfigurationError('A metric tag needs to specify an OID or a MIB: {}'.format(tag))
+
+            if 'tag' not in tag:
+                if not ('tags' in tag and 'match' in tag):
+                    raise ConfigurationError(
+                        'A metric tag needs to specify either a tag, '
+                        'or a mapping of tags and a regular expression: {}'.format(tag)
+                    )
+                tags = tag['tags']
+                if not isinstance(tags, dict):
+                    raise ConfigurationError(
+                        'Specified tags needs to be a mapping of tag name to regular '
+                        'expression matching: {}'.format(tag)
+                    )
+                match = tag['match']
+                try:
+                    compiled_match = re.compile(match)
+                except re.error as e:
+                    raise ConfigurationError('Failed compile regular expression {}: {}'.format(match, e))
+                else:
+                    parsed = ParsedMatchMetricTags(tags, symbol, compiled_match)
+            else:
+                tag_name = tag['tag']
+                parsed = ParsedMetricTag(tag_name, symbol)  # type: ignore
+
             if 'MIB' in tag:
                 mib = tag['MIB']
                 identity = ObjectIdentity(mib, symbol)
             else:
                 oid = tag['OID']
                 identity = ObjectIdentity(oid)
                 self._resolver.register(OID(oid).as_tuple(), symbol)
+
             object_type = ObjectType(identity)
             oids.append(object_type)
-            parsed_metric_tags.append(ParsedMetricTag(tag_name, symbol))
+            parsed_metric_tags.append(parsed)
+
         return oids, parsed_metric_tags
 
     def add_uptime_metric(self):

snmp/datadog_checks/snmp/data/conf.yaml.example

@@ -332,6 +332,8 @@ instances:
 
     ## @param metric_tags - list of elements - optional
     ## Specify tags that you want applied to all metrics. A tag can be applied from a symbol or an OID.
+    ## Regular expressions can be used to separate the resulting value into several tags, or get a
+    ## substring using the regular Python engine: https://docs.python.org/3/library/re.html
     #
     # metric_tags:
     #  - # From a symbol
@@ -342,3 +344,10 @@ instances:
     #    OID: 1.3.6.1.2.1.1.5
     #    symbol: sysName
     #    tag: snmp_host
+    #  - # With regular expression matching
+    #    MIB: SNMPv2-MIB
+    #    symbol: sysName
+    #    match: (.*)-(.*)
+    #    tags:
+    #        host: \\2
+    #        device_type: \\1

snmp/datadog_checks/snmp/snmp.py

@@ -6,6 +6,7 @@
 import functools
 import ipaddress
 import json
+import re
 import threading
 import time
 from collections import defaultdict
@@ -19,7 +20,7 @@
 
 from .commands import snmp_bulk, snmp_get, snmp_getnext
 from .compat import read_persistent_cache, write_persistent_cache
-from .config import InstanceConfig, ParsedMetric, ParsedMetricTag, ParsedTableMetric
+from .config import InstanceConfig, ParsedMatchMetricTags, ParsedMetric, ParsedMetricTag, ParsedTableMetric
 from .exceptions import PySnmpError
 from .metrics import as_metric_with_forced_type, as_metric_with_inferred_type
 from .pysnmp_types import ObjectIdentity, ObjectType, noSuchInstance, noSuchObject
@@ -388,7 +389,7 @@ def _check_with_config(self, config):
         return error
 
     def extract_metric_tags(self, metric_tags, results):
-        # type: (List[ParsedMetricTag], Dict[str, dict]) -> List[str]
+        # type: (List[Union[ParsedMetricTag, ParsedMatchMetricTags]], Dict[str, dict]) -> List[str]
         extracted_tags = []
         for tag in metric_tags:
             if tag.symbol not in results:
@@ -400,7 +401,10 @@ def extract_metric_tags(self, metric_tags, results):
                     'You are trying to use a table column (OID `{}`) as a metric tag. This is not supported as '
                     '`metric_tags` can only refer to scalar OIDs.'.format(tag.symbol)
                 )
-            extracted_tags.append('{}:{}'.format(tag.name, tag_values[0]))
+            try:
+                extracted_tags.extend(tag.matched_tags(tag_values[0]))
+            except re.error as e:
+                self.log.debug('Failed to match %s for %s: %s', tag_values[0], tag.symbol, e)
         return extracted_tags
 
     def report_metrics(

snmp/tests/test_check.py

@@ -842,6 +842,18 @@ def test_metric_tags_misconfiguration():
     with pytest.raises(ConfigurationError):
         common.create_check(instance)
 
+    instance['metric_tags'] = [{'tags': {'foo': 'bar'}, 'symbol': 'sysName', 'MIB': 'SNMPv2-MIB'}]
+    with pytest.raises(ConfigurationError):
+        common.create_check(instance)
+
+    instance['metric_tags'] = [{'tags': 'foo', 'match': 'bar', 'symbol': 'sysName', 'MIB': 'SNMPv2-MIB'}]
+    with pytest.raises(ConfigurationError):
+        common.create_check(instance)
+
+    instance['metric_tags'] = [{'tags': {'foo': 'bar'}, 'match': '(', 'symbol': 'sysName', 'MIB': 'SNMPv2-MIB'}]
+    with pytest.raises(ConfigurationError):
+        common.create_check(instance)
+
 
 def test_metric_tag_multiple(aggregator, caplog):
     metrics = common.SUPPORTED_METRIC_TYPES
@@ -864,3 +876,32 @@ def test_metric_tag_multiple(aggregator, caplog):
             break
     else:
         raise AssertionError('Expected WARNING log with message `{}`'.format(expected_message))
+
+
+def test_metric_tag_matching(aggregator):
+    metrics = common.SUPPORTED_METRIC_TYPES
+    instance = common.generate_instance_config(metrics)
+    instance['metric_tags'] = [
+        {
+            'MIB': 'SNMPv2-MIB',
+            'symbol': 'sysName',
+            'match': '(\\d\\d)(.*)',
+            'tags': {'host_prefix': '\\1', 'host': '\\2'},
+        }
+    ]
+    check = common.create_check(instance)
+
+    check.check(instance)
+
+    tags = list(common.CHECK_TAGS)
+    tags.append('host:ba948911b9')
+    tags.append('host_prefix:41')
+
+    for metric in common.SUPPORTED_METRIC_TYPES:
+        metric_name = "snmp." + metric['name']
+        aggregator.assert_metric(metric_name, tags=tags, count=1)
+    aggregator.assert_metric('snmp.sysUpTimeInstance', count=1)
+
+    aggregator.assert_service_check("snmp.can_check", status=SnmpCheck.OK, tags=tags, at_least=1)
+
+    aggregator.all_metrics_asserted()