[AD] Add docker labels as tags in AD (#3564)

* [AD] Add docker labels as tags

* Add tests

* address review comments

datadog.conf.example

@@ -108,7 +108,7 @@ gce_updated_hostname: yes
 
 # ========================================================================== #
 # Service Discovery                                                          #
-# See http://docs.datadoghq.com/guides/servicediscovery/ for details #
+# See https://docs.datadoghq.com/guides/autodiscovery/ for details #
 # ========================================================================== #
 #
 # Service discovery allows the agent to look for running services
@@ -141,6 +141,17 @@ gce_updated_hostname: yes
 # Enable JMX checks for service discovery
 # sd_jmx_enable: no
 #
+# Docker labels as tags
+# We can extract docker labels and add them as tags to all metrics reported by service discovery.
+# All you have to do is supply a comma-separated list of label names to extract from containers when found.
+# Note: these tags won't be applied to docker metrics, only service discovery metrics.
+# For applying labels to docker metrics, refer to https://github.com/DataDog/integrations-core/blob/bfdb3b0cf34ca6b6c92cc0e467a4f343e7f96ff2/docker_daemon/conf.yaml.example#L187-L190
+#
+# docker_labels_as_tags: label_name
+#
+# Example:
+# docker_labels_as_tags: com.docker.compose.service, com.docker.compose.project
+#
 # ========================================================================== #
 # Other                                                                      #
 # ========================================================================== #

tests/core/test_dockerutil.py

@@ -72,16 +72,72 @@ def test_image_name_from_image_repodigests(self):
         self.assertEqual('alpine', du.image_name_extractor(co))
 
     def test_extract_container_tags(self):
-        test_data = [
+        no_label_test_data = [
             # Nominal case
             [{'Image': 'redis:3.2'}, ['docker_image:redis:3.2', 'image_name:redis', 'image_tag:3.2']],
             # No tag
             [{'Image': 'redis'}, ['docker_image:redis', 'image_name:redis']],
             # No image
             [{}, []],
         ]
-        for test in test_data:
-            self.assertEqual(test[1], DockerUtil().extract_container_tags(test[0]))
+        labeled_test_data = [
+            # No labels
+            (
+                # ctr inspect
+                {
+                    'Image': 'redis:3.2',
+                    'Config': {
+                        'Labels': {}
+                    }
+                },
+                # labels as tags
+                [],
+                # expected result
+                ['docker_image:redis:3.2', 'image_name:redis', 'image_tag:3.2']
+            ),
+            # Un-monitored labels
+            (
+                {
+                    'Image': 'redis:3.2',
+                    'Config': {
+                        'Labels': {
+                            'foo': 'bar'
+                        }
+                    }
+                },
+                [],
+                ['docker_image:redis:3.2', 'image_name:redis', 'image_tag:3.2']
+            ),
+            # no labels, with labels_as_tags list
+            (
+                {
+                    'Image': 'redis:3.2',
+                    'Config': {
+                        'Labels': {}
+                    }
+                },
+                ['foo'],
+                ['docker_image:redis:3.2', 'image_name:redis', 'image_tag:3.2']
+            ),
+            # labels and labels_as_tags list
+            (
+                {
+                    'Image': 'redis:3.2',
+                    'Config': {
+                        'Labels': {'foo': 'bar', 'f00': 'b4r'}
+                    }
+                },
+                ['foo'],
+                ['docker_image:redis:3.2', 'image_name:redis', 'image_tag:3.2', 'foo:bar']
+            ),
+
+        ]
+        for test in no_label_test_data:
+            self.assertEqual(test[1], DockerUtil().extract_container_tags(test[0], []))
+
+        for test in labeled_test_data:
+            self.assertEqual(test[2], DockerUtil().extract_container_tags(test[0], test[1]))
+
 
     def test_docker_host_tags_ok(self):
         mock_version = mock.MagicMock(name='version', return_value={'Version': '1.13.1'})

utils/dockerutil.py

@@ -513,23 +513,27 @@ def find_cgroup_filename_pattern(cls, mountpoints, container_id):
 
         raise MountException("Cannot find Docker cgroup directory. Be sure your system is supported.")
 
-    def extract_container_tags(self, co):
+    def extract_container_tags(self, co, labels_as_tags):
         """
         Retrives docker_image, image_name and image_tag tags as a list for a
         container. If the container or image is invalid, will gracefully
-        return an empty list
+        return an empty list.
+        Also extract container labels on demand.
         """
         tags = []
         docker_image = self.image_name_extractor(co)
         image_name_array = self.image_tag_extractor(co, 0)
         image_tag_array = self.image_tag_extractor(co, 1)
+        label_tags = self.label_extractor(co, labels_as_tags)
 
         if docker_image:
             tags.append('docker_image:%s' % docker_image)
         if image_name_array and len(image_name_array) > 0:
             tags.append('image_name:%s' % image_name_array[0])
         if image_tag_array and len(image_tag_array) > 0:
             tags.append('image_tag:%s' % image_tag_array[0])
+        if label_tags:
+            tags += label_tags
         return tags
 
     def image_tag_extractor(self, entity, key):
@@ -603,6 +607,17 @@ def image_name_resolver(self, image):
         else:
             return image
 
+    def label_extractor(self, ctr, lbl_to_tags):
+        """Returns a list of tags based on a container and a label name list"""
+        tags = []
+        labels = ctr.get('Config', {}).get('Labels', {})
+        if not labels:
+            return tags
+        for lbl_name, lbl_val in labels.iteritems():
+            if lbl_name in lbl_to_tags:
+                tags.append('{}:{}'.format(lbl_name, lbl_val))
+        return tags
+
     @classmethod
     def container_name_extractor(cls, co):
         names = co.get('Names', [])

utils/service_discovery/sd_docker_backend.py

@@ -107,6 +107,13 @@ def __init__(self, agentConfig):
             'tags': self._get_additional_tags,
         }
 
+        # docker labels we'll add as tags to all instances SD configures
+        self.docker_labels_as_tags = agentConfig.get('docker_labels_as_tags', '')
+        if self.docker_labels_as_tags:
+            self.docker_labels_as_tags = [label.strip() for label in self.docker_labels_as_tags.split(',')]
+        else:
+            self.docker_labels_as_tags = []
+
         AbstractSDBackend.__init__(self, agentConfig)
 
     def _make_fetch_state(self):
@@ -288,7 +295,7 @@ def _extract_port_from_list(self, ports, tpl_var):
     def get_tags(self, state, c_id):
         """Extract useful tags from docker or platform APIs. These are collected by default."""
         c_inspect = state.inspect_container(c_id)
-        tags = self.dockerutil.extract_container_tags(c_inspect)
+        tags = self.dockerutil.extract_container_tags(c_inspect, self.docker_labels_as_tags)
 
         if Platform.is_k8s():
             if not self.kubeutil.init_success: