From e0655cfc8ac3bf0523ec2d9f4a44d66db1486795 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 2 May 2017 14:45:45 +0200 Subject: [PATCH] fix docker image name getting when using sha256 for specs Some orchestrators init containers with the image sha256 instead of the image name, and docker inspect returns this. If it's the case, make a docker image inspect call to get the image name and cache the result --- tests/core/test_dockerutil.py | 31 ++++++++++++ utils/dockerutil.py | 50 ++++++++++++++++++-- utils/service_discovery/sd_docker_backend.py | 5 +- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/tests/core/test_dockerutil.py b/tests/core/test_dockerutil.py index be1c71ed4f..9ad75015ed 100644 --- a/tests/core/test_dockerutil.py +++ b/tests/core/test_dockerutil.py @@ -1,6 +1,9 @@ # stdlib import unittest +# 3rd party +import mock + # project from utils.dockerutil import DockerUtil @@ -28,3 +31,31 @@ def test_parse_subsystem(self): du = DockerUtil() for line, exp_res in lines: self.assertEquals(du._parse_subsystem(line), exp_res) + + def test_image_name_from_container(self): + co = {'Image': 'redis:3.2'} + self.assertEqual('redis:3.2', DockerUtil().image_name_extractor(co)) + pass + + @mock.patch('docker.Client.inspect_image') + @mock.patch('docker.Client.__init__') + def test_image_name_from_image_repotags(self, mock_init, mock_image): + mock_image.return_value = {'RepoTags': ["redis:3.2"], 'RepoDigests': []} + mock_init.return_value = None + sha = 'sha256:e48e77eee11b6d9ac9fc35a23992b4158355a8ec3fd3725526eba3f467e4b6c9' + co = {'Image': sha} + self.assertEqual('redis:3.2', DockerUtil().image_name_extractor(co)) + mock_image.assert_called_once_with(sha) + + # Make sure cache is used insead of call again inspect_image + DockerUtil().image_name_extractor(co) + mock_image.assert_called_once() + + @mock.patch('docker.Client.inspect_image') + @mock.patch('docker.Client.__init__') + def test_image_name_from_image_repodigests(self, mock_init, mock_image): + mock_image.return_value = {'RepoTags': [], + 'RepoDigests': ['alpine@sha256:4f2d8bbad359e3e6f23c0498e009aaa3e2f31996cbea7269b78f92ee43647811']} + mock_init.return_value = None + co = {'Image': 'sha256:e48e77eee11b6d9ac9fc35a23992b4158355a8ec3fd3725526eba3f467e4b6d9'} + self.assertEqual('alpine', DockerUtil().image_name_extractor(co)) diff --git a/utils/dockerutil.py b/utils/dockerutil.py index 6571bc051e..a3d3d5b682 100644 --- a/utils/dockerutil.py +++ b/utils/dockerutil.py @@ -70,6 +70,9 @@ def __init__(self, **kwargs): # At first run we'll just collect the events from the latest 60 secs self._latest_event_collection_ts = int(time.time()) - 60 + # Memory cache for sha256 to image name mapping + self._image_sha_to_name_mapping = {} + # Try to detect if we are on Swarm self.fetch_swarm_state() @@ -428,10 +431,10 @@ def find_cgroup_filename_pattern(cls, mountpoints, container_id): raise MountException("Cannot find Docker cgroup directory. Be sure your system is supported.") - @classmethod - def image_tag_extractor(cls, entity, key): - if "Image" in entity: - split = entity["Image"].split(":") + def image_tag_extractor(self, entity, key): + name = self.image_name_extractor(entity) + if len(name): + split = name.split(":") if len(split) <= key: return None elif len(split) > 2: @@ -439,7 +442,8 @@ def image_tag_extractor(cls, entity, key): # the split will be like [repo_url, repo_port/image_name, image_tag]. Let's avoid that split = [':'.join(split[:-1]), split[-1]] return [split[key]] - if entity.get('RepoTags'): + # Entity is an image. TODO: deprecate? + elif entity.get('RepoTags'): splits = [el.split(":") for el in entity["RepoTags"]] tags = set() for split in splits: @@ -459,6 +463,42 @@ def image_tag_extractor(cls, entity, key): return None + def image_name_extractor(self, co): + """ + Returns the image name for a container, either directly from the + container's Image property or by inspecting the image entity if + the reference is its sha256 sum and not its name. + Result is cached for performance, no invalidation planned as image + churn is low on typical hosts. + """ + if "Image" in co: + image = co.get('Image', '') + if image.startswith('sha256:'): + # Some orchestrators setup containers with image checksum instead of image name + try: + if image in self._image_sha_to_name_mapping: + return self._image_sha_to_name_mapping[image] + else: + image_spec = self.client.inspect_image(image) + try: + name = image_spec.get('RepoTags')[0] + self._image_sha_to_name_mapping[image] = name + return name + except Exception: + pass + try: + name = image_spec.get('RepoDigests')[0] + name = name.split('@')[0] # Last resort, we get the name with no tag + self._image_sha_to_name_mapping[image] = name + return name + except Exception: + pass + except Exception: + pass + else: + return image + return None + @classmethod def container_name_extractor(cls, co): names = co.get('Names', []) diff --git a/utils/service_discovery/sd_docker_backend.py b/utils/service_discovery/sd_docker_backend.py index 1e716b7a7e..2dc49e2ba3 100644 --- a/utils/service_discovery/sd_docker_backend.py +++ b/utils/service_discovery/sd_docker_backend.py @@ -87,7 +87,8 @@ def __init__(self, agentConfig): agentConfig['sd_config_backend'] = None self.config_store = get_config_store(agentConfig=agentConfig) - self.docker_client = DockerUtil(config_store=self.config_store).client + self.dockerutil = DockerUtil(config_store=self.config_store) + self.docker_client = self.dockerutil.client if Platform.is_k8s(): try: self.kubeutil = KubeUtil() @@ -347,7 +348,7 @@ def get_configs(self): configs = {} state = self._make_fetch_state() containers = [( - container.get('Image'), + self.dockerutil.image_name_extractor(container), container.get('Id'), container.get('Labels') ) for container in self.docker_client.containers()]