From e51a13ca4e31e476097809a54df375b6f5b95fa2 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 9 Jun 2020 14:18:57 -0400 Subject: [PATCH 1/3] Support multiple root layouts --- .../datadog_checks/downloader/cli.py | 15 +++++++++++-- .../datadog_checks/downloader/download.py | 22 ++++++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/datadog_checks_downloader/datadog_checks/downloader/cli.py b/datadog_checks_downloader/datadog_checks/downloader/cli.py index a3d31237b06e2..e5f42043a5a97 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/cli.py +++ b/datadog_checks_downloader/datadog_checks/downloader/cli.py @@ -8,7 +8,7 @@ import re # 2nd party. -from .download import REPOSITORY_URL_PREFIX, TUFDownloader +from .download import DEFAULT_ROOT_LAYOUT_TYPE, REPOSITORY_URL_PREFIX, ROOT_LAYOUTS, TUFDownloader from .exceptions import NonCanonicalVersion, NonDatadogPackage # Private module functions. @@ -39,6 +39,14 @@ def download(): parser.add_argument('--version', type=str, default=None, help='The version number of the desired Datadog check.') + parser.add_argument( + '--type', + type=str, + default=DEFAULT_ROOT_LAYOUT_TYPE, + choices=list(ROOT_LAYOUTS), + help='The type of integration.', + ) + parser.add_argument( '-v', '--verbose', action='count', default=0, help='Show verbose information about TUF and in-toto.' ) @@ -47,6 +55,7 @@ def download(): repository_url_prefix = args.repository standard_distribution_name = args.standard_distribution_name version = args.version + root_layout_type = args.type verbose = args.verbose if not standard_distribution_name.startswith('datadog-'): @@ -55,7 +64,9 @@ def download(): if version and not __is_canonical(version): raise NonCanonicalVersion(version) - tuf_downloader = TUFDownloader(repository_url_prefix=repository_url_prefix, verbose=verbose) + tuf_downloader = TUFDownloader( + repository_url_prefix=repository_url_prefix, root_layout_type=root_layout_type, verbose=verbose + ) wheel_relpath = tuf_downloader.get_wheel_relpath(standard_distribution_name, version=version) wheel_abspath = tuf_downloader.download(wheel_relpath) print(wheel_abspath) # pylint: disable=print-statement diff --git a/datadog_checks_downloader/datadog_checks/downloader/download.py b/datadog_checks_downloader/datadog_checks/downloader/download.py index f0b5e23b8215d..0a2135e8ef43e 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/download.py +++ b/datadog_checks_downloader/datadog_checks/downloader/download.py @@ -48,7 +48,8 @@ REPOSITORY_URL_PREFIX = 'https://dd-integrations-core-wheels-build-stable.datadoghq.com' # Where to find our in-toto root layout. IN_TOTO_METADATA_DIR = 'in-toto-metadata' -IN_TOTO_ROOT_LAYOUT = '2.root.layout' +ROOT_LAYOUTS = {'core': '2.root.layout', 'extras': '1.extras.root.layout'} +DEFAULT_ROOT_LAYOUT_TYPE = 'core' # Global variables. @@ -56,7 +57,9 @@ class TUFDownloader: - def __init__(self, repository_url_prefix=REPOSITORY_URL_PREFIX, verbose=0): + def __init__( + self, repository_url_prefix=REPOSITORY_URL_PREFIX, root_layout_type=DEFAULT_ROOT_LAYOUT_TYPE, verbose=0 + ): # 0 => 60 (effectively /dev/null) # 1 => 50 (CRITICAL) # 2 => 40 (ERROR) @@ -71,6 +74,9 @@ def __init__(self, repository_url_prefix=REPOSITORY_URL_PREFIX, verbose=0): tuf_settings.repositories_directory = REPOSITORIES_DIR + self.root_layout_type = root_layout_type + self.root_layout = ROOT_LAYOUTS[self.root_layout_type] + # NOTE: The directory where the targets for *this* repository is # cached. We hard-code this keep this to a subdirectory dedicated to # this repository. @@ -124,7 +130,7 @@ def __download_in_toto_root_layout(self): # expected version of the root layout. This is so that, for example, we # can introduce new parameters w/o breaking old downloaders that don't # know how to substitute them. - target_relpath = os.path.join(IN_TOTO_METADATA_DIR, IN_TOTO_ROOT_LAYOUT) + target_relpath = os.path.join(IN_TOTO_METADATA_DIR, self.root_layout) return self.__download_with_tuf(target_relpath) def __download_custom(self, target, extension): @@ -133,6 +139,10 @@ def __download_custom(self, target, extension): fileinfo = target.get('fileinfo', {}) custom = fileinfo.get('custom', {}) + + if custom.get('root-layout-type', DEFAULT_ROOT_LAYOUT_TYPE) != self.root_layout_type: + return target_abspaths + in_toto_metadata = custom.get('in-toto', []) for target_relpath in in_toto_metadata: @@ -177,7 +187,7 @@ def __download_in_toto_links(self, target, target_relpath): return link_abspaths def __load_root_layout(self, target_relpath): - root_layout = Metablock.load(IN_TOTO_ROOT_LAYOUT) + root_layout = Metablock.load(self.root_layout) root_layout_pubkeys = glob.glob('*.pub') root_layout_pubkeys = import_public_keys_from_files_as_dict(root_layout_pubkeys) # Parameter substitution. @@ -188,9 +198,9 @@ def __handle_in_toto_verification_exception(self, target_relpath, e): logger.exception('in-toto failed to verify %s', target_relpath) if isinstance(e, LinkNotFoundError) and str(e) == RevokedDeveloper.MSG: - raise RevokedDeveloper(target_relpath, IN_TOTO_ROOT_LAYOUT) + raise RevokedDeveloper(target_relpath, self.root_layout) else: - raise + raise e def __in_toto_verify(self, inspection_packet, target_relpath): # Make a temporary directory in a parent directory we control. From 03b0f395930bad9daa617642b2a4ee02b7206744 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Tue, 9 Jun 2020 15:25:07 -0400 Subject: [PATCH 2/3] address --- .../datadog_checks/downloader/download.py | 6 ++++-- .../datadog_checks/downloader/exceptions.py | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/datadog_checks_downloader/datadog_checks/downloader/download.py b/datadog_checks_downloader/datadog_checks/downloader/download.py index 0a2135e8ef43e..e4347985ba435 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/download.py +++ b/datadog_checks_downloader/datadog_checks/downloader/download.py @@ -23,6 +23,7 @@ from .exceptions import ( DuplicatePackage, InconsistentSimpleIndex, + IncorrectRootLayoutType, MissingVersions, NoInTotoLinkMetadataFound, NoInTotoRootLayoutPublicKeysFound, @@ -140,8 +141,9 @@ def __download_custom(self, target, extension): fileinfo = target.get('fileinfo', {}) custom = fileinfo.get('custom', {}) - if custom.get('root-layout-type', DEFAULT_ROOT_LAYOUT_TYPE) != self.root_layout_type: - return target_abspaths + root_layout_type = custom.get('root-layout-type', DEFAULT_ROOT_LAYOUT_TYPE) + if root_layout_type != self.root_layout_type: + raise IncorrectRootLayoutType(root_layout_type, self.root_layout_type) in_toto_metadata = custom.get('in-toto', []) diff --git a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py index 6051a52234cb8..22da99dd13cad 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py +++ b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py @@ -29,6 +29,15 @@ def __str__(self): # Exceptions for the download module. +class IncorrectRootLayoutType(Exception): + def __init__(self, found, expected): + self.found = found + self.expected = expected + + def __str__(self): + return 'found {}, expected {}'.format(self.found, self.expected) + + class SimpleIndexError(Exception): def __init__(self, standard_distribution_name): self.standard_distribution_name = standard_distribution_name From 1020658566e858a4d7173ddff4eec51757271b40 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 10 Jun 2020 14:48:24 -0400 Subject: [PATCH 3/3] address --- .../datadog_checks/downloader/download.py | 18 +++++++++--------- .../datadog_checks/downloader/exceptions.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/datadog_checks_downloader/datadog_checks/downloader/download.py b/datadog_checks_downloader/datadog_checks/downloader/download.py index e4347985ba435..065b664f97c2d 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/download.py +++ b/datadog_checks_downloader/datadog_checks/downloader/download.py @@ -30,7 +30,7 @@ NoSuchDatadogPackage, NoSuchDatadogPackageVersion, PythonVersionMismatch, - RevokedDeveloper, + RevokedDeveloperOrMachine, ) from .parameters import substitute @@ -75,8 +75,8 @@ def __init__( tuf_settings.repositories_directory = REPOSITORIES_DIR - self.root_layout_type = root_layout_type - self.root_layout = ROOT_LAYOUTS[self.root_layout_type] + self.__root_layout_type = root_layout_type + self.__root_layout = ROOT_LAYOUTS[self.__root_layout_type] # NOTE: The directory where the targets for *this* repository is # cached. We hard-code this keep this to a subdirectory dedicated to @@ -131,7 +131,7 @@ def __download_in_toto_root_layout(self): # expected version of the root layout. This is so that, for example, we # can introduce new parameters w/o breaking old downloaders that don't # know how to substitute them. - target_relpath = os.path.join(IN_TOTO_METADATA_DIR, self.root_layout) + target_relpath = os.path.join(IN_TOTO_METADATA_DIR, self.__root_layout) return self.__download_with_tuf(target_relpath) def __download_custom(self, target, extension): @@ -142,8 +142,8 @@ def __download_custom(self, target, extension): custom = fileinfo.get('custom', {}) root_layout_type = custom.get('root-layout-type', DEFAULT_ROOT_LAYOUT_TYPE) - if root_layout_type != self.root_layout_type: - raise IncorrectRootLayoutType(root_layout_type, self.root_layout_type) + if root_layout_type != self.__root_layout_type: + raise IncorrectRootLayoutType(root_layout_type, self.__root_layout_type) in_toto_metadata = custom.get('in-toto', []) @@ -189,7 +189,7 @@ def __download_in_toto_links(self, target, target_relpath): return link_abspaths def __load_root_layout(self, target_relpath): - root_layout = Metablock.load(self.root_layout) + root_layout = Metablock.load(self.__root_layout) root_layout_pubkeys = glob.glob('*.pub') root_layout_pubkeys = import_public_keys_from_files_as_dict(root_layout_pubkeys) # Parameter substitution. @@ -199,8 +199,8 @@ def __load_root_layout(self, target_relpath): def __handle_in_toto_verification_exception(self, target_relpath, e): logger.exception('in-toto failed to verify %s', target_relpath) - if isinstance(e, LinkNotFoundError) and str(e) == RevokedDeveloper.MSG: - raise RevokedDeveloper(target_relpath, self.root_layout) + if isinstance(e, LinkNotFoundError) and str(e) == RevokedDeveloperOrMachine.MSG: + raise RevokedDeveloperOrMachine(target_relpath, self.__root_layout) else: raise e diff --git a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py index 22da99dd13cad..f9d1ffa3ac564 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/exceptions.py +++ b/datadog_checks_downloader/datadog_checks/downloader/exceptions.py @@ -111,11 +111,11 @@ class NoInTotoRootLayoutPublicKeysFound(TUFInTotoError): pass -class RevokedDeveloper(TUFInTotoError): +class RevokedDeveloperOrMachine(TUFInTotoError): MSG = "Step 'tag' requires '1' link metadata file(s), found '0'." def __init__(self, target_relpath, in_toto_root_layout): - super(RevokedDeveloper, self).__init__(target_relpath) + super(RevokedDeveloperOrMachine, self).__init__(target_relpath) self.in_toto_root_layout = in_toto_root_layout def __str__(self):