From 44745e84af223385a003a3b758d537455988e699 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 13 Nov 2019 17:58:22 +0300 Subject: [PATCH 01/33] Copy pasting changes from branch venafi-fix --- doc/topics/venafi/index.rst | 257 ++----- requirements/static/py2.7/linux.txt | 16 + requirements/static/py3.4/linux.txt | 16 + requirements/static/py3.5/linux.txt | 16 + requirements/static/py3.6/linux.txt | 16 + requirements/static/py3.7/linux.txt | 16 + salt/pillar/venafi.py | 9 +- salt/runners/venafiapi.py | 700 +++--------------- .../integration/externalapi/test_venafiapi.py | 164 ++-- tests/integration/files/conf/master | 5 + 10 files changed, 384 insertions(+), 831 deletions(-) diff --git a/doc/topics/venafi/index.rst b/doc/topics/venafi/index.rst index b756d4f271ea..729a0ae91aee 100644 --- a/doc/topics/venafi/index.rst +++ b/doc/topics/venafi/index.rst @@ -5,256 +5,147 @@ Venafi Tools for Salt Introduction ~~~~~~~~~~~~ -Before using these modules you need to register an account with Venafi, and -configure it in your ``master`` configuration file. +First, you need to configure the ``master`` file. This is because +all module functions require either a configured ``api_key`` (for Cloud) or +``a ttp_user`` with a ``tpp_password`` and a ``base_url`` (for Trust Platform). -First, you need to add a placeholder to the ``master`` file. This is because -the module will not load unless it finds an ``api_key`` setting, valid or not. -Open up ``/etc/salt/master`` and add: +For Venafi Cloud: .. code-block:: yaml venafi: - api_key: None - -Then register your email address with Venafi using the following command: + api_key: abcdef01-2345-6789-abcd-ef0123456789 + base_url: "https://cloud.venafi.example.com/" (optional) -.. code-block:: bash +If you don't have a Venafi Cloud account, you can sign up for one on the `enrollment page`_. - salt-run venafi.register +.. _enrollment page: https://www.venafi.com/platform/cloud/devops -This command will not return an ``api_key`` to you; that will be send to you -via email from Venafi. Once you have received that key, open up your ``master`` -file and set the ``api_key`` to it: +For Venafi Platform: .. code-block:: yaml venafi: - api_key: abcdef01-2345-6789-abcd-ef0123456789 + base_url: "https://tpp.example.com/" + tpp_user: admin + tpp_password: "Str0ngPa$$w0rd" + trust_bundle: "/opt/venafi/bundle.pem" + +*It is not common for the Venafi Platform's REST API (WebSDK) to be secured using a certificate issued by a publicly trusted CA, therefore establishing trust for that server certificate is a critical part of your configuration. Ideally this is done by obtaining the root CA certificate in the issuing chain in PEM format and copying that file to your Salt Master (e.g. /opt/venafi/bundle.pem). You then reference that file using the 'trust_bundle' parameter as shown above.* -To enable the ability for creating keys and certificates it is necessary to enable the -external pillars. Open the ``/etc/salt/master`` file and add: +For the Venafi module to create keys and certificates it is necessary to enable external pillars. This is done by adding the following to the ``/etc/salt/master`` file: .. code-block:: yaml ext_pillar: - venafi: True -To modify the URL being used for the Venafi Certificate issuance modify the file -in ``/etc/salt/master`` and add the base_url information following under the venafi tag: - -.. code-block:: yaml - - venafi: - base_url: http://newurl.venafi.com - - -Example Usage -~~~~~~~~~~~~~ -Generate a CSR and submit it to Venafi for issuance, using the 'Internet' zone: -salt-run venafi.request minion.example.com minion.example.com zone=Internet - -Retrieve a certificate for a previously submitted request with request ID -aaa-bbb-ccc-dddd: -salt-run venafi.pickup aaa-bbb-ccc-dddd Runner Functions ~~~~~~~~~~~~~~~~ -gen_key -------- - -Generate and return a ``private_key``. If a ``dns_name`` is passed in, the -``private_key`` will be cached under that name. - -The key will be generated based on the policy values that were configured -by the Venafi administrator. A default Certificate Use Policy is associated -with a zone; the key type and key length parameters associated with this value -will be used. - -.. code-block:: bash - - salt-run venafi.gen_key minion.example.com minion.example.com zone=Internet \ - password=SecretSauce - -:param str minion_id: Required. The name of the minion which hosts the domain - name in question. - -:param str dns_name: Required. The FQDN of the domain that will be hosted on - the minion. - -:param str zone: Required. Default value is "default". The zone on Venafi that - the domain belongs to. - -:param str password: Optional. If specified, the password to use to access the - generated key. - - -gen_csr -------- - -Generate a csr using the host's private_key. Analogous to: - -.. code-block:: bash - - salt-run venafi.gen_csr minion.example.com minion.example.com country=US \ - state=California loc=Sacramento org=CompanyName org_unit=DevOps \ - zone=Internet password=SecretSauce - -:param str minion_id: Required. - -:param str dns_name: Required. - -:param str zone: Optional. Default value is "default". The zone on Venafi that - the domain belongs to. - -:param str country=None: Optional. The two-letter ISO abbreviation for your - country. - -:param str state=None: Optional. The state/county/region where your - organisation is legally located. Must not be abbreviated. - -:param str loc=None: Optional. The city where your organisation is legally - located. - -:param str org=None: Optional. The exact legal name of your organisation. Do - not abbreviate your organisation name. - -:param str org_unit=None: Optional. Section of the organisation, can be left - empty if this does not apply to your case. - -:param str password=None: Optional. Password for the CSR. - - request ------- +This command is used to enroll a certificate from Venafi Cloud or Venafi Platform. -Request a new certificate. Analogous to: - -.. code-block:: bash - - salt-run venafi.request minion.example.com minion.example.com country=US \ - state=California loc=Sacramento org=CompanyName org_unit=DevOps \ - zone=Internet password=SecretSauce +``minion_id`` + ID of the minion for which the certificate is being issued. Required. -:param str minion_id: Required. +``dns_name`` + DNS subject name for the certificate. Required if ``csr_path`` is not specified. -:param str dns_name: Required. +``csr_path`` + Full path name of certificate signing request file to enroll. Required if ``dns_name`` is not specified. -:param str zone: Required. Default value is "default". The zone on Venafi that - the certificate request will be submitted to. +``zone`` + Venafi Cloud zone ID or Venafi Platform folder that specify key and certificate policy. Defaults to "Default". For Venafi Cloud, the Zone ID can be found in the Zone page for your Venafi Cloud project. -:param str country=None: Optional. The two-letter ISO abbreviation for your - country. +``org_unit`` + Business Unit, Department, etc. Do not specify if it does not apply. -:param str state=None: Optional. The state/county/region where your - organisation is legally located. Must not be abbreviated. +``org`` + Exact legal name of your organization. Do not abbreviate. -:param str loc=None: Optional. The city where your organisation is legally - located. +``loc`` + City/locality where your organization is legally located. -:param str org=None: Optional. The exact legal name of your organisation. Do - not abbreviate your organisation name. +``state`` + State or province where your organization is legally located. Must not be abbreviated. -:param str org_unit=None: Optional. Section of the organisation, can be left - empty if this does not apply to your case. +``country`` + Country where your organization is legally located; two-letter ISO code. -:param str password=None: Optional. Password for the CSR. +``key_password`` + Password for encrypting the private key. -:param str company_id=None: Optional, but may be configured in ``master`` file - instead. - -register --------- - -Register a new user account +The syntax for requesting a new certificate with private key generation looks like this: .. code-block:: bash - salt-run venafi.register username@example.com - -:param str email: Required. The email address to use for the new Venafi account. - - -show_company ------------- + salt-run venafi.request minion.example.com dns_name=www.example.com \ + country=US state=California loc=Sacramento org="Company Name" org_unit=DevOps \ + zone=Internet key_password=SecretSauce -Show company information, especially the company id +And the syntax for requesting a new certificate using a previously generated CSR looks like this: .. code-block:: bash - salt-run venafi.show_company example.com + salt-run venafi.request minion.example.com csr_path=/tmp/minion.req zone=Internet -:param str domain: Required. The domain name to look up information for. - -show_csrs +show_cert --------- +This command is used to show last issued certificate for domain. -Show certificate requests for the configured API key. +``dns_name`` + DNS subject name of the certificate to look up. .. code-block:: bash - salt-run venafi.show_csrs - - -show_zones ----------- + salt-run venafi.show_cert www.example.com -Show zones for the specified company id. - -.. code-block:: bash - salt-run venafi.show_zones - -:param str company_id: Optional. The company id to show the zones for. - - -pickup, show_cert +list_domain_cache ----------------- - -Show certificate requests for the specified certificate id. Analogous to the -VCert pickup command. +This command lists domains that have been cached on this Salt Master. .. code-block:: bash - salt-run venafi.pickup 4295ebc0-14bf-11e7-b965-1df050017ec1 - -:param str id\_: Required. The id of the certificate to look up. + salt-run venafi.list_domain_cache -show_rsa --------- +del_cached_domain +----------------- +This command deletes a domain from the Salt Master's cache. -Show a private RSA key. +``domains`` + A domain name, or a comma-separated list of domain names, to delete from this master's cache. .. code-block:: bash - salt-run venafi.show_rsa minion.example.com minion.example.com - -:param str minion_id: The name of the minion to display the key for. - -:param str dns_name: The domain name to display the key for. - - -list_domain_cache ------------------ + salt-run venafi.del_cached_domain www.example.com -List domains that have been cached on this master. -.. code-block:: bash +Transfer certificate to a minion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - salt-run venafi.list_domain_cache +To transfer a cached certificate to a minion, you can use Venafi pillar. +Example state (SLS) file: -del_cached_domain ------------------ - -Delete a domain from this master's cache. +.. code-block:: yaml -.. code-block:: bash + /etc/ssl/cert/www.example.com.crt: + file.managed: + - contents_pillar: venafi:www.example.com:cert + - replace: True - salt-run venafi.delete_domain_cache example.com + /etc/ssl/cert/www.example.com.key: + file.managed: + - contents_pillar: venafi:www.example.com:pkey + - replace: True -:param str domains: A domain name, or a comma-separated list of domain names, - to delete from this master's cache. + /etc/ssl/cert/www.example.com-chain.pem: + file.managed: + - contents_pillar: venafi:www.example.com:chain + - replace: True \ No newline at end of file diff --git a/requirements/static/py2.7/linux.txt b/requirements/static/py2.7/linux.txt index 3c46959fd755..5bc465427019 100644 --- a/requirements/static/py2.7/linux.txt +++ b/requirements/static/py2.7/linux.txt @@ -127,3 +127,19 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy + +# venafi +asn1crypto==1.2.0 # via certvalidator, oscrypto +certifi==2019.9.11 # via requests +certvalidator==0.11.1 # via vcert +cffi==1.13.2 # via cryptography +chardet==3.0.4 # via requests +cryptography==2.8 # via vcert +idna==2.8 # via requests +oscrypto==1.1.0 # via certvalidator +pycparser==2.19 # via cffi +python-dateutil==2.8.1 # via vcert +requests==2.22.0 # via vcert +six==1.13.0 # via cryptography, python-dateutil, vcert +urllib3==1.25.7 # via requests +vcert==0.6.8 \ No newline at end of file diff --git a/requirements/static/py3.4/linux.txt b/requirements/static/py3.4/linux.txt index 8351bb83b7d3..9c6f59dae1e3 100644 --- a/requirements/static/py3.4/linux.txt +++ b/requirements/static/py3.4/linux.txt @@ -118,3 +118,19 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy + +# venafi +asn1crypto==1.2.0 # via certvalidator, oscrypto +certifi==2019.9.11 # via requests +certvalidator==0.11.1 # via vcert +cffi==1.13.2 # via cryptography +chardet==3.0.4 # via requests +cryptography==2.8 # via vcert +idna==2.8 # via requests +oscrypto==1.1.0 # via certvalidator +pycparser==2.19 # via cffi +python-dateutil==2.8.1 # via vcert +requests==2.22.0 # via vcert +six==1.13.0 # via cryptography, python-dateutil, vcert +urllib3==1.25.7 # via requests +vcert==0.6.8 \ No newline at end of file diff --git a/requirements/static/py3.5/linux.txt b/requirements/static/py3.5/linux.txt index cd8d3c396e73..8d0aff6a132b 100644 --- a/requirements/static/py3.5/linux.txt +++ b/requirements/static/py3.5/linux.txt @@ -116,3 +116,19 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy + +# venafi +asn1crypto==1.2.0 # via certvalidator, oscrypto +certifi==2019.9.11 # via requests +certvalidator==0.11.1 # via vcert +cffi==1.13.2 # via cryptography +chardet==3.0.4 # via requests +cryptography==2.8 # via vcert +idna==2.8 # via requests +oscrypto==1.1.0 # via certvalidator +pycparser==2.19 # via cffi +python-dateutil==2.8.1 # via vcert +requests==2.22.0 # via vcert +six==1.13.0 # via cryptography, python-dateutil, vcert +urllib3==1.25.7 # via requests +vcert==0.6.8 \ No newline at end of file diff --git a/requirements/static/py3.6/linux.txt b/requirements/static/py3.6/linux.txt index ffb3c0a9a2f1..d84f24cc69ab 100644 --- a/requirements/static/py3.6/linux.txt +++ b/requirements/static/py3.6/linux.txt @@ -116,3 +116,19 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy + +# venafi +asn1crypto==1.2.0 # via certvalidator, oscrypto +certifi==2019.9.11 # via requests +certvalidator==0.11.1 # via vcert +cffi==1.13.2 # via cryptography +chardet==3.0.4 # via requests +cryptography==2.8 # via vcert +idna==2.8 # via requests +oscrypto==1.1.0 # via certvalidator +pycparser==2.19 # via cffi +python-dateutil==2.8.1 # via vcert +requests==2.22.0 # via vcert +six==1.13.0 # via cryptography, python-dateutil, vcert +urllib3==1.25.7 # via requests +vcert==0.6.8 \ No newline at end of file diff --git a/requirements/static/py3.7/linux.txt b/requirements/static/py3.7/linux.txt index 0c4615c563a3..b6339f43a19b 100644 --- a/requirements/static/py3.7/linux.txt +++ b/requirements/static/py3.7/linux.txt @@ -116,3 +116,19 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy + +# venafi +asn1crypto==1.2.0 # via certvalidator, oscrypto +certifi==2019.9.11 # via requests +certvalidator==0.11.1 # via vcert +cffi==1.13.2 # via cryptography +chardet==3.0.4 # via requests +cryptography==2.8 # via vcert +idna==2.8 # via requests +oscrypto==1.1.0 # via certvalidator +pycparser==2.19 # via cffi +python-dateutil==2.8.1 # via vcert +requests==2.22.0 # via vcert +six==1.13.0 # via cryptography, python-dateutil, vcert +urllib3==1.25.7 # via requests +vcert==0.6.8 \ No newline at end of file diff --git a/salt/pillar/venafi.py b/salt/pillar/venafi.py index 5b4d728eb40b..baefd2a01cfe 100644 --- a/salt/pillar/venafi.py +++ b/salt/pillar/venafi.py @@ -36,9 +36,8 @@ def ext_pillar(minion_id, pillar, conf): cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) ret = {} - dns_names = cache.fetch('venafi/minions', minion_id) - for dns_name in dns_names: + for dns_name in cache.list('venafi/domains'): data = cache.fetch('venafi/domains', dns_name) - ret[dns_name] = data - del ret[dns_name]['csr'] - return {'venafi': ret} + if data['minion_id'] == minion_id: + ret[dns_name] = data + return {'venafi': ret} \ No newline at end of file diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 1da6b7d59b65..8b05e06d1fcb 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -2,49 +2,35 @@ ''' Support for Venafi -Before using this module you need to register an account with Venafi, and -configure it in your ``master`` configuration file. +:depends: - vcert Python module -First, you need to add a placeholder to the ``master`` file. This is because -the module will not load unless it finds an ``api_key`` setting, valid or not. -Open up ``/etc/salt/master`` and add: +:configuration: In order to connect to Venafi services you need to specify it in + Salt master configuration. + Example for Venafi Cloud (using env variables): -.. code-block:: yaml + .. code-block:: yaml venafi: - api_key: None + api_key: "sdb://osenv/CLOUDAPIKEY" -Then register your email address with Venafi using the following command: + Example for Venafi Platform (using env variables): -.. code-block:: bash - - salt-run venafi.register - -This command will not return an ``api_key`` to you; that will be sent to you -via email from Venafi. Once you have received that key, open up your ``master`` -file and set the ``api_key`` to it: - -.. code-block:: yaml + .. code-block:: yaml venafi: - api_key: abcdef01-2345-6789-abcd-ef0123456789 + base_url: "https://tpp.example.com/" + tpp_user: admin + tpp_password: "sdb://osenv/TPP_PASSWORD" + trust_bundle: "/opt/venafi/bundle.pem" + ''' from __future__ import absolute_import, print_function, unicode_literals import logging -import os -import tempfile +import time -try: - from M2Crypto import RSA - HAS_M2 = True -except ImportError: - HAS_M2 = False - try: - from Cryptodome.PublicKey import RSA - except ImportError: - from Crypto.PublicKey import RSA # Import Salt libs +import sys import salt.cache import salt.syspaths as syspaths import salt.utils.files @@ -54,311 +40,114 @@ # Import 3rd-party libs from salt.ext import six +try: + import vcert + from vcert.common import CertificateRequest + HAS_VCERT = True +except ImportError: + HAS_VCERT = False +CACHE_BANK_NAME = 'venafi/domains' __virtualname__ = 'venafi' log = logging.getLogger(__name__) -def __virtual__(): - ''' - Only load the module if venafi is installed - ''' - if __opts__.get('venafi', {}).get('api_key'): - return __virtualname__ - return False - +def _init_connection(): + log.info("Initializing Venafi Trust Platform or Venafi Cloud connection") + api_key = __opts__.get('venafi', {}).get('api_key', '') + base_url = __opts__.get('venafi', {}).get('base_url', '') + log.info("Using base_url: %s", base_url) + tpp_user = __opts__.get('venafi', {}).get('tpp_user', '') + log.info("Using tpp_user: %s", tpp_user) + tpp_password = __opts__.get('venafi', {}).get('tpp_password', '') + trust_bundle = __opts__.get('venafi', {}).get('trust_bundle', '') + fake = __opts__.get('venafi', {}).get('fake', '') + log.info("Finished config processing") + if fake: + return vcert.Connection(fake=True) + elif trust_bundle: + log.info("Will use trust bundle from file %s", trust_bundle) + return vcert.Connection(url=base_url, token=api_key, user=tpp_user, password=tpp_password, + http_request_kwargs={"verify": trust_bundle}) + else: + return vcert.Connection(url=base_url, token=api_key, user=tpp_user, password=tpp_password) -def _base_url(): - ''' - Return the base_url - ''' - return __opts__.get('venafi', {}).get( - 'base_url', 'https://api.venafi.cloud/v1' - ) - - -def _api_key(): - ''' - Return the API key - ''' - return __opts__.get('venafi', {}).get('api_key', '') - -def gen_key(minion_id, dns_name=None, zone='default', password=None): - ''' - Generate and return an private_key. If a ``dns_name`` is passed in, the - private_key will be cached under that name. The type of key and the - parameters used to generate the key are based on the default certificate - use policy associated with the specified zone. - - CLI Example: - - .. code-block:: bash - - salt-run venafi.gen_key [dns_name] [zone] [password] +def __virtual__(): ''' - # Get the default certificate use policy associated with the zone - # so we can generate keys that conform with policy - - # The /v1/zones/tag/{name} API call is a shortcut to get the zoneID - # directly from the name - - qdata = __utils__['http.query']( - '{0}/zones/tag/{1}'.format(_base_url(), zone), - method='GET', - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - 'Content-Type': 'application/json', - }, - ) - - zone_id = qdata['dict']['id'] - - # the /v1/certificatepolicies?zoneId API call returns the default - # certificate use and certificate identity policies - - qdata = __utils__['http.query']( - '{0}/certificatepolicies?zoneId={1}'.format(_base_url(), zone_id), - method='GET', - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - 'Content-Type': 'application/json', - }, - ) - - policies = qdata['dict']['certificatePolicies'] - - # Extract the key length and key type from the certificate use policy - # and generate the private key accordingly - - for policy in policies: - if policy['certificatePolicyType'] == "CERTIFICATE_USE": - keyTypes = policy['keyTypes'] - # in case multiple keytypes and key lengths are supported - # always use the first key type and key length - keygen_type = keyTypes[0]['keyType'] - key_len = keyTypes[0]['keyLengths'][0] - - if int(key_len) < 2048: - key_len = 2048 - - if keygen_type == "RSA": - if HAS_M2: - gen = RSA.gen_key(key_len, 65537) - private_key = gen.as_pem(cipher='des_ede3_cbc', callback=lambda x: six.b(password)) - else: - gen = RSA.generate(bits=key_len) - private_key = gen.exportKey('PEM', password) - if dns_name is not None: - bank = 'venafi/domains' - cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) - try: - data = cache.fetch(bank, dns_name) - data['private_key'] = private_key - data['minion_id'] = minion_id - except TypeError: - data = {'private_key': private_key, - 'minion_id': minion_id} - cache.store(bank, dns_name, data) - return private_key - - -def gen_csr( - minion_id, - dns_name, - zone='default', - country=None, - state=None, - loc=None, - org=None, - org_unit=None, - password=None, - ): + Only load the module if vcert module is installed ''' - Generate a csr using the host's private_key. - Analogous to: - - .. code-block:: bash - - VCert gencsr -cn [CN Value] -o "Beta Organization" -ou "Beta Group" \ - -l "Palo Alto" -st "California" -c US - - CLI Example: - - .. code-block:: bash - - salt-run venafi.gen_csr - ''' - tmpdir = tempfile.mkdtemp() - os.chmod(tmpdir, 0o700) - - bank = 'venafi/domains' - cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) - data = cache.fetch(bank, dns_name) - if data is None: - data = {} - if 'private_key' not in data: - data['private_key'] = gen_key(minion_id, dns_name, zone, password) - - tmppriv = '{0}/priv'.format(tmpdir) - tmpcsr = '{0}/csr'.format(tmpdir) - with salt.utils.files.fopen(tmppriv, 'w') as if_: - if_.write(salt.utils.stringutils.to_str(data['private_key'])) - - if country is None: - country = __opts__.get('venafi', {}).get('country') - - if state is None: - state = __opts__.get('venafi', {}).get('state') - - if loc is None: - loc = __opts__.get('venafi', {}).get('loc') - - if org is None: - org = __opts__.get('venafi', {}).get('org') - - if org_unit is None: - org_unit = __opts__.get('venafi', {}).get('org_unit') - - subject = '/C={0}/ST={1}/L={2}/O={3}/OU={4}/CN={5}'.format( - country, - state, - loc, - org, - org_unit, - dns_name, - ) - - cmd = "openssl req -new -sha256 -key {0} -out {1} -subj '{2}'".format( - tmppriv, - tmpcsr, - subject - ) - if password is not None: - cmd += ' -passin pass:{0}'.format(password) - output = __salt__['salt.cmd']('cmd.run', cmd) - - if 'problems making Certificate Request' in output: - raise CommandExecutionError( - 'There was a problem generating the CSR. Please ensure that you ' - 'have the following variables set either on the command line, or ' - 'in the venafi section of your master configuration file: ' - 'country, state, loc, org, org_unit' - ) - - with salt.utils.files.fopen(tmpcsr, 'r') as of_: - csr = salt.utils.stringutils.to_unicode(of_.read()) - - data['minion_id'] = minion_id - data['csr'] = csr - cache.store(bank, dns_name, data) - return csr + if not HAS_VCERT: + return False + return __virtualname__ def request( - minion_id, - dns_name=None, - zone='default', - request_id=None, - country='US', - state='California', - loc='Palo Alto', - org='Beta Organization', - org_unit='Beta Group', - password=None, - zone_id=None, - ): + minion_id, + dns_name=None, + zone=None, + country=None, + state=None, + loc=None, + org=None, + org_unit=None, + key_password=None, + csr_path=None, +): ''' Request a new certificate - Uses the following command: - - .. code-block:: bash - - VCert enroll -z -k -cn - CLI Example: .. code-block:: bash salt-run venafi.request ''' - if password is not None: - if password.startswith('sdb://'): - password = __salt__['sdb.get'](password) - - if zone_id is None: - zone_id = __opts__.get('venafi', {}).get('zone_id') - if zone_id is None and zone is not None: - zone_id = get_zone_id(zone) + if zone is None: + log.error(msg=str("Missing zone parameter")) + sys.exit(1) - if zone_id is None: - raise CommandExecutionError( - 'Either a zone or a zone_id must be passed in or ' - 'configured in the master file. This id can be retreived using ' - 'venafi.show_company ' - ) + if key_password is not None: + if key_password.startswith('sdb://'): + key_password = __salt__['sdb.get'](key_password) + conn = _init_connection() - private_key = gen_key(minion_id, dns_name, zone, password) - - csr = gen_csr( - minion_id, - dns_name, - zone=zone, - country=country, - state=state, - loc=loc, - org=org, - org_unit=org_unit, - password=password, - ) - - pdata = salt.utils.json.dumps({ - 'zoneId': zone_id, - 'certificateSigningRequest': csr, - }) - - qdata = __utils__['http.query']( - '{0}/certificaterequests'.format(_base_url()), - method='POST', - data=pdata, - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - 'Content-Type': 'application/json', - }, - ) - - request_id = qdata['dict']['certificateRequests'][0]['id'] - ret = { - 'request_id': request_id, - 'private_key': private_key, - 'csr': csr, - 'zone': zone, - } + if csr_path is not None: + log.info("Will use generated CSR from %s", csr_path) + log.info("Using CN %s", dns_name) + try: + with salt.utils.files.fopen(csr_path) as csr_file: + csr = csr_file.read() + request = CertificateRequest(csr=csr, common_name=dns_name) + except Exception as e: + log.error(msg=str(e)) + sys.exit(1) + else: + request = CertificateRequest(common_name=dns_name, country=country, province=state, locality=loc, + organization=org, organizational_unit=org_unit, key_password=key_password) + zone_config = conn.read_zone_conf(zone) + request.update_from_zone_config(zone_config) + conn.request_cert(request, zone) + if csr_path is None: + private_key = request.private_key_pem + else: + private_key = None + while True: + time.sleep(5) + cert = conn.retrieve_cert(request) + if cert: + break - bank = 'venafi/domains' cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) - data = cache.fetch(bank, dns_name) - if data is None: - data = {} - data.update({ + data = { 'minion_id': minion_id, - 'request_id': request_id, - 'private_key': private_key, - 'zone': zone, - 'csr': csr, - }) - cache.store(bank, dns_name, data) - _id_map(minion_id, dns_name) - - return ret + 'cert': cert.cert, + 'chain': cert.chain, + 'pkey': private_key + } + cache.store(CACHE_BANK_NAME, dns_name, data) + return cert.cert, private_key # Request and renew are the same, so far as this module is concerned @@ -367,263 +156,33 @@ def request( def _id_map(minion_id, dns_name): ''' - Maintain a relationship between a minion and a dns name + Maintain a relationship between a minion and a DNS name ''' - bank = 'venafi/minions' + cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) - dns_names = cache.fetch(bank, minion_id) + dns_names = cache.fetch(CACHE_BANK_NAME, minion_id) if not isinstance(dns_names, list): dns_names = [] if dns_name not in dns_names: dns_names.append(dns_name) - cache.store(bank, minion_id, dns_names) - - -def register(email): - ''' - Register a new user account - - CLI Example: - - .. code-block:: bash - - salt-run venafi.register email@example.com - ''' - data = __utils__['http.query']( - '{0}/useraccounts'.format(_base_url()), - method='POST', - data=salt.utils.json.dumps({ - 'username': email, - 'userAccountType': 'API', - }), - status=True, - decode=True, - decode_type='json', - header_dict={ - 'Content-Type': 'application/json', - }, - ) - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - return data.get('dict', {}) - - -def show_company(domain): - ''' - Show company information, especially the company id - - CLI Example: + cache.store(CACHE_BANK_NAME, minion_id, dns_names) - .. code-block:: bash - - salt-run venafi.show_company example.com - ''' - data = __utils__['http.query']( - '{0}/companies/domain/{1}'.format(_base_url(), domain), - status=True, - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - }, - ) - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - return data.get('dict', {}) - - -def show_csrs(): - ''' - Show certificate requests for this API key - - CLI Example: - - .. code-block:: bash - salt-run venafi.show_csrs +def show_cert(dns_name): ''' - data = __utils__['http.query']( - '{0}/certificaterequests'.format(_base_url()), - status=True, - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - }, - ) - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - return data.get('dict', {}) - - -def get_zone_id(zone_name): - ''' - Get the zone ID for the given zone name + Show issued certificate for domain CLI Example: .. code-block:: bash - salt-run venafi.get_zone_id default + salt-run venafi.show_cert example.com ''' - data = __utils__['http.query']( - '{0}/zones/tag/{1}'.format(_base_url(), zone_name), - status=True, - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - }, - ) - - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - return data['dict']['id'] - - -def show_policies(): - ''' - Show zone details for the API key owner's company - - CLI Example: - - .. code-block:: bash - - salt-run venafi.show_zones - ''' - data = __utils__['http.query']( - '{0}/certificatepolicies'.format(_base_url()), - status=True, - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - }, - ) - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - return data['dict'] - - -def show_zones(): - ''' - Show zone details for the API key owner's company - - CLI Example: - - .. code-block:: bash - - salt-run venafi.show_zones - ''' - data = __utils__['http.query']( - '{0}/zones'.format(_base_url()), - status=True, - decode=True, - decode_type='json', - header_dict={ - 'tppl-api-key': _api_key(), - }, - ) - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - return data['dict'] - - -def show_cert(id_): - ''' - Show certificate requests for this API key - - CLI Example: - - .. code-block:: bash - - salt-run venafi.show_cert 01234567-89ab-cdef-0123-456789abcdef - ''' - data = __utils__['http.query']( - '{0}/certificaterequests/{1}/certificate'.format(_base_url(), id_), - params={ - 'format': 'PEM', - 'chainOrder': 'ROOT_FIRST' - }, - status=True, - text=True, - header_dict={'tppl-api-key': _api_key()}, - ) - status = data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(data['error']) - ) - data = data.get('body', '') - csr_data = __utils__['http.query']( - '{0}/certificaterequests/{1}'.format(_base_url(), id_), - status=True, - decode=True, - decode_type='json', - header_dict={'tppl-api-key': _api_key()}, - ) - status = csr_data['status'] - if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): - raise CommandExecutionError( - 'There was an API error: {0}'.format(csr_data['error']) - ) - csr_data = csr_data.get('dict', {}) - certs = _parse_certs(data) - dns_name = '' - for item in csr_data['certificateName'].split(','): - if item.startswith('cn='): - dns_name = item.split('=')[1] - #certs['CSR Data'] = csr_data cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) - domain_data = cache.fetch('venafi/domains', dns_name) - if domain_data is None: - domain_data = {} - certs['private_key'] = domain_data.get('private_key') - domain_data.update(certs) - cache.store('venafi/domains', dns_name, domain_data) - - certs['request_id'] = id_ - return certs - - -pickup = show_cert - - -def show_rsa(minion_id, dns_name): - ''' - Show a private RSA key - - CLI Example: - - .. code-block:: bash - - salt-run venafi.show_rsa myminion domain.example.com - ''' - cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) - bank = 'venafi/domains' - data = cache.fetch( - bank, dns_name - ) - return data['private_key'] + domain_data = cache.fetch(CACHE_BANK_NAME, dns_name) or {} + cert = domain_data.get('cert') + return cert def list_domain_cache(): @@ -662,51 +221,8 @@ def del_cached_domain(domains): failed = [] for domain in domains: try: - cache.flush('venafi/domains', domain) + cache.flush(CACHE_BANK_NAME, domain) success.append(domain) except CommandExecutionError: failed.append(domain) - return {'Succeeded': success, 'Failed': failed} - - -def _parse_certs(data): - cert_mode = False - cert = '' - certs = [] - rsa_key = '' - for line in data.splitlines(): - if not line.strip(): - continue - if 'Successfully posted request' in line: - comps = line.split(' for ') - request_id = comps[-1].strip() - continue - if 'END CERTIFICATE' in line or 'END RSA private_key' in line: - if 'RSA' in line: - rsa_key = rsa_key + line - else: - cert = cert + line - certs.append(cert) - cert_mode = False - continue - if 'BEGIN CERTIFICATE' in line or 'BEGIN RSA private_key' in line: - if 'RSA' in line: - rsa_key = line + '\n' - else: - cert = line + '\n' - cert_mode = True - continue - if cert_mode is True: - cert = cert + line + '\n' - continue - - rcert = certs.pop(0) - eecert = certs.pop(-1) - ret = { - 'end_entity_certificate': eecert, - 'private_key': rsa_key, - 'root_certificate': rcert, - 'intermediate_certificates': certs - } - - return ret + return {'Succeeded': success, 'Failed': failed} \ No newline at end of file diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index ee1a9edd495d..ea40dd5054bf 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -4,14 +4,23 @@ ''' # Import Python libs from __future__ import absolute_import +from __future__ import print_function import functools import random import string # Import Salt Testing libs from tests.support.case import ShellCase -from tests.support.helpers import destructiveTest, expensiveTest from salt.ext.six.moves import range +from salt.utils.files import fopen +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import serialization +import pytest +import tempfile +from os import path +from os import environ def _random_name(prefix=''): @@ -25,70 +34,123 @@ def with_random_name(func): ''' generate a randomized name for a container ''' + @functools.wraps(func) def wrapper(self, *args, **kwargs): name = _random_name(prefix='salt_') - return func(self, _random_name(prefix='salt_test_'), *args, **kwargs) + return func(self, _random_name(prefix='salt-test-'), *args, **kwargs) + return wrapper -@destructiveTest -@expensiveTest +# @destructiveTest +# @expensiveTest class VenafiTest(ShellCase): ''' Test the venafi runner ''' @with_random_name - def test_gen_key_password(self, name): - ''' - venafi.gen_key - ''' - ret = self.run_run_plus(fun='venafi.gen_key', - minion_id='{0}.test.saltstack.com'.format(name), - dns_name='{0}.test.saltstack.com'.format(name), - zone='Internet', - password='SecretSauce') - self.assertEqual(ret['out'][0], '-----BEGIN RSA PRIVATE KEY-----') - self.assertEqual(ret['out'][1], 'Proc-Type: 4,ENCRYPTED') - self.assertEqual(ret['out'][-1], '-----END RSA PRIVATE KEY-----') + def test_request(self, name): + print("Testing Venafi request cert") + print("Using venafi config:", self.master_opts['venafi']) + cn = '{0}.example.com'.format(name) + ret = self.run_run_plus(fun='venafi.request', + minion_id=cn, + dns_name=cn, + key_password='secretPassword', + zone=environ.get('CLOUDZONE')) + print("Ret is:\n", ret) + cert_output = ret['return'][0] + if not cert_output: + pytest.fail('venafi_certificate not found in output_value') - @with_random_name - def test_gen_key_without_password(self, name): - ''' - venafi.gen_key - ''' - ret = self.run_run_plus(fun='venafi.gen_key', - minion_id='{0}.test.saltstack.com'.format(name), - dns_name='{0}.test.saltstack.com'.format(name), - zone='Internet') - self.assertEqual(ret['out'][0], '-----BEGIN RSA PRIVATE KEY-----') - self.assertNotEqual(ret['out'][1], 'Proc-Type: 4,ENCRYPTED') - self.assertEqual(ret['out'][-1], '-----END RSA PRIVATE KEY-----') + print("Testing certificate:\n", cert_output) + cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) + assert isinstance(cert, x509.Certificate) + assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ + x509.NameAttribute( + NameOID.COMMON_NAME, cn + ) + ] - @with_random_name - def test_gen_csr(self, name): - ''' - venafi.gen_csr - ''' - ret = self.run_run_plus(fun='venafi.gen_csr', - minion_id='{0}.test.saltstack.com'.format(name), - dns_name='{0}.test.saltstack.com'.format(name), - country='US', state='Utah', loc='Salt Lake City', - org='Salt Stack Inc.', org_unit='Testing', - zone='Internet', password='SecretSauce') - self.assertEqual(ret['out'][0], '-----BEGIN CERTIFICATE REQUEST-----') - self.assertEqual(ret['out'][-1], '-----END CERTIFICATE REQUEST-----') + pkey_output = ret['return'][1] + print("Testing pkey:\n", pkey_output) + if not pkey_output: + pytest.fail('venafi_private key not found in output_value') + + pkey = serialization.load_pem_private_key(pkey_output.encode(), password=b'secretPassword', + backend=default_backend()) + + pkey_public_key_pem = pkey.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + cert_public_key_pem = cert.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + assert pkey_public_key_pem == cert_public_key_pem @with_random_name - def test_request(self, name): - ''' - venafi.request - ''' + def test_sign(self, name): + print("Testing Venafi sign CSR") + + csr_pem = """-----BEGIN CERTIFICATE REQUEST----- +MIIFbDCCA1QCAQAwgbQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRIwEAYD +VQQHDAlTYWx0IExha2UxFDASBgNVBAoMC1ZlbmFmaSBJbmMuMRQwEgYDVQQLDAtJ +bnRlZ3JhdGlvbjEnMCUGCSqGSIb3DQEJARYYZW1haWxAdmVuYWZpLmV4YW1wbGUu +Y29tMS0wKwYDVQQDDCR0ZXN0LWNzci0zMjMxMzEzMS52ZW5hZmkuZXhhbXBsZS5j +b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC4T0bdjq+mF+DABhF+ +XWCwOXXUWbPNWa72VVhxoelbyTS0iIeZEe64AvNGykytFdOuT/F9pdkZa+Io07R1 +ZMp6Ak8dp2Wjt4c5rayVZus6ZK+0ZwBRJO7if/cqhEpxy8Wz1RMfVLf2AE1u/xZS +QSYY0BTRWGmPqrFJrIGbnyQfvmGVPk3cA0RfdrwYJZXtZ2/4QNrbNCoSoSmqTHzt +NAtZhvT2dPU9U48Prx4b2460x+ck3xA1OdJNXV7n5u53QbxOIcjdGT0lJ62ml70G +5gvEHmdPcg+t5cw/Sm5cfDSUEDtNEXvD4oJXfP98ty6f1cYsZpcrgxRwk9RfGain +hvoweXhZP3NWnU5nRdn2nOfExv+xMeQOyB/rYv98zqzK6LvwKhwI5UB1l/n9KTpg +jgaNCP4x/KAsrPecbHK91oiqGSbPn4wtTYOmPkDxSzATN317u7fE20iqvVAUy/O+ +7SCNNKEDPX2NP9LLz0IPK0roQxLiwd2CVyN6kEXuzs/3psptkNRMSlhyeAZdfrOE +CNOp46Pam9f9HGBqzXxxoIlfzLqHHL584kgFlBm7qmivVrgp6zdLPDa+UayXEl2N +O17SnGS8nkOTmfg3cez7lzX/LPLO9X/Y1xKYqx5hoGZhh754K8mzDWCVCYThWgou +yBOYY8uNXiX6ldqzQUHpbxxQgwIDAQABoHIwcAYJKoZIhvcNAQkOMWMwYTBfBgNV +HREEWDBWgilhbHQxLXRlc3QtY3NyLTMyMzEzMTMxLnZlbmFmaS5leGFtcGxlLmNv +bYIpYWx0Mi10ZXN0LWNzci0zMjMxMzEzMS52ZW5hZmkuZXhhbXBsZS5jb20wDQYJ +KoZIhvcNAQELBQADggIBAJd87BIdeh0WWoyQ4IX+ENpNqmm/sLmdfmUB/hj9NpBL +qbr2UTWaSr1jadoZ+mrDxtm1Z0YJDTTIrEWxkBOW5wQ039lYZNe2tfDXSJZwJn7u +2keaXtWQ2SdduK1wOPDO9Hra6WnH7aEq5D1AyoghvPsZwTqZkNynt/A1BZW5C/ha +J9/mwgWfL4qXBGBOhLwKN5GUo3erUkJIdH0TlMqI906D/c/YAuJ86SRdQtBYci6X +bJ7C+OnoiV6USn1HtQE6dfOMeS8voJuixpSIvHZ/Aim6kSAN1Za1f6FQAkyqbF+o +oKTJHDS1CPWikCeLdpPUcOCDIbsiISTsMZkEvIkzZ7dKBIlIugauxw3vaEpk47jN +Wq09r639RbSv/Qs8D6uY66m1IpL4zHm4lTAknrjM/BqihPxc8YiN76ssajvQ4SFT +DHPrDweEVe4KL1ENw8nv4wdkIFKwJTDarV5ZygbETzIhfa2JSBZFTdN+Wmd2Mh5h +OTu+vuHrJF2TO8g1G48EB/KWGt+yvVUpWAanRMwldnFX80NcUlM7GzNn6IXTeE+j +BttIbvAAVJPG8rVCP8u3DdOf+vgm5macj9oLoVP8RBYo/z0E3e+H50nXv3uS6JhN +xlAKgaU6i03jOm5+sww5L2YVMi1eeBN+kx7o94ogpRemC/EUidvl1PUJ6+e7an9V +-----END CERTIFICATE REQUEST----- + """ + + tmp_dir = tempfile.gettempdir() + with fopen(path.join(tmp_dir, 'venafi-temp-test-csr.pem'), 'w+') as f: + print("Saving test CSR to temp file", f.name) + f.write(csr_pem) + csr_path = f.name + + print("Using venafi config:", self.master_opts['venafi']) + cn = "test-csr-32313131.venafi.example.com" ret = self.run_run_plus(fun='venafi.request', - minion_id='{0}.example.com'.format(name), - dns_name='{0}.example.com'.format(name), - country='US', state='Utah', loc='Salt Lake City', - org='Salt Stack Inc.', org_unit='Testing', - zone='Internet', password='SecretSauce') - self.assertTrue('request_id' in ret['return']) + minion_id=cn, + csr_path=csr_path, + zone=environ.get('CLOUDZONE')) + print("Ret is:\n", ret) + cert_output = ret['return'][0] + if not cert_output: + pytest.fail('venafi_certificate not found in output_value') + + print("Testing certificate:\n", cert_output) + cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) + assert isinstance(cert, x509.Certificate) + assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ + x509.NameAttribute( + NameOID.COMMON_NAME, cn + ) + ] \ No newline at end of file diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 1b639a522d63..d7730a4f22ac 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -128,3 +128,8 @@ peer_run: - vault.generate_token sdbvault: driver: vault +venafi: + #For Venafi Cloud + #api_key: "sdb://osenv/CLOUDAPIKEY" + #base_url: "sdb://osenv/CLOUDURL" + fake: "true" \ No newline at end of file From d6c85654d0f092e5990a54c8dc7f98e476d7d112 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 13 Nov 2019 18:10:34 +0300 Subject: [PATCH 02/33] Adding new line to the end of file --- salt/pillar/venafi.py | 2 +- salt/runners/venafiapi.py | 2 +- tests/integration/externalapi/test_venafiapi.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/pillar/venafi.py b/salt/pillar/venafi.py index baefd2a01cfe..74cb89514c46 100644 --- a/salt/pillar/venafi.py +++ b/salt/pillar/venafi.py @@ -40,4 +40,4 @@ def ext_pillar(minion_id, pillar, conf): data = cache.fetch('venafi/domains', dns_name) if data['minion_id'] == minion_id: ret[dns_name] = data - return {'venafi': ret} \ No newline at end of file + return {'venafi': ret} diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 8b05e06d1fcb..87efd75c3188 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -225,4 +225,4 @@ def del_cached_domain(domains): success.append(domain) except CommandExecutionError: failed.append(domain) - return {'Succeeded': success, 'Failed': failed} \ No newline at end of file + return {'Succeeded': success, 'Failed': failed} diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index ea40dd5054bf..a2e4545d4b22 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -153,4 +153,4 @@ def test_sign(self, name): x509.NameAttribute( NameOID.COMMON_NAME, cn ) - ] \ No newline at end of file + ] From 570183a9a0c515ff14cbdad815fd3ad1ef0a07b2 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Tue, 19 Nov 2019 16:29:40 +0300 Subject: [PATCH 03/33] Updating dependencies and removing commented @destructiveTest --- requirements/static/py2.7/darwin.txt | 2 ++ requirements/static/py3.5/darwin.txt | 2 +- requirements/static/py3.5/linux.txt | 34 ++++--------------- requirements/static/py3.6/darwin.txt | 2 +- requirements/static/py3.6/linux.txt | 34 ++++--------------- requirements/static/py3.7/darwin.txt | 2 +- requirements/static/py3.7/linux.txt | 34 ++++--------------- .../integration/externalapi/test_venafiapi.py | 2 -- 8 files changed, 26 insertions(+), 86 deletions(-) diff --git a/requirements/static/py2.7/darwin.txt b/requirements/static/py2.7/darwin.txt index 59a9f5f0efea..cb5c8ad2870d 100644 --- a/requirements/static/py2.7/darwin.txt +++ b/requirements/static/py2.7/darwin.txt @@ -138,3 +138,5 @@ yamlordereddictloader==0.4.0 zc.lockfile==1.4 # via cherrypy # Passthrough dependencies from pkg/osx/req.txt pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 diff --git a/requirements/static/py3.5/darwin.txt b/requirements/static/py3.5/darwin.txt index a065e89c0cc0..c3049c3b80d1 100644 --- a/requirements/static/py3.5/darwin.txt +++ b/requirements/static/py3.5/darwin.txt @@ -10,9 +10,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 backports.functools-lru-cache==1.5 # via cheroot backports.ssl_match_hostname==3.7.0.1 +backports_abc==0.5 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 diff --git a/requirements/static/py3.5/linux.txt b/requirements/static/py3.5/linux.txt index 8d0aff6a132b..6d94115eee79 100644 --- a/requirements/static/py3.5/linux.txt +++ b/requirements/static/py3.5/linux.txt @@ -6,27 +6,24 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via websocket-client bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 -cffi==1.12.2 +certifi==2019.9.11 +cffi==1.13.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.8 # via moto, paramiko, pyopenssl dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -84,7 +81,7 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -92,7 +89,7 @@ pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==5.1.2 pyzmq==18.0.1 ; python_version != "3.4" -requests==2.21.0 +requests==2.22.0 responses==0.10.6 # via moto rfc3987==1.3.8 rsa==4.0 # via google-auth @@ -101,14 +98,13 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" -urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +urllib3==1.24.3 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 watchdog==0.9.0 websocket-client==0.40.0 # via docker, kubernetes @@ -116,19 +112,3 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy - -# venafi -asn1crypto==1.2.0 # via certvalidator, oscrypto -certifi==2019.9.11 # via requests -certvalidator==0.11.1 # via vcert -cffi==1.13.2 # via cryptography -chardet==3.0.4 # via requests -cryptography==2.8 # via vcert -idna==2.8 # via requests -oscrypto==1.1.0 # via certvalidator -pycparser==2.19 # via cffi -python-dateutil==2.8.1 # via vcert -requests==2.22.0 # via vcert -six==1.13.0 # via cryptography, python-dateutil, vcert -urllib3==1.25.7 # via requests -vcert==0.6.8 \ No newline at end of file diff --git a/requirements/static/py3.6/darwin.txt b/requirements/static/py3.6/darwin.txt index 38979afc3bc9..61b08e5321ec 100644 --- a/requirements/static/py3.6/darwin.txt +++ b/requirements/static/py3.6/darwin.txt @@ -10,9 +10,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 backports.functools-lru-cache==1.5 # via cheroot backports.ssl_match_hostname==3.7.0.1 +backports_abc==0.5 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 diff --git a/requirements/static/py3.6/linux.txt b/requirements/static/py3.6/linux.txt index d84f24cc69ab..627f7bd0fa59 100644 --- a/requirements/static/py3.6/linux.txt +++ b/requirements/static/py3.6/linux.txt @@ -6,27 +6,24 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via websocket-client bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 -cffi==1.12.2 +certifi==2019.9.11 +cffi==1.13.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.8 # via moto, paramiko, pyopenssl dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -84,7 +81,7 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -92,7 +89,7 @@ pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==5.1.2 pyzmq==18.0.1 ; python_version != "3.4" -requests==2.21.0 +requests==2.22.0 responses==0.10.6 # via moto rfc3987==1.3.8 rsa==4.0 # via google-auth @@ -101,14 +98,13 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" -urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +urllib3==1.24.3 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 watchdog==0.9.0 websocket-client==0.40.0 # via docker, kubernetes @@ -116,19 +112,3 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy - -# venafi -asn1crypto==1.2.0 # via certvalidator, oscrypto -certifi==2019.9.11 # via requests -certvalidator==0.11.1 # via vcert -cffi==1.13.2 # via cryptography -chardet==3.0.4 # via requests -cryptography==2.8 # via vcert -idna==2.8 # via requests -oscrypto==1.1.0 # via certvalidator -pycparser==2.19 # via cffi -python-dateutil==2.8.1 # via vcert -requests==2.22.0 # via vcert -six==1.13.0 # via cryptography, python-dateutil, vcert -urllib3==1.25.7 # via requests -vcert==0.6.8 \ No newline at end of file diff --git a/requirements/static/py3.7/darwin.txt b/requirements/static/py3.7/darwin.txt index 401861d0d349..d1d068ff0356 100644 --- a/requirements/static/py3.7/darwin.txt +++ b/requirements/static/py3.7/darwin.txt @@ -10,9 +10,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 backports.functools-lru-cache==1.5 # via cheroot backports.ssl_match_hostname==3.7.0.1 +backports_abc==0.5 bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 diff --git a/requirements/static/py3.7/linux.txt b/requirements/static/py3.7/linux.txt index b6339f43a19b..f590477b3458 100644 --- a/requirements/static/py3.7/linux.txt +++ b/requirements/static/py3.7/linux.txt @@ -6,27 +6,24 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via websocket-client bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth -certifi==2019.3.9 -cffi==1.12.2 +certifi==2019.9.11 +cffi==1.13.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.8 # via moto, paramiko, pyopenssl dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -84,7 +81,7 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -92,7 +89,7 @@ pytz==2019.1 # via moto, tempora pyvmomi==6.7.1.2018.12 pyyaml==5.1.2 pyzmq==18.0.1 ; python_version != "3.4" -requests==2.21.0 +requests==2.22.0 responses==0.10.6 # via moto rfc3987==1.3.8 rsa==4.0 # via google-auth @@ -101,14 +98,13 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" -urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +urllib3==1.24.3 # via botocore, kubernetes, python-etcd, requests virtualenv==16.4.3 watchdog==0.9.0 websocket-client==0.40.0 # via docker, kubernetes @@ -116,19 +112,3 @@ werkzeug==0.15.6 # via moto wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zc.lockfile==1.4 # via cherrypy - -# venafi -asn1crypto==1.2.0 # via certvalidator, oscrypto -certifi==2019.9.11 # via requests -certvalidator==0.11.1 # via vcert -cffi==1.13.2 # via cryptography -chardet==3.0.4 # via requests -cryptography==2.8 # via vcert -idna==2.8 # via requests -oscrypto==1.1.0 # via certvalidator -pycparser==2.19 # via cffi -python-dateutil==2.8.1 # via vcert -requests==2.22.0 # via vcert -six==1.13.0 # via cryptography, python-dateutil, vcert -urllib3==1.25.7 # via requests -vcert==0.6.8 \ No newline at end of file diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index a2e4545d4b22..0839acad0102 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -43,8 +43,6 @@ def wrapper(self, *args, **kwargs): return wrapper -# @destructiveTest -# @expensiveTest class VenafiTest(ShellCase): ''' Test the venafi runner From 4e713a33605d94f5f92517bd30d354d3ed56f2fe Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 20 Nov 2019 18:02:29 +0300 Subject: [PATCH 04/33] Updating code --- salt/runners/venafiapi.py | 33 ++++++++++--------- .../integration/externalapi/test_venafiapi.py | 18 +++++----- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 87efd75c3188..dedc605d5256 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -77,9 +77,9 @@ def __virtual__(): ''' Only load the module if vcert module is installed ''' - if not HAS_VCERT: - return False - return __virtualname__ + if HAS_VCERT: + return __virtualname___ + return False def request( @@ -113,9 +113,16 @@ def request( key_password = __salt__['sdb.get'](key_password) conn = _init_connection() - if csr_path is not None: + if csr_path is None: + request = CertificateRequest(common_name=dns_name, country=country, province=state, locality=loc, + organization=org, organizational_unit=org_unit, key_password=key_password) + zone_config = conn.read_zone_conf(zone) + request.update_from_zone_config(zone_config) + private_key = request.private_key_pem + else: log.info("Will use generated CSR from %s", csr_path) log.info("Using CN %s", dns_name) + private_key = None try: with salt.utils.files.fopen(csr_path) as csr_file: csr = csr_file.read() @@ -123,21 +130,17 @@ def request( except Exception as e: log.error(msg=str(e)) sys.exit(1) - else: - request = CertificateRequest(common_name=dns_name, country=country, province=state, locality=loc, - organization=org, organizational_unit=org_unit, key_password=key_password) - zone_config = conn.read_zone_conf(zone) - request.update_from_zone_config(zone_config) conn.request_cert(request, zone) - if csr_path is None: - private_key = request.private_key_pem - else: - private_key = None - while True: - time.sleep(5) + + + #TODO: add timeout parameter here + t = time.time() + 300 + while time.time() < t: cert = conn.retrieve_cert(request) if cert: break + else: + time.sleep(5) cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) data = { diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 0839acad0102..792f6ecf38bc 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -8,6 +8,7 @@ import functools import random import string +import logging # Import Salt Testing libs from tests.support.case import ShellCase @@ -22,6 +23,7 @@ from os import path from os import environ +log = logging.getLogger(__name__) def _random_name(prefix=''): ret = prefix @@ -50,20 +52,20 @@ class VenafiTest(ShellCase): @with_random_name def test_request(self, name): - print("Testing Venafi request cert") - print("Using venafi config:", self.master_opts['venafi']) + log.info("Testing Venafi request cert") + log.info("Using venafi config:", self.master_opts['venafi']) cn = '{0}.example.com'.format(name) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, dns_name=cn, key_password='secretPassword', zone=environ.get('CLOUDZONE')) - print("Ret is:\n", ret) + log.info("Ret is:\n", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') - print("Testing certificate:\n", cert_output) + log.info("Testing certificate:\n", cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ @@ -73,7 +75,7 @@ def test_request(self, name): ] pkey_output = ret['return'][1] - print("Testing pkey:\n", pkey_output) + log.info("Testing pkey:\n", pkey_output) if not pkey_output: pytest.fail('venafi_private key not found in output_value') @@ -92,7 +94,7 @@ def test_request(self, name): @with_random_name def test_sign(self, name): - print("Testing Venafi sign CSR") + log.info("Testing Venafi sign CSR") csr_pem = """-----BEGIN CERTIFICATE REQUEST----- MIIFbDCCA1QCAQAwgbQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRIwEAYD @@ -129,22 +131,18 @@ def test_sign(self, name): tmp_dir = tempfile.gettempdir() with fopen(path.join(tmp_dir, 'venafi-temp-test-csr.pem'), 'w+') as f: - print("Saving test CSR to temp file", f.name) f.write(csr_pem) csr_path = f.name - print("Using venafi config:", self.master_opts['venafi']) cn = "test-csr-32313131.venafi.example.com" ret = self.run_run_plus(fun='venafi.request', minion_id=cn, csr_path=csr_path, zone=environ.get('CLOUDZONE')) - print("Ret is:\n", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') - print("Testing certificate:\n", cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ From acc6b21573d672b20504dc15c52762f89c59b572 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 20 Nov 2019 19:24:42 +0300 Subject: [PATCH 05/33] Updating code --- salt/runners/venafiapi.py | 2 +- .../integration/externalapi/test_venafiapi.py | 44 ++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index dedc605d5256..4fb51053b815 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -78,7 +78,7 @@ def __virtual__(): Only load the module if vcert module is installed ''' if HAS_VCERT: - return __virtualname___ + return __virtualname__ return False diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 792f6ecf38bc..4d1daec60f33 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -13,14 +13,12 @@ # Import Salt Testing libs from tests.support.case import ShellCase from salt.ext.six.moves import range -from salt.utils.files import fopen from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import serialization import pytest import tempfile -from os import path from os import environ log = logging.getLogger(__name__) @@ -129,24 +127,28 @@ def test_sign(self, name): -----END CERTIFICATE REQUEST----- """ - tmp_dir = tempfile.gettempdir() - with fopen(path.join(tmp_dir, 'venafi-temp-test-csr.pem'), 'w+') as f: + # tmp_dir = tempfile.gettempdir() + # with fopen(path.join(tmp_dir, 'venafi-temp-test-csr.pem'), 'w+') as f: + # f.write(csr_pem) + # csr_path = f.name + + with tempfile.NamedTemporaryFile('w+') as f: f.write(csr_pem) + f.flush() csr_path = f.name - - cn = "test-csr-32313131.venafi.example.com" - ret = self.run_run_plus(fun='venafi.request', - minion_id=cn, - csr_path=csr_path, - zone=environ.get('CLOUDZONE')) - cert_output = ret['return'][0] - if not cert_output: - pytest.fail('venafi_certificate not found in output_value') - - cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) - assert isinstance(cert, x509.Certificate) - assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ - x509.NameAttribute( - NameOID.COMMON_NAME, cn - ) - ] + cn = "test-csr-32313131.venafi.example.com" + ret = self.run_run_plus(fun='venafi.request', + minion_id=cn, + csr_path=csr_path, + zone=environ.get('CLOUDZONE')) + cert_output = ret['return'][0] + if not cert_output: + pytest.fail('venafi_certificate not found in output_value') + + cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) + assert isinstance(cert, x509.Certificate) + assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ + x509.NameAttribute( + NameOID.COMMON_NAME, cn + ) + ] From b1f904fce011e73eefc6588123d9809c93960200 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Thu, 21 Nov 2019 23:03:54 +0300 Subject: [PATCH 06/33] Update vcert, fixing private key recieving which was changed. --- requirements/static/py2.7/linux.txt | 2 +- requirements/static/py3.4/linux.txt | 2 +- salt/runners/venafiapi.py | 3 +-- tests/integration/externalapi/test_venafiapi.py | 9 ++++++--- tests/integration/files/conf/master | 12 ++++++++---- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/requirements/static/py2.7/linux.txt b/requirements/static/py2.7/linux.txt index 5bc465427019..c4082ebf3990 100644 --- a/requirements/static/py2.7/linux.txt +++ b/requirements/static/py2.7/linux.txt @@ -142,4 +142,4 @@ python-dateutil==2.8.1 # via vcert requests==2.22.0 # via vcert six==1.13.0 # via cryptography, python-dateutil, vcert urllib3==1.25.7 # via requests -vcert==0.6.8 \ No newline at end of file +vcert==0.7.0 \ No newline at end of file diff --git a/requirements/static/py3.4/linux.txt b/requirements/static/py3.4/linux.txt index 9c6f59dae1e3..d06160d33b84 100644 --- a/requirements/static/py3.4/linux.txt +++ b/requirements/static/py3.4/linux.txt @@ -133,4 +133,4 @@ python-dateutil==2.8.1 # via vcert requests==2.22.0 # via vcert six==1.13.0 # via cryptography, python-dateutil, vcert urllib3==1.25.7 # via requests -vcert==0.6.8 \ No newline at end of file +vcert==0.7.0 \ No newline at end of file diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 4fb51053b815..93d24603e4ff 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -118,11 +118,9 @@ def request( organization=org, organizational_unit=org_unit, key_password=key_password) zone_config = conn.read_zone_conf(zone) request.update_from_zone_config(zone_config) - private_key = request.private_key_pem else: log.info("Will use generated CSR from %s", csr_path) log.info("Using CN %s", dns_name) - private_key = None try: with salt.utils.files.fopen(csr_path) as csr_file: csr = csr_file.read() @@ -142,6 +140,7 @@ def request( else: time.sleep(5) + private_key = request.private_key_pem cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) data = { 'minion_id': minion_id, diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 4d1daec60f33..7e6f074f7bb2 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -21,6 +21,7 @@ import tempfile from os import environ +import logging log = logging.getLogger(__name__) def _random_name(prefix=''): @@ -51,7 +52,7 @@ class VenafiTest(ShellCase): @with_random_name def test_request(self, name): log.info("Testing Venafi request cert") - log.info("Using venafi config:", self.master_opts['venafi']) + log.info("Using venafi config: %s", self.master_opts['venafi']) cn = '{0}.example.com'.format(name) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, @@ -59,11 +60,13 @@ def test_request(self, name): key_password='secretPassword', zone=environ.get('CLOUDZONE')) log.info("Ret is:\n", ret) + print("Ret is:\n", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') - log.info("Testing certificate:\n", cert_output) + log.info("Testing certificate:\n %s", cert_output) + print("Testing certificate:\n %s", cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ @@ -73,7 +76,7 @@ def test_request(self, name): ] pkey_output = ret['return'][1] - log.info("Testing pkey:\n", pkey_output) + log.info("Testing pkey:\n %s", pkey_output) if not pkey_output: pytest.fail('venafi_private key not found in output_value') diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index d7730a4f22ac..45e3d598dcb9 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -129,7 +129,11 @@ peer_run: sdbvault: driver: vault venafi: - #For Venafi Cloud - #api_key: "sdb://osenv/CLOUDAPIKEY" - #base_url: "sdb://osenv/CLOUDURL" - fake: "true" \ No newline at end of file + #For TPP + #base_url: "sdb://osenv/TPPURL" + #tpp_user: "sdb://osenv/TPPUSER" + #tpp_password: "sdb://osenv/TPPPASSWORD" + base_url: "https://ha-tpp1.sqlha.com:5008/vedsdk" + tpp_user: "admin" + tpp_password: "newPassw0rd!" + trust_bundle: "/opt/venafi/bundle.pem" \ No newline at end of file From 7e990c0e3988e79fd09b09132d685d9cfa7ce44e Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 22 Nov 2019 14:58:34 +0300 Subject: [PATCH 07/33] Fixing code for vcert refactoring --- salt/runners/venafiapi.py | 16 ++++++++++++++-- tests/integration/externalapi/test_venafiapi.py | 15 +++++++-------- tests/integration/files/conf/master | 9 +-------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 93d24603e4ff..f3cdc2cbcf1d 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -58,7 +58,6 @@ def _init_connection(): base_url = __opts__.get('venafi', {}).get('base_url', '') log.info("Using base_url: %s", base_url) tpp_user = __opts__.get('venafi', {}).get('tpp_user', '') - log.info("Using tpp_user: %s", tpp_user) tpp_password = __opts__.get('venafi', {}).get('tpp_password', '') trust_bundle = __opts__.get('venafi', {}).get('trust_bundle', '') fake = __opts__.get('venafi', {}).get('fake', '') @@ -93,6 +92,7 @@ def request( org_unit=None, key_password=None, csr_path=None, + pkey_path=None, ): ''' Request a new certificate @@ -140,7 +140,19 @@ def request( else: time.sleep(5) - private_key = request.private_key_pem + if csr_path is None: + private_key = request.private_key_pem + else: + if pkey_path: + try: + with salt.utils.files.fopen(pkey_path) as pkey_file: + private_key = pkey_file.read() + except Exception as e: + log.error(msg=str(e)) + sys.exit(1) + else: + private_key = None + cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) data = { 'minion_id': minion_id, diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 7e6f074f7bb2..fa6556b783e4 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -4,7 +4,6 @@ ''' # Import Python libs from __future__ import absolute_import -from __future__ import print_function import functools import random import string @@ -51,22 +50,21 @@ class VenafiTest(ShellCase): @with_random_name def test_request(self, name): - log.info("Testing Venafi request cert") - log.info("Using venafi config: %s", self.master_opts['venafi']) + log.debug("Testing Venafi request cert") + log.debug("Using venafi config: %s", self.master_opts['venafi']) cn = '{0}.example.com'.format(name) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, dns_name=cn, key_password='secretPassword', zone=environ.get('CLOUDZONE')) - log.info("Ret is:\n", ret) + log.debug("Ret is:\n", ret) print("Ret is:\n", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') - log.info("Testing certificate:\n %s", cert_output) - print("Testing certificate:\n %s", cert_output) + log.debug("Testing certificate:\n %s", cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ @@ -76,7 +74,7 @@ def test_request(self, name): ] pkey_output = ret['return'][1] - log.info("Testing pkey:\n %s", pkey_output) + log.debug("Testing pkey:\n %s", pkey_output) if not pkey_output: pytest.fail('venafi_private key not found in output_value') @@ -95,7 +93,7 @@ def test_request(self, name): @with_random_name def test_sign(self, name): - log.info("Testing Venafi sign CSR") + log.debug("Testing Venafi sign CSR") csr_pem = """-----BEGIN CERTIFICATE REQUEST----- MIIFbDCCA1QCAQAwgbQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRIwEAYD @@ -144,6 +142,7 @@ def test_sign(self, name): minion_id=cn, csr_path=csr_path, zone=environ.get('CLOUDZONE')) + log.debug("Ret is:\n", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 45e3d598dcb9..25b4378bef51 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -129,11 +129,4 @@ peer_run: sdbvault: driver: vault venafi: - #For TPP - #base_url: "sdb://osenv/TPPURL" - #tpp_user: "sdb://osenv/TPPUSER" - #tpp_password: "sdb://osenv/TPPPASSWORD" - base_url: "https://ha-tpp1.sqlha.com:5008/vedsdk" - tpp_user: "admin" - tpp_password: "newPassw0rd!" - trust_bundle: "/opt/venafi/bundle.pem" \ No newline at end of file + fake: "true" \ No newline at end of file From e3918a76de0f01e6a52991e284bc73ef31b2c373 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 22 Nov 2019 15:16:26 +0300 Subject: [PATCH 08/33] removing print --- tests/integration/externalapi/test_venafiapi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index fa6556b783e4..4866c21f679e 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -59,7 +59,6 @@ def test_request(self, name): key_password='secretPassword', zone=environ.get('CLOUDZONE')) log.debug("Ret is:\n", ret) - print("Ret is:\n", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') From 4277e274345155a59651b07fb9ba2b3ad02cc65d Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 22 Nov 2019 15:17:16 +0300 Subject: [PATCH 09/33] removing commented code --- tests/integration/externalapi/test_venafiapi.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 4866c21f679e..4b0474ff2628 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -127,11 +127,6 @@ def test_sign(self, name): -----END CERTIFICATE REQUEST----- """ - # tmp_dir = tempfile.gettempdir() - # with fopen(path.join(tmp_dir, 'venafi-temp-test-csr.pem'), 'w+') as f: - # f.write(csr_pem) - # csr_path = f.name - with tempfile.NamedTemporaryFile('w+') as f: f.write(csr_pem) f.flush() From d2b04ce61a5a54b9115757ce272c60749149c718 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 22 Nov 2019 15:33:09 +0300 Subject: [PATCH 10/33] Fixing lint --- salt/runners/venafiapi.py | 1 - tests/integration/externalapi/test_venafiapi.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index f3cdc2cbcf1d..bcfed9181f57 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -130,7 +130,6 @@ def request( sys.exit(1) conn.request_cert(request, zone) - #TODO: add timeout parameter here t = time.time() + 300 while time.time() < t: diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 4b0474ff2628..3dc7e18b2b3d 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -23,6 +23,7 @@ import logging log = logging.getLogger(__name__) + def _random_name(prefix=''): ret = prefix for _ in range(8): @@ -58,7 +59,7 @@ def test_request(self, name): dns_name=cn, key_password='secretPassword', zone=environ.get('CLOUDZONE')) - log.debug("Ret is:\n", ret) + log.debug("Ret is:\n %s", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') @@ -136,7 +137,7 @@ def test_sign(self, name): minion_id=cn, csr_path=csr_path, zone=environ.get('CLOUDZONE')) - log.debug("Ret is:\n", ret) + log.debug("Ret is:\n %s", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') From bb93e8e517b29afb3c957b4b5243fdf0ea6f2bcf Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 22 Nov 2019 18:32:45 +0300 Subject: [PATCH 11/33] Running pre-commit run -av --- requirements/static/linux.in | 1 + requirements/static/py2.7/darwin.txt | 6 ++++++ requirements/static/py2.7/linux.txt | 3 +-- requirements/static/py3.4/linux.txt | 3 +-- requirements/static/py3.5/linux.txt | 10 +++++++--- requirements/static/py3.6/linux.txt | 10 +++++++--- requirements/static/py3.7/linux.txt | 10 +++++++--- 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/requirements/static/linux.in b/requirements/static/linux.in index feafe459dc49..7ffab8f5b4ad 100644 --- a/requirements/static/linux.in +++ b/requirements/static/linux.in @@ -41,3 +41,4 @@ timelib tornado<5.0 virtualenv watchdog +vcert==0.7.1 diff --git a/requirements/static/py2.7/darwin.txt b/requirements/static/py2.7/darwin.txt index cb5c8ad2870d..3cddc67b5d5c 100644 --- a/requirements/static/py2.7/darwin.txt +++ b/requirements/static/py2.7/darwin.txt @@ -140,3 +140,9 @@ zc.lockfile==1.4 # via cherrypy pyobjc==5.1.2 # Passthrough dependencies from pkg/osx/req.txt pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 diff --git a/requirements/static/py2.7/linux.txt b/requirements/static/py2.7/linux.txt index c4082ebf3990..0ce6af8b7b4f 100644 --- a/requirements/static/py2.7/linux.txt +++ b/requirements/static/py2.7/linux.txt @@ -141,5 +141,4 @@ pycparser==2.19 # via cffi python-dateutil==2.8.1 # via vcert requests==2.22.0 # via vcert six==1.13.0 # via cryptography, python-dateutil, vcert -urllib3==1.25.7 # via requests -vcert==0.7.0 \ No newline at end of file +urllib3==1.25.7 # via requests \ No newline at end of file diff --git a/requirements/static/py3.4/linux.txt b/requirements/static/py3.4/linux.txt index d06160d33b84..7896096e9e5a 100644 --- a/requirements/static/py3.4/linux.txt +++ b/requirements/static/py3.4/linux.txt @@ -132,5 +132,4 @@ pycparser==2.19 # via cffi python-dateutil==2.8.1 # via vcert requests==2.22.0 # via vcert six==1.13.0 # via cryptography, python-dateutil, vcert -urllib3==1.25.7 # via requests -vcert==0.7.0 \ No newline at end of file +urllib3==1.25.7 # via requests \ No newline at end of file diff --git a/requirements/static/py3.5/linux.txt b/requirements/static/py3.5/linux.txt index 6d94115eee79..ae7a5077abb6 100644 --- a/requirements/static/py3.5/linux.txt +++ b/requirements/static/py3.5/linux.txt @@ -6,6 +6,7 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog +asn1crypto==1.2.0 # via certvalidator, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto @@ -16,6 +17,7 @@ boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.9.11 +certvalidator==0.11.1 # via vcert cffi==1.13.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy @@ -23,7 +25,7 @@ cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.8 # via moto, paramiko, pyopenssl +cryptography==2.8 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -55,6 +57,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator paramiko==2.4.2 pathlib2==2.3.3 # via pytest pathtools==0.1.2 # via watchdog @@ -81,7 +84,7 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -98,13 +101,14 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client +six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.3 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.1 virtualenv==16.4.3 watchdog==0.9.0 websocket-client==0.40.0 # via docker, kubernetes diff --git a/requirements/static/py3.6/linux.txt b/requirements/static/py3.6/linux.txt index 627f7bd0fa59..c84e32c54a5f 100644 --- a/requirements/static/py3.6/linux.txt +++ b/requirements/static/py3.6/linux.txt @@ -6,6 +6,7 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog +asn1crypto==1.2.0 # via certvalidator, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto @@ -16,6 +17,7 @@ boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.9.11 +certvalidator==0.11.1 # via vcert cffi==1.13.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy @@ -23,7 +25,7 @@ cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.8 # via moto, paramiko, pyopenssl +cryptography==2.8 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -55,6 +57,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator paramiko==2.4.2 pathtools==0.1.2 # via watchdog pbr==5.1.3 # via mock @@ -81,7 +84,7 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -98,13 +101,14 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client +six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.3 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.1 virtualenv==16.4.3 watchdog==0.9.0 websocket-client==0.40.0 # via docker, kubernetes diff --git a/requirements/static/py3.7/linux.txt b/requirements/static/py3.7/linux.txt index f590477b3458..d9967cc05dd1 100644 --- a/requirements/static/py3.7/linux.txt +++ b/requirements/static/py3.7/linux.txt @@ -6,6 +6,7 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog +asn1crypto==1.2.0 # via certvalidator, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto @@ -16,6 +17,7 @@ boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.9.11 +certvalidator==0.11.1 # via vcert cffi==1.13.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy @@ -23,7 +25,7 @@ cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy coverage==4.5.3 # via pytest-cov croniter==0.3.29 -cryptography==2.8 # via moto, paramiko, pyopenssl +cryptography==2.8 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -55,6 +57,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator paramiko==2.4.2 pathtools==0.1.2 # via watchdog pbr==5.1.3 # via mock @@ -81,7 +84,7 @@ pytest-salt==2018.12.8 pytest-tempdir==2018.8.11 pytest-timeout==1.3.3 pytest==4.4.1 -python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.1 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -98,13 +101,14 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client +six==1.13.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.3 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.1 virtualenv==16.4.3 watchdog==0.9.0 websocket-client==0.40.0 # via docker, kubernetes From 3ef6013bbb99df19be7f226d542c45425ce6f105 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Mon, 9 Dec 2019 15:45:09 +0300 Subject: [PATCH 12/33] Removing log from test --- salt/runners/venafiapi.py | 2 ++ tests/integration/externalapi/test_venafiapi.py | 12 +----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index bcfed9181f57..9d7af49d8094 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -104,6 +104,7 @@ def request( salt-run venafi.request ''' + log.info("Requesting Venafi certificate") if zone is None: log.error(msg=str("Missing zone parameter")) sys.exit(1) @@ -117,6 +118,7 @@ def request( request = CertificateRequest(common_name=dns_name, country=country, province=state, locality=loc, organization=org, organizational_unit=org_unit, key_password=key_password) zone_config = conn.read_zone_conf(zone) + log.info("Updating request from zone %s",zone_config) request.update_from_zone_config(zone_config) else: log.info("Will use generated CSR from %s", csr_path) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 3dc7e18b2b3d..34826a7bb987 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -7,7 +7,7 @@ import functools import random import string -import logging + # Import Salt Testing libs from tests.support.case import ShellCase @@ -20,9 +20,6 @@ import tempfile from os import environ -import logging -log = logging.getLogger(__name__) - def _random_name(prefix=''): ret = prefix @@ -51,20 +48,16 @@ class VenafiTest(ShellCase): @with_random_name def test_request(self, name): - log.debug("Testing Venafi request cert") - log.debug("Using venafi config: %s", self.master_opts['venafi']) cn = '{0}.example.com'.format(name) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, dns_name=cn, key_password='secretPassword', zone=environ.get('CLOUDZONE')) - log.debug("Ret is:\n %s", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') - log.debug("Testing certificate:\n %s", cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ @@ -74,7 +67,6 @@ def test_request(self, name): ] pkey_output = ret['return'][1] - log.debug("Testing pkey:\n %s", pkey_output) if not pkey_output: pytest.fail('venafi_private key not found in output_value') @@ -93,7 +85,6 @@ def test_request(self, name): @with_random_name def test_sign(self, name): - log.debug("Testing Venafi sign CSR") csr_pem = """-----BEGIN CERTIFICATE REQUEST----- MIIFbDCCA1QCAQAwgbQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRIwEAYD @@ -137,7 +128,6 @@ def test_sign(self, name): minion_id=cn, csr_path=csr_path, zone=environ.get('CLOUDZONE')) - log.debug("Ret is:\n %s", ret) cert_output = ret['return'][0] if not cert_output: pytest.fail('venafi_certificate not found in output_value') From d82bcb3d6a5f25d74a2c225f22780dc4a2848f5e Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Mon, 9 Dec 2019 15:47:18 +0300 Subject: [PATCH 13/33] Avoid using break --- salt/runners/venafiapi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 9d7af49d8094..959024c73f64 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -133,12 +133,12 @@ def request( conn.request_cert(request, zone) #TODO: add timeout parameter here - t = time.time() + 300 - while time.time() < t: + timeout_seconds = 300 + timeout = time.time() + timeout_seconds + cert = None + while cert is None and time.time() < timeout: cert = conn.retrieve_cert(request) - if cert: - break - else: + if cert is None: time.sleep(5) if csr_path is None: From 2216faae6cdf2d87da112687206148c4fad9f122 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Mon, 9 Dec 2019 15:53:03 +0300 Subject: [PATCH 14/33] dib't use pytest.fail --- tests/integration/externalapi/test_venafiapi.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 34826a7bb987..3c862d461398 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -55,8 +55,7 @@ def test_request(self, name): key_password='secretPassword', zone=environ.get('CLOUDZONE')) cert_output = ret['return'][0] - if not cert_output: - pytest.fail('venafi_certificate not found in output_value') + assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) @@ -67,8 +66,7 @@ def test_request(self, name): ] pkey_output = ret['return'][1] - if not pkey_output: - pytest.fail('venafi_private key not found in output_value') + assert pkey_output is not None, 'venafi_private key not found in output_value' pkey = serialization.load_pem_private_key(pkey_output.encode(), password=b'secretPassword', backend=default_backend()) @@ -129,8 +127,7 @@ def test_sign(self, name): csr_path=csr_path, zone=environ.get('CLOUDZONE')) cert_output = ret['return'][0] - if not cert_output: - pytest.fail('venafi_certificate not found in output_value') + assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) From 94e6948119aa3c1aa51b0c42763347ecdd3c5038 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 11 Dec 2019 12:10:48 +0300 Subject: [PATCH 15/33] running pre-commit run -av with vcert --- requirements/static/linux.in | 2 +- requirements/static/py2.7/darwin.txt | 4 ++++ requirements/static/py3.4/linux.txt | 15 ++++++++------- requirements/static/py3.5/linux.txt | 14 +++++++------- requirements/static/py3.6/linux.txt | 14 +++++++------- requirements/static/py3.7/linux.txt | 14 +++++++------- 6 files changed, 34 insertions(+), 29 deletions(-) diff --git a/requirements/static/linux.in b/requirements/static/linux.in index 7ffab8f5b4ad..052cb2c9ca14 100644 --- a/requirements/static/linux.in +++ b/requirements/static/linux.in @@ -41,4 +41,4 @@ timelib tornado<5.0 virtualenv watchdog -vcert==0.7.1 +vcert~=0.7.0 diff --git a/requirements/static/py2.7/darwin.txt b/requirements/static/py2.7/darwin.txt index f4073b375ccc..0b175903547c 100644 --- a/requirements/static/py2.7/darwin.txt +++ b/requirements/static/py2.7/darwin.txt @@ -144,3 +144,7 @@ pyobjc==5.1.2 pyobjc==5.1.2 # Passthrough dependencies from pkg/osx/req.txt pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 diff --git a/requirements/static/py3.4/linux.txt b/requirements/static/py3.4/linux.txt index dafa99fa412f..a33bdd384afb 100644 --- a/requirements/static/py3.4/linux.txt +++ b/requirements/static/py3.4/linux.txt @@ -6,26 +6,26 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.2.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via docker, websocket-client +backports.ssl-match-hostname==3.7.0.1 # via docker bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.6.1 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -58,6 +58,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 pathlib2==2.3.3 # via importlib-metadata, pytest @@ -84,7 +85,7 @@ pytest-salt-runtests-bridge==2019.7.10 pytest-salt==2019.11.27 pytest-tempdir==2019.10.12 pytest==4.6.6 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -103,14 +104,14 @@ scp==0.13.2 # via junos-eznc selectors2==2.0.1 # via ncclient setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 watchdog==0.9.0 wcwidth==0.1.7 # via pytest diff --git a/requirements/static/py3.5/linux.txt b/requirements/static/py3.5/linux.txt index 53251aba132d..fbdc7c87f3d8 100644 --- a/requirements/static/py3.5/linux.txt +++ b/requirements/static/py3.5/linux.txt @@ -6,26 +6,25 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.2.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via websocket-client bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.6.1 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -58,6 +57,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 pathlib2==2.3.3 # via pytest @@ -84,7 +84,7 @@ pytest-salt-runtests-bridge==2019.7.10 pytest-salt==2019.11.27 pytest-tempdir==2019.10.12 pytest==4.6.6 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -101,14 +101,14 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 watchdog==0.9.0 wcwidth==0.1.7 # via pytest diff --git a/requirements/static/py3.6/linux.txt b/requirements/static/py3.6/linux.txt index d7ee75252cc5..b2b4c6ab3b87 100644 --- a/requirements/static/py3.6/linux.txt +++ b/requirements/static/py3.6/linux.txt @@ -6,26 +6,25 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.2.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via websocket-client bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.6.1 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -58,6 +57,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 pathtools==0.1.2 # via watchdog @@ -84,7 +84,7 @@ pytest-salt-runtests-bridge==2019.7.10 pytest-salt==2019.11.27 pytest-tempdir==2019.10.12 pytest==4.6.6 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -101,14 +101,14 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 watchdog==0.9.0 wcwidth==0.1.7 # via pytest diff --git a/requirements/static/py3.7/linux.txt b/requirements/static/py3.7/linux.txt index 21049e3309f0..bd675dfaa99b 100644 --- a/requirements/static/py3.7/linux.txt +++ b/requirements/static/py3.7/linux.txt @@ -6,26 +6,25 @@ # apache-libcloud==2.0.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.2.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -backports-abc==0.5 # via tornado backports.functools-lru-cache==1.5 # via cheroot -backports.ssl-match-hostname==3.7.0.1 # via websocket-client bcrypt==3.1.6 # via paramiko boto3==1.9.132 boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheroot==6.5.4 # via cherrypy cherrypy==17.3.0 contextlib2==0.5.5 # via cherrypy croniter==0.3.29 -cryptography==2.6.1 # via moto, paramiko, pyopenssl +cryptography==2.6.1 # via moto, paramiko, pyopenssl, vcert dnspython==1.16.0 docker-pycreds==0.4.0 # via docker docker==3.7.2 @@ -58,6 +57,7 @@ moto==1.3.7 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.1.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 pathtools==0.1.2 # via watchdog @@ -84,7 +84,7 @@ pytest-salt-runtests-bridge==2019.7.10 pytest-salt==2019.11.27 pytest-tempdir==2019.10.12 pytest==4.6.6 -python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto +python-dateutil==2.8.0 # via botocore, croniter, kubernetes, moto, vcert python-etcd==0.4.5 python-gnupg==0.4.4 python-jose==2.0.2 # via moto @@ -101,14 +101,14 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 setuptools-scm==3.2.0 -singledispatch==3.4.0.3 # via tornado -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 strict-rfc3339==0.7 tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 watchdog==0.9.0 wcwidth==0.1.7 # via pytest From f03337b85032ce2864852d9338754c9c27309d07 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 11 Dec 2019 18:20:36 +0300 Subject: [PATCH 16/33] fixing PEP8 E231: missing whitespace after ',' --- salt/runners/venafiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 959024c73f64..0d38d41bb511 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -118,7 +118,7 @@ def request( request = CertificateRequest(common_name=dns_name, country=country, province=state, locality=loc, organization=org, organizational_unit=org_unit, key_password=key_password) zone_config = conn.read_zone_conf(zone) - log.info("Updating request from zone %s",zone_config) + log.info("Updating request from zone %s", zone_config) request.update_from_zone_config(zone_config) else: log.info("Will use generated CSR from %s", csr_path) From 28b43bbe6317a04bc76a17d9b8928eece5bdc3ff Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Thu, 12 Dec 2019 17:55:19 +0300 Subject: [PATCH 17/33] removing unused import --- tests/integration/externalapi/test_venafiapi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 3c862d461398..b43307810145 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -16,7 +16,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import serialization -import pytest import tempfile from os import environ From 23bc4bec1d0325ee0f5941ffc68ab74ef661e837 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 18 Dec 2019 15:38:53 +0300 Subject: [PATCH 18/33] Adding temporary debug --- tests/integration/externalapi/test_venafiapi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index b43307810145..d2e455c8c770 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -56,6 +56,7 @@ def test_request(self, name): cert_output = ret['return'][0] assert cert_output is not None, 'venafi_certificate not found in `output_value`' + print("temp debug:"+cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ @@ -126,6 +127,7 @@ def test_sign(self, name): csr_path=csr_path, zone=environ.get('CLOUDZONE')) cert_output = ret['return'][0] + print("temp debug:" + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) From 601e9150b2908a44ac495670a52146a438778ddd Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 18 Dec 2019 16:14:24 +0300 Subject: [PATCH 19/33] Set zone in file instead of getting it from env --- tests/integration/externalapi/test_venafiapi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index d2e455c8c770..b6d1d1624512 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -52,7 +52,7 @@ def test_request(self, name): minion_id=cn, dns_name=cn, key_password='secretPassword', - zone=environ.get('CLOUDZONE')) + zone='fake') cert_output = ret['return'][0] assert cert_output is not None, 'venafi_certificate not found in `output_value`' @@ -125,7 +125,7 @@ def test_sign(self, name): ret = self.run_run_plus(fun='venafi.request', minion_id=cn, csr_path=csr_path, - zone=environ.get('CLOUDZONE')) + zone='fake') cert_output = ret['return'][0] print("temp debug:" + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' From cf025726fd4e9d4ec53bedf89128281831b3c2f4 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Wed, 18 Dec 2019 18:56:15 +0300 Subject: [PATCH 20/33] removing debug print --- tests/integration/externalapi/test_venafiapi.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index b6d1d1624512..654019ce61b2 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -56,7 +56,6 @@ def test_request(self, name): cert_output = ret['return'][0] assert cert_output is not None, 'venafi_certificate not found in `output_value`' - print("temp debug:"+cert_output) cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ @@ -127,7 +126,6 @@ def test_sign(self, name): csr_path=csr_path, zone='fake') cert_output = ret['return'][0] - print("temp debug:" + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) From c43c7805af65aa3b135764e5318cde99412fdd65 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Thu, 19 Dec 2019 13:00:51 +0300 Subject: [PATCH 21/33] removing unused import --- tests/integration/externalapi/test_venafiapi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 654019ce61b2..df3574031cdf 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -17,7 +17,6 @@ from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import serialization import tempfile -from os import environ def _random_name(prefix=''): From 9367e3ce81e04a4a46ca20c84a3147684166435e Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 20 Dec 2019 13:05:44 +0300 Subject: [PATCH 22/33] adding debug print --- tests/integration/externalapi/test_venafiapi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index df3574031cdf..d071fe9ff3e0 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -47,6 +47,7 @@ class VenafiTest(ShellCase): @with_random_name def test_request(self, name): cn = '{0}.example.com'.format(name) + print("requesting cn "+cn) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, dns_name=cn, From ffe918eeb4d103568e6265c1365ddd06144adbf1 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Tue, 31 Dec 2019 02:12:48 +0300 Subject: [PATCH 23/33] adding debug print to understand why tests are failing --- tests/integration/externalapi/test_venafiapi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index d071fe9ff3e0..3918af21bd39 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -54,10 +54,12 @@ def test_request(self, name): key_password='secretPassword', zone='fake') cert_output = ret['return'][0] + print("cert out is: "+cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) + print("cn in cert is"+cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ x509.NameAttribute( NameOID.COMMON_NAME, cn From eb1b35604b48f94f24d5c4cbc679588dbdd0c224 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Tue, 31 Dec 2019 02:39:54 +0300 Subject: [PATCH 24/33] removing breaking print --- tests/integration/externalapi/test_venafiapi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 3918af21bd39..512b48dbd7ac 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -59,7 +59,6 @@ def test_request(self, name): cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) assert isinstance(cert, x509.Certificate) - print("cn in cert is"+cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)) assert cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) == [ x509.NameAttribute( NameOID.COMMON_NAME, cn From 10adbcf89eca35e9b1296bff00a8ac23574a1207 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Tue, 31 Dec 2019 10:01:43 +0300 Subject: [PATCH 25/33] decode uncode CN in python 2.7 --- tests/integration/externalapi/test_venafiapi.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 512b48dbd7ac..cfc1f9bd6475 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -7,6 +7,7 @@ import functools import random import string +from six import text_type # Import Salt Testing libs @@ -47,6 +48,11 @@ class VenafiTest(ShellCase): @with_random_name def test_request(self, name): cn = '{0}.example.com'.format(name) + + # Provide python27 compatibility + if not isinstance(cn, text_type): + cn = cn.decode() + print("requesting cn "+cn) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, From 26c7fbb1107f92183d046f200815d116c3c031ae Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Tue, 31 Dec 2019 11:17:11 +0300 Subject: [PATCH 26/33] decode uncode CN in python 2.7 for second test, remove print --- tests/integration/externalapi/test_venafiapi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index cfc1f9bd6475..c46ca20a9342 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -53,14 +53,12 @@ def test_request(self, name): if not isinstance(cn, text_type): cn = cn.decode() - print("requesting cn "+cn) ret = self.run_run_plus(fun='venafi.request', minion_id=cn, dns_name=cn, key_password='secretPassword', zone='fake') cert_output = ret['return'][0] - print("cert out is: "+cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) @@ -128,6 +126,11 @@ def test_sign(self, name): f.flush() csr_path = f.name cn = "test-csr-32313131.venafi.example.com" + + # Provide python27 compatibility + if not isinstance(cn, text_type): + cn = cn.decode() + ret = self.run_run_plus(fun='venafi.request', minion_id=cn, csr_path=csr_path, From 1f706c4969da46984a271af4f0862201b51b7ea9 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Tue, 31 Dec 2019 12:48:12 +0300 Subject: [PATCH 27/33] use from salt.ext.six import text_type --- tests/integration/externalapi/test_venafiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index c46ca20a9342..9c2bb8e4d2af 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -7,12 +7,12 @@ import functools import random import string -from six import text_type # Import Salt Testing libs from tests.support.case import ShellCase from salt.ext.six.moves import range +from salt.ext.six import text_type from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.x509.oid import NameOID From 6c3cd6846400f37bcca2e4105919bf32417ac119 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Sun, 5 Jan 2020 14:20:30 +0300 Subject: [PATCH 28/33] Adding print to debug test on macos --- tests/integration/externalapi/test_venafiapi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 9c2bb8e4d2af..f9ad414a1338 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -59,6 +59,7 @@ def test_request(self, name): key_password='secretPassword', zone='fake') cert_output = ret['return'][0] + print("cert out is: " + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) @@ -136,6 +137,7 @@ def test_sign(self, name): csr_path=csr_path, zone='fake') cert_output = ret['return'][0] + print("cert out is: " + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) From 325ac51b6e05ce17e0a586dc3495856e28061dab Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 6 Jan 2020 16:50:58 -0600 Subject: [PATCH 29/33] more debugging --- tests/integration/externalapi/test_venafiapi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index f9ad414a1338..747c78caa9de 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -59,6 +59,7 @@ def test_request(self, name): key_password='secretPassword', zone='fake') cert_output = ret['return'][0] + print('return is', ret) print("cert out is: " + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' @@ -137,6 +138,7 @@ def test_sign(self, name): csr_path=csr_path, zone='fake') cert_output = ret['return'][0] + print('return is', ret) print("cert out is: " + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' From b825433fcb077e7a5262639b16f1d00f806fc084 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Thu, 9 Jan 2020 16:14:01 +0300 Subject: [PATCH 30/33] Running pre-commit run -av for darwin --- requirements/static/darwin.in | 2 ++ requirements/static/py2.7/darwin.txt | 2 ++ requirements/static/py3.4/linux-crypto.txt | 1 - requirements/static/py3.5/darwin-crypto.txt | 1 - requirements/static/py3.5/darwin.txt | 8 ++++++-- requirements/static/py3.5/linux-crypto.txt | 1 - requirements/static/py3.5/windows-crypto.txt | 1 - requirements/static/py3.6/darwin-crypto.txt | 1 - requirements/static/py3.6/darwin.txt | 7 +++++-- requirements/static/py3.6/linux-crypto.txt | 1 - requirements/static/py3.6/windows-crypto.txt | 1 - requirements/static/py3.7/darwin-crypto.txt | 1 - requirements/static/py3.7/darwin.txt | 7 +++++-- requirements/static/py3.7/linux-crypto.txt | 1 - requirements/static/py3.7/windows-crypto.txt | 1 - 15 files changed, 20 insertions(+), 16 deletions(-) diff --git a/requirements/static/darwin.in b/requirements/static/darwin.in index d3f777e8c6c8..290ca0aa35fa 100644 --- a/requirements/static/darwin.in +++ b/requirements/static/darwin.in @@ -33,3 +33,5 @@ Genshi Cheetah3==3.1.0 Mako wempy; python_version <'3' + +vcert~=0.7.0 \ No newline at end of file diff --git a/requirements/static/py2.7/darwin.txt b/requirements/static/py2.7/darwin.txt index d6372a4830cb..82c9d06759c6 100644 --- a/requirements/static/py2.7/darwin.txt +++ b/requirements/static/py2.7/darwin.txt @@ -149,3 +149,5 @@ pyobjc==5.1.2 pyobjc==5.1.2 # Passthrough dependencies from pkg/osx/req.txt pyobjc==5.1.2 +# Passthrough dependencies from pkg/osx/req.txt +pyobjc==5.1.2 diff --git a/requirements/static/py3.4/linux-crypto.txt b/requirements/static/py3.4/linux-crypto.txt index a83659e76786..d0634bbd83f4 100644 --- a/requirements/static/py3.4/linux-crypto.txt +++ b/requirements/static/py3.4/linux-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.3 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.5/darwin-crypto.txt b/requirements/static/py3.5/darwin-crypto.txt index a6a86036f2e7..492a8bd18558 100644 --- a/requirements/static/py3.5/darwin-crypto.txt +++ b/requirements/static/py3.5/darwin-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.0 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.5/darwin.txt b/requirements/static/py3.5/darwin.txt index 0b60fe3bb031..036bcb0f924e 100644 --- a/requirements/static/py3.5/darwin.txt +++ b/requirements/static/py3.5/darwin.txt @@ -6,7 +6,7 @@ # apache-libcloud==2.4.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.3.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto @@ -19,6 +19,8 @@ boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheetah3==3.1.0 @@ -65,6 +67,7 @@ msgpack-python==0.5.6 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.2.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 # via junos-eznc, ncclient, scp pathlib2==2.3.3 # via pytest @@ -104,7 +107,7 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 singledispatch==3.4.0.3 -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 smmap==0.9.0 strict-rfc3339==0.7 @@ -112,6 +115,7 @@ tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 vultr==1.0.1 watchdog==0.9.0 diff --git a/requirements/static/py3.5/linux-crypto.txt b/requirements/static/py3.5/linux-crypto.txt index 59c45171e55c..6cdeaecccc4d 100644 --- a/requirements/static/py3.5/linux-crypto.txt +++ b/requirements/static/py3.5/linux-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.3 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.5/windows-crypto.txt b/requirements/static/py3.5/windows-crypto.txt index 1e209d42238b..091858543a9c 100644 --- a/requirements/static/py3.5/windows-crypto.txt +++ b/requirements/static/py3.5/windows-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.0 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.6/darwin-crypto.txt b/requirements/static/py3.6/darwin-crypto.txt index 9c11d937f084..3cc68e7cab08 100644 --- a/requirements/static/py3.6/darwin-crypto.txt +++ b/requirements/static/py3.6/darwin-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.0 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.6/darwin.txt b/requirements/static/py3.6/darwin.txt index d944e0e6301d..835c31596b7f 100644 --- a/requirements/static/py3.6/darwin.txt +++ b/requirements/static/py3.6/darwin.txt @@ -6,7 +6,7 @@ # apache-libcloud==2.4.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.3.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto @@ -19,6 +19,7 @@ boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheetah3==3.1.0 @@ -65,6 +66,7 @@ msgpack-python==0.5.6 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.2.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 # via junos-eznc, ncclient, scp pathtools==0.1.2 # via watchdog @@ -104,7 +106,7 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 singledispatch==3.4.0.3 -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 smmap==0.9.0 strict-rfc3339==0.7 @@ -112,6 +114,7 @@ tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 vultr==1.0.1 watchdog==0.9.0 diff --git a/requirements/static/py3.6/linux-crypto.txt b/requirements/static/py3.6/linux-crypto.txt index 3b92d88ffb20..efb7b5dc6eaa 100644 --- a/requirements/static/py3.6/linux-crypto.txt +++ b/requirements/static/py3.6/linux-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.3 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.6/windows-crypto.txt b/requirements/static/py3.6/windows-crypto.txt index 2e4eaf54bcc0..6f6886495685 100644 --- a/requirements/static/py3.6/windows-crypto.txt +++ b/requirements/static/py3.6/windows-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.0 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.7/darwin-crypto.txt b/requirements/static/py3.7/darwin-crypto.txt index 370308dc087b..0a4ffcf4c8d4 100644 --- a/requirements/static/py3.7/darwin-crypto.txt +++ b/requirements/static/py3.7/darwin-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.0 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.7/darwin.txt b/requirements/static/py3.7/darwin.txt index d6aa4e4cad66..952f96bdb70c 100644 --- a/requirements/static/py3.7/darwin.txt +++ b/requirements/static/py3.7/darwin.txt @@ -6,7 +6,7 @@ # apache-libcloud==2.4.0 argh==0.26.2 # via watchdog -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.3.0 # via certvalidator, cryptography, oscrypto atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto @@ -19,6 +19,7 @@ boto==2.49.0 botocore==1.12.132 # via boto3, moto, s3transfer cachetools==3.1.0 # via google-auth certifi==2019.3.9 +certvalidator==0.11.1 # via vcert cffi==1.12.2 chardet==3.0.4 # via requests cheetah3==3.1.0 @@ -65,6 +66,7 @@ msgpack-python==0.5.6 msgpack==0.5.6 ncclient==0.6.4 # via junos-eznc netaddr==0.7.19 # via junos-eznc +oscrypto==1.2.0 # via certvalidator packaging==19.2 # via pytest paramiko==2.4.2 # via junos-eznc, ncclient, scp pathtools==0.1.2 # via watchdog @@ -104,7 +106,7 @@ salttesting==2017.6.1 scp==0.13.2 # via junos-eznc setproctitle==1.1.10 singledispatch==3.4.0.3 -six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client +six==1.12.0 # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kubernetes, mock, more-itertools, moto, ncclient, packaging, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, vcert, websocket-client smmap2==2.0.5 # via gitdb2 smmap==0.9.0 strict-rfc3339==0.7 @@ -112,6 +114,7 @@ tempora==1.14.1 # via portend timelib==0.2.4 tornado==4.5.3 ; python_version >= "3.4" urllib3==1.24.2 # via botocore, kubernetes, python-etcd, requests +vcert==0.7.3 virtualenv==16.4.3 vultr==1.0.1 watchdog==0.9.0 diff --git a/requirements/static/py3.7/linux-crypto.txt b/requirements/static/py3.7/linux-crypto.txt index a376a05e786c..1139152146b9 100644 --- a/requirements/static/py3.7/linux-crypto.txt +++ b/requirements/static/py3.7/linux-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.3 -typing==3.7.4.1 # via m2crypto diff --git a/requirements/static/py3.7/windows-crypto.txt b/requirements/static/py3.7/windows-crypto.txt index ee941ca2d08e..82aeec9d3bfd 100644 --- a/requirements/static/py3.7/windows-crypto.txt +++ b/requirements/static/py3.7/windows-crypto.txt @@ -6,4 +6,3 @@ # m2crypto==0.35.2 pycryptodomex==3.9.0 -typing==3.7.4.1 # via m2crypto From 16cea5f986e793c5c2b94c72ab65b8858d86a445 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Fri, 10 Jan 2020 00:20:36 +0300 Subject: [PATCH 31/33] removing print --- tests/integration/externalapi/test_venafiapi.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/externalapi/test_venafiapi.py b/tests/integration/externalapi/test_venafiapi.py index 747c78caa9de..9c2bb8e4d2af 100644 --- a/tests/integration/externalapi/test_venafiapi.py +++ b/tests/integration/externalapi/test_venafiapi.py @@ -59,8 +59,6 @@ def test_request(self, name): key_password='secretPassword', zone='fake') cert_output = ret['return'][0] - print('return is', ret) - print("cert out is: " + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) @@ -138,8 +136,6 @@ def test_sign(self, name): csr_path=csr_path, zone='fake') cert_output = ret['return'][0] - print('return is', ret) - print("cert out is: " + cert_output) assert cert_output is not None, 'venafi_certificate not found in `output_value`' cert = x509.load_pem_x509_certificate(cert_output.encode(), default_backend()) From 01bdec4f5bd605eff650dbd879440d67e6fa0e5f Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Sat, 11 Jan 2020 10:13:37 +0300 Subject: [PATCH 32/33] raise exception instead of sys.exit --- salt/runners/venafiapi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 0d38d41bb511..6b28bb686c76 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -128,8 +128,9 @@ def request( csr = csr_file.read() request = CertificateRequest(csr=csr, common_name=dns_name) except Exception as e: - log.error(msg=str(e)) - sys.exit(1) + raise Exception( + 'Unable to open file {file}: {excp}'.format(file=csr_path,excp=e) + ) conn.request_cert(request, zone) #TODO: add timeout parameter here @@ -149,8 +150,9 @@ def request( with salt.utils.files.fopen(pkey_path) as pkey_file: private_key = pkey_file.read() except Exception as e: - log.error(msg=str(e)) - sys.exit(1) + raise Exception( + 'Unable to open file {file}: {excp}'.format(file=pkey_path, excp=e) + ) else: private_key = None From 7421fda2b89967b510b5b526ae0f633ad3fcd4f5 Mon Sep 17 00:00:00 2001 From: Aleksandr Rykalin Date: Mon, 13 Jan 2020 12:11:00 +0300 Subject: [PATCH 33/33] Fixing pep warnings --- salt/runners/venafiapi.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 6b28bb686c76..bc584165372a 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Support for Venafi :depends: - vcert Python module @@ -23,7 +23,7 @@ tpp_password: "sdb://osenv/TPP_PASSWORD" trust_bundle: "/opt/venafi/bundle.pem" -''' +""" from __future__ import absolute_import, print_function, unicode_literals import logging import time @@ -73,9 +73,9 @@ def _init_connection(): def __virtual__(): - ''' + """ Only load the module if vcert module is installed - ''' + """ if HAS_VCERT: return __virtualname__ return False @@ -94,7 +94,7 @@ def request( csr_path=None, pkey_path=None, ): - ''' + """ Request a new certificate CLI Example: @@ -102,7 +102,7 @@ def request( .. code-block:: bash salt-run venafi.request - ''' + """ log.info("Requesting Venafi certificate") if zone is None: @@ -129,11 +129,11 @@ def request( request = CertificateRequest(csr=csr, common_name=dns_name) except Exception as e: raise Exception( - 'Unable to open file {file}: {excp}'.format(file=csr_path,excp=e) + 'Unable to open file {file}: {excp}'.format(file=csr_path, excp=e) ) conn.request_cert(request, zone) - #TODO: add timeout parameter here + # TODO: add timeout parameter here timeout_seconds = 300 timeout = time.time() + timeout_seconds cert = None @@ -172,9 +172,9 @@ def request( def _id_map(minion_id, dns_name): - ''' + """ Maintain a relationship between a minion and a DNS name - ''' + """ cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) dns_names = cache.fetch(CACHE_BANK_NAME, minion_id) @@ -186,7 +186,7 @@ def _id_map(minion_id, dns_name): def show_cert(dns_name): - ''' + """ Show issued certificate for domain CLI Example: @@ -194,7 +194,7 @@ def show_cert(dns_name): .. code-block:: bash salt-run venafi.show_cert example.com - ''' + """ cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) domain_data = cache.fetch(CACHE_BANK_NAME, dns_name) or {} @@ -203,7 +203,7 @@ def show_cert(dns_name): def list_domain_cache(): - ''' + """ List domains that have been cached CLI Example: @@ -211,13 +211,13 @@ def list_domain_cache(): .. code-block:: bash salt-run venafi.list_domain_cache - ''' + """ cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) return cache.list('venafi/domains') def del_cached_domain(domains): - ''' + """ Delete cached domains from the master CLI Example: @@ -225,7 +225,7 @@ def del_cached_domain(domains): .. code-block:: bash salt-run venafi.del_cached_domain domain1.example.com,domain2.example.com - ''' + """ cache = salt.cache.Cache(__opts__, syspaths.CACHE_DIR) if isinstance(domains, six.string_types): domains = domains.split(',')