From 8c9a10c46fa59454b378850ce35d3b6f19cebb5a Mon Sep 17 00:00:00 2001 From: sommersoft Date: Sun, 21 Mar 2021 22:59:17 -0500 Subject: [PATCH 1/2] verify .readthedocs.yml with parsed yaml vs. the file's sha --- .../lib/circuitpython_library_validators.py | 58 ++++++++++++++++--- requirements.txt | 1 + 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/adabot/lib/circuitpython_library_validators.py b/adabot/lib/circuitpython_library_validators.py index d90ebbc..10c7874 100644 --- a/adabot/lib/circuitpython_library_validators.py +++ b/adabot/lib/circuitpython_library_validators.py @@ -33,6 +33,7 @@ from pylint import lint from pylint.reporters import JSONReporter from sh.contrib import git +import yaml from adabot import github_requests as github from adabot import pypi_requests as pypi @@ -187,11 +188,45 @@ def __init__(self, validators, bundle_submodules, latest_pylint, keep_repos=Fals self.validators = validators self.bundle_submodules = bundle_submodules self.latest_pylint = pkg_version_parse(latest_pylint) + self._rtd_yaml_base = None self.output_file_data = [] self.validate_contents_quiet = kw_args.get("validate_contents_quiet", False) self.has_setup_py_disabled = set() self.keep_repos = keep_repos + + @property + def rtd_yml_base(self): + """ The parsed YAML from `.readthedocs.yml` in the cookiecutter-adafruit-circuitpython repo. + Used to verify that a library's `.readthedocs.yml` matches this version. + """ + if self._rtd_yaml_base is None: + rtd_yml_dl_url = ( + "https://raw.githubusercontent.com/adafruit/cookiecutter-adafruit-" + "circuitpython/master/%7B%25%20if%20cookiecutter.library_prefix" + "%20%25%7D%7B%7B%20cookiecutter.library_prefix%20%7C%20capitalize" + "%20%7D%7D_%7B%25%20endif%20%25%7DCircuitPython_%7B%7B%20cookiecutter" + ".library_name%7D%7D/%7B%25%20if%20cookiecutter.sphinx_docs%20in" + "%20%5B'y'%2C%20'yes'%5D%20%25%7D.readthedocs.yml%7B%25%20endif" + "%20%25%7D" + ) + rtd_yml = requests.get(rtd_yml_dl_url) + if rtd_yml.ok: + try: + self._rtd_yaml_base = yaml.safe_load(rtd_yml.text) + except yaml.YAMLError: + print( + "Error parsing cookiecutter .readthedocs.yml." + ) + self._rtd_yaml_base = "" + else: + print( + "Error retrieving cookiecutter .readthedocs.yml" + ) + self._rtd_yaml_base = "" + + return self._rtd_yaml_base + def run_repo_validation(self, repo): """Run all the current validation functions on the provided repository and return their results as a list of string errors. @@ -652,13 +687,22 @@ def validate_contents(self, repo): errors.append(ERROR_UNABLE_PULL_REPO_CONTENTS) if "readthedocs.yml" in files or ".readthedocs.yml" in files: - fn = "readthedocs.yml" - if ".readthedocs.yml" in files: - fn = ".readthedocs.yml" - file_info = content_list[files.index(fn)] - if (file_info["sha"] != "f4243ad548bc5e4431f2d3c5d486f6c9c863888b" and - file_info["sha"] != "78a4671650248f4382e6eb72dab71c2d86824ca2"): - errors.append(ERROR_MISMATCHED_READTHEDOCS) + if self.rtd_yml_base != "": + fn = "readthedocs.yml" + if ".readthedocs.yml" in files: + fn = ".readthedocs.yml" + file_info = content_list[files.index(fn)] + rtd_contents = requests.get(file_info["download_url"]) + if rtd_contents.ok: + try: + rtd_yml = yaml.safe_load(rtd_contents.text) + if rtd_yml != self.rtd_yml_base: + errors.append(ERROR_MISMATCHED_READTHEDOCS) + except yaml.YAMLError: + self.output_file_data.append( + "Error parsing {} .readthedocs.yml.".format(repo["name"]) + ) + errors.append(ERROR_OUTPUT_HANDLER) else: errors.append(ERROR_MISSING_READTHEDOCS) diff --git a/requirements.txt b/requirements.txt index 453a06a..7e6e30e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ chardet==3.0.4 idna==2.6 packaging==20.3 pylint +pyyaml==5.4.1 redis==2.10.6 requests==2.20.0 sh==1.12.14 From 7bdf358ddf6dee2bf15748a6dd1bf6f6d34e5de6 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Sun, 21 Mar 2021 23:02:39 -0500 Subject: [PATCH 2/2] misc pylint cleanup --- adabot/lib/circuitpython_library_validators.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/adabot/lib/circuitpython_library_validators.py b/adabot/lib/circuitpython_library_validators.py index 10c7874..b7ea0cf 100644 --- a/adabot/lib/circuitpython_library_validators.py +++ b/adabot/lib/circuitpython_library_validators.py @@ -757,8 +757,6 @@ def __check_lib_name(repo_name, file_name): if name_rebuilt: # avoid adding things like 'simpletest.py' -> '' file_names.add(name_rebuilt) - found = False - return any( name.startswith(repo_name) for name in file_names ) @@ -786,7 +784,7 @@ def __check_lib_name(repo_name, file_name): errors.append(ERROR_MISSING_EXAMPLE_FOLDER) # first location .py files whose names begin with "adafruit_" - re_str = re.compile('adafruit\_[\w]*\.py') + re_str = re.compile(r'adafruit\_[\w]*\.py') pyfiles = ([x["download_url"] for x in content_list if re_str.fullmatch(x["name"])]) for pyfile in pyfiles: @@ -794,7 +792,7 @@ def __check_lib_name(repo_name, file_name): errors.extend(self._validate_py_for_u_modules(repo, pyfile)) # now location any directories whose names begin with "adafruit_" - re_str = re.compile('adafruit\_[\w]*') + re_str = re.compile(r'adafruit\_[\w]*') for adir in dirs: if re_str.fullmatch(adir): # retrieve the files in that directory