diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a287038b4..575d2986e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Linting - Check that the `.prettierignore` file exists and that starts with the same content. +- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) ### General @@ -18,12 +19,9 @@ ### Modules +- Add `--fix-version` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) - Fix a bug in the regex extracting the version from biocontainers URLs [#1598](https://github.com/nf-core/tools/pull/1598) -### Linting - -- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) - ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] - Patch release to try to fix the template sync ([#1585](https://github.com/nf-core/tools/pull/1585)) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 50a65f4778..f3f0701fba 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -596,7 +596,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("-a", "--all", is_flag=True, help="Run on all modules") @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") -def lint(ctx, tool, dir, key, all, local, passed): +@click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") +def lint(ctx, tool, dir, key, all, local, passed, fix_version): """ Lint one or more modules in a directory. @@ -609,7 +610,15 @@ def lint(ctx, tool, dir, key, all, local, passed): try: module_lint = nf_core.modules.ModuleLint(dir=dir) module_lint.modules_repo = ctx.obj["modules_repo_obj"] - module_lint.lint(module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed) + module_lint.lint( + module=tool, + key=key, + all_modules=all, + print_results=True, + local=local, + show_passed=passed, + fix_version=fix_version, + ) if len(module_lint.failed) > 0: sys.exit(1) except nf_core.modules.lint.ModuleLintException as e: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index ed9db99371..9bbe173ef5 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -101,7 +101,16 @@ def __init__(self, dir): def _get_all_lint_tests(): return ["main_nf", "meta_yml", "module_todos", "module_deprecations"] - def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False): + def lint( + self, + module=None, + key=(), + all_modules=False, + print_results=True, + show_passed=False, + local=False, + fix_version=False, + ): """ Lint all or one specific module @@ -118,6 +127,7 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ :param module: A specific module to lint :param print_results: Whether to print the linting results :param show_passed: Whether passed tests should be shown as well + :param fix_version: Update the module version if a newer version is available :returns: A ModuleLint object containing information of the passed, warned and failed tests @@ -174,11 +184,11 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ # Lint local modules if local and len(local_modules) > 0: - self.lint_modules(local_modules, local=True) + self.lint_modules(local_modules, local=True, fix_version=fix_version) # Lint nf-core modules if len(nfcore_modules) > 0: - self.lint_modules(nfcore_modules, local=False) + self.lint_modules(nfcore_modules, local=False, fix_version=fix_version) if print_results: self._print_results(show_passed=show_passed) @@ -282,19 +292,21 @@ def get_installed_modules(self): return local_modules, nfcore_modules - def lint_modules(self, modules, local=False): + def lint_modules(self, modules, local=False, fix_version=False): """ Lint a list of modules Args: modules ([NFCoreModule]): A list of module objects local (boolean): Whether the list consist of local or nf-core modules + fix_version (boolean): Fix the module version if a newer version is available """ progress_bar = rich.progress.Progress( "[bold blue]{task.description}", rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[test_name]}", transient=True, + console=console, ) with progress_bar: lint_progress = progress_bar.add_task( @@ -305,9 +317,9 @@ def lint_modules(self, modules, local=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, local=local) + self.lint_module(mod, progress_bar, local=local, fix_version=fix_version) - def lint_module(self, mod, local=False): + def lint_module(self, mod, progress_bar, local=False, fix_version=False): """ Perform linting on one module @@ -326,14 +338,17 @@ def lint_module(self, mod, local=False): # Only check the main script in case of a local module if local: - self.main_nf(mod) + self.main_nf(mod, fix_version, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] # Otherwise run all the lint tests else: for test_name in self.lint_tests: - getattr(self, test_name)(mod) + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version, progress_bar) + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] self.warned += [LintResult(mod, *m) for m in mod.warned] diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 8d745cd186..44fd4e8353 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -3,12 +3,19 @@ Lint the main.nf file of a module """ +import logging import re +import requests +from galaxy.tool_util.deps.mulled.util import build_target + import nf_core +import nf_core.modules.module_utils + +log = logging.getLogger(__name__) -def main_nf(module_lint_object, module): +def main_nf(module_lint_object, module, fix_version, progress_bar): """ Lint a ``main.nf`` module file @@ -100,7 +107,7 @@ def main_nf(module_lint_object, module): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines): + if check_process_section(module, process_lines, fix_version, progress_bar): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -190,7 +197,7 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines): +def check_process_section(self, lines, fix_version, progress_bar): """ Lint the section of a module between the process definition and the 'input:' definition @@ -236,14 +243,14 @@ def check_process_section(self, lines): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for l in lines: - if re.search("bioconda::", l): + if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] l = l.strip(" '\"") - if l.startswith("https://containers") or l.startswith("https://depot"): + if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 singularity_tag = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l).group(1) - if l.startswith("biocontainers/") or l.startswith("quay.io/"): + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 docker_tag = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l).group(1) @@ -272,9 +279,30 @@ def check_process_section(self, lines): last_ver = response.get("latest_version") if last_ver is not None and last_ver != bioconda_version: package, ver = bp.split("=", 1) - self.warned.append( - ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) - ) + # If a new version is available and fix is True, update the version + if fix_version: + if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response): + progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}") + log.debug(f"Updating package {package} {ver} -> {last_ver}") + self.passed.append( + ( + "bioconda_latest", + f"Conda package has been updated to the latest available: `{bp}`", + self.main_nf, + ) + ) + else: + progress_bar.print( + f"[blue]INFO[/blue]\t Tried to update package. Unable to update package '{package}' {ver} -> {last_ver}" + ) + log.debug(f"Unable to update package {package} {ver} -> {last_ver}") + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) + else: + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) @@ -341,3 +369,70 @@ def _is_empty(self, line): if line.strip().replace(" ", "") == "": empty = True return empty + + +def _fix_module_version(self, current_version, latest_version, singularity_tag, response): + """Updates the module version + + Changes the bioconda current version by the latest version. + Obtains the latest build from bioconda response + Checks that the new URLs for docker and singularity with the tag [version]--[build] are valid + Changes the docker and singularity URLs + """ + # Get latest build + build = _get_build(response) + + with open(self.main_nf, "r") as source: + lines = source.readlines() + + # Check if the new version + build exist and replace + new_lines = [] + for line in lines: + l = line.strip(" '\"") + build_type = _container_type(l) + if build_type == "bioconda": + new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) + elif build_type == "singularity" or build_type == "docker": + # Check that the new url is valid + new_url = re.search( + "(?:')(.+)(?:')", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line) + ).group(1) + response_new_container = requests.get( + "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True + ) + log.debug( + f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}" + ) + if response_new_container.status_code != 200: + return False + new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) + else: + new_lines.append(line) + + # Replace outdated versions by the latest one + with open(self.main_nf, "w") as source: + for line in new_lines: + source.write(line) + + return True + + +def _get_build(response): + """Get the latest build of the container version""" + build_times = [] + latest_v = response.get("latest_version") + files = response.get("files") + for f in files: + if f.get("version") == latest_v: + build_times.append((f.get("upload_time"), f.get("attrs").get("build"))) + return sorted(build_times, key=lambda tup: tup[0], reverse=True)[0][1] + + +def _container_type(line): + """Returns the container type of a build.""" + if re.search("bioconda::", line): + return "bioconda" + if line.startswith("https://containers") or line.startswith("https://depot"): + return "singularity" + if line.startswith("biocontainers/") or line.startswith("quay.io/"): + return "docker"