Merge pull request #1683 from ErikDanielsson/lint-singularity-image

Lint for container names

CHANGELOG.md

@@ -18,6 +18,7 @@
 - Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538))
 - Use black also to format python files in workflows ([#1563](https://github.com/nf-core/tools/pull/1563))
 - Add check for mimetype in the `input` parameter. ([#1647](https://github.com/nf-core/tools/issues/1647))
+- Check that the singularity and docker tags are parsable. Add `--fail-warned` flag to `nf-core modules lint` ([#1654](https://github.com/nf-core/tools/issues/1654))
 
 ### General
 

nf_core/__main__.py

@@ -643,10 +643,11 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="<pipeline/modules directory>")
 @click.option("-k", "--key", type=str, metavar="<test>", multiple=True, help="Run only these lint tests")
 @click.option("-a", "--all", is_flag=True, help="Run on all modules")
+@click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures")
 @click.option("--local", is_flag=True, help="Run additional lint tests for local modules")
 @click.option("--passed", is_flag=True, help="Show passed tests")
 @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available")
-def lint(ctx, tool, dir, key, all, local, passed, fix_version):
+def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version):
     """
     Lint one or more modules in a directory.
 
@@ -659,6 +660,7 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     try:
         module_lint = nf_core.modules.ModuleLint(
             dir,
+            fail_warned,
             ctx.obj["modules_repo_url"],
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],

nf_core/download.py

@@ -479,7 +479,7 @@ def find_container_images(self):
 
                                     # Don't recognise this, throw a warning
                                     else:
-                                        log.error(f"[red]Cannot parse container string, skipping: [green]{match}")
+                                        log.error(f"[red]Cannot parse container string, skipping: [green]'{file}'")
 
                         if this_container:
                             containers_raw.append(this_container)

nf_core/modules/lint/__init__.py

@@ -70,13 +70,14 @@ class ModuleLint(ModuleCommand):
     from .module_todos import module_todos
     from .module_version import module_version
 
-    def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=None):
+    def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull=False, base_path=None):
         self.dir = dir
         try:
             self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
         except LookupError as e:
             raise UserWarning(e)
 
+        self.fail_warned = fail_warned
         self.passed = []
         self.warned = []
         self.failed = []
@@ -245,7 +246,7 @@ def get_installed_modules(self):
 
             # Filter local modules
             if os.path.exists(local_modules_dir):
-                local_modules = sorted([x for x in local_modules if x.endswith(".nf")])
+                local_modules = sorted([x for x in os.listdir(local_modules_dir) if x.endswith(".nf")])
 
         # nf-core/modules
         if self.repo_type == "modules":
@@ -342,7 +343,11 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False):
         if local:
             self.main_nf(mod, fix_version, progress_bar)
             self.passed += [LintResult(mod, *m) for m in mod.passed]
-            self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)]
+            warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)]
+            if not self.fail_warned:
+                self.warned += warned
+            else:
+                self.failed += warned
 
         # Otherwise run all the lint tests
         else:
@@ -353,7 +358,12 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False):
                     getattr(self, test_name)(mod)
 
             self.passed += [LintResult(mod, *m) for m in mod.passed]
-            self.warned += [LintResult(mod, *m) for m in mod.warned]
+            warned = [LintResult(mod, *m) for m in mod.warned]
+            if not self.fail_warned:
+                self.warned += warned
+            else:
+                self.failed += warned
+
             self.failed += [LintResult(mod, *m) for m in mod.failed]
 
     def _print_results(self, show_passed=False):

nf_core/modules/lint/main_nf.py

@@ -241,19 +241,30 @@ def check_process_section(self, lines, fix_version, progress_bar):
             self.passed.append(("process_standard_label", "Correct process label", self.main_nf))
     else:
         self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf))
-
     for l in lines:
         if _container_type(l) == "bioconda":
             bioconda_packages = [b for b in l.split() if "bioconda::" in b]
         l = l.strip(" '\"")
         if _container_type(l) == "singularity":
             # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1
             # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0
-            singularity_tag = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l).group(1)
+            match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l)
+            if match is not None:
+                singularity_tag = match.group(1)
+                self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf))
+            else:
+                self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf))
+                singularity_tag = None
         if _container_type(l) == "docker":
             # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5
             # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1
-            docker_tag = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l).group(1)
+            match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l)
+            if match is not None:
+                docker_tag = match.group(1)
+                self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf))
+            else:
+                self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf))
+                docker_tag = None
 
     # Check that all bioconda packages have build numbers
     # Also check for newer versions
@@ -434,6 +445,15 @@ def _container_type(line):
     if re.search("bioconda::", line):
         return "bioconda"
     if line.startswith("https://containers") or line.startswith("https://depot"):
-        return "singularity"
+        # Look for a http download URL.
+        # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980
+        url_regex = (
+            r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)"
+        )
+        url_match = re.search(url_regex, line, re.S)
+        if url_match:
+            return "singularity"
+        else:
+            return None
     if line.startswith("biocontainers/") or line.startswith("quay.io/"):
         return "docker"

nf_core/utils.py

@@ -327,6 +327,7 @@ def setup_requests_cachedir():
         "backend": "sqlite",
     }
 
+    logging.getLogger("requests_cache").setLevel(logging.WARNING)
     try:
         if not os.path.exists(cachedir):
             os.makedirs(cachedir)