Merge pull request #1716 from ErikDanielsson/lint-patch

Support patch file in `nf-core modules lint`

CHANGELOG.md

@@ -62,6 +62,7 @@
 - Add short option for `--no-pull` option in `nf-core modules`
 - Add `nf-core modules patch` command ([#1312](https://github.com/nf-core/tools/issues/1312))
 - Add support for patch in `nf-core modules update` command ([#1312](https://github.com/nf-core/tools/issues/1312))
+- Add support for patch in `nf-core modules lint` command ([#1312](https://github.com/nf-core/tools/issues/1312))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 

README.md

@@ -1219,6 +1219,8 @@ The generated patches work with `nf-core modules update`: when you install a new
 the patch automatically. The patch application fails if the new version of the module modifies the same lines as the patch. In this case,
 the patch new version is installed but the old patch file is preversed.
 
+When linting a patched module, the patch is applied in reverse to recover the original files and then the module is linted as usual.
+
 ### Create a new module
 
 This command creates a new nf-core module from the nf-core module template.

docs/api/make_lint_md.py

@@ -44,7 +44,11 @@ def make_docs(docs_basedir, lint_tests, md_template):
 modules_docs_basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_src", "module_lint_tests")
 make_docs(
     modules_docs_basedir,
-    nf_core.modules.lint.ModuleLint._get_all_lint_tests(),
+    list(
+        set(nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=True)).union(
+            nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=False)
+        )
+    ),
     """# {0}
 
 ```{{eval-rst}}

nf_core/lint/__init__.py

@@ -56,7 +56,7 @@ def run_linting(
     # Verify that the requested tests exist
     if key:
         all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union(
-            set(nf_core.modules.lint.ModuleLint._get_all_lint_tests())
+            set(nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=True))
         )
         bad_keys = [k for k in key if k not in all_tests]
         if len(bad_keys) > 0:
@@ -92,7 +92,9 @@ def run_linting(
     # Run only the tests we want
     if key:
         # Select only the module lint tests
-        module_lint_tests = list(set(key).intersection(set(nf_core.modules.lint.ModuleLint._get_all_lint_tests())))
+        module_lint_tests = list(
+            set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=True)))
+        )
     else:
         # If no key is supplied, run the default modules tests
         module_lint_tests = ("module_changes", "module_version")

nf_core/modules/lint/__init__.py

@@ -9,31 +9,23 @@
 
 from __future__ import print_function
 
-import json
 import logging
 import operator
 import os
-import re
-import sys
 
 import questionary
-import requests
 import rich
-import yaml
 from rich.markdown import Markdown
-from rich.panel import Panel
 from rich.table import Table
 
 import nf_core.modules.module_utils
 import nf_core.utils
-from nf_core.lint.pipeline_todos import pipeline_todos
 from nf_core.lint_utils import console
 from nf_core.modules.modules_command import ModuleCommand
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
 from nf_core.modules.nfcore_module import NFCoreModule
 from nf_core.utils import plural_s as _s
-from nf_core.utils import rich_force_colors
 
 log = logging.getLogger(__name__)
 
@@ -66,6 +58,7 @@ class ModuleLint(ModuleCommand):
     from .meta_yml import meta_yml
     from .module_changes import module_changes
     from .module_deprecations import module_deprecations
+    from .module_patch import module_patch
     from .module_tests import module_tests
     from .module_todos import module_todos
     from .module_version import module_version
@@ -82,26 +75,27 @@ def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull
         self.warned = []
         self.failed = []
         self.modules_repo = ModulesRepo(remote_url, branch, no_pull, base_path)
-        self.lint_tests = self._get_all_lint_tests()
+        self.lint_tests = self.get_all_lint_tests(self.repo_type == "pipeline")
         # Get lists of modules install in directory
         self.all_local_modules, self.all_nfcore_modules = self.get_installed_modules()
 
         self.lint_config = None
         self.modules_json = None
 
-        # Add tests specific to nf-core/modules or pipelines
-        if self.repo_type == "modules":
-            self.lint_tests.append("module_tests")
-
-        if self.repo_type == "pipeline":
-            # Add as first test to load git_sha before module_changes
-            self.lint_tests.insert(0, "module_version")
-            # Only check if modules have been changed in pipelines
-            self.lint_tests.append("module_changes")
-
     @staticmethod
-    def _get_all_lint_tests():
-        return ["main_nf", "meta_yml", "module_todos", "module_deprecations"]
+    def get_all_lint_tests(is_pipeline):
+        if is_pipeline:
+            return [
+                "module_patch",
+                "module_version",
+                "main_nf",
+                "meta_yml",
+                "module_todos",
+                "module_deprecations",
+                "module_changes",
+            ]
+        else:
+            return ["main_nf", "meta_yml", "module_todos", "module_deprecations", "module_tests"]
 
     def lint(
         self,

nf_core/modules/lint/main_nf.py

@@ -6,11 +6,13 @@
 import logging
 import re
 import sqlite3
+from pathlib import Path
 
 import requests
 
 import nf_core
 import nf_core.modules.module_utils
+from nf_core.modules.modules_differ import ModulesDiffer
 
 log = logging.getLogger(__name__)
 
@@ -38,14 +40,26 @@ def main_nf(module_lint_object, module, fix_version, progress_bar):
     inputs = []
     outputs = []
 
-    # Check whether file exists and load it
-    try:
-        with open(module.main_nf, "r") as fh:
-            lines = fh.readlines()
-        module.passed.append(("main_nf_exists", "Module file exists", module.main_nf))
-    except FileNotFoundError as e:
-        module.failed.append(("main_nf_exists", "Module file does not exist", module.main_nf))
-        return
+    # Check if we have a patch file affecting the 'main.nf' file
+    # otherwise read the lines directly from the module
+    lines = None
+    if module.is_patched:
+        lines = ModulesDiffer.try_apply_patch(
+            module.module_name,
+            "nf-core/modules",
+            module.patch_path,
+            Path(module.module_dir).relative_to(module.base_dir),
+            reverse=True,
+        ).get("main.nf")
+    if lines is None:
+        try:
+            # Check whether file exists and load it
+            with open(module.main_nf, "r") as fh:
+                lines = fh.readlines()
+            module.passed.append(("main_nf_exists", "Module file exists", module.main_nf))
+        except FileNotFoundError as e:
+            module.failed.append(("main_nf_exists", "Module file does not exist", module.main_nf))
+            return
 
     deprecated_i = ["initOptions", "saveFiles", "getSoftwareName", "getProcessName", "publishDir"]
     lines_j = "\n".join(lines)

nf_core/modules/lint/meta_yml.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python
 
+from pathlib import Path
+
 import yaml
 
+from nf_core.modules.modules_differ import ModulesDiffer
+
 
 def meta_yml(module_lint_object, module):
     """
@@ -19,19 +23,32 @@ def meta_yml(module_lint_object, module):
     """
     required_keys = ["name", "output"]
     required_keys_lists = ["input", "output"]
-    try:
-        with open(module.meta_yml, "r") as fh:
-            meta_yaml = yaml.safe_load(fh)
-        module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml))
-    except FileNotFoundError:
-        module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml))
-        return
+    # Check if we have a patch file, get original file in that case
+    meta_yaml = None
+    if module.is_patched:
+        lines = ModulesDiffer.try_apply_patch(
+            module.module_name,
+            "nf-core/modules",
+            module.patch_path,
+            Path(module.module_dir).relative_to(module.base_dir),
+            reverse=True,
+        ).get("meta.yml")
+        if lines is not None:
+            meta_yaml = yaml.safe_load("".join(lines))
+    if meta_yaml is None:
+        try:
+            with open(module.meta_yml, "r") as fh:
+                meta_yaml = yaml.safe_load(fh)
+            module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml))
+        except FileNotFoundError:
+            module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml))
+            return
 
     # Confirm that all required keys are given
     contains_required_keys = True
     all_list_children = True
     for rk in required_keys:
-        if not rk in meta_yaml.keys():
+        if rk not in meta_yaml.keys():
             module.failed.append(("meta_required_keys", f"`{rk}` not specified in YAML", module.meta_yml))
             contains_required_keys = False
         elif rk in meta_yaml.keys() and not isinstance(meta_yaml[rk], list) and rk in required_keys_lists:

nf_core/modules/lint/module_changes.py

@@ -2,6 +2,11 @@
 Check whether the content of a module has changed compared to the original repository
 """
 import os
+import shutil
+import tempfile
+from pathlib import Path
+
+from nf_core.modules.modules_differ import ModulesDiffer
 
 
 def module_changes(module_lint_object, module):
@@ -17,22 +22,40 @@ def module_changes(module_lint_object, module):
 
     Only runs when linting a pipeline, not the modules repository
     """
+    if module.is_patched:
+        # If the module is patched, we need to apply
+        # the patch in reverse before comparing with the remote
+        tempdir = Path(tempfile.mkdtemp())
+        shutil.copytree(module.module_dir, tempdir, dirs_exist_ok=True)
+        try:
+            new_lines = ModulesDiffer.try_apply_patch(
+                module.module_name, "nf-core/modules", module.patch_path, tempdir, reverse=True
+            )
+            for file, lines in new_lines.items():
+                with open(tempdir / file, "w") as fh:
+                    fh.writelines(lines)
+        except LookupError:
+            # This error is already reported by module_patch, so just return
+            return
+    else:
+        tempdir = module.module_dir
+
     for f, same in module_lint_object.modules_repo.module_files_identical(
-        module.module_name, module.module_dir, module.git_sha
+        module.module_name, tempdir, module.git_sha
     ).items():
         if same:
             module.passed.append(
                 (
                     "check_local_copy",
                     "Local copy of module up to date",
-                    f"{os.path.join(module.module_dir, f)}",
+                    f"{Path(module.module_dir, f)}",
                 )
             )
         else:
             module.failed.append(
                 (
                     "check_local_copy",
                     "Local copy of module does not match remote",
-                    f"{os.path.join(module.module_dir, f)}",
+                    f"{Path(module.module_dir, f)}",
                 )
             )