Merge pull request #1724 from ErikDanielsson/git-remote-agnostic

Make `nf-core modules` commands work with arbitrary git remotes

CHANGELOG.md

@@ -64,6 +64,7 @@
 - Add support for patch in `nf-core modules update` command ([#1312](https://github.com/nf-core/tools/issues/1312))
 - Add support for patch in `nf-core modules lint` command ([#1312](https://github.com/nf-core/tools/issues/1312))
 - Add support for custom remotes in `nf-core modules lint` ([#1715](https://github.com/nf-core/tools/issues/1715))
+- Make `nf-core modules` commands work with arbitrary git remotes ([#1721](https://github.com/nf-core/tools/issues/1721))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 

nf_core/lint/modules_json.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from pathlib import Path
+
 from nf_core.modules.modules_command import ModuleCommand
 from nf_core.modules.modules_json import ModulesJson
 
@@ -21,10 +23,9 @@ def modules_json(self):
     modules_json = ModulesJson(self.wf_path)
     modules_json.load()
     modules_json_dict = modules_json.modules_json
+    modules_dir = Path(self.wf_path, "modules")
 
     if modules_json:
-        modules_command.get_pipeline_modules()
-
         all_modules_passed = True
 
         for repo in modules_json_dict["repos"].keys():
@@ -35,9 +36,11 @@ def modules_json(self):
                 )
                 continue
 
-            for key in modules_json_dict["repos"][repo]["modules"]:
-                if not key in modules_command.module_names[repo]:
-                    failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
+            for module in modules_json_dict["repos"][repo]["modules"]:
+                if not Path(modules_dir, repo, module).exists():
+                    failed.append(
+                        f"Entry for `{Path(repo, module)}` found in `modules.json` but module is not installed in pipeline."
+                    )
                     all_modules_passed = False
 
         if all_modules_passed:

nf_core/modules/info.py

@@ -11,6 +11,7 @@
 from rich.text import Text
 
 import nf_core.utils
+from nf_core.modules.modules_json import ModulesJson
 
 from .module_utils import get_repo_type
 from .modules_command import ModuleCommand
@@ -35,7 +36,11 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull, base_path):
                 log.debug(f"Only showing remote info: {e}")
                 pipeline_dir = None
 
-        self.get_pipeline_modules()
+        if self.repo_type == "pipeline":
+            self.modules_json = ModulesJson(self.dir)
+            self.modules_json.check_up_to_date()
+        else:
+            self.modules_json = None
         self.module = self.init_mod_name(tool)
 
     def init_mod_name(self, module):
@@ -51,9 +56,9 @@ def init_mod_name(self, module):
             ).unsafe_ask()
             if local:
                 if self.repo_type == "modules":
-                    modules = self.module_names["modules"]
+                    modules = self.get_modules_clone_modules()
                 else:
-                    modules = self.module_names.get(self.modules_repo.fullname)
+                    modules = self.modules_json.get_all_modules().get(self.modules_repo.fullname)
                     if modules is None:
                         raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'")
             else:
@@ -98,7 +103,7 @@ def get_local_yaml(self):
             repo_name = self.modules_repo.fullname
             module_base_path = os.path.join(self.dir, "modules", repo_name)
             # Check that we have any modules installed from this repo
-            modules = self.module_names.get(repo_name)
+            modules = self.modules_json.get_all_modules().get(repo_name)
             if modules is None:
                 raise LookupError(f"No modules installed from {self.modules_repo.remote_url}")
 

nf_core/modules/install.py

@@ -118,8 +118,7 @@ def install(self, module):
                 return False
         else:
             # Fetch the latest commit for the module
-            git_log = list(self.modules_repo.get_module_git_log(module, depth=1))
-            version = git_log[0]["git_sha"]
+            version = self.modules_repo.get_latest_module_version(module)
 
         if self.force:
             log.info(f"Removing installed version of '{self.modules_repo.fullname}/{module}'")

nf_core/modules/lint/__init__.py

@@ -77,20 +77,23 @@ def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull
         self.failed = []
         self.modules_repo = ModulesRepo(remote_url, branch, no_pull, base_path)
         self.lint_tests = self.get_all_lint_tests(self.repo_type == "pipeline")
-        # Get lists of modules install in directory
-        self.get_pipeline_modules(local=True)
 
         if self.repo_type == "pipeline":
-            if self.modules_repo.fullname in self.module_names:
+            modules_json = ModulesJson(self.dir)
+            modules_json.check_up_to_date()
+            all_pipeline_modules = modules_json.get_all_modules()
+            if self.modules_repo.fullname in all_pipeline_modules:
                 module_dir = Path(self.dir, "modules", self.modules_repo.fullname)
                 self.all_remote_modules = [
                     NFCoreModule(m, self.modules_repo.fullname, module_dir / m, self.repo_type, Path(self.dir))
-                    for m in self.module_names[self.modules_repo.fullname]
+                    for m in all_pipeline_modules[self.modules_repo.fullname]
                 ]
+                if not self.all_remote_modules:
+                    raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.")
                 local_module_dir = Path(self.dir, "modules", "local")
                 self.all_local_modules = [
                     NFCoreModule(m, None, local_module_dir / m, self.repo_type, Path(self.dir), nf_core_module=False)
-                    for m in self.module_names.get("local", [])
+                    for m in self.get_local_modules()
                 ]
 
             else:
@@ -99,9 +102,11 @@ def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull
             module_dir = Path(self.dir, "modules")
             self.all_remote_modules = [
                 NFCoreModule(m, None, module_dir / m, self.repo_type, Path(self.dir))
-                for m in self.module_names["modules"]
+                for m in self.get_modules_clone_modules()
             ]
             self.all_local_modules = []
+            if not self.all_remote_modules:
+                raise LookupError("No modules in 'modules' directory")
 
         self.lint_config = None
         self.modules_json = None

nf_core/modules/list.py

@@ -68,14 +68,10 @@ def pattern_msg(keywords):
             modules_json = ModulesJson(self.dir)
             modules_json.check_up_to_date()
 
-            # Get installed modules
-            self.get_pipeline_modules()
-
             # Filter by keywords
             repos_with_mods = {
-                repo_name: [mod for mod in self.module_names[repo_name] if all(k in mod for k in keywords)]
-                for repo_name in self.module_names
-                if repo_name != "local"
+                repo_name: [mod for mod in modules if all(k in mod for k in keywords)]
+                for repo_name, modules in modules_json.get_all_modules().items()
             }
 
             # Nothing found

nf_core/modules/module_test.py

@@ -16,6 +16,7 @@
 import nf_core.modules.module_utils
 import nf_core.utils
 from nf_core.modules.modules_command import ModuleCommand
+from nf_core.modules.modules_json import ModulesJson
 
 log = logging.getLogger(__name__)
 
@@ -61,7 +62,6 @@ def __init__(
         self.pytest_args = pytest_args
 
         super().__init__(".", remote_url, branch, no_pull)
-        self.get_pipeline_modules()
 
     def run(self):
         """Run test steps"""
@@ -79,17 +79,19 @@ def _check_inputs(self):
 
         # Retrieving installed modules
         if self.repo_type == "modules":
-            installed_modules = self.module_names["modules"]
+            installed_modules = self.get_modules_clone_modules()
         else:
-            installed_modules = self.module_names.get(self.modules_repo.fullname)
+            modules_json = ModulesJson(self.dir)
+            modules_json.check_up_to_date()
+            installed_modules = modules_json.get_all_modules().get(self.modules_repo.fullname)
 
         # Get the tool name if not specified
         if self.module_name is None:
             if self.no_prompts:
                 raise UserWarning(
                     "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL."
                 )
-            if installed_modules is None:
+            if not installed_modules:
                 raise UserWarning(
                     f"No installed modules were found from '{self.modules_repo.remote_url}'.\n"
                     f"Are you running the tests inside the nf-core/modules main directory?\n"

nf_core/modules/modules_command.py

@@ -26,7 +26,6 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=N
         """
         self.modules_repo = ModulesRepo(remote_url, branch, no_pull, base_path)
         self.dir = dir
-        self.module_names = None
         try:
             if self.dir:
                 self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
@@ -35,53 +34,23 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=N
         except LookupError as e:
             raise UserWarning(e)
 
-    def get_pipeline_modules(self, local=False):
+    def get_modules_clone_modules(self):
         """
-        Get the modules installed in the current directory.
-
-        If the current directory is a pipeline, the `module_names`
-        field is set to a dictionary indexed by the different
-        installation repositories in the directory. If the directory
-        is a clone of nf-core/modules the filed is set to
-        `{"modules": modules_in_dir}`
-
+        Get the modules available in a clone of nf-core/modules
         """
-
-        self.module_names = {}
-
         module_base_path = Path(self.dir, "modules")
+        return [
+            str(Path(dir).relative_to(module_base_path))
+            for dir, _, files in os.walk(module_base_path)
+            if "main.nf" in files
+        ]
 
-        if self.repo_type == "pipeline":
-            module_relpaths = (
-                Path(dir).relative_to(module_base_path)
-                for dir, _, files in os.walk(module_base_path)
-                if "main.nf" in files and not str(Path(dir).relative_to(module_base_path)).startswith("local")
-            )
-            # The two highest directories are the repo owner and repo name. The rest is the module name
-            repos_and_modules = (("/".join(dir.parts[:2]), "/".join(dir.parts[2:])) for dir in module_relpaths)
-            for repo, module in repos_and_modules:
-                if repo not in self.module_names:
-                    self.module_names[repo] = []
-                self.module_names[repo].append(module)
-
-            local_module_dir = Path(module_base_path, "local")
-            if local and local_module_dir.exists():
-                # Get the local modules
-                self.module_names["local"] = [
-                    str(path.relative_to(local_module_dir))
-                    for path in local_module_dir.iterdir()
-                    if path.suffix == ".nf"
-                ]
-
-        elif self.repo_type == "modules":
-            self.module_names["modules"] = [
-                str(Path(dir).relative_to(module_base_path))
-                for dir, _, files in os.walk(module_base_path)
-                if "main.nf" in files
-            ]
-        else:
-            log.error("Directory is neither a clone of nf-core/modules nor a pipeline")
-            raise SystemError
+    def get_local_modules(self):
+        """
+        Get the local modules in a pipeline
+        """
+        local_module_dir = Path(self.dir, "modules", "local")
+        return [str(path.relative_to(local_module_dir)) for path in local_module_dir.iterdir() if path.suffix == ".nf"]
 
     def has_valid_directory(self):
         """Check that we were given a pipeline or clone of nf-core/modules"""
@@ -123,6 +92,24 @@ def clear_module_dir(self, module_name, module_dir):
             log.error(f"Could not remove module: {e}")
             return False
 
+    def modules_from_repo(self, repo_name):
+        """
+        Gets the modules installed from a certain repository
+
+        Args:
+            repo_name (str): The name of the repository
+
+        Returns:
+            [str]: The names of the modules
+        """
+        repo_dir = Path(self.dir, "modules", repo_name)
+        if not repo_dir.exists():
+            raise LookupError(f"Nothing installed from {repo_name} in pipeline")
+
+        return [
+            str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files
+        ]
+
     def install_module_files(self, module_name, module_version, modules_repo, install_dir):
         """
         Installs a module into the given directory