Make nf-core modules commands work with arbitrary git remotes

CHANGELOG.md

@@ -63,6 +63,7 @@
 - Add `nf-core modules patch` command ([#1312](https://github.com/nf-core/tools/issues/1312))
 - Add support for patch in `nf-core modules update` command ([#1312](https://github.com/nf-core/tools/issues/1312))
 - Add support for patch in `nf-core modules lint` command ([#1312](https://github.com/nf-core/tools/issues/1312))
+- Make `nf-core modules` commands work with arbitrary git remotes ([#1721](https://github.com/nf-core/tools/issues/1721))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 

nf_core/lint/modules_json.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from pathlib import Path
+
 from nf_core.modules.modules_command import ModuleCommand
 from nf_core.modules.modules_json import ModulesJson
 
@@ -21,10 +23,9 @@ def modules_json(self):
     modules_json = ModulesJson(self.wf_path)
     modules_json.load()
     modules_json_dict = modules_json.modules_json
+    modules_dir = Path(self.wf_path, "modules")
 
     if modules_json:
-        modules_command.get_pipeline_modules()
-
         all_modules_passed = True
 
         for repo in modules_json_dict["repos"].keys():
@@ -35,9 +36,11 @@ def modules_json(self):
                 )
                 continue
 
-            for key in modules_json_dict["repos"][repo]["modules"]:
-                if not key in modules_command.module_names[repo]:
-                    failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
+            for module in modules_json_dict["repos"][repo]["modules"]:
+                if not Path(modules_dir, repo, module).exists():
+                    failed.append(
+                        f"Entry for `{Path(repo, module)}` found in `modules.json` but module is not installed in pipeline."
+                    )
                     all_modules_passed = False
 
         if all_modules_passed:

nf_core/modules/info.py

@@ -11,6 +11,7 @@
 from rich.text import Text
 
 import nf_core.utils
+from nf_core.modules.modules_json import ModulesJson
 
 from .module_utils import get_repo_type
 from .modules_command import ModuleCommand
@@ -35,8 +36,12 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull, base_path):
                 log.debug(f"Only showing remote info: {e}")
                 pipeline_dir = None
 
-        self.get_pipeline_modules()
         self.module = self.init_mod_name(tool)
+        if self.repo_type == "pipeline":
+            self.modules_json = ModulesJson(self.dir)
+            self.modules_json.check_up_to_date()
+        else:
+            self.modules_json = None
 
     def init_mod_name(self, module):
         """
@@ -51,9 +56,9 @@ def init_mod_name(self, module):
             ).unsafe_ask()
             if local:
                 if self.repo_type == "modules":
-                    modules = self.module_names["modules"]
+                    modules = self.get_modules_clone_modules()
                 else:
-                    modules = self.module_names.get(self.modules_repo.fullname)
+                    modules = self.modules_json.get_all_modules().get(self.modules_repo.fullname)
                     if modules is None:
                         raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'")
             else:
@@ -98,7 +103,7 @@ def get_local_yaml(self):
             repo_name = self.modules_repo.fullname
             module_base_path = os.path.join(self.dir, "modules", repo_name)
             # Check that we have any modules installed from this repo
-            modules = self.module_names.get(repo_name)
+            modules = self.modules_json.get_all_modules().get(repo_name)
             if modules is None:
                 raise LookupError(f"No modules installed from {self.modules_repo.remote_url}")
 

nf_core/modules/install.py

@@ -118,8 +118,7 @@ def install(self, module):
                 return False
         else:
             # Fetch the latest commit for the module
-            git_log = list(self.modules_repo.get_module_git_log(module, depth=1))
-            version = git_log[0]["git_sha"]
+            version = self.modules_repo.get_latest_module_version(module)
 
         if self.force:
             log.info(f"Removing installed version of '{self.modules_repo.fullname}/{module}'")

nf_core/modules/list.py

@@ -68,13 +68,10 @@ def pattern_msg(keywords):
             modules_json = ModulesJson(self.dir)
             modules_json.check_up_to_date()
 
-            # Get installed modules
-            self.get_pipeline_modules()
-
             # Filter by keywords
             repos_with_mods = {
-                repo_name: [mod for mod in self.module_names[repo_name] if all(k in mod for k in keywords)]
-                for repo_name in self.module_names
+                repo_name: [mod for mod in modules if all(k in mod for k in keywords)]
+                for repo_name, modules in modules_json.get_all_modules().items()
             }
 
             # Nothing found

nf_core/modules/module_test.py

@@ -16,6 +16,7 @@
 import nf_core.modules.module_utils
 import nf_core.utils
 from nf_core.modules.modules_command import ModuleCommand
+from nf_core.modules.modules_json import ModulesJson
 
 log = logging.getLogger(__name__)
 
@@ -61,7 +62,6 @@ def __init__(
         self.pytest_args = pytest_args
 
         super().__init__(".", remote_url, branch, no_pull)
-        self.get_pipeline_modules()
 
     def run(self):
         """Run test steps"""
@@ -79,17 +79,19 @@ def _check_inputs(self):
 
         # Retrieving installed modules
         if self.repo_type == "modules":
-            installed_modules = self.module_names["modules"]
+            installed_modules = self.get_modules_clone_modules()
         else:
-            installed_modules = self.module_names.get(self.modules_repo.fullname)
+            modules_json = ModulesJson(self.dir)
+            modules_json.check_up_to_date()
+            installed_modules = modules_json.get_all_modules().get(self.modules_repo.fullname)
 
         # Get the tool name if not specified
         if self.module_name is None:
             if self.no_prompts:
                 raise UserWarning(
                     "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL."
                 )
-            if installed_modules is None:
+            if not installed_modules:
                 raise UserWarning(
                     f"No installed modules were found from '{self.modules_repo.remote_url}'.\n"
                     f"Are you running the tests inside the nf-core/modules main directory?\n"

nf_core/modules/modules_command.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import shutil
+from pathlib import Path
 
 import yaml
 
@@ -34,47 +35,16 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=N
         except LookupError as e:
             raise UserWarning(e)
 
-    def get_pipeline_modules(self):
+    def get_modules_clone_modules(self):
         """
-        Get the modules installed in the current directory.
-
-        If the current directory is a pipeline, the `module_names`
-        field is set to a dictionary indexed by the different
-        installation repositories in the directory. If the directory
-        is a clone of nf-core/modules the filed is set to
-        `{"modules": modules_in_dir}`
-
+        Get the modules available in a clone of nf-core/modules
         """
-
-        self.module_names = {}
-
-        module_base_path = f"{self.dir}/modules/"
-
-        if self.repo_type == "pipeline":
-            repo_owners = (owner for owner in os.listdir(module_base_path) if owner != "local")
-            repo_names = (
-                f"{repo_owner}/{name}"
-                for repo_owner in repo_owners
-                for name in os.listdir(f"{module_base_path}/{repo_owner}")
-            )
-            for repo_name in repo_names:
-                repo_path = os.path.join(module_base_path, repo_name)
-                module_mains_path = f"{repo_path}/**/main.nf"
-                module_mains = glob.glob(module_mains_path, recursive=True)
-                if len(module_mains) > 0:
-                    self.module_names[repo_name] = [
-                        os.path.dirname(os.path.relpath(mod, repo_path)) for mod in module_mains
-                    ]
-
-        elif self.repo_type == "modules":
-            module_mains_path = f"{module_base_path}/**/main.nf"
-            module_mains = glob.glob(module_mains_path, recursive=True)
-            self.module_names["modules"] = [
-                os.path.dirname(os.path.relpath(mod, module_base_path)) for mod in module_mains
-            ]
-        else:
-            log.error("Directory is neither a clone of nf-core/modules nor a pipeline")
-            raise SystemError
+        module_base_path = Path(self.dir, "modules")
+        return [
+            str(Path(dir).relative_to(module_base_path))
+            for dir, _, files in os.walk(module_base_path)
+            if "main.nf" in files
+        ]
 
     def has_valid_directory(self):
         """Check that we were given a pipeline or clone of nf-core/modules"""
@@ -116,6 +86,24 @@ def clear_module_dir(self, module_name, module_dir):
             log.error(f"Could not remove module: {e}")
             return False
 
+    def modules_from_repo(self, repo_name):
+        """
+        Gets the modules installed from a certain repository
+
+        Args:
+            repo_name (str): The name of the repository
+
+        Returns:
+            [str]: The names of the modules
+        """
+        repo_dir = Path(self.dir, "modules", repo_name)
+        if not repo_dir.exists():
+            raise LookupError(f"Nothing installed from {repo_name} in pipeline")
+
+        return [
+            str(Path(dir_path).relative_to(repo_dir) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files)
+        ]
+
     def install_module_files(self, module_name, module_version, modules_repo, install_dir):
         """
         Installs a module into the given directory

nf_core/modules/modules_json.py

@@ -32,6 +32,7 @@ def __init__(self, pipeline_dir):
         self.dir = pipeline_dir
         self.modules_dir = os.path.join(self.dir, "modules")
         self.modules_json = None
+        self.pipeline_modules = None
 
     def create(self):
         """
@@ -49,7 +50,7 @@ def create(self):
         if not os.path.exists(modules_dir):
             raise UserWarning("Can't find a ./modules directory. Is this a DSL2 pipeline?")
 
-        repos = self.get_pipeline_module_repositories(modules_dir)
+        repos = self.get_pipeline_module_repositories(Path(modules_dir))
 
         # Get all module names in the repos
         repo_module_names = [
@@ -106,29 +107,30 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None):
         """
         Finds all module repositories in the modules directory. Ignores the local modules.
         Args:
-            modules_dir (str): base directory for the module files
+            modules_dir (Path): base directory for the module files
         Returns
             repos [ (str, str, str) ]: List of tuples of repo name, repo remote URL and path to modules in repo
         """
         if repos is None:
             repos = {}
 
         # Check if there are any nf-core modules installed
-        if os.path.exists(os.path.join(modules_dir, nf_core.modules.modules_repo.NF_CORE_MODULES_NAME)):
+        if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists():
             repos[nf_core.modules.modules_repo.NF_CORE_MODULES_NAME] = (
                 nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE,
                 nf_core.modules.modules_repo.NF_CORE_MODULES_BASE_PATH,
             )
 
         # Check if there are any untracked repositories
-        dirs_not_covered = self.dir_tree_uncovered(modules_dir, [name for name in repos])
+        dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos])
         if len(dirs_not_covered) > 0:
             log.info("Found custom module repositories when creating 'modules.json'")
             # Loop until all directories in the base directory are covered by a remote
             while len(dirs_not_covered) > 0:
                 log.info(
                     "The following director{s} in the modules directory are untracked: '{l}'".format(
-                        s="ies" if len(dirs_not_covered) > 0 else "y", l="', '".join(dirs_not_covered)
+                        s="ies" if len(dirs_not_covered) > 0 else "y",
+                        l="', '".join(str(dir.relative_to(modules_dir)) for dir in dirs_not_covered),
                     )
                 )
                 nrepo_remote = questionary.text(
@@ -146,7 +148,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None):
 
                 # Verify that there is a directory corresponding the remote
                 nrepo_name = nf_core.modules.module_utils.path_from_remote(nrepo_remote)
-                if not os.path.exists(os.path.join(modules_dir, nrepo_name)):
+                if not (modules_dir / nrepo_name).exists():
                     log.info(
                         "The provided remote does not seem to correspond to a local directory. "
                         "The directory structure should be the same as in the remote."
@@ -155,7 +157,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None):
                         "Please provide the correct directory, it will be renamed. If left empty, the remote will be ignored."
                     ).unsafe_ask()
                     if dir_name:
-                        os.rename(os.path.join(modules_dir, dir_name), os.path.join(modules_dir, nrepo_name))
+                        (modules_dir, dir_name).rename(modules_dir / nrepo_name)
                     else:
                         continue
 
@@ -168,7 +170,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None):
                     nrepo_base_path = nf_core.modules.modules_repo.NF_CORE_MODULES_BASE_PATH
 
                 repos[nrepo_name] = (nrepo_remote, nrepo_base_path)
-                dirs_not_covered = self.dir_tree_uncovered(modules_dir, [name for name in repos])
+                dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos])
         return repos
 
     def find_correct_commit_sha(self, module_name, module_path, modules_repo):
@@ -199,26 +201,27 @@ def dir_tree_uncovered(self, modules_dir, repos):
         subdirectories are therefore ignore.
 
         Args:
-            module_dir (str): Base path of modules in pipeline
-            repos ([ str ]): List of repos that are covered by a remote
+            module_dir (Path): Base path of modules in pipeline
+            repos ([ Path ]): List of repos that are covered by a remote
 
         Returns:
-            dirs_not_covered ([ str ]): A list of directories that are currently not covered by any remote.
+            dirs_not_covered ([ Path ]): A list of directories that are currently not covered by any remote.
         """
+
         # Initialise the FIFO queue. Note that we assume the directory to be correctly
         # configured, i.e. no files etc.
-        fifo = [os.path.join(modules_dir, subdir) for subdir in os.listdir(modules_dir) if subdir != "local"]
+        fifo = [subdir for subdir in modules_dir.iterdir() if subdir.stem != "local"]
         depth = 1
         dirs_not_covered = []
         while len(fifo) > 0:
             temp_queue = []
-            repos_at_level = {os.path.join(*os.path.split(repo)[:depth]): len(os.path.split(repo)) for repo in repos}
+            repos_at_level = {Path(*repo.parts[:depth]): len(repo.parts) for repo in repos}
             for dir in fifo:
-                rel_dir = os.path.relpath(dir, modules_dir)
+                rel_dir = dir.relative_to(modules_dir)
                 if rel_dir in repos_at_level.keys():
                     # Go the next depth if this directory is not one of the repos
                     if depth < repos_at_level[rel_dir]:
-                        temp_queue.extend([os.path.join(dir, subdir) for subdir in os.listdir(dir)])
+                        temp_queue.extend(dir.iterdir())
                 else:
                     # Otherwise add the directory to the ones not covered
                     dirs_not_covered.append(dir)
@@ -649,6 +652,24 @@ def get_base_path(self, repo_name):
             self.load()
         return self.modules_json.get("repos", {}).get(repo_name, {}).get("base_path", None)
 
+    def get_all_modules(self):
+        """
+        Retrieves all pipeline modules that are reported in the modules.json
+
+        Returns:
+            (dict[str, [str]]): Dictionary indexed with the repo names, with a
+                                list of modules as values
+        """
+        if self.modules_json is None:
+            self.load()
+        if self.pipeline_modules is None:
+            self.pipeline_modules = {}
+            for repo, repo_entry in self.modules_json.get("repos", {}).items():
+                if "modules" in repo_entry:
+                    self.pipeline_modules[repo] = list(repo_entry["modules"])
+
+        return self.pipeline_modules
+
     def dump(self):
         """
         Sort the modules.json, and write it to file

nf_core/modules/patch.py

@@ -20,16 +20,16 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=N
         super().__init__(dir, remote_url, branch, no_pull, base_path)
 
         self.modules_json = ModulesJson(dir)
-        self.get_pipeline_modules()
 
     def param_check(self, module):
         if not self.has_valid_directory():
             raise UserWarning()
 
-        if module is not None and module not in self.module_names[self.modules_repo.fullname]:
+        if module is not None and module not in self.modules_json.get_all_modules().get(self.modules_repo.fullname, {}):
             raise UserWarning(f"Module '{Path(self.modules_repo.fullname, module)}' does not exist in the pipeline")
 
     def patch(self, module=None):
+        self.modules_json.check_up_to_date()
         self.param_check(module)
 
         if module is None: