Merge branch 'dev' into update-gitpod

CHANGELOG.md

@@ -6,18 +6,23 @@
 
 - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133]
 - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub]
-- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`)
+- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170))
 
 ### Linting
 
 ### Modules
 
-- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175))
+- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177))
 
 ### Subworkflows
 
+- Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177))
+- Fix `nf-core subworkflows test` not running subworkflow tests ([#2181](https://github.com/nf-core/tools/pull/2181))
+
 ### General
 
+- `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)).
+
 ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19]
 
 ### Template

nf_core/components/components_test.py

@@ -180,7 +180,10 @@ def _run_pytests(self):
         console.rule(self.component_name, style="black")
 
         # Set pytest arguments
-        command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"]
+        tag = self.component_name
+        if self.component_type == "subworkflows":
+            tag = "subworkflows/" + tag
+        command_args = ["--tag", f"{tag}", "--symlink", "--keep-workflow-wd", "--git-aware"]
         command_args += self.pytest_args
 
         # Run pytest

nf_core/components/info.py

@@ -8,6 +8,7 @@
 from rich.console import Group
 from rich.markdown import Markdown
 from rich.panel import Panel
+from rich.syntax import Syntax
 from rich.table import Table
 from rich.text import Text
 
@@ -128,6 +129,19 @@ def init_mod_name(self, component):
                     choices=components,
                     style=nf_core.utils.nfcore_question_style,
                 ).unsafe_ask()
+        else:
+            if self.repo_type == "pipeline":
+                # check if the module is locally installed
+                local_paths = self.modules_json.get_all_components(self.component_type).get(
+                    self.modules_repo.remote_url, {}
+                )
+                for directory, comp in local_paths:
+                    if comp == component:
+                        component_base_path = Path(self.dir, self.component_type)
+                        self.local_path = Path(component_base_path, directory, component)
+                        break
+                if self.local_path:
+                    self.local = True
 
         return component
 
@@ -283,12 +297,37 @@ def generate_component_info_help(self):
             renderables.append(outputs_table)
 
         # Installation command
-        if self.remote_location:
+        if self.remote_location and not self.local:
             cmd_base = f"nf-core {self.component_type}"
             if self.remote_location != NF_CORE_MODULES_REMOTE:
                 cmd_base = f"nf-core {self.component_type} --git-remote {self.remote_location}"
             renderables.append(
                 Text.from_markup(f"\n :computer:  Installation command: [magenta]{cmd_base} install {self.component}\n")
             )
 
+        # Print include statement
+        if self.local_path:
+            install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path)
+            component_name = "_".join(self.component.upper().split("/"))
+            renderables.append(
+                Text.from_markup(f"\n [blue]Use the following statement to include this {self.component_type[:-1]}:")
+            )
+            renderables.append(
+                Syntax(
+                    f"include {{ {component_name} }} from '../{Path(install_folder, self.component).relative_to(self.dir)}/main'",
+                    "groovy",
+                    theme="ansi_dark",
+                    padding=1,
+                )
+            )
+            if self.component_type == "subworkflows":
+                subworkflow_config = Path(install_folder, self.component, "nextflow.config").relative_to(self.dir)
+                if os.path.isfile(subworkflow_config):
+                    renderables.append(
+                        Text.from_markup("\n [blue]Add the following config statement to use this subworkflow:")
+                    )
+                    renderables.append(
+                        Syntax(f"includeConfig '{subworkflow_config}'", "groovy", theme="ansi_dark", padding=1)
+                    )
+
         return Group(*renderables)

nf_core/components/install.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import re
 from pathlib import Path
 
 import questionary
@@ -74,10 +73,10 @@ def install(self, component, silent=False):
         )
 
         # Set the install folder based on the repository name
-        install_folder = os.path.join(self.dir, self.component_type, self.modules_repo.repo_path)
+        install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path)
 
         # Compute the component directory
-        component_dir = os.path.join(install_folder, component)
+        component_dir = Path(install_folder, component)
 
         # Check that the component is not already installed
         component_not_installed = self.check_component_installed(
@@ -169,19 +168,19 @@ def collect_and_verify_name(self, component, modules_repo):
         if component is None:
             component = questionary.autocomplete(
                 f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:",
-                choices=sorted(modules_repo.get_avail_components(self.component_type)),
+                choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.sha)),
                 style=nf_core.utils.nfcore_question_style,
             ).unsafe_ask()
 
         # Check that the supplied name is an available module/subworkflow
-        if component and component not in modules_repo.get_avail_components(self.component_type):
+        if component and component not in modules_repo.get_avail_components(self.component_type, commit=self.sha):
             log.error(
                 f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}."
             )
             log.info(f"Use the command 'nf-core {self.component_type} list' to view available software")
             return False
 
-        if not modules_repo.component_exists(component, self.component_type):
+        if not modules_repo.component_exists(component, self.component_type, commit=self.sha):
             warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})"
             log.warning(warn_msg)
             return False

nf_core/components/update.py

@@ -386,7 +386,7 @@ def get_single_component_info(self, component):
             )
 
         # Check that the supplied name is an available module/subworkflow
-        if component and component not in self.modules_repo.get_avail_components(self.component_type):
+        if component and component not in self.modules_repo.get_avail_components(self.component_type, commit=self.sha):
             raise LookupError(
                 f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}."
                 f"Use the command 'nf-core {self.component_type} list remote' to view available software"
@@ -879,25 +879,23 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up
             if m_update in updated:
                 continue
             original_component_type, original_update_all = self._change_component_type("modules")
-            self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist)
-            self._reset_component_type(original_component_type, original_update_all)
+            try:
+                self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist)
+            except LookupError as e:
+                # If the module to be updated is not available, check if there has been a name change
+                if "not found in list of available" in str(e):
+                    # Skip update, we check for name changes with manage_changes_in_linked_components
+                    pass
+                else:
+                    raise
+            finally:
+                self._reset_component_type(original_component_type, original_update_all)
 
     def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update):
         """Check for linked components added or removed in the new subworkflow version"""
         if self.component_type == "subworkflows":
             subworkflow_directory = Path(self.dir, self.component_type, self.modules_repo.repo_path, component)
             included_modules, included_subworkflows = get_components_to_install(subworkflow_directory)
-            # If a new module/subworkflow is included in the subworklfow and wasn't included before
-            for module in included_modules:
-                if module not in modules_to_update:
-                    log.info(f"Installing newly included module '{module}' for '{component}'")
-                    install_module_object = ComponentInstall(self.dir, "modules", installed_by=component)
-                    install_module_object.install(module, silent=True)
-            for subworkflow in included_subworkflows:
-                if subworkflow not in subworkflows_to_update:
-                    log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'")
-                    install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component)
-                    install_subworkflow_object.install(subworkflow, silent=True)
             # If a module/subworkflow has been removed from the subworkflow
             for module in modules_to_update:
                 if module not in included_modules:
@@ -909,6 +907,17 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw
                     log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.")
                     remove_subworkflow_object = ComponentRemove("subworkflows", self.dir)
                     remove_subworkflow_object.remove(subworkflow, removed_by=component)
+            # If a new module/subworkflow is included in the subworklfow and wasn't included before
+            for module in included_modules:
+                if module not in modules_to_update:
+                    log.info(f"Installing newly included module '{module}' for '{component}'")
+                    install_module_object = ComponentInstall(self.dir, "modules", installed_by=component)
+                    install_module_object.install(module, silent=True)
+            for subworkflow in included_subworkflows:
+                if subworkflow not in subworkflows_to_update:
+                    log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'")
+                    install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component)
+                    install_subworkflow_object.install(subworkflow, silent=True)
 
     def _change_component_type(self, new_component_type):
         original_component_type = self.component_type

nf_core/module-template/modules/meta.yml

@@ -1,7 +1,7 @@
 name: "{{ component_name_underscore }}"
 {% if not_empty_template -%}
 ## TODO nf-core: Add a description of the module and list keywords
-{%- endif -%}
+{% endif -%}
 description: write your description here
 keywords:
   - sort

nf_core/modules/modules_repo.py

@@ -302,7 +302,7 @@ def checkout(self, commit):
         """
         self.repo.git.checkout(commit)
 
-    def component_exists(self, component_name, component_type, checkout=True):
+    def component_exists(self, component_name, component_type, checkout=True, commit=None):
         """
         Check if a module/subworkflow exists in the branch of the repo
 
@@ -312,7 +312,7 @@ def component_exists(self, component_name, component_type, checkout=True):
         Returns:
             (bool): Whether the module/subworkflow exists in this branch of the repository
         """
-        return component_name in self.get_avail_components(component_type, checkout=checkout)
+        return component_name in self.get_avail_components(component_type, checkout=checkout, commit=commit)
 
     def get_component_dir(self, component_name, component_type):
         """
@@ -449,7 +449,7 @@ def get_commit_info(self, sha):
                 return message, date
         raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'")
 
-    def get_avail_components(self, component_type, checkout=True):
+    def get_avail_components(self, component_type, checkout=True, commit=None):
         """
         Gets the names of the modules/subworkflows in the repository. They are detected by
         checking which directories have a 'main.nf' file
@@ -459,6 +459,8 @@ def get_avail_components(self, component_type, checkout=True):
         """
         if checkout:
             self.checkout_branch()
+        if commit is not None:
+            self.checkout(commit)
         # Get directory
         if component_type == "modules":
             directory = self.modules_dir

tests/modules/info.py

@@ -38,7 +38,6 @@ def test_modules_info_local(self):
     """Test getting info about a locally installed module"""
     self.mods_install.install("trimgalore")
     mods_info = nf_core.modules.ModuleInfo(self.pipeline_dir, "trimgalore")
-    mods_info.local = True
     mods_info_output = mods_info.get_component_info()
     console = Console(record=True)
     console.print(mods_info_output)
@@ -47,6 +46,7 @@ def test_modules_info_local(self):
     assert "Module: trimgalore" in output
     assert "Inputs" in output
     assert "Outputs" in output
+    assert "Location" in output
 
 
 def test_modules_info_in_modules_repo(self):

tests/subworkflows/update.py

@@ -8,6 +8,7 @@
 import nf_core.utils
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
+from nf_core.modules.remove import ModuleRemove
 from nf_core.modules.update import ModuleUpdate
 from nf_core.subworkflows.update import SubworkflowUpdate
 
@@ -321,6 +322,10 @@ def test_update_change_of_included_modules(self):
     # Check that tabix/tabix is there
     assert "tabix/tabix" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
     assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/tabix").is_dir()
+    # Check that ensemblevep is not there but instead we have ensemblevep/vep (due to a file re-naming)
+    assert "ensemblvep" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
+    assert "ensemblvep/vep" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
+    assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "ensemblvep/vep").is_dir()
 
 
 def cmp_component(dir1, dir2):