diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 7fd049f6fc..629c1562da 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,5 +1,7 @@ import logging import os +import re +from pathlib import Path import questionary import rich.prompt @@ -113,3 +115,25 @@ def prompt_component_version_sha(component_name, component_type, modules_repo, i ).unsafe_ask() page_nbr += 1 return git_sha + + +def get_components_to_install(subworkflow_dir): + """ + Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows. + """ + modules = [] + subworkflows = [] + with open(Path(subworkflow_dir, "main.nf"), "r") as fh: + for line in fh: + regex = re.compile( + r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" + ) + match = regex.match(line) + if match and len(match.groups()) == 2: + name, link = match.groups() + if link.startswith("../../../"): + name_split = name.lower().split("_") + modules.append("/".join(name_split)) + elif link.startswith("../"): + subworkflows.append(name.lower()) + return modules, subworkflows diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 77533fc503..2d4a1b19df 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -10,7 +10,10 @@ import nf_core.modules.modules_utils import nf_core.utils from nf_core.components.components_command import ComponentCommand -from nf_core.components.components_utils import prompt_component_version_sha +from nf_core.components.components_utils import ( + get_components_to_install, + prompt_component_version_sha, +) from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME @@ -53,7 +56,8 @@ def install(self, component, silent=False): # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() + if not silent: + modules_json.check_up_to_date() # Verify SHA if not self.modules_repo.verify_sha(self.prompt, self.sha): @@ -138,32 +142,11 @@ def install(self, component, silent=False): ) return True - def get_modules_subworkflows_to_install(self, subworkflow_dir): - """ - Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows. - """ - modules = [] - subworkflows = [] - with open(Path(subworkflow_dir, "main.nf"), "r") as fh: - for line in fh: - regex = re.compile( - r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" - ) - match = regex.match(line) - if match and len(match.groups()) == 2: - name, link = match.groups() - if link.startswith("../../../"): - name_split = name.lower().split("_") - modules.append("/".join(name_split)) - elif link.startswith("../"): - subworkflows.append(name.lower()) - return modules, subworkflows - def install_included_components(self, subworkflow_dir): """ Install included modules and subworkflows """ - modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) + modules_to_install, subworkflows_to_install = get_components_to_install(subworkflow_dir) for s_install in subworkflows_to_install: original_installed = self.installed_by self.installed_by = Path(subworkflow_dir).parts[-1] diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 22aef3b5da..c246fbfef5 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -96,7 +96,8 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr self.check_modules_structure() # Verify that 'modules.json' is consistent with the installed modules - self.modules_json.check_up_to_date() + if not silent: + self.modules_json.check_up_to_date() if not self.update_all and component is None: choices = [f"All {self.component_type}", f"Named {self.component_type[:-1]}"] @@ -233,9 +234,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr else: updated.append(component) recursive_update = True - modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update( - component, modules_repo - ) + modules_to_update, subworkflows_to_update = self.get_components_to_update(component, modules_repo) if not silent and len(modules_to_update + subworkflows_to_update) > 0: log.warning( f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be added to the same diff file.\n" @@ -282,9 +281,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type) updated.append(component) recursive_update = True - modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update( - component, modules_repo - ) + modules_to_update, subworkflows_to_update = self.get_components_to_update(component, modules_repo) if not silent and not self.update_all and len(modules_to_update + subworkflows_to_update) > 0: log.warning( f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" @@ -349,14 +346,14 @@ def get_single_component_info(self, component): # Check if there are any modules/subworkflows installed from the repo repo_url = self.modules_repo.remote_url components = self.modules_json.get_all_components(self.component_type).get(repo_url) - choices = [component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components] - if repo_url not in self.modules_json.get_all_components(self.component_type): + if components is None: raise LookupError(f"No {self.component_type} installed from '{repo_url}'") + choices = [component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components] if component is None: component = questionary.autocomplete( f"{self.component_type[:-1].title()} name:", - choices=choices.sort(), + choices=sorted(choices), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -819,8 +816,13 @@ def try_apply_patch( return True - def get_modules_subworkflows_to_update(self, component, modules_repo): - """Get all modules and subworkflows linked to the updated component.""" + def get_components_to_update(self, component, modules_repo): + """ + Get all modules and subworkflows linked to the updated component. + + Returns: + (list,list): A tuple of lists with the modules and subworkflows to update + """ mods_json = self.modules_json.get_modules_json() modules_to_update = [] subworkflows_to_update = [] diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index fcce0e982f..a38cea2cd9 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -90,7 +90,7 @@ def __init__( modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() all_pipeline_modules = modules_json.get_all_components(self.component_type) - if self.modules_repo.remote_url in all_pipeline_modules: + if all_pipeline_modules is not None and self.modules_repo.remote_url in all_pipeline_modules: module_dir = Path(self.dir, "modules", "nf-core") self.all_remote_modules = [ NFCoreModule(m[1], self.modules_repo.remote_url, module_dir / m[1], self.repo_type, Path(self.dir)) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index bad80631d3..0c6ed019ad 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -14,6 +14,7 @@ import nf_core.modules.modules_repo import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_utils import get_components_to_install from .modules_differ import ModulesDiffer @@ -323,7 +324,7 @@ def determine_branches_and_shas(self, component_type, install_dir, remote_url, c repo_entry[component] = { "branch": modules_repo.branch, "git_sha": correct_commit_sha, - "installed_by": "modules", + "installed_by": [component_type], } # Clean up the modules/subworkflows we were unable to find the sha for @@ -549,9 +550,10 @@ def check_up_to_date(self): if not self.has_git_url_and_modules(): raise UserWarning except UserWarning: - log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") + log.info("The 'modules.json' file is not up to date. Recreating the 'modules.json' file.") self.create() + # Get unsynced components ( modules_missing_from_modules_json, subworkflows_missing_from_modules_json, @@ -587,6 +589,16 @@ def check_up_to_date(self): component_type ] + # Recreate "installed_by" entry + original_pipeline_components = self.pipeline_components + self.pipeline_components = None + subworkflows_dict = self.get_all_components("subworkflows") + if subworkflows_dict: + for repo, subworkflows in subworkflows_dict.items(): + for org, subworkflow in subworkflows: + self.recreate_dependencies(repo, org, subworkflow) + self.pipeline_components = original_pipeline_components + self.dump() def load(self): @@ -917,6 +929,8 @@ def get_all_components(self, component_type): if component_type in repo_entry: for dir, components in repo_entry[component_type].items(): self.pipeline_components[repo] = [(dir, m) for m in components] + if self.pipeline_components == {}: + self.pipeline_components = None return self.pipeline_components @@ -1106,3 +1120,28 @@ def components_with_repos(): } } ) + + def recreate_dependencies(self, repo, org, subworkflow): + """ + Try to recreate the installed_by entries for subworkflows. + Remove self installation entry from dependencies, assuming that the modules.json has been freshly created, + i.e., no module or subworkflow has been installed by the user in the meantime + """ + + sw_path = Path(self.subworkflows_dir, org, subworkflow) + dep_mods, dep_subwfs = get_components_to_install(sw_path) + + for dep_mod in dep_mods: + installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] + if installed_by == ["modules"]: + self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] + if subworkflow not in installed_by: + self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(subworkflow) + + for dep_subwf in dep_subwfs: + installed_by = self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] + if installed_by == ["subworkflows"]: + self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] + if subworkflow not in installed_by: + self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"].append(subworkflow) + self.recreate_dependencies(repo, org, dep_subwf)