diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd4e4d330..4ae6325455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971)) - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out` - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001)) +- Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999)) ### Modules diff --git a/docs/api/make_lint_md.py b/docs/api/make_lint_md.py index e0265c707d..9b5a706473 100644 --- a/docs/api/make_lint_md.py +++ b/docs/api/make_lint_md.py @@ -21,7 +21,6 @@ def make_docs(docs_basedir, lint_tests, md_template): else: with open(fn, "w") as fh: fh.write(md_template.format(test_name)) - print(test_name) for fn in existing_docs: os.remove(fn) diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index 1206c7c415..3452665f55 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -99,7 +99,6 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su # Check whether component file already exists component_file = os.path.join(local_component_dir, f"{name}.nf") if os.path.exists(component_file) and not force_overwrite: - print(f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite") raise UserWarning( f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" ) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index ed730edeff..5ad00695bc 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -1,12 +1,7 @@ -import glob -import json import logging import os -import re -import jinja2 import questionary -import rich import nf_core.modules.modules_utils import nf_core.utils @@ -77,7 +72,7 @@ def get_version(component, component_type, sha, prompt, current_version, modules version = sha elif prompt: try: - version = nf_core.modules.modules_utils.prompt_component_version_sha( + version = prompt_component_version_sha( component, component_type, installed_sha=current_version, @@ -98,11 +93,11 @@ def clean_modules_json(component, component_type, modules_repo, modules_json): """ for repo_url, repo_content in modules_json.modules_json["repos"].items(): for dir, dir_components in repo_content[component_type].items(): - for name, _ in dir_components.items(): + for name, component_values in dir_components.items(): if name == component and dir == modules_repo.repo_path: repo_to_remove = repo_url log.info( f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" ) - modules_json.remove_entry(component, repo_to_remove, modules_repo.repo_path) - break + modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path) + return component_values["installed_by"] diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index ad93049f56..e0919e07dd 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -1,15 +1,12 @@ import logging import os -import questionary - import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.modules_json import ModulesJson from .modules_command import ModuleCommand -from .modules_repo import NF_CORE_MODULES_NAME log = logging.getLogger(__name__) @@ -24,11 +21,16 @@ def __init__( remote_url=None, branch=None, no_pull=False, + installed_by=False, ): super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha + if installed_by: + self.installed_by = installed_by + else: + self.installed_by = self.component_type def install(self, module, silent=False): if self.repo_type == "modules": @@ -71,6 +73,11 @@ def install(self, module, silent=False): if not nf_core.components.components_install.check_component_installed( self.component_type, module, current_version, module_dir, self.modules_repo, self.force, self.prompt ): + log.debug( + f"Module is already installed and force is not set.\nAdding the new installation source {self.installed_by} for module {module} to 'modules.json' without installing the module." + ) + modules_json.load() + modules_json.update(self.modules_repo, module, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( @@ -80,10 +87,11 @@ def install(self, module, silent=False): return False # Remove module if force is set + install_track = None if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") self.clear_component_dir(module, module_dir) - nf_core.components.components_install.clean_modules_json( + install_track = nf_core.components.components_install.clean_modules_json( module, self.component_type, self.modules_repo, modules_json ) @@ -103,5 +111,5 @@ def install(self, module, silent=False): # Update module.json with newly installed module modules_json.load() - modules_json.update(self.modules_repo, module, version) + modules_json.update(self.modules_repo, module, version, self.installed_by, install_track) return True diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 151914b38b..e2c45b935b 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -274,7 +274,11 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): found_sha = True break if found_sha: - repo_entry[module] = {"branch": modules_repo.branch, "git_sha": correct_commit_sha} + repo_entry[module] = { + "branch": modules_repo.branch, + "git_sha": correct_commit_sha, + "installed_by": "modules", + } # Clean up the modules we were unable to find the sha for for module in sb_local: @@ -373,25 +377,32 @@ def parse_dirs(self, dirs, missing_installation, component_type): component_in_file = False git_url = None for repo in missing_installation: - for dir_name in missing_installation[repo][component_type]: - if component in missing_installation[repo][component_type][dir_name]: - component_in_file = True - git_url = repo - break + if component_type in missing_installation[repo]: + for dir_name in missing_installation[repo][component_type]: + if component in missing_installation[repo][component_type][dir_name]: + component_in_file = True + git_url = repo + break if not component_in_file: - # If it is not, add it to the list of missing subworkflow + # If it is not, add it to the list of missing components untracked_dirs.append(component) else: - # If it does, remove the subworkflow from missing_installation + # If it does, remove the component from missing_installation module_repo = missing_installation[git_url] # Check if the entry has a git sha and branch before removing components_dict = module_repo[component_type][install_dir] if "git_sha" not in components_dict[component] or "branch" not in components_dict[component]: self.determine_module_branches_and_shas(component, git_url, module_repo["base_path"], [component]) - # Remove the subworkflow from subworkflows without installation + # Remove the component from components without installation module_repo[component_type][install_dir].pop(component) if len(module_repo[component_type][install_dir]) == 0: + # If no modules/subworkflows with missing installation left, remove the install_dir from missing_installation + missing_installation[git_url][component_type].pop(install_dir) + if len(module_repo[component_type]) == 0: + # If no modules/subworkflows with missing installation left, remove the component_type from missing_installation + missing_installation[git_url].pop(component_type) + if len(module_repo) == 0: # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation missing_installation.pop(git_url) @@ -469,6 +480,9 @@ def check_up_to_date(self): If a module/subworkflow is installed but the entry in 'modules.json' is missing we iterate through the commit log in the remote to try to determine the SHA. + + Check that we have the "installed_by" value in 'modules.json', otherwise add it. + Assume that the modules/subworkflows were installed by an nf-core command (don't track installed by subworkflows). """ try: self.load() @@ -503,6 +517,16 @@ def check_up_to_date(self): if len(subworkflows_missing_from_modules_json) > 0: self.resolve_missing_from_modules_json(subworkflows_missing_from_modules_json, "subworkflows") + # If the "installed_by" value is not present for modules/subworkflows, add it. + for repo, repo_content in self.modules_json["repos"].items(): + for component_type, dir_content in repo_content.items(): + for install_dir, installed_components in dir_content.items(): + for component, component_features in installed_components.items(): + if "installed_by" not in component_features: + self.modules_json["repos"][repo][component_type][install_dir][component]["installed_by"] = [ + component_type + ] + self.dump() def load(self): @@ -521,7 +545,7 @@ def load(self): except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") - def update(self, modules_repo, module_name, module_version, write_file=True): + def update(self, modules_repo, module_name, module_version, installed_by, installed_by_log=None, write_file=True): """ Updates the 'module.json' file with new module info @@ -529,8 +553,12 @@ def update(self, modules_repo, module_name, module_version, write_file=True): modules_repo (ModulesRepo): A ModulesRepo object configured for the new module module_name (str): Name of new module module_version (str): git SHA for the new module entry + installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' write_file (bool): whether to write the updated modules.json to a file. """ + if installed_by_log is None: + installed_by_log = [] + if self.modules_json is None: self.load() repo_name = modules_repo.repo_path @@ -543,13 +571,22 @@ def update(self, modules_repo, module_name, module_version, write_file=True): repo_modules_entry[module_name] = {} repo_modules_entry[module_name]["git_sha"] = module_version repo_modules_entry[module_name]["branch"] = branch + try: + if installed_by not in repo_modules_entry[module_name]["installed_by"]: + repo_modules_entry[module_name]["installed_by"].append(installed_by) + except KeyError: + repo_modules_entry[module_name]["installed_by"] = [installed_by] + finally: + repo_modules_entry[module_name]["installed_by"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if write_file: self.dump() - def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version, write_file=True): + def update_subworkflow( + self, modules_repo, subworkflow_name, subworkflow_version, installed_by, installed_by_log=None, write_file=True + ): """ Updates the 'module.json' file with new subworkflow info @@ -557,8 +594,12 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version modules_repo (ModulesRepo): A ModulesRepo object configured for the new subworkflow subworkflow_name (str): Name of new subworkflow subworkflow_version (str): git SHA for the new subworkflow entry + installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' write_file (bool): whether to write the updated modules.json to a file. """ + if installed_by_log is None: + installed_by_log = [] + if self.modules_json is None: self.load() repo_name = modules_repo.repo_path @@ -574,18 +615,26 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version repo_subworkflows_entry[subworkflow_name] = {} repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version repo_subworkflows_entry[subworkflow_name]["branch"] = branch + try: + if installed_by not in repo_subworkflows_entry[subworkflow_name]["installed_by"]: + repo_subworkflows_entry[subworkflow_name]["installed_by"].append(installed_by) + except KeyError: + repo_subworkflows_entry[subworkflow_name]["installed_by"] = [installed_by] + finally: + repo_subworkflows_entry[subworkflow_name]["installed_by"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if write_file: self.dump() - def remove_entry(self, module_name, repo_url, install_dir): + def remove_entry(self, component_type, name, repo_url, install_dir): """ Removes an entry from the 'modules.json' file. Args: - module_name (str): Name of the module to be removed + component_type (Str): Type of component [modules, subworkflows] + name (str): Name of the module to be removed repo_url (str): URL of the repository containing the module install_dir (str): Name of the directory where modules are installed Returns: @@ -595,15 +644,17 @@ def remove_entry(self, module_name, repo_url, install_dir): return False if repo_url in self.modules_json.get("repos", {}): repo_entry = self.modules_json["repos"][repo_url] - if module_name in repo_entry["modules"].get(install_dir, {}): - repo_entry["modules"][install_dir].pop(module_name) + if name in repo_entry[component_type].get(install_dir, {}): + repo_entry[component_type][install_dir].pop(name) else: - log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.") + log.warning( + f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file." + ) return False - if len(repo_entry["modules"][install_dir]) == 0: + if len(repo_entry[component_type][install_dir]) == 0: self.modules_json["repos"].pop(repo_url) else: - log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.") + log.warning(f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file.") return False self.dump() diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 1284c6d59c..8afe634257 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -50,13 +50,13 @@ def remove(self, module): if modules_json.module_present(module, self.modules_repo.remote_url, repo_path): log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") - modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) + modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path) return False log.info(f"Removing {module}") # Remove entry from modules.json - modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) + modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path) # Remove the module return self.clear_component_dir(module, module_dir) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index dcdde1c67b..223637e73b 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -152,7 +152,7 @@ def update(self, module=None): if sha is not None: version = sha elif self.prompt: - version = nf_core.modules.modules_utils.prompt_component_version_sha( + version = prompt_component_version_sha( module, "modules", modules_repo=modules_repo, installed_sha=current_version ) else: @@ -223,10 +223,10 @@ def update(self, module=None): # Clear the module directory and move the installed files there self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version) # Update modules.json with newly installed module - self.modules_json.update(modules_repo, module, version) + self.modules_json.update(modules_repo, module, version, self.component_type) else: # Don't save to a file, just iteratively update the variable - self.modules_json.update(modules_repo, module, version, write_file=False) + self.modules_json.update(modules_repo, module, version, self.component_type, write_file=False) if self.save_diff_fn: # Write the modules.json diff to the file diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 4037d5905f..8618bacab6 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,15 +7,18 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed_by": ["modules"] } } } diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 44d7505f50..750007f294 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -3,14 +3,11 @@ import re from pathlib import Path -import questionary - import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME from .subworkflows_command import SubworkflowCommand @@ -27,11 +24,16 @@ def __init__( remote_url=None, branch=None, no_pull=False, + installed_by=False, ): super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha + if installed_by: + self.installed_by = installed_by + else: + self.installed_by = self.component_type def install(self, subworkflow, silent=False): if self.repo_type == "modules": @@ -77,6 +79,11 @@ def install(self, subworkflow, silent=False): self.force, self.prompt, ): + log.debug( + f"Subworkflow is already installed and force is not set.\nAdding the new installation source {self.installed_by} for subworkflow {subworkflow} to 'modules.json' without installing the subworkflow." + ) + modules_json.load() + modules_json.update(self.modules_repo, subworkflow, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( @@ -86,12 +93,19 @@ def install(self, subworkflow, silent=False): return False # Remove subworkflow if force is set and component is installed + install_track = None if self.force and nf_core.components.components_install.check_component_installed( - self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force + self.component_type, + subworkflow, + current_version, + subworkflow_dir, + self.modules_repo, + self.force, + self.prompt, ): log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") self.clear_component_dir(subworkflow, subworkflow_dir) - nf_core.components.components_install.clean_modules_json( + install_track = nf_core.components.components_install.clean_modules_json( subworkflow, self.component_type, self.modules_repo, modules_json ) @@ -117,7 +131,7 @@ def install(self, subworkflow, silent=False): # Update module.json with newly installed subworkflow modules_json.load() - modules_json.update_subworkflow(self.modules_repo, subworkflow, version) + modules_json.update_subworkflow(self.modules_repo, subworkflow, version, self.installed_by, install_track) return True def get_modules_subworkflows_to_install(self, subworkflow_dir): @@ -147,7 +161,10 @@ def install_included_components(self, subworkflow_dir): """ modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) for s_install in subworkflows_to_install: + original_installed = self.installed_by + self.installed_by = Path(subworkflow_dir).parts[-1] self.install(s_install, silent=True) + self.installed_by = original_installed for m_install in modules_to_install: module_install = ModuleInstall( self.dir, @@ -156,5 +173,6 @@ def install_included_components(self, subworkflow_dir): sha=self.sha, remote_url=self.modules_repo.remote_url, branch=self.modules_repo.branch, + installed_by=Path(subworkflow_dir).parts[-1], ) module_install.install(m_install, silent=True) diff --git a/tests/modules/list.py b/tests/modules/list.py index 2cd1333faf..d92cd58dd5 100644 --- a/tests/modules/list.py +++ b/tests/modules/list.py @@ -19,7 +19,6 @@ def test_modules_list_remote_gitlab(self): """Test listing the modules in the remote gitlab repo""" mods_list = nf_core.modules.ModuleList(None, remote=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) listed_mods = mods_list.list_components() - print(f"listed modules are {listed_mods}") console = Console(record=True) console.print(listed_mods) output = console.export_text() diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 60fb3006c1..20eee54e30 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -32,7 +32,7 @@ def test_mod_json_update(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the modules.json file mod_repo_obj = ModulesRepo() - mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", False) + mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", "modules", write_file=False) mod_json = mod_json_obj.get_modules_json() assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"] @@ -41,6 +41,7 @@ def test_mod_json_update(self): NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["branch"] ) + assert "modules" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed_by"] def test_mod_json_create(self): @@ -154,7 +155,7 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the fastqc module entry to an invalid git_sha - mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", True) + mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", write_file=True) # Remove the fastqc module fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 0f7a7c9101..57af1c499e 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -1,4 +1,3 @@ -import json import os import tempfile from pathlib import Path @@ -234,15 +233,11 @@ def test_create_patch_update_success(self): "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) - with open(os.path.join(module_path, "main.nf"), "r") as fh: - print(fh.readlines()) # Update the module update_obj = nf_core.modules.ModuleUpdate( self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH ) assert update_obj.update(BISMARK_ALIGN) - with open(os.path.join(module_path, "main.nf"), "r") as fh: - print(fh.readlines()) # Check that a patch file with the correct name has been created assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} @@ -257,7 +252,6 @@ def test_create_patch_update_success(self): with open(module_path / patch_fn, "r") as fh: patch_lines = fh.readlines() module_relpath = module_path.relative_to(self.pipeline_dir) - print(patch_lines) assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines assert f"+++ {module_relpath / 'main.nf'}\n" in patch_lines assert "- tuple val(meta), path(reads)\n" in patch_lines diff --git a/tests/modules/update.py b/tests/modules/update.py index 9ca293add8..05f026fbbd 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -214,7 +214,7 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" - install_obj = nf_core.modules.ModuleInstall( + install_obj = ModuleInstall( self.pipeline_dir, prompt=False, force=False,