From 248e267ce491469ec04e443f7b69e66706ee73d2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 10:33:15 +0200 Subject: [PATCH 01/16] remove option base-path from main --- nf_core/__main__.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 2762b26885..34e2e95dc7 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -354,14 +354,8 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdow default=False, help="Do not pull in latest changes to local clone of modules repository.", ) -@click.option( - "--base-path", - type=str, - default=None, - help="Specify where the modules are stored in the remote", -) @click.pass_context -def modules(ctx, git_remote, branch, no_pull, base_path): +def modules(ctx, git_remote, branch, no_pull): """ Commands to manage Nextflow DSL2 modules (tool wrappers). """ @@ -373,7 +367,6 @@ def modules(ctx, git_remote, branch, no_pull, base_path): ctx.obj["modules_repo_url"] = git_remote ctx.obj["modules_repo_branch"] = branch ctx.obj["modules_repo_no_pull"] = no_pull - ctx.obj["modules_repo_base_path"] = base_path # nf-core modules list subcommands @@ -402,7 +395,6 @@ def remote(ctx, keywords, json): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) print(module_list.list_modules(keywords, json)) except (UserWarning, LookupError) as e: @@ -433,7 +425,6 @@ def local(ctx, keywords, json, dir): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) print(module_list.list_modules(keywords, json)) except (UserWarning, LookupError) as e: @@ -470,7 +461,6 @@ def install(ctx, tool, dir, prompt, force, sha): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) exit_status = module_install.install(tool) if not exit_status and all: @@ -528,7 +518,6 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) exit_status = module_install.update(tool) if not exit_status and all: @@ -562,7 +551,6 @@ def patch(ctx, tool, dir): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) module_patch.patch(tool) except (UserWarning, LookupError) as e: @@ -591,7 +579,6 @@ def remove(ctx, dir, tool): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) module_remove.remove(tool) except (UserWarning, LookupError) as e: @@ -695,7 +682,6 @@ def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) module_lint.lint( module=tool, @@ -746,7 +732,6 @@ def info(ctx, tool, dir): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) print(module_info.get_module_info()) except (UserWarning, LookupError) as e: @@ -772,7 +757,6 @@ def bump_versions(ctx, tool, dir, all, show_all): ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - ctx.obj["modules_repo_base_path"], ) version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all) except nf_core.modules.module_utils.ModuleException as e: From 46c8ed0b05f32efe25559cd45ad4e2a5b65ccf01 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 10:36:41 +0200 Subject: [PATCH 02/16] remove from bump_versions --- nf_core/modules/bump_versions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index b6be65857f..a30f93eda6 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -26,8 +26,8 @@ class ModuleVersionBumper(ModuleCommand): - def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False, base_path=None): - super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path) + def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.up_to_date = None self.updated = None From ff2695dff95a8a950adf56d66afcf3e06f5706f4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 10:39:06 +0200 Subject: [PATCH 03/16] remove from info --- nf_core/modules/info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 94dddbf51c..37e0b4db43 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -55,8 +55,8 @@ class ModuleInfo(ModuleCommand): Take the parsed meta.yml and generate rich help """ - def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull, base_path): - super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path) + def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.meta = None self.local_path = None self.remote_location = None From 573eb1477df811405c80b4b0d9cf2e1dada6b223 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 10:41:30 +0200 Subject: [PATCH 04/16] remove from modules install --- nf_core/modules/install.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 6f013ce3b9..926d8e93a6 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -23,19 +23,8 @@ def __init__( remote_url=None, branch=None, no_pull=False, - base_path=None, ): - # Check if we are given a base path, otherwise look in the modules.json - if base_path is None: - try: - modules_json = ModulesJson(pipeline_dir) - repo_name = nf_core.modules.module_utils.path_from_remote(remote_url) - base_path = modules_json.get_base_path(repo_name) - except: - # We don't want to fail yet if the modules.json is not found - pass - - super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path) + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha From dd62d0c04c2c9ffd29be9f903718a178273eec85 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 10:54:44 +0200 Subject: [PATCH 05/16] remove from list and modules_repo --- nf_core/modules/list.py | 5 ++--- nf_core/modules/modules_repo.py | 18 +++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index cc99c406d5..1c6a151126 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -11,8 +11,8 @@ class ModuleList(ModuleCommand): - def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False, base_path=None): - super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path) + def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.remote = remote def list_modules(self, keywords=None, print_json=False): @@ -103,7 +103,6 @@ def pattern_msg(keywords): # pass repo_name to get info on modules even outside nf-core/modules message, date = ModulesRepo( remote_url=repo_entry["git_url"], - base_path=repo_entry["base_path"], branch=module_entry["branch"], ).get_commit_info(version_sha) except LookupError as e: diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 8b00d1cb3d..68e9fda3e5 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -17,7 +17,6 @@ # Constants for the nf-core/modules repo used throughout the module files NF_CORE_MODULES_NAME = "nf-core/modules" NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" -NF_CORE_MODULES_BASE_PATH = "modules" NF_CORE_MODULES_DEFAULT_BRANCH = "master" @@ -112,7 +111,7 @@ def get_remote_branches(remote_url): branches[sha] = branch_name return set(branches.values()) - def __init__(self, remote_url=None, branch=None, no_pull=False, base_path=None, no_progress=False): + def __init__(self, remote_url=None, branch=None, no_pull=False, no_progress=False): """ Initializes the object and clones the git repository if it is not already present """ @@ -128,11 +127,6 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, base_path=None, self.fullname = nf_core.modules.module_utils.path_from_remote(self.remote_url) - if base_path is None: - base_path = NF_CORE_MODULES_BASE_PATH - - self.base_path = base_path - self.setup_local_repo(remote_url, branch, no_progress) # Verify that the repo seems to be correctly configured @@ -140,7 +134,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, base_path=None, self.verify_branch() # Convenience variable - self.modules_dir = os.path.join(self.local_repo_dir, self.base_path) + self.modules_dir = os.path.join(self.local_repo_dir, "modules") self.avail_module_names = None @@ -252,8 +246,10 @@ def verify_branch(self): Verifies the active branch conforms do the correct directory structure """ dir_names = os.listdir(self.local_repo_dir) - if self.base_path not in dir_names: - err_str = f"Repository '{self.fullname}' ({self.branch}) does not contain the '{self.base_path}' directory" + if "modules" not in dir_names: + err_str = f"Repository '{self.fullname}' ({self.branch}) does not contain the 'modules/' directory" + if "software" in dir_names: + err_str += ".\nAs of version 2.0, the 'software/' directory should be renamed to 'modules/'" raise LookupError(err_str) def checkout_branch(self): @@ -368,7 +364,7 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 ( dict ): Iterator of commit SHAs and associated (truncated) message """ self.checkout_branch() - module_path = os.path.join(self.base_path, module_name) + module_path = os.path.join("modules", module_name) commits = self.repo.iter_commits(max_count=depth, paths=module_path) commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits From cf04fb2e9741a72a48de4e5d06e5a0a478f43bd2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 10:57:12 +0200 Subject: [PATCH 06/16] remove from module_command --- nf_core/modules/modules_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 9f82c5bfa5..9d2798e4be 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -19,11 +19,11 @@ class ModuleCommand: Base class for the 'nf-core modules' commands """ - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=None): + def __init__(self, dir, remote_url=None, branch=None, no_pull=False): """ Initialise the ModulesCommand object """ - self.modules_repo = ModulesRepo(remote_url, branch, no_pull, base_path) + self.modules_repo = ModulesRepo(remote_url, branch, no_pull) self.dir = dir try: if self.dir: From a00659066baacf77f9d17de0535c7c9170dd7410 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 11:07:07 +0200 Subject: [PATCH 07/16] remove from modules_json --- nf_core/modules/modules_json.py | 69 +++++++++------------------------ nf_core/modules/update.py | 1 - 2 files changed, 18 insertions(+), 52 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 18b7c29a7b..e222b26add 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -62,18 +62,16 @@ def create(self): if "main.nf" in file_names ], repo_remote, - base_path, ) - for repo_name, (repo_remote, base_path) in repos.items() + for repo_name, (repo_remote) in repos.items() ] - for repo_name, module_names, remote_url, base_path in sorted(repo_module_names): + for repo_name, module_names, remote_url in sorted(repo_module_names): modules_json["repos"][repo_name] = {} modules_json["repos"][repo_name]["git_url"] = remote_url modules_json["repos"][repo_name]["modules"] = {} - modules_json["repos"][repo_name]["base_path"] = base_path modules_json["repos"][repo_name]["modules"] = self.determine_module_branches_and_shas( - repo_name, remote_url, base_path, module_names + repo_name, remote_url, module_names ) modules_json_path = Path(self.dir, "modules.json") @@ -101,7 +99,6 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): repos[nf_core.modules.modules_repo.NF_CORE_MODULES_NAME] = ( nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE, - nf_core.modules.modules_repo.NF_CORE_MODULES_BASE_PATH, ) # The function might rename some directories, keep track of them renamed_dirs = {} @@ -150,16 +147,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): else: continue - # Prompt the user for the modules base path in the remote - nrepo_base_path = questionary.text( - f"Please provide the path of the modules directory in the remote. " - f"Will default to '{nf_core.modules.modules_repo.NF_CORE_MODULES_BASE_PATH}' if left empty.", - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - if not nrepo_base_path: - nrepo_base_path = nf_core.modules.modules_repo.NF_CORE_MODULES_BASE_PATH - - repos[nrepo_name] = (nrepo_remote, nrepo_base_path) + repos[nrepo_name] = (nrepo_remote, "modules") dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos]) return repos, renamed_dirs @@ -198,7 +186,7 @@ def dir_tree_uncovered(self, modules_dir, repos): depth += 1 return dirs_not_covered - def determine_module_branches_and_shas(self, repo_name, remote_url, base_path, modules): + def determine_module_branches_and_shas(self, repo_name, remote_url, modules): """ Determines what branch and commit sha each module in the pipeline belong to @@ -208,7 +196,6 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, base_path, m Args: repo_name (str): The name of the module repository remote_url (str): The url to the remote repository - base_path (Path): The base path in the remote modules_base_path (Path): The path to the modules directory in the pipeline modules ([str]): List of names of installed modules from the repository @@ -216,7 +203,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, base_path, m (dict[str, dict[str, str]]): The module.json entries for the modules from the repository """ - default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url, base_path=base_path) + default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url) repo_path = self.modules_dir / repo_name # Get the branches present in the repository, as well as the default branch available_branches = nf_core.modules.modules_repo.ModulesRepo.get_remote_branches(remote_url) @@ -257,7 +244,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, base_path, m break # Create a new modules repo with the selected branch, and retry find the sha modules_repo = nf_core.modules.modules_repo.ModulesRepo( - remote_url=remote_url, base_path=base_path, branch=branch, no_pull=True, no_progress=True + remote_url=remote_url, branch=branch, no_pull=True, no_progress=True ) else: found_sha = True @@ -366,27 +353,26 @@ def unsynced_modules(self): return untracked_dirs, missing_installation - def has_git_url_and_base_path(self): + def has_git_url(self): """ Check that that all repo entries in the modules.json - has a git url and a base_path + has a git url Returns: (bool): True if they are found for all repos, False otherwise """ for repo_entry in self.modules_json.get("repos", {}).values(): - if "git_url" not in repo_entry or "base_path" not in repo_entry: + if "git_url" not in repo_entry: return False return True - def reinstall_repo(self, repo_name, remote_url, base_path, module_entries): + def reinstall_repo(self, repo_name, remote_url, module_entries): """ Reinstall modules from a repository Args: repo_name (str): The name of the repository remote_url (str): The git url of the remote repository - base_path (Path): The base path in the repository modules ([ dict[str, dict[str, str]] ]): Module entries with branch and git sha info @@ -407,9 +393,7 @@ def reinstall_repo(self, repo_name, remote_url, base_path, module_entries): for branch, modules in branches_and_mods.items(): try: - modules_repo = nf_core.modules.modules_repo.ModulesRepo( - remote_url=remote_url, branch=branch, base_path=base_path - ) + modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url, branch=branch) except LookupError as e: log.error(e) failed_to_install.extend(modules) @@ -432,7 +416,7 @@ def check_up_to_date(self): the commit log in the remote to try to determine the SHA. """ self.load() - if not self.has_git_url_and_base_path(): + if not self.has_git_url(): raise UserWarning( "The 'modules.json' file is not up to date. " "Please reinstall it by removing it and rerunning the command." @@ -456,8 +440,7 @@ def check_up_to_date(self): for repo, contents in missing_installation.items(): module_entries = contents["modules"] remote_url = contents["git_url"] - base_path = contents["base_path"] - remove_from_mod_json[repo] = self.reinstall_repo(repo, remote_url, base_path, module_entries) + remove_from_mod_json[repo] = self.reinstall_repo(repo, remote_url, module_entries) # If the reinstall fails, we remove those entries in 'modules.json' if sum(map(len, remove_from_mod_json.values())) > 0: @@ -490,8 +473,7 @@ def check_up_to_date(self): # Get the remotes we are missing tracked_repos = { - repo_name: (repo_entry["git_url"], repo_entry["base_path"]) - for repo_name, repo_entry in self.modules_json["repos"].items() + repo_name: (repo_entry["git_url"]) for repo_name, repo_entry in self.modules_json["repos"].items() } repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) @@ -509,14 +491,13 @@ def check_up_to_date(self): repos_with_modules[repo_name].append(module) for repo_name, modules in repos_with_modules.items(): - remote_url, base_path = repos[repo_name] - repo_entry = self.determine_module_branches_and_shas(repo_name, remote_url, base_path, modules) + remote_url = repos[repo_name] + repo_entry = self.determine_module_branches_and_shas(repo_name, remote_url, modules) if repo_name in self.modules_json["repos"]: self.modules_json["repos"][repo_name]["modules"].update(repo_entry) else: self.modules_json["repos"][repo_name] = { "git_url": remote_url, - "base_path": base_path, "modules": repo_entry, } @@ -552,10 +533,9 @@ def update(self, modules_repo, module_name, module_version, write_file=True): self.load() repo_name = modules_repo.fullname remote_url = modules_repo.remote_url - base_path = modules_repo.base_path branch = modules_repo.branch if repo_name not in self.modules_json["repos"]: - self.modules_json["repos"][repo_name] = {"modules": {}, "git_url": remote_url, "base_path": base_path} + self.modules_json["repos"][repo_name] = {"modules": {}, "git_url": remote_url} repo_modules_entry = self.modules_json["repos"][repo_name]["modules"] if module_name not in repo_modules_entry: repo_modules_entry[module_name] = {} @@ -696,19 +676,6 @@ def get_git_url(self, repo_name): self.load() return self.modules_json.get("repos", {}).get(repo_name, {}).get("git_url", None) - def get_base_path(self, repo_name): - """ - Returns the modules base path of a repo - Args: - repo_name (str): Name of the repository - - Returns: - (str): The base path of the repository if it exists, None otherwise - """ - if self.modules_json is None: - self.load() - return self.modules_json.get("repos", {}).get(repo_name, {}).get("base_path", None) - def get_all_modules(self): """ Retrieves all pipeline modules that are reported in the modules.json diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index a1841d24bf..2c2c265f94 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -421,7 +421,6 @@ def get_all_modules_info(self, branch=None): repo_name, self.modules_json.get_git_url(repo_name), branch, - self.modules_json.get_base_path(repo_name), mods_shas, ) for (repo_name, branch), mods_shas in repos_and_branches.items() From 8f5a8791231974eca5e2e673a2be54cd62e691b2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 11:08:41 +0200 Subject: [PATCH 08/16] remove from patch --- nf_core/modules/patch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 8827ed711a..b907256bcf 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -16,8 +16,8 @@ class ModulePatch(ModuleCommand): - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=None): - super().__init__(dir, remote_url, branch, no_pull, base_path) + def __init__(self, dir, remote_url=None, branch=None, no_pull=False): + super().__init__(dir, remote_url, branch, no_pull) self.modules_json = ModulesJson(dir) From 430d86f5db88033a2b9dc2a231baad49c2516e67 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 11:12:16 +0200 Subject: [PATCH 09/16] remove from update --- nf_core/modules/update.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 2c2c265f94..4644c2398e 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -31,9 +31,8 @@ def __init__( remote_url=None, branch=None, no_pull=False, - base_path=None, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path) + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha @@ -428,9 +427,9 @@ def get_all_modules_info(self, branch=None): # Create ModulesRepo objects repo_objs_mods = [] - for repo_name, repo_url, branch, base_path, mods_shas in modules_info: + for repo_name, repo_url, branch, mods_shas in modules_info: try: - modules_repo = ModulesRepo(remote_url=repo_url, branch=branch, base_path=base_path) + modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) except LookupError as e: log.warning(e) log.info(f"Skipping modules in '{repo_name}'") From fde59196969f506ff928775388bb4ac158201e5b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 11:12:39 +0200 Subject: [PATCH 10/16] remove from odules init --- nf_core/modules/lint/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 8fb6fecc2c..3910ef829e 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -64,7 +64,7 @@ class ModuleLint(ModuleCommand): from .module_todos import module_todos from .module_version import module_version - def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull=False, base_path=None): + def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull=False): self.dir = dir try: self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) @@ -75,7 +75,7 @@ def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull self.passed = [] self.warned = [] self.failed = [] - self.modules_repo = ModulesRepo(remote_url, branch, no_pull, base_path) + self.modules_repo = ModulesRepo(remote_url, branch, no_pull) self.lint_tests = self.get_all_lint_tests(self.repo_type == "pipeline") if self.repo_type == "pipeline": From 898707e5d88d77901675fab52bd85da5b516493b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 11:13:00 +0200 Subject: [PATCH 11/16] remove from modules.json from template --- nf_core/pipeline-template/modules.json | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index a14671c6ed..9c8d724aef 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -3,7 +3,6 @@ "homePage": "https://github.com/{{ name }}", "repos": { "nf-core/modules": { - "base_path": "modules", "git_url": "https://github.com/nf-core/modules.git", "modules": { "custom/dumpsoftwareversions": { From 048aa0732639c47f619090bc1a711d4e41a239e2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 11:13:55 +0200 Subject: [PATCH 12/16] remove get_base_path test --- tests/modules/modules_json.py | 8 -------- tests/test_modules.py | 1 - 2 files changed, 9 deletions(-) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 35808c529b..4427b7a425 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -4,7 +4,6 @@ from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ( - NF_CORE_MODULES_BASE_PATH, NF_CORE_MODULES_DEFAULT_BRANCH, NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, @@ -153,13 +152,6 @@ def test_mod_json_get_git_url(self): assert mod_json_obj.get_git_url("INVALID_REPO") is None -def test_mod_json_get_base_path(self): - """Tests the get_base_path function""" - mod_json_obj = ModulesJson(self.pipeline_dir) - assert mod_json_obj.get_base_path(NF_CORE_MODULES_NAME) == NF_CORE_MODULES_BASE_PATH - assert mod_json_obj.get_base_path("INVALID_REPO") is None - - def test_mod_json_dump(self): """Tests the dump function""" mod_json_obj = ModulesJson(self.pipeline_dir) diff --git a/tests/test_modules.py b/tests/test_modules.py index 2ceb2ab08f..fda7e2842c 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -133,7 +133,6 @@ def test_modulesrepo_class(self): test_get_modules_json, test_mod_json_create, test_mod_json_dump, - test_mod_json_get_base_path, test_mod_json_get_git_url, test_mod_json_get_module_version, test_mod_json_module_present, From e0b6b0211fdb033914e68ea922ea0267e5618852 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 12:52:50 +0200 Subject: [PATCH 13/16] fix bug in repos --- nf_core/modules/modules_json.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index e222b26add..7cd9aa0031 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -63,7 +63,7 @@ def create(self): ], repo_remote, ) - for repo_name, (repo_remote) in repos.items() + for repo_name, repo_remote in repos.items() ] for repo_name, module_names, remote_url in sorted(repo_module_names): @@ -97,9 +97,9 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): # Check if there are any nf-core modules installed if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): - repos[nf_core.modules.modules_repo.NF_CORE_MODULES_NAME] = ( - nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE, - ) + repos[ + nf_core.modules.modules_repo.NF_CORE_MODULES_NAME + ] = nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE # The function might rename some directories, keep track of them renamed_dirs = {} # Check if there are any untracked repositories From 354f7faaff5bf4a3362988cb198f25624673ad12 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 19 Aug 2022 13:14:29 +0200 Subject: [PATCH 14/16] fix doccumentation --- CHANGELOG.md | 2 +- README.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4908d0000..49c190b6e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,7 +56,7 @@ - Add prompt for module name to `nf-core modules info` ([#1644](https://github.com/nf-core/tools/issues/1644)) - Update docs with example of custom git remote ([#1645](https://github.com/nf-core/tools/issues/1645)) - Command `nf-core modules test` obtains module name suggestions from installed modules ([#1624](https://github.com/nf-core/tools/pull/1624)) -- Add `--base-path` flag to `nf-core modules` to specify the base path for the modules in a remote. Also refactored `modules.json` code. ([#1643](https://github.com/nf-core/tools/issues/1643)) +- Add `--base-path` flag to `nf-core modules` to specify the base path for the modules in a remote. Also refactored `modules.json` code. ([#1643](https://github.com/nf-core/tools/issues/1643)) Removed after ([#1754](https://github.com/nf-core/tools/pull/1754)) - Rename methods in `ModulesJson` to remove explicit reference to `modules.json` - Fix inconsistencies in the `--save-diff` flag `nf-core modules update`. Refactor `nf-core modules update` ([#1536](https://github.com/nf-core/tools/pull/1536)) - Fix bug in `ModulesJson.check_up_to_date` causing it to ask for the remote of local modules diff --git a/README.md b/README.md index 278bbbaf34..8fc64978b9 100644 --- a/README.md +++ b/README.md @@ -945,8 +945,6 @@ For example, if you want to install the `fastqc` module from the repository `nf- nf-core modules --git-remote git@gitlab.com:nf-core/modules-test.git install fastqc ``` -If the modules in your custom remote are stored in another directory than `modules`, you can specify the path by using the `--base-path ` flag. This will default to `modules`. Note that all branches in a remote must use the same base path, otherwise the commands will fail. - Note that a custom remote must follow a similar directory structure to that of `nf-core/moduleś` for the `nf-core modules` commands to work properly. The modules commands will during initalisation try to pull changes from the remote repositories. If you want to disable this, for example From a571223c2588888fff7148b356161a14393ccebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 19 Aug 2022 14:03:09 +0200 Subject: [PATCH 15/16] Apply suggestion from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/modules/modules_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 68e9fda3e5..5654ca76da 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -249,7 +249,7 @@ def verify_branch(self): if "modules" not in dir_names: err_str = f"Repository '{self.fullname}' ({self.branch}) does not contain the 'modules/' directory" if "software" in dir_names: - err_str += ".\nAs of version 2.0, the 'software/' directory should be renamed to 'modules/'" + err_str += ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" raise LookupError(err_str) def checkout_branch(self): From 1e6513d3ddfac27fdbfa34f7dd62e6d895e18afb Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 19 Aug 2022 12:05:13 +0000 Subject: [PATCH 16/16] [automated] Fix linting with Prettier --- nf_core/modules/modules_repo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 5654ca76da..2a559b1112 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -249,7 +249,9 @@ def verify_branch(self): if "modules" not in dir_names: err_str = f"Repository '{self.fullname}' ({self.branch}) does not contain the 'modules/' directory" if "software" in dir_names: - err_str += ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" + err_str += ( + ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" + ) raise LookupError(err_str) def checkout_branch(self):