From e529ae04927ce36f38ebee9dd81a6f70fbc83b00 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 17:20:26 +0200 Subject: [PATCH 01/24] fix bug when updating modules from old version in old folder structure --- nf_core/modules/modules_repo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 55a0d3412c..2c03c9019b 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -432,6 +432,10 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 module_path = os.path.join("modules", self.repo_path, module_name) commits = self.repo.iter_commits(max_count=depth, paths=module_path) commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + # Grab commits also from previous modules structure + module_path = os.path.join("modules", module_name) + commits = self.repo.iter_commits(max_count=depth, paths=module_path) + commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): From ba36c76649f6042786599cbfce2f8b7b6db4ce4a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 17:20:52 +0200 Subject: [PATCH 02/24] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b31fedd92..6104a8e92b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix error in tagging GitPod docker images during releases - Don't remove local copy of modules repo, only update it with fetch ([#1879](https://github.com/nf-core/tools/pull/1879)) +- Fix bug when updating modules from old version in old folder structure ### Modules From e85549f2ba42ff1fc73d9bcab596c4b22c82e132 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 17:34:04 +0200 Subject: [PATCH 03/24] fix a small mistake --- nf_core/modules/modules_repo.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 2c03c9019b..3db4040932 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -430,12 +430,17 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 """ self.checkout_branch() module_path = os.path.join("modules", self.repo_path, module_name) - commits = self.repo.iter_commits(max_count=depth, paths=module_path) - commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + commits_new = self.repo.iter_commits(max_count=depth, paths=module_path) + commits_new = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_new + ] # Grab commits also from previous modules structure module_path = os.path.join("modules", module_name) - commits = self.repo.iter_commits(max_count=depth, paths=module_path) - commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + commits_old = self.repo.iter_commits(max_count=depth, paths=module_path) + commits_old = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old + ] + commits = commits_new + commits_old return commits def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): From fb46a2d4d351d64259f5237276f25b92b005e3c8 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:36:26 +0200 Subject: [PATCH 04/24] Add simple log statement if a config exists for the subworkflow to print the necessary include statement --- nf_core/modules/modules_repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 55a0d3412c..4e45eab59f 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -381,6 +381,8 @@ def install_subworkflow(self, subworkflow_name, install_dir, commit): # Copy the files from the repo to the install folder shutil.copytree(self.get_subworkflow_dir(subworkflow_name), Path(install_dir, subworkflow_name)) + if Path(install_dir, subworkflow_name, 'nextflow.config').is_file(): + log.info(f"Subworkflow config include statement: includeConfig '{Path(install_dir, subworkflow_name, 'nextflow.config')}'") # Switch back to the tip of the branch self.checkout_branch() From 0c00bcefeb9c70a24ccf9fe5306a96ee7ddbaf1f Mon Sep 17 00:00:00 2001 From: Snorkan Date: Mon, 10 Oct 2022 18:05:00 +0200 Subject: [PATCH 05/24] Remove tick in the wrong place Removed tick that lead to wrong words were highlighted --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ad2846b23..c9548c6a40 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ To help developers working with pipeline schema, nf-core tools has three `schema Nextflow can take input parameters in a JSON or YAML file when running a pipeline using the `-params-file` option. This command validates such a file against the pipeline schema. -`Usage is `nf-core schema validate `. eg with the pipeline downloaded [above](#download-pipeline), you can run: +Usage is `nf-core schema validate `. eg with the pipeline downloaded [above](#download-pipeline), you can run: - [`nf-core` tools installation](#installation) +- [`nf-core` tools update](#update-tools) - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) - [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) @@ -186,6 +187,22 @@ If you would prefer to skip this check, set the environment variable `NFCORE_NO_ export NFCORE_NO_VERSION_CHECK=1 ``` +### Update tools + +It is advisable to keep nf-core/tools updated to the most recent version. The command to update depends on the system used to install it, for example if you have installed it with conda you can use: +```bash +conda update nf-core +``` +if you used pip: +```bash +pip install --upgrade nf-core +``` +or +```bash +pip install -U nf-core +``` +Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages). + ## Listing pipelines The command `nf-core list` shows all available nf-core pipelines along with their latest version, when that was published and how recently the pipeline code was pulled to your local system (if at all). From 31f1e69096d7128cb69529968d4386314ca173f4 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Tue, 11 Oct 2022 15:51:16 +0200 Subject: [PATCH 16/24] accommodate review --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index c30189eb4f..9160d7b231 100644 --- a/README.md +++ b/README.md @@ -196,11 +196,6 @@ conda update nf-core if you used pip: ```bash pip install --upgrade nf-core -``` -or -```bash -pip install -U nf-core -``` Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages). ## Listing pipelines From 518c9cb701233b26a089f3751a590ad09b8c50e9 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Tue, 11 Oct 2022 15:55:32 +0200 Subject: [PATCH 17/24] fix the code block markers --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9160d7b231..ce16f65a9c 100644 --- a/README.md +++ b/README.md @@ -193,9 +193,12 @@ It is advisable to keep nf-core/tools updated to the most recent version. The co ```bash conda update nf-core ``` + if you used pip: ```bash pip install --upgrade nf-core +``` + Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages). ## Listing pipelines From 8f414ad314a1af9e048306f89a0b0cf33f246b5c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 11 Oct 2022 16:17:58 +0200 Subject: [PATCH 18/24] check_up_to_date is also checking subworkflows --- nf_core/modules/modules_json.py | 176 +++++++++++++++++--------------- 1 file changed, 94 insertions(+), 82 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 918901a5d2..26cb755a43 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -455,14 +455,14 @@ def reinstall_repo(self, install_dir, remote_url, module_entries): def check_up_to_date(self): """ - Checks whether the modules installed in the directory + Checks whether the modules and subworkflows installed in the directory are consistent with the entries in the 'modules.json' file and vice versa. - If a module has an entry in the 'modules.json' file but is missing in the directory, - we first try to reinstall the module from the remote and if that fails we remove the entry + If a module/subworkflow has an entry in the 'modules.json' file but is missing in the directory, + we first try to reinstall the module/subworkflow from the remote and if that fails we remove the entry in 'modules.json'. - If a module is installed but the entry in 'modules.json' is missing we iterate through + If a module/subworkflow is installed but the entry in 'modules.json' is missing we iterate through the commit log in the remote to try to determine the SHA. """ try: @@ -479,88 +479,18 @@ def check_up_to_date(self): missing_installation, ) = self.unsynced_modules() - # If there are any modules left in 'modules.json' after all installed are removed, + # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them if len(missing_installation) > 0: - missing_but_in_mod_json = [ - f"'modules/{install_dir}/{module}'" - for repo_url, contents in missing_installation.items() - for install_dir, dir_contents in contents["modules"].items() - for module in dir_contents - ] - log.info( - f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" - ) - - remove_from_mod_json = {} - for repo_url, contents in missing_installation.items(): - for install_dir, module_entries in contents["modules"].items(): - remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( - install_dir, repo_url, module_entries - ) - - # If the reinstall fails, we remove those entries in 'modules.json' - if sum(map(len, remove_from_mod_json.values())) > 0: - uninstallable_mods = [ - f"'{install_dir}/{module}'" - for (repo_url, install_dir), modules in remove_from_mod_json.items() - for module in modules - ] - if len(uninstallable_mods) == 1: - log.info(f"Was unable to reinstall {uninstallable_mods[0]}. Removing 'modules.json' entry") - else: - log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_mods)}" - ) - - for (repo_url, install_dir), module_entries in remove_from_mod_json.items(): - for module in module_entries: - self.modules_json["repos"][repo_url]["modules"][install_dir].pop(module) - if len(self.modules_json["repos"][repo_url]["modules"][install_dir]) == 0: - self.modules_json["repos"].pop(repo_url) + resolve_missing_installation(self, missing_installation, "modules") + resolve_missing_installation(self, missing_installation, "subworkflows") - # If some modules didn't have an entry in the 'modules.json' file + # If some modules/subworkflows didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote - if len(missing_from_modules_json) > 0: - format_missing = [f"'{dir}'" for dir in missing_from_modules_json] - if len(format_missing) == 1: - log.info(f"Recomputing commit SHA for module {format_missing[0]} which was missing from 'modules.json'") - else: - log.info( - f"Recomputing commit SHAs for modules which were missing from 'modules.json': {', '.join(format_missing)}" - ) - - # Get the remotes we are missing - tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} - repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) - - modules_with_repos = ( - ( - nf_core.modules.module_utils.path_from_remote(repo_url), - str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))), - ) - for dir in missing_from_modules_json - for repo_url in repos - if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url)) - ) - - repos_with_modules = {} - for install_dir, module in modules_with_repos: - if install_dir not in repos_with_modules: - repos_with_modules[install_dir] = [] - repos_with_modules[install_dir].append(module) - - for install_dir, modules in repos_with_modules.items(): - remote_url = [url for url, content in repos.items() if install_dir in content][0] - repo_entry = self.determine_module_branches_and_shas(install_dir, remote_url, modules) - if remote_url in self.modules_json["repos"]: - self.modules_json["repos"][remote_url]["modules"][install_dir].update(repo_entry) - else: - self.modules_json["repos"][remote_url] = { - "modules": { - install_dir: repo_entry, - } - } + if len(modules_missing_from_modules_json) > 0: + self.resolve_missing_from_modules_json(modules_missing_from_modules_json, "modules") + if len(subworkflows_missing_from_modules_json) > 0: + self.resolve_missing_from_modules_json(subworkflows_missing_from_modules_json, "subworkflows") self.dump() @@ -909,3 +839,85 @@ def get_installed_subworkflows(self): self.pipeline_subworkflows[repo] = [(dir, name) for name in subworkflow] return self.pipeline_subworkflows + + +def resolve_missing_installation(self, missing_installation, component_type): + missing_but_in_mod_json = [ + f"'{component_type}/{install_dir}/{component}'" + for repo_url, contents in missing_installation.items() + for install_dir, dir_contents in contents[component_type].items() + for component in dir_contents + ] + log.info( + f"Reinstalling {component_type} found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" + ) + + remove_from_mod_json = {} + for repo_url, contents in missing_installation.items(): + for install_dir, component_entries in contents[component_type].items(): + remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( + install_dir, repo_url, component_entries + ) + + # If the reinstall fails, we remove those entries in 'modules.json' + if sum(map(len, remove_from_mod_json.values())) > 0: + uninstallable_components = [ + f"'{install_dir}/{component}'" + for (repo_url, install_dir), components in remove_from_mod_json.items() + for component in components + ] + if len(uninstallable_components) == 1: + log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") + else: + log.info( + f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" + ) + + for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): + for component in component_entries: + self.modules_json["repos"][repo_url][component_type][install_dir].pop(component) + if len(self.modules_json["repos"][repo_url][component_type][install_dir]) == 0: + self.modules_json["repos"].pop(repo_url) + + def resolve_missing_from_modules_json(self, missing_from_modules_json, component_type): + format_missing = [f"'{dir}'" for dir in missing_from_modules_json] + if len(format_missing) == 1: + log.info( + f"Recomputing commit SHA for {component_type[:-1]} {format_missing[0]} which was missing from 'modules.json'" + ) + else: + log.info( + f"Recomputing commit SHAs for {component_type} which were missing from 'modules.json': {', '.join(format_missing)}" + ) + + # Get the remotes we are missing + tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} + repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) + + components_with_repos = ( + ( + nf_core.modules.module_utils.path_from_remote(repo_url), + str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))), + ) + for dir in missing_from_modules_json + for repo_url in repos + if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url)) + ) + + repos_with_components = {} + for install_dir, component in components_with_repos: + if install_dir not in repos_with_components: + repos_with_components[install_dir] = [] + repos_with_components[install_dir].append(component) + + for install_dir, components in repos_with_components.items(): + remote_url = [url for url, content in repos.items() if install_dir in content][0] + repo_entry = self.determine_module_branches_and_shas(install_dir, remote_url, components) + if remote_url in self.modules_json["repos"]: + self.modules_json["repos"][remote_url][component_type][install_dir].update(repo_entry) + else: + self.modules_json["repos"][remote_url] = { + component_type: { + install_dir: repo_entry, + } + } From 098518af49547c5f8f5ae394509306e1adc94fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jennifer=20M=C3=BCller?= <43847598+jenmuell@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:20:05 +0200 Subject: [PATCH 19/24] Added missing dot Added a missing dot in the `Launch a Pipeline` section of Tools https://github.com/nf-core/tools/issues/1932 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ad2846b23..0e0c518479 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ Archived pipelines are not returned by default. To include them, use the `--show ## Launch a pipeline Some nextflow pipelines have a considerable number of command line flags that can be used. -To help with this, you can use the `nf-core launch` command +To help with this, you can use the `nf-core launch` command. You can choose between a web-based graphical interface or an interactive command-line wizard tool to enter the pipeline parameters for your run. Both interfaces show documentation alongside each parameter and validate your inputs. From 8b1d8f861546081f6c4f3113f93f135e4cd2f644 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 11 Oct 2022 16:30:05 +0200 Subject: [PATCH 20/24] fix silly bugs :) --- nf_core/modules/modules_json.py | 69 ++++++++++++++++----------------- nf_core/subworkflows/install.py | 10 ++--- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 26cb755a43..c26f835731 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -840,44 +840,43 @@ def get_installed_subworkflows(self): return self.pipeline_subworkflows + def resolve_missing_installation(self, missing_installation, component_type): + missing_but_in_mod_json = [ + f"'{component_type}/{install_dir}/{component}'" + for repo_url, contents in missing_installation.items() + for install_dir, dir_contents in contents[component_type].items() + for component in dir_contents + ] + log.info( + f"Reinstalling {component_type} found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" + ) -def resolve_missing_installation(self, missing_installation, component_type): - missing_but_in_mod_json = [ - f"'{component_type}/{install_dir}/{component}'" - for repo_url, contents in missing_installation.items() - for install_dir, dir_contents in contents[component_type].items() - for component in dir_contents - ] - log.info( - f"Reinstalling {component_type} found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" - ) - - remove_from_mod_json = {} - for repo_url, contents in missing_installation.items(): - for install_dir, component_entries in contents[component_type].items(): - remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( - install_dir, repo_url, component_entries - ) + remove_from_mod_json = {} + for repo_url, contents in missing_installation.items(): + for install_dir, component_entries in contents[component_type].items(): + remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( + install_dir, repo_url, component_entries + ) - # If the reinstall fails, we remove those entries in 'modules.json' - if sum(map(len, remove_from_mod_json.values())) > 0: - uninstallable_components = [ - f"'{install_dir}/{component}'" - for (repo_url, install_dir), components in remove_from_mod_json.items() - for component in components - ] - if len(uninstallable_components) == 1: - log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") - else: - log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" - ) + # If the reinstall fails, we remove those entries in 'modules.json' + if sum(map(len, remove_from_mod_json.values())) > 0: + uninstallable_components = [ + f"'{install_dir}/{component}'" + for (repo_url, install_dir), components in remove_from_mod_json.items() + for component in components + ] + if len(uninstallable_components) == 1: + log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") + else: + log.info( + f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" + ) - for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): - for component in component_entries: - self.modules_json["repos"][repo_url][component_type][install_dir].pop(component) - if len(self.modules_json["repos"][repo_url][component_type][install_dir]) == 0: - self.modules_json["repos"].pop(repo_url) + for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): + for component in component_entries: + self.modules_json["repos"][repo_url][component_type][install_dir].pop(component) + if len(self.modules_json["repos"][repo_url][component_type][install_dir]) == 0: + self.modules_json["repos"].pop(repo_url) def resolve_missing_from_modules_json(self, missing_from_modules_json, component_type): format_missing = [f"'{dir}'" for dir in missing_from_modules_json] diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index de286e0e72..f17067fa2e 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -53,11 +53,11 @@ def install(self, subworkflow, silent=False): # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: - # It's the first subworkflow installed in the pipeline! - modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { - self.modules_repo.repo_path: {} - } + # if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: + # # It's the first subworkflow installed in the pipeline! + # modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { + # self.modules_repo.repo_path: {} + # } if self.prompt and self.sha is not None: log.error("Cannot use '--sha' and '--prompt' at the same time!") From 9c517b4aff4b049a68b38374a291e6e3b928fd8e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 11 Oct 2022 16:30:58 +0200 Subject: [PATCH 21/24] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 272b5398a9..e5d0479b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Don't remove local copy of modules repo, only update it with fetch ([#1881](https://github.com/nf-core/tools/pull/1881)) - Add subworkflow commands create-test-yml, create and install ([#1897](https://github.com/nf-core/tools/pull/1897)) - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) +- `check_up_to_date()` function from `modules_json` also checks for subworkflows. ### Modules From 8fbbef4dbd9aad30713551c54c15022b2d759cb2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 11 Oct 2022 16:53:04 +0200 Subject: [PATCH 22/24] fix linting in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ce16f65a9c..ce1e27f9ce 100644 --- a/README.md +++ b/README.md @@ -190,11 +190,13 @@ export NFCORE_NO_VERSION_CHECK=1 ### Update tools It is advisable to keep nf-core/tools updated to the most recent version. The command to update depends on the system used to install it, for example if you have installed it with conda you can use: + ```bash conda update nf-core ``` if you used pip: + ```bash pip install --upgrade nf-core ``` From 4247a792bd6f5b67d68919b203ef5c8cb202af55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 12 Oct 2022 10:29:20 +0200 Subject: [PATCH 23/24] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/modules/modules_json.py | 8 ++++---- nf_core/subworkflows/install.py | 5 ----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index c26f835731..e538f3ef27 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -364,7 +364,7 @@ def unsynced_modules(self): def parse_dirs(self, dirs, missing_installation, component_type): untracked_dirs = [] for dir in dirs: - # Check if the module/ssubworkflows directory exists in modules.json + # Check if the module/subworkflows directory exists in modules.json install_dir = dir.parts[0] component = str(Path(*dir.parts[1:])) component_in_file = False @@ -388,7 +388,7 @@ def parse_dirs(self, dirs, missing_installation, component_type): # Remove the subworkflow from subworkflows without installation module_repo[component_type][install_dir].pop(component) if len(module_repo[component_type][install_dir]) == 0: - # If no subworkflows with missing installation left, remove the git_url from missing_installation + # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation missing_installation.pop(git_url) return untracked_dirs, missing_installation @@ -482,8 +482,8 @@ def check_up_to_date(self): # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them if len(missing_installation) > 0: - resolve_missing_installation(self, missing_installation, "modules") resolve_missing_installation(self, missing_installation, "subworkflows") + resolve_missing_installation(self, missing_installation, "modules") # If some modules/subworkflows didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote @@ -869,7 +869,7 @@ def resolve_missing_installation(self, missing_installation, component_type): log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") else: log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" + f"Was unable to reinstall some {component_type}. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" ) for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index f17067fa2e..f7e495b53b 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -53,11 +53,6 @@ def install(self, subworkflow, silent=False): # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - # if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: - # # It's the first subworkflow installed in the pipeline! - # modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { - # self.modules_repo.repo_path: {} - # } if self.prompt and self.sha is not None: log.error("Cannot use '--sha' and '--prompt' at the same time!") From a4b4645951ee6d0a7a377e518712df6a36f6f365 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 10:52:58 +0200 Subject: [PATCH 24/24] make tests pass --- nf_core/modules/modules_json.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index e538f3ef27..bfeac885b1 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -482,8 +482,14 @@ def check_up_to_date(self): # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them if len(missing_installation) > 0: - resolve_missing_installation(self, missing_installation, "subworkflows") - resolve_missing_installation(self, missing_installation, "modules") + if "subworkflows" in [ + c_type for _, repo_content in missing_installation.items() for c_type in repo_content.keys() + ]: + self.resolve_missing_installation(missing_installation, "subworkflows") + if "modules" in [ + c_type for _, repo_content in missing_installation.items() for c_type in repo_content.keys() + ]: + self.resolve_missing_installation(missing_installation, "modules") # If some modules/subworkflows didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote