From c8e48be1602e12d9443e2197527660bdbb54f5bf Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:00:34 +0100 Subject: [PATCH 1/7] Rewrite modules create repo type check. * Use .nf-core.yml file if we can find it, with 'repository_type' key * Traverse parent directories to look for this file, reset target directory if found * Add flag to specify if no .nf-core.yml file available --- nf_core/__main__.py | 5 ++- nf_core/modules/bump_versions.py | 2 +- nf_core/modules/create.py | 57 ++++++++++++++++++-------- nf_core/pipeline-template/.nf-core.yml | 1 + 4 files changed, 44 insertions(+), 21 deletions(-) create mode 100644 nf_core/pipeline-template/.nf-core.yml diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a9500509a4..398495e000 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -498,7 +498,8 @@ def remove(ctx, dir, tool): @click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name): +@click.option("-r", "--repo-type", type=click.Choice(["pipeline", "modules"]), default=None, help="Type of repository") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, repo_type): """ Create a new DSL2 module from the nf-core template. @@ -519,7 +520,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: - module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name) + module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name, repo_type) module_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 4d24faa52b..f7e0a595c3 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -64,7 +64,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): # Get list of all modules _, nfcore_modules = nf_core.modules.module_utils.get_installed_modules(self.dir) - # Load the .nf-core-tools.config + # Load the .nf-core.yml config self.tools_config = nf_core.utils.load_tools_config(self.dir) # Prompt for module or all diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index a9c071c4ce..8bbbf41440 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -25,7 +25,15 @@ class ModuleCreate(object): def __init__( - self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False, conda_name=None + self, + directory=".", + tool="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + repo_type=None, ): self.directory = directory self.tool = tool @@ -36,7 +44,7 @@ def __init__( self.subtool = None self.tool_conda_name = conda_name self.tool_licence = None - self.repo_type = None + self.repo_type = repo_type self.tool_licence = "" self.tool_description = "" self.tool_doc_url = "" @@ -75,9 +83,12 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.repo_type = self.get_repo_type(self.directory) + self.get_repo_type() except LookupError as e: raise UserWarning(e) + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " @@ -272,27 +283,37 @@ def render_template(self): template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) - def get_repo_type(self, directory): + def get_repo_type(self): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ # Verify that the pipeline dir exists - if dir is None or not os.path.exists(directory): - raise UserWarning(f"Could not find directory: {directory}") - - readme = os.path.join(directory, "README.md") - # Determine repository type - if os.path.exists(readme): - with open(readme) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: + if dir is None or not os.path.exists(self.directory): + raise UserWarning(f"Could not find directory: {self.directory}") + + # Try to find the root directory + base_dir = os.path.abspath(self.directory) + config_path = os.path.join(base_dir, ".nf-core.yml") + while not os.path.exists(config_path) and base_dir != os.path.dirname(base_dir): + base_dir = os.path.dirname(base_dir) + config_path = os.path.join(base_dir, ".nf-core.yml") + # Reset self.directory if we found the config file (will be an absolute path) + if os.path.exists(config_path): + self.directory = base_dir + + # Figure out the repository type from the .nf-core.yml config file if we can + if os.path.exists(config_path): + tools_config = nf_core.utils.load_tools_config(self.directory) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + self.repo_type = tools_config["repository_type"] + return + + # Could be set on the command line - throw an error if not + if not self.repo_type: raise UserWarning( - f"This directory does not look like a clone of nf-core/modules or an nf-core pipeline: '{directory}'" - " Please point to a valid directory." + f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{self.directory}'" + "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" ) def get_module_dirs(self): diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml new file mode 100644 index 0000000000..3805dc81c1 --- /dev/null +++ b/nf_core/pipeline-template/.nf-core.yml @@ -0,0 +1 @@ +repository_type: pipeline From 42795ec61f7f68a8c144c0e46e09aa3f82ad28d5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:38:15 +0100 Subject: [PATCH 2/7] .yml or .yaml, check cli flag doesn't conflict with .nf-core.yml --- nf_core/modules/create.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 8bbbf41440..72b106844f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -294,20 +294,29 @@ def get_repo_type(self): # Try to find the root directory base_dir = os.path.abspath(self.directory) - config_path = os.path.join(base_dir, ".nf-core.yml") - while not os.path.exists(config_path) and base_dir != os.path.dirname(base_dir): + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + while ( + not os.path.exists(config_path_yml) + and not os.path.exists(config_path_yaml) + and base_dir != os.path.dirname(base_dir) + ): base_dir = os.path.dirname(base_dir) - config_path = os.path.join(base_dir, ".nf-core.yml") + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") # Reset self.directory if we found the config file (will be an absolute path) - if os.path.exists(config_path): + if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): self.directory = base_dir # Figure out the repository type from the .nf-core.yml config file if we can - if os.path.exists(config_path): - tools_config = nf_core.utils.load_tools_config(self.directory) - if tools_config.get("repository_type") in ["pipeline", "modules"]: - self.repo_type = tools_config["repository_type"] - return + tools_config = nf_core.utils.load_tools_config(self.directory) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + if self.repo_type is not None and self.repo_type != tools_config["repository_type"]: + raise UserWarning( + f"'--repo-type {self.repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" + ) + self.repo_type = tools_config["repository_type"] + return # Could be set on the command line - throw an error if not if not self.repo_type: From f527df628cfd8ee8b1081392bfb74d321d8abac5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:45:22 +0100 Subject: [PATCH 3/7] Changelog and docs --- CHANGELOG.md | 2 ++ README.md | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c2b650d45..86ae6db51c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,12 @@ * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +* Added `.nf-core.yml` file with `repository_type: pipeline` for modules commands ## General * Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379)) +* Made `nf-core modules create` detect repository type with explicit `.nf-core.yml` or `--repo-type`, instead of random readme stuff ([#1391](https://github.com/nf-core/tools/pull/1391)) ### Modules diff --git a/README.md b/README.md index 65fb66b004..404f990eba 100644 --- a/README.md +++ b/README.md @@ -925,7 +925,7 @@ github.com: git_protocol: ``` -The easiest way to create this configuration file is through *GitHub CLI*: follow +The easiest way to create this configuration file is through _GitHub CLI_: follow its [installation instructions](https://cli.github.com/manual/installation) and then call: @@ -1116,11 +1116,17 @@ This command creates a new nf-core module from the nf-core module template. This ensures that your module follows the nf-core guidelines. The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. -You can create a new module using `nf-core modules create`. This will create the new module in the current working directory. To specify another directory, use `--dir `. +You can create a new module using `nf-core modules create`. -If writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, the `` argument should be the path to the clone of your fork of the modules repository. +This command can be used both when writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, +and also when creating local modules for a pipeline. -Alternatively, if writing a more niche module that does not make sense to share, `` should be the path to your pipeline. +Which type of repository you are working in is detected by the `repository_type` flag in a `.nf-core.yml` file in the root directory, +set to either `pipeline` or `modules`. +The command will automatically look through parent directories for this file to set the root path, so that you can run the command in a subdirectory. +It will start in the current working directory, or whatever is specified with `--dir `. + +If you do not have a `.nf-core.yml` file you can specify the repository type with the `--repo-type` flag. The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files. From 5141c558d13d7410045ca05d81347ea4cf85adf4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:54:49 +0100 Subject: [PATCH 4/7] Fix tests, hopefully --- tests/test_modules.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index fb59537c62..ce95c7543b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -19,8 +19,8 @@ def create_modules_repo_dummy(tmp_dir): os.makedirs(os.path.join(root_dir, "tests", "config")) with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, "README.md"), "w") as fh: - fh.writelines(["# ![nf-core/modules](docs/images/nfcore-modules_logo.png)", "\n"]) + with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + fh.writelines(["repository_type: modules", "\n"]) # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) @@ -47,15 +47,10 @@ def setUp(self): self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_install_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") - # Set up remove objects print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) - self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_remove_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") + self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) From fe45705d8e658ac811862c06f626a46265c17512 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:56:46 +0100 Subject: [PATCH 5/7] Fix syntax error typo --- tests/test_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index ce95c7543b..cfa3408e69 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -50,7 +50,7 @@ def setUp(self): # Set up remove objects print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) - self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir + self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) From 29d681cbfcb8f1f87fedbadb722c4e2d907bf22b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 17:08:17 +0100 Subject: [PATCH 6/7] Found duplicated method, standardised + updated --- nf_core/modules/bump_versions.py | 4 +-- nf_core/modules/create.py | 45 ++-------------------------- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/module_utils.py | 47 +++++++++++++++++++++++------- nf_core/modules/modules_command.py | 2 +- 5 files changed, 41 insertions(+), 59 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index f7e0a595c3..d8b0450a69 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -15,8 +15,6 @@ from rich.markdown import Markdown import rich from nf_core.utils import rich_force_colors -import sys -import yaml import nf_core.utils import nf_core.modules.module_utils @@ -55,7 +53,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.show_up_to_date = show_uptodate # Verify that this is not a pipeline - repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) if not repo_type == "modules": raise nf_core.modules.module_utils.ModuleException( "This command only works on the nf-core/modules repository, not on pipelines!" diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 72b106844f..c4371853ed 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -19,6 +19,7 @@ import yaml import nf_core.utils +import nf_core.modules.module_utils log = logging.getLogger(__name__) @@ -83,7 +84,7 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.get_repo_type() + self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type) except LookupError as e: raise UserWarning(e) log.info(f"Repository type: [blue]{self.repo_type}") @@ -283,48 +284,6 @@ def render_template(self): template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) - def get_repo_type(self): - """ - Determine whether this is a pipeline repository or a clone of - nf-core/modules - """ - # Verify that the pipeline dir exists - if dir is None or not os.path.exists(self.directory): - raise UserWarning(f"Could not find directory: {self.directory}") - - # Try to find the root directory - base_dir = os.path.abspath(self.directory) - config_path_yml = os.path.join(base_dir, ".nf-core.yml") - config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") - while ( - not os.path.exists(config_path_yml) - and not os.path.exists(config_path_yaml) - and base_dir != os.path.dirname(base_dir) - ): - base_dir = os.path.dirname(base_dir) - config_path_yml = os.path.join(base_dir, ".nf-core.yml") - config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") - # Reset self.directory if we found the config file (will be an absolute path) - if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): - self.directory = base_dir - - # Figure out the repository type from the .nf-core.yml config file if we can - tools_config = nf_core.utils.load_tools_config(self.directory) - if tools_config.get("repository_type") in ["pipeline", "modules"]: - if self.repo_type is not None and self.repo_type != tools_config["repository_type"]: - raise UserWarning( - f"'--repo-type {self.repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" - ) - self.repo_type = tools_config["repository_type"] - return - - # Could be set on the command line - throw an error if not - if not self.repo_type: - raise UserWarning( - f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{self.directory}'" - "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" - ) - def get_module_dirs(self): """Given a directory and a tool/subtool, set the file paths and check if they already exist diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 892cacf8de..af432ec269 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand): def __init__(self, dir): self.dir = dir try: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir, self.repo_type) except LookupError as e: raise UserWarning(e) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index dd9afc9582..a7c600a996 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -337,24 +337,49 @@ def get_installed_modules(dir, repo_type="modules"): return local_modules, nfcore_modules -def get_repo_type(dir): +def get_repo_type(dir, repo_type=None): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ # Verify that the pipeline dir exists if dir is None or not os.path.exists(dir): - raise LookupError("Could not find directory: {}".format(dir)) + raise UserWarning(f"Could not find directory: {dir}") + + # Try to find the root directory + base_dir = os.path.abspath(dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + while ( + not os.path.exists(config_path_yml) + and not os.path.exists(config_path_yaml) + and base_dir != os.path.dirname(base_dir) + ): + base_dir = os.path.dirname(base_dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + # Reset dir if we found the config file (will be an absolute path) + if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): + dir = base_dir + + # Figure out the repository type from the .nf-core.yml config file if we can + tools_config = nf_core.utils.load_tools_config(dir) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + if repo_type is not None and repo_type != tools_config["repository_type"]: + raise UserWarning( + f"'--repo-type {repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" + ) + return [dir, tools_config["repository_type"]] - # Determine repository type - if os.path.exists(os.path.join(dir, "README.md")): - with open(os.path.join(dir, "README.md")) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: - raise LookupError("Could not determine repository type of '{}'".format(dir)) + # Could be set on the command line - throw an error if not + if not repo_type: + raise UserWarning( + f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{dir}'" + "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" + ) + + # It was set on the command line, return what we were given + return [dir, repo_type] def verify_pipeline_dir(dir): diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 866bffabb9..58198740b6 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -29,7 +29,7 @@ def __init__(self, dir): self.module_names = [] try: if self.dir: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) else: self.repo_type = None except LookupError as e: From c5697402d4e16541fd3a76e312ee36a3e07e95e2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 17:12:17 +0100 Subject: [PATCH 7/7] Modules lint: don't supply repo type, find it --- nf_core/modules/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af432ec269..e81eea7f85 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand): def __init__(self, dir): self.dir = dir try: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir, self.repo_type) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) except LookupError as e: raise UserWarning(e)