From ad0a6a94ed31966ef639c829387120e7cf99502f Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Tue, 4 Jul 2023 11:43:53 +0200 Subject: [PATCH 01/14] add params-template subcommand --- README.md | 17 ++ docs/api/_src/api/params-template.md | 9 + nf_core/__main__.py | 36 +++- nf_core/params_template.py | 276 +++++++++++++++++++++++++++ tests/test_params_template.py | 79 ++++++++ 5 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 docs/api/_src/api/params-template.md create mode 100644 nf_core/params_template.py create mode 100644 tests/test_params_template.py diff --git a/README.md b/README.md index 012e4c4b12..2679124796 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ A python package with helper tools for the nf-core community. - [`nf-core` tools update](#update-tools) - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) +- [`nf-core params-template` - Create a parameter file template](#create-a-parameter-file-template) - [`nf-core download` - Download a pipeline for offline use](#downloading-pipelines-for-offline-use) - [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) - [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) @@ -311,6 +312,22 @@ Do you want to run this command now? [y/n]: - `--url` - Change the URL used for the graphical interface, useful for development work on the website. +## Create a parameter file template + +Sometimes it is easier to manually edit a parameter file than to use the web interface or interactive commandline wizard +provided by `nf-core launch`, for example when running a pipeline with many options on a remote server without a graphical interface. + +You can create a parameter file template with the `nf-core params-template` command. +This file can then be passed to `nextflow` with the `-params-file` flag. + +This command takes one argument - either the name of a nf-core pipeline which will be pulled automatically, +or the path to a directory containing a Nextflow pipeline _(can be any pipeline, doesn't have to be nf-core)_. + +The generated YAML file contains all parameters set to the pipeline default value along with their description in comments. +This template can then be used by uncommenting and modifying the value of parameters you want to pass to a pipline run. + +Hidden options are not shown by default but can be included using the `-x|--show-hidden` flag. + ## Downloading pipelines for offline use Sometimes you may need to run an nf-core pipeline on a server or HPC system that has no internet connection. diff --git a/docs/api/_src/api/params-template.md b/docs/api/_src/api/params-template.md new file mode 100644 index 0000000000..af9608b003 --- /dev/null +++ b/docs/api/_src/api/params-template.md @@ -0,0 +1,9 @@ +# nf_core.params_template + +```{eval-rst} +.. automodule:: nf_core.params_template + :members: + :undoc-members: + :show-inheritance: + :private-members: +``` diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d57d27f1e6..a364c46c21 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -13,6 +13,7 @@ from nf_core import __version__ from nf_core.download import DownloadError from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE +from nf_core.params_template import ParamsFileTemplateBuilder from nf_core.utils import check_if_outdated, rich_force_colors, setup_nfcore_dir # Set up logging as the root logger @@ -29,7 +30,7 @@ "nf-core": [ { "name": "Commands for users", - "commands": ["list", "launch", "download", "licences"], + "commands": ["list", "launch", "params-template", "download", "licences"], }, { "name": "Commands for developers", @@ -221,6 +222,39 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all sys.exit(1) +# nf-core params-template +@nf_core_cli.command() +@click.argument("pipeline", required=False, metavar="") +@click.option("-r", "--revision", help="Release/branch/SHA of the pipeline (if remote)") +@click.option( + "-o", + "--output", + type=str, + default="nf-params.yml", + metavar="", + help="Output filename. Defaults to `nf-params.yml`", +) +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +@click.option( + "-x", "--show-hidden", is_flag=True, default=False, help="Show hidden params which don't normally need changing" +) +def params_template(pipeline, revision, output, force, show_hidden): + """ + Build a parameter file template for a pipeline. + + Uses the pipeline schema file to generate a YAML file that can be passed + to Nextflow using the `-params-file` option. + Parameters descriptions are shown in comments and the all parameters are set + to the pipeline defaults. + + This output file is intended as a template and should be edited before use. + """ + builder = ParamsFileTemplateBuilder(pipeline, revision) + + if not builder.write_template(output, show_hidden=show_hidden, force=force): + sys.exit(1) + + # nf-core download @nf_core_cli.command() @click.argument("pipeline", required=False, metavar="") diff --git a/nf_core/params_template.py b/nf_core/params_template.py new file mode 100644 index 0000000000..09ab2f063d --- /dev/null +++ b/nf_core/params_template.py @@ -0,0 +1,276 @@ +""" Create YAML parameter file template """ + +from __future__ import print_function + +import json +import logging +import os +import textwrap +from typing import Literal, Optional + +import questionary +import rich +import rich.columns + +import nf_core.list +import nf_core.utils +from nf_core.schema import PipelineSchema + +log = logging.getLogger(__name__) + +INTRO = ( + "This is an example parameter file to pass to the `-params-file` option " + "of nextflow run with the {pipeline_name} pipeline." +) + +USAGE = "Uncomment lines with a single '#' if you want to pass the parameter " "to the pipeline." + +H1_SEPERATOR = "## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +H2_SEPERATOR = "## ----------------------------------------------------------------------------" + +ModeLiteral = Literal["both", "start", "end", "none"] + + +def _print_wrapped(text, fill_char="-", mode="both", width=80, indent=0, drop_whitespace=True): + """Helper function to format text for the params-file template. + + Args: + text (str): Text to print + fill_char (str, optional): + Character to use for creating dividers. Defaults to '-'. + mode (str, optional): + Where to place dividers. Defaults to "both". + width (int, optional): + Maximum line-width of the output text. Defaults to 80. + indent (int, optional): + Number of spaces to indent the text. Defaults to 0. + drop_whitespace (bool, optional): + Whether to drop whitespace from the start and end of lines. + """ + if len(fill_char) != 1: + raise ValueError("fill_char must be a single character") + + prefix = "## " + out = "" + + if mode in ("both", "start"): + out += prefix.ljust(width, fill_char) + "\n" + + wrap_indent = f"{prefix}{' ' * indent}" + + textlines = textwrap.wrap( + text, + width=width - len(prefix), + initial_indent=wrap_indent, + subsequent_indent=wrap_indent, + drop_whitespace=drop_whitespace, + ) + + for line in textlines: + out += line + "\n" + + if mode in ("both", "end"): + out += prefix.ljust(width, fill_char) + "\n" + + return out + + +class ParamsFileTemplateBuilder: + """Class to hold config option to launch a pipeline. + + Args: + pipeline (str, optional): + Path to a local pipeline path or a remote pipeline. + revision (str, optional): + Revision of the pipeline to use. + """ + + def __init__( + self, + pipeline=None, + revision=None, + ): + """Initialise the ParamFileTemplateBuilder class + + Args: + pipeline (str, optional): Path to a local pipeline path or a remote pipeline. + revision (str, optional): Revision of the pipeline to use. + """ + self.pipeline = pipeline + self.pipeline_revision = revision + self.schema_obj: Optional[PipelineSchema] = None + + # Fetch remote workflows + self.wfs = nf_core.list.Workflows() + self.wfs.get_remote_workflows() + + def get_pipeline(self): + """ + Prompt the user for a pipeline name and get the schema + """ + # Prompt for pipeline if not supplied + if self.pipeline is None: + launch_type = questionary.select( + "Generate parameter file template for local pipeline " "or remote GitHub pipeline?", + choices=["Remote pipeline", "Local path"], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + if launch_type == "Remote pipeline": + try: + self.pipeline = nf_core.utils.prompt_remote_pipeline_name(self.wfs) + except AssertionError as e: + log.error(e.args[0]) + return False + else: + self.pipeline = questionary.path( + "Path to workflow:", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + # Get the schema + self.schema_obj = nf_core.schema.PipelineSchema() + self.schema_obj.get_schema_path(self.pipeline, local_only=False, revision=self.pipeline_revision) + self.schema_obj.get_wf_params() + + def format_group(self, definition, show_hidden=False): + """Format a group of parameters of the schema as commented YAML. + + Args: + definition (dict): Definition of the group from the schema + show_hidden (bool): Whether to include hidden parameters + + Returns: + str: Formatted output for a group + """ + out = "" + title = definition.get("title", definition) + description = definition.get("description", "") + properties = definition.get("properties") + + hidden_props = set() + for param_key, param_props in properties.items(): + if param_props.get("hidden", False): + hidden_props.add(param_key) + + out += _print_wrapped(title, "=", mode="both") + if description: + out += _print_wrapped(description, mode="none") + + if len(hidden_props) > 0: + out += "\n" + out += _print_wrapped(f"({len(hidden_props)} hidden parameters are not shown)", mode="none") + out += "\n\n" + + required_props = definition.get("required", []) + + for prop_key, props in properties.items(): + param_out = self.format_param(prop_key, props, required_props, show_hidden=show_hidden) + if param_out is not None: + out += param_out + out += "\n" + + return out + + def format_param(self, name, properties, required_properties=(), show_hidden=False): + """ + Format a single parameter of the schema as commented YAML + + Args: + name (str): Name of the parameter + properties (dict): Properties of the parameter + required_properties (list): List of required properties + show_hidden (bool): Whether to include hidden parameters + + Returns: + str: Section of a params-file.yml for given parameter + None: + If the parameter is skipped because it is hidden and + show_hidden is not set + """ + out = "" + hidden = properties.get("hidden", False) + + if not show_hidden and hidden: + return None + + description = properties.get("description", "") + self.schema_obj.get_schema_defaults() + default = properties.get("default") + typ = properties.get("type") + required = name in required_properties + + out += _print_wrapped(name, "-", mode="both") + + if description: + out += _print_wrapped(description + "\n", mode="none", indent=4) + + if typ: + out += _print_wrapped(f"Type: {typ}", mode="none", indent=4) + + out += _print_wrapped("\n", mode="end") + out += f"# {name} = {json.dumps(default)}\n" + + return out + + def generate_template_file(self, show_hidden=False): + """Generate the contents of a parameter template file. + + Assumes the pipeline has been fetched (if remote) and the schema loaded. + + Args: + show_hidden (bool): Whether to include hidden parameters + + Returns: + str: Formatted output for the pipeline schema + """ + schema = self.schema_obj.schema + pipeline_name = self.schema_obj.pipeline_manifest.get("name", self.pipeline) + + # Build the header section + out = "" + + out += _print_wrapped(pipeline_name, "~", mode="both", indent=4) + out += _print_wrapped(INTRO.format(pipeline_name=pipeline_name), " ", mode="none", indent=4) + out += _print_wrapped("\n", " ", mode="none", indent=4, drop_whitespace=False) + out += _print_wrapped(USAGE, "-", mode="end", indent=4) + + # Add all parameter groups + for definition in schema.get("definitions", {}).values(): + out += self.format_group(definition, show_hidden=show_hidden) + out += "\n" + + return out + + def write_template(self, output_fn="nf-params.yaml", show_hidden=False, force=False): + """Build a template file for the pipeline schema. + + Args: + output_fn (str, optional): Filename to write the template to. + show_hidden (bool, optional): + Include parameters marked as hidden in the output + force (bool, optional): Whether to overwrite existing output file. + + Returns: + bool: True if the template was written successfully, False otherwise + """ + + self.get_pipeline() + + try: + self.schema_obj.load_schema() + self.schema_obj.validate_schema() + except AssertionError as e: + log.error(f'Pipeline schema file is invalid ("{self.schema_obj.schema_filename}"): {e}') + log.info("Please fix this file, then try again.") + return False + + schema_out = self.generate_template_file(show_hidden=show_hidden) + + if os.path.exists(output_fn) and not force: + log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") + return False + with open(output_fn, "w") as fh: + fh.write(schema_out) + log.info(f"Parameter schema file written to '{output_fn}'") + + return True diff --git a/tests/test_params_template.py b/tests/test_params_template.py new file mode 100644 index 0000000000..92003d3c7b --- /dev/null +++ b/tests/test_params_template.py @@ -0,0 +1,79 @@ +import json +import os +import shutil +import tempfile +from pathlib import Path + +import nf_core.create +import nf_core.schema +from nf_core.params_template import ParamsFileTemplateBuilder + + +class TestParamsTemplateBuilder: + """Class for schema tests""" + + def setup_class(self): + """Create a new PipelineSchema object""" + self.schema_obj = nf_core.schema.PipelineSchema() + self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + # Create a test pipeline in temp directory + self.tmp_dir = tempfile.mkdtemp() + self.template_dir = os.path.join(self.tmp_dir, "wf") + create_obj = nf_core.create.PipelineCreate( + "testpipeline", "", "", outdir=self.template_dir, no_git=True, plain=True + ) + create_obj.init_pipeline() + + self.template_schema = os.path.join(self.template_dir, "nextflow_schema.json") + self.params_template_builder = ParamsFileTemplateBuilder(self.template_dir) + + def teardown_class(self): + if os.path.exists(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + + def test_build_template(self): + outfile = os.path.join(self.tmp_dir, "params-file.yml") + self.params_template_builder.write_template(outfile) + + assert os.path.exists(outfile) + + with open(outfile, "r") as fh: + out = fh.read() + + assert "nf-core/testpipeline" in out + + def test_build_template_invalid_schema(self, caplog): + """Build a schema from a template""" + outfile = os.path.join(self.tmp_dir, "params-file.yml") + schema_path = Path(self.template_schema) + invalid_schema_file = shutil.copy(self.template_schema, Path(self.template_schema).name + "_invalid.json") + + # Remove the allOf section to make the schema invalid + with open(invalid_schema_file, "r") as fh: + o = json.load(fh) + del o["allOf"] + + with open(invalid_schema_file, "w") as fh: + json.dump(o, fh) + + builder = ParamsFileTemplateBuilder(invalid_schema_file) + res = builder.write_template(outfile) + + assert res is False + assert "Pipeline schema file is invalid" in caplog.text + + def test_build_template_file_exists(self, caplog): + """Build a schema from a template""" + + # Creates a new empty file + outfile = Path(self.tmp_dir) / "params-file.yml" + with open(outfile, "w") as fp: + pass + + res = self.params_template_builder.write_template(outfile) + + assert res is False + assert f"File '{outfile}' exists!" in caplog.text + + outfile.unlink() From e4f78ca4df048b38bcdaf3435f6054946959e2c6 Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Tue, 4 Jul 2023 14:34:18 +0200 Subject: [PATCH 02/14] add whitespace around header and groups --- nf_core/params_template.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/params_template.py b/nf_core/params_template.py index 09ab2f063d..638c557a9e 100644 --- a/nf_core/params_template.py +++ b/nf_core/params_template.py @@ -156,8 +156,8 @@ def format_group(self, definition, show_hidden=False): if description: out += _print_wrapped(description, mode="none") + out += "\n" if len(hidden_props) > 0: - out += "\n" out += _print_wrapped(f"({len(hidden_props)} hidden parameters are not shown)", mode="none") out += "\n\n" @@ -233,6 +233,7 @@ def generate_template_file(self, show_hidden=False): out += _print_wrapped(INTRO.format(pipeline_name=pipeline_name), " ", mode="none", indent=4) out += _print_wrapped("\n", " ", mode="none", indent=4, drop_whitespace=False) out += _print_wrapped(USAGE, "-", mode="end", indent=4) + out += "\n" # Add all parameter groups for definition in schema.get("definitions", {}).values(): From e0229c86ef7c3ebcdfca16cdc3eb55e9d052f2fa Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Tue, 4 Jul 2023 14:34:48 +0200 Subject: [PATCH 03/14] reword cli help message --- nf_core/__main__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a364c46c21..e09ed62953 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -242,12 +242,13 @@ def params_template(pipeline, revision, output, force, show_hidden): """ Build a parameter file template for a pipeline. - Uses the pipeline schema file to generate a YAML file that can be passed - to Nextflow using the `-params-file` option. - Parameters descriptions are shown in comments and the all parameters are set - to the pipeline defaults. + Uses the pipeline schema file to generate a YAML parameters file. + Parameters are set to the pipeline defaults and descriptions are shown in comments. + After the output file is generated, it can then be edited as needed before + passing to nextflow using the `-params-file` option. - This output file is intended as a template and should be edited before use. + Run using a remote pipeline name (such as GitHub `user/repo` or a URL), + a local pipeline directory. """ builder = ParamsFileTemplateBuilder(pipeline, revision) From 542a4531f14280c024034692808303b07bdae228 Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Tue, 4 Jul 2023 14:41:22 +0200 Subject: [PATCH 04/14] fix missing period --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e09ed62953..cd79a3269d 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -232,7 +232,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all type=str, default="nf-params.yml", metavar="", - help="Output filename. Defaults to `nf-params.yml`", + help="Output filename. Defaults to `nf-params.yml`.", ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") @click.option( From be2d827ecb5144ba89f6281a32adf30eb2f00e0d Mon Sep 17 00:00:00 2001 From: Florian De Temmerman <69114541+fbdtemme@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:59:02 +0200 Subject: [PATCH 05/14] Update README.md Co-authored-by: Phil Ewels --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2679124796..735c3bc68c 100644 --- a/README.md +++ b/README.md @@ -326,7 +326,7 @@ or the path to a directory containing a Nextflow pipeline _(can be any pipeline, The generated YAML file contains all parameters set to the pipeline default value along with their description in comments. This template can then be used by uncommenting and modifying the value of parameters you want to pass to a pipline run. -Hidden options are not shown by default but can be included using the `-x|--show-hidden` flag. +Hidden options are not included by default, but can be included using the `-x`/`--show-hidden` flag. ## Downloading pipelines for offline use From 52d088e8108d172895283cbe52ae0ec795d9cafc Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Tue, 4 Jul 2023 17:06:34 +0200 Subject: [PATCH 06/14] rename command to create-params-file --- README.md | 6 +++--- docs/api/_src/api/{params-template.md => params-file.md} | 4 ++-- nf_core/__main__.py | 8 ++++---- nf_core/{params_template.py => params_file.py} | 8 ++++---- tests/test_params_template.py | 8 ++++---- 5 files changed, 17 insertions(+), 17 deletions(-) rename docs/api/_src/api/{params-template.md => params-file.md} (59%) rename nf_core/{params_template.py => params_file.py} (97%) diff --git a/README.md b/README.md index 735c3bc68c..98c7bfed8c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ A python package with helper tools for the nf-core community. - [`nf-core` tools update](#update-tools) - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) -- [`nf-core params-template` - Create a parameter file template](#create-a-parameter-file-template) +- [`nf-core create-params-file` - Create a parameter file](#create-a-parameter-file) - [`nf-core download` - Download a pipeline for offline use](#downloading-pipelines-for-offline-use) - [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) - [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) @@ -312,12 +312,12 @@ Do you want to run this command now? [y/n]: - `--url` - Change the URL used for the graphical interface, useful for development work on the website. -## Create a parameter file template +## Create a parameter file Sometimes it is easier to manually edit a parameter file than to use the web interface or interactive commandline wizard provided by `nf-core launch`, for example when running a pipeline with many options on a remote server without a graphical interface. -You can create a parameter file template with the `nf-core params-template` command. +You can create a parameter file template with the `nf-core create-params-file` command. This file can then be passed to `nextflow` with the `-params-file` flag. This command takes one argument - either the name of a nf-core pipeline which will be pulled automatically, diff --git a/docs/api/_src/api/params-template.md b/docs/api/_src/api/params-file.md similarity index 59% rename from docs/api/_src/api/params-template.md rename to docs/api/_src/api/params-file.md index af9608b003..c5bbfc0f1f 100644 --- a/docs/api/_src/api/params-template.md +++ b/docs/api/_src/api/params-file.md @@ -1,7 +1,7 @@ -# nf_core.params_template +# nf_core.params_file ```{eval-rst} -.. automodule:: nf_core.params_template +.. automodule:: nf_core.params_file :members: :undoc-members: :show-inheritance: diff --git a/nf_core/__main__.py b/nf_core/__main__.py index cd79a3269d..9baf4cec52 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -13,7 +13,7 @@ from nf_core import __version__ from nf_core.download import DownloadError from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE -from nf_core.params_template import ParamsFileTemplateBuilder +from nf_core.params_file import ParamsFileBuilder from nf_core.utils import check_if_outdated, rich_force_colors, setup_nfcore_dir # Set up logging as the root logger @@ -238,9 +238,9 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all @click.option( "-x", "--show-hidden", is_flag=True, default=False, help="Show hidden params which don't normally need changing" ) -def params_template(pipeline, revision, output, force, show_hidden): +def create_params_file(pipeline, revision, output, force, show_hidden): """ - Build a parameter file template for a pipeline. + Build a parameter file for a pipeline. Uses the pipeline schema file to generate a YAML parameters file. Parameters are set to the pipeline defaults and descriptions are shown in comments. @@ -250,7 +250,7 @@ def params_template(pipeline, revision, output, force, show_hidden): Run using a remote pipeline name (such as GitHub `user/repo` or a URL), a local pipeline directory. """ - builder = ParamsFileTemplateBuilder(pipeline, revision) + builder = ParamsFileBuilder(pipeline, revision) if not builder.write_template(output, show_hidden=show_hidden, force=force): sys.exit(1) diff --git a/nf_core/params_template.py b/nf_core/params_file.py similarity index 97% rename from nf_core/params_template.py rename to nf_core/params_file.py index 638c557a9e..207bfb65be 100644 --- a/nf_core/params_template.py +++ b/nf_core/params_file.py @@ -1,4 +1,4 @@ -""" Create YAML parameter file template """ +""" Create a YAML parameter file """ from __future__ import print_function @@ -75,7 +75,7 @@ def _print_wrapped(text, fill_char="-", mode="both", width=80, indent=0, drop_wh return out -class ParamsFileTemplateBuilder: +class ParamsFileBuilder: """Class to hold config option to launch a pipeline. Args: @@ -111,7 +111,7 @@ def get_pipeline(self): # Prompt for pipeline if not supplied if self.pipeline is None: launch_type = questionary.select( - "Generate parameter file template for local pipeline " "or remote GitHub pipeline?", + "Generate parameter file for local pipeline " "or remote GitHub pipeline?", choices=["Remote pipeline", "Local path"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -272,6 +272,6 @@ def write_template(self, output_fn="nf-params.yaml", show_hidden=False, force=Fa return False with open(output_fn, "w") as fh: fh.write(schema_out) - log.info(f"Parameter schema file written to '{output_fn}'") + log.info(f"Parameter file written to '{output_fn}'") return True diff --git a/tests/test_params_template.py b/tests/test_params_template.py index 92003d3c7b..409311f5d7 100644 --- a/tests/test_params_template.py +++ b/tests/test_params_template.py @@ -6,10 +6,10 @@ import nf_core.create import nf_core.schema -from nf_core.params_template import ParamsFileTemplateBuilder +from nf_core.params_file import ParamsFileBuilder -class TestParamsTemplateBuilder: +class TestParamsFileBuilder: """Class for schema tests""" def setup_class(self): @@ -26,7 +26,7 @@ def setup_class(self): create_obj.init_pipeline() self.template_schema = os.path.join(self.template_dir, "nextflow_schema.json") - self.params_template_builder = ParamsFileTemplateBuilder(self.template_dir) + self.params_template_builder = ParamsFileBuilder(self.template_dir) def teardown_class(self): if os.path.exists(self.tmp_dir): @@ -57,7 +57,7 @@ def test_build_template_invalid_schema(self, caplog): with open(invalid_schema_file, "w") as fh: json.dump(o, fh) - builder = ParamsFileTemplateBuilder(invalid_schema_file) + builder = ParamsFileBuilder(invalid_schema_file) res = builder.write_template(outfile) assert res is False From e24bdb1097b031d9d70f6e729a8bd84ca82e3b6e Mon Sep 17 00:00:00 2001 From: Florian De Temmerman <69114541+fbdtemme@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:11:55 +0200 Subject: [PATCH 07/14] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98c7bfed8c..e92e31516f 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ Do you want to run this command now? [y/n]: Sometimes it is easier to manually edit a parameter file than to use the web interface or interactive commandline wizard provided by `nf-core launch`, for example when running a pipeline with many options on a remote server without a graphical interface. -You can create a parameter file template with the `nf-core create-params-file` command. +You can create a parameter file with all parameters of a pipeline with the `nf-core create-params-file` command. This file can then be passed to `nextflow` with the `-params-file` flag. This command takes one argument - either the name of a nf-core pipeline which will be pulled automatically, From 826cf13af0a959226b3a85e0d3086d365781db0d Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Tue, 4 Jul 2023 17:18:55 +0200 Subject: [PATCH 08/14] improve test setup --- ...params_template.py => test_params_file.py} | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) rename tests/{test_params_template.py => test_params_file.py} (60%) diff --git a/tests/test_params_template.py b/tests/test_params_file.py similarity index 60% rename from tests/test_params_template.py rename to tests/test_params_file.py index 409311f5d7..459072c9e3 100644 --- a/tests/test_params_template.py +++ b/tests/test_params_file.py @@ -12,25 +12,36 @@ class TestParamsFileBuilder: """Class for schema tests""" - def setup_class(self): + @classmethod + def setup_class(cls): """Create a new PipelineSchema object""" - self.schema_obj = nf_core.schema.PipelineSchema() - self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + cls.schema_obj = nf_core.schema.PipelineSchema() + cls.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Create a test pipeline in temp directory - self.tmp_dir = tempfile.mkdtemp() - self.template_dir = os.path.join(self.tmp_dir, "wf") + cls.tmp_dir = tempfile.mkdtemp() + cls.template_dir = os.path.join(cls.tmp_dir, "wf") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "", "", outdir=self.template_dir, no_git=True, plain=True + "testpipeline", "", "", outdir=cls.template_dir, no_git=True, plain=True ) create_obj.init_pipeline() - self.template_schema = os.path.join(self.template_dir, "nextflow_schema.json") - self.params_template_builder = ParamsFileBuilder(self.template_dir) + cls.template_schema = os.path.join(cls.template_dir, "nextflow_schema.json") + cls.params_template_builder = ParamsFileBuilder(cls.template_dir) + cls.invalid_template_schema = os.path.join(cls.template_dir, "nextflow_schema_invalid.json") - def teardown_class(self): - if os.path.exists(self.tmp_dir): - shutil.rmtree(self.tmp_dir) + # Remove the allOf section to make the schema invalid + with open(cls.template_schema, "r") as fh: + o = json.load(fh) + del o["allOf"] + + with open(cls.invalid_template_schema, "w") as fh: + json.dump(o, fh) + + @classmethod + def teardown_class(cls): + if os.path.exists(cls.tmp_dir): + shutil.rmtree(cls.tmp_dir) def test_build_template(self): outfile = os.path.join(self.tmp_dir, "params-file.yml") @@ -45,19 +56,8 @@ def test_build_template(self): def test_build_template_invalid_schema(self, caplog): """Build a schema from a template""" - outfile = os.path.join(self.tmp_dir, "params-file.yml") - schema_path = Path(self.template_schema) - invalid_schema_file = shutil.copy(self.template_schema, Path(self.template_schema).name + "_invalid.json") - - # Remove the allOf section to make the schema invalid - with open(invalid_schema_file, "r") as fh: - o = json.load(fh) - del o["allOf"] - - with open(invalid_schema_file, "w") as fh: - json.dump(o, fh) - - builder = ParamsFileBuilder(invalid_schema_file) + outfile = os.path.join(self.tmp_dir, "params-file-invalid.yml") + builder = ParamsFileBuilder(self.invalid_template_schema) res = builder.write_template(outfile) assert res is False From d2c8db09742359aab8cd9689f8f12a90c22a47de Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Wed, 5 Jul 2023 09:32:14 +0200 Subject: [PATCH 09/14] fix old command name in cli command group --- nf_core/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9baf4cec52..c88cd18c20 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -30,7 +30,7 @@ "nf-core": [ { "name": "Commands for users", - "commands": ["list", "launch", "params-template", "download", "licences"], + "commands": ["list", "launch", "create-params-file", "download", "licences"], }, { "name": "Commands for developers", @@ -222,7 +222,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all sys.exit(1) -# nf-core params-template +# nf-core create-params-file @nf_core_cli.command() @click.argument("pipeline", required=False, metavar="") @click.option("-r", "--revision", help="Release/branch/SHA of the pipeline (if remote)") From 75c6ab66e392eab97b188dfc1ee66308e4c33b16 Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Wed, 5 Jul 2023 09:32:23 +0200 Subject: [PATCH 10/14] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a9431b07..a8680b1ee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Initialise `docker_image_name` to fix `UnboundLocalError` error ([#2374](https://github.com/nf-core/tools/pull/2374)) - Fix prompt pipeline revision during launch ([#2375](https://github.com/nf-core/tools/pull/2375)) +- Add a `create-params-file` command to create a YAML parameter file for a pipeline containing parameter documentation and defaults. ([#2362](https://github.com/nf-core/tools/pull/2362)) # [v2.9 - Chromium Falcon](https://github.com/nf-core/tools/releases/tag/2.9) + [2023-06-29] From 84327204d16c681b7145b720cefc67e9eab67c66 Mon Sep 17 00:00:00 2001 From: Florian De Temmerman <69114541+fbdtemme@users.noreply.github.com> Date: Wed, 5 Jul 2023 13:37:13 +0200 Subject: [PATCH 11/14] remove `template` from function names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/__main__.py | 2 +- nf_core/params_file.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index c88cd18c20..d745a896f7 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -252,7 +252,7 @@ def create_params_file(pipeline, revision, output, force, show_hidden): """ builder = ParamsFileBuilder(pipeline, revision) - if not builder.write_template(output, show_hidden=show_hidden, force=force): + if not builder.write_params_file(output, show_hidden=show_hidden, force=force): sys.exit(1) diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 207bfb65be..8bbc57aab2 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -90,7 +90,7 @@ def __init__( pipeline=None, revision=None, ): - """Initialise the ParamFileTemplateBuilder class + """Initialise the ParamFileBuilder class Args: pipeline (str, optional): Path to a local pipeline path or a remote pipeline. @@ -212,7 +212,7 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal return out - def generate_template_file(self, show_hidden=False): + def generate_params_file(self, show_hidden=False): """Generate the contents of a parameter template file. Assumes the pipeline has been fetched (if remote) and the schema loaded. @@ -265,7 +265,7 @@ def write_template(self, output_fn="nf-params.yaml", show_hidden=False, force=Fa log.info("Please fix this file, then try again.") return False - schema_out = self.generate_template_file(show_hidden=show_hidden) + schema_out = self.generate_params_file(show_hidden=show_hidden) if os.path.exists(output_fn) and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") From 2d17e43d48b2e24846524c71be03be9fd1a20bd2 Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Thu, 6 Jul 2023 11:31:04 +0200 Subject: [PATCH 12/14] fix incomplete renaming --- nf_core/params_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 8bbc57aab2..a6d7bd6ee4 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -242,7 +242,7 @@ def generate_params_file(self, show_hidden=False): return out - def write_template(self, output_fn="nf-params.yaml", show_hidden=False, force=False): + def write_params_file(self, output_fn="nf-params.yaml", show_hidden=False, force=False): """Build a template file for the pipeline schema. Args: From 5e78eaf2b1aa02f125796e4dd7d8d5d7a2be85d9 Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Thu, 6 Jul 2023 11:31:27 +0200 Subject: [PATCH 13/14] add pipeline version to params file --- nf_core/params_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/params_file.py b/nf_core/params_file.py index a6d7bd6ee4..39986b95c2 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -225,11 +225,12 @@ def generate_params_file(self, show_hidden=False): """ schema = self.schema_obj.schema pipeline_name = self.schema_obj.pipeline_manifest.get("name", self.pipeline) + pipeline_version = self.schema_obj.pipeline_manifest.get("version", "0.0.0") # Build the header section out = "" - out += _print_wrapped(pipeline_name, "~", mode="both", indent=4) + out += _print_wrapped(f"{pipeline_name} {pipeline_version}", "~", mode="both", indent=4) out += _print_wrapped(INTRO.format(pipeline_name=pipeline_name), " ", mode="none", indent=4) out += _print_wrapped("\n", " ", mode="none", indent=4, drop_whitespace=False) out += _print_wrapped(USAGE, "-", mode="end", indent=4) From aa887503cc1694b05bff931dc3d31958e0d3000c Mon Sep 17 00:00:00 2001 From: Florian De Temmerman Date: Wed, 19 Jul 2023 10:18:39 +0200 Subject: [PATCH 14/14] update tests after renaming --- tests/test_params_file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_params_file.py b/tests/test_params_file.py index 459072c9e3..824e8fe345 100644 --- a/tests/test_params_file.py +++ b/tests/test_params_file.py @@ -45,7 +45,7 @@ def teardown_class(cls): def test_build_template(self): outfile = os.path.join(self.tmp_dir, "params-file.yml") - self.params_template_builder.write_template(outfile) + self.params_template_builder.write_params_file(outfile) assert os.path.exists(outfile) @@ -58,7 +58,7 @@ def test_build_template_invalid_schema(self, caplog): """Build a schema from a template""" outfile = os.path.join(self.tmp_dir, "params-file-invalid.yml") builder = ParamsFileBuilder(self.invalid_template_schema) - res = builder.write_template(outfile) + res = builder.write_params_file(outfile) assert res is False assert "Pipeline schema file is invalid" in caplog.text @@ -71,7 +71,7 @@ def test_build_template_file_exists(self, caplog): with open(outfile, "w") as fp: pass - res = self.params_template_builder.write_template(outfile) + res = self.params_template_builder.write_params_file(outfile) assert res is False assert f"File '{outfile}' exists!" in caplog.text