diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a9431b07..a8680b1ee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Initialise `docker_image_name` to fix `UnboundLocalError` error ([#2374](https://github.com/nf-core/tools/pull/2374)) - Fix prompt pipeline revision during launch ([#2375](https://github.com/nf-core/tools/pull/2375)) +- Add a `create-params-file` command to create a YAML parameter file for a pipeline containing parameter documentation and defaults. ([#2362](https://github.com/nf-core/tools/pull/2362)) # [v2.9 - Chromium Falcon](https://github.com/nf-core/tools/releases/tag/2.9) + [2023-06-29] diff --git a/README.md b/README.md index 012e4c4b12..e92e31516f 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ A python package with helper tools for the nf-core community. - [`nf-core` tools update](#update-tools) - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) +- [`nf-core create-params-file` - Create a parameter file](#create-a-parameter-file) - [`nf-core download` - Download a pipeline for offline use](#downloading-pipelines-for-offline-use) - [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) - [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) @@ -311,6 +312,22 @@ Do you want to run this command now? [y/n]: - `--url` - Change the URL used for the graphical interface, useful for development work on the website. +## Create a parameter file + +Sometimes it is easier to manually edit a parameter file than to use the web interface or interactive commandline wizard +provided by `nf-core launch`, for example when running a pipeline with many options on a remote server without a graphical interface. + +You can create a parameter file with all parameters of a pipeline with the `nf-core create-params-file` command. +This file can then be passed to `nextflow` with the `-params-file` flag. + +This command takes one argument - either the name of a nf-core pipeline which will be pulled automatically, +or the path to a directory containing a Nextflow pipeline _(can be any pipeline, doesn't have to be nf-core)_. + +The generated YAML file contains all parameters set to the pipeline default value along with their description in comments. +This template can then be used by uncommenting and modifying the value of parameters you want to pass to a pipline run. + +Hidden options are not included by default, but can be included using the `-x`/`--show-hidden` flag. + ## Downloading pipelines for offline use Sometimes you may need to run an nf-core pipeline on a server or HPC system that has no internet connection. diff --git a/docs/api/_src/api/params-file.md b/docs/api/_src/api/params-file.md new file mode 100644 index 0000000000..c5bbfc0f1f --- /dev/null +++ b/docs/api/_src/api/params-file.md @@ -0,0 +1,9 @@ +# nf_core.params_file + +```{eval-rst} +.. automodule:: nf_core.params_file + :members: + :undoc-members: + :show-inheritance: + :private-members: +``` diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d57d27f1e6..d745a896f7 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -13,6 +13,7 @@ from nf_core import __version__ from nf_core.download import DownloadError from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE +from nf_core.params_file import ParamsFileBuilder from nf_core.utils import check_if_outdated, rich_force_colors, setup_nfcore_dir # Set up logging as the root logger @@ -29,7 +30,7 @@ "nf-core": [ { "name": "Commands for users", - "commands": ["list", "launch", "download", "licences"], + "commands": ["list", "launch", "create-params-file", "download", "licences"], }, { "name": "Commands for developers", @@ -221,6 +222,40 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all sys.exit(1) +# nf-core create-params-file +@nf_core_cli.command() +@click.argument("pipeline", required=False, metavar="") +@click.option("-r", "--revision", help="Release/branch/SHA of the pipeline (if remote)") +@click.option( + "-o", + "--output", + type=str, + default="nf-params.yml", + metavar="", + help="Output filename. Defaults to `nf-params.yml`.", +) +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +@click.option( + "-x", "--show-hidden", is_flag=True, default=False, help="Show hidden params which don't normally need changing" +) +def create_params_file(pipeline, revision, output, force, show_hidden): + """ + Build a parameter file for a pipeline. + + Uses the pipeline schema file to generate a YAML parameters file. + Parameters are set to the pipeline defaults and descriptions are shown in comments. + After the output file is generated, it can then be edited as needed before + passing to nextflow using the `-params-file` option. + + Run using a remote pipeline name (such as GitHub `user/repo` or a URL), + a local pipeline directory. + """ + builder = ParamsFileBuilder(pipeline, revision) + + if not builder.write_params_file(output, show_hidden=show_hidden, force=force): + sys.exit(1) + + # nf-core download @nf_core_cli.command() @click.argument("pipeline", required=False, metavar="") diff --git a/nf_core/params_file.py b/nf_core/params_file.py new file mode 100644 index 0000000000..39986b95c2 --- /dev/null +++ b/nf_core/params_file.py @@ -0,0 +1,278 @@ +""" Create a YAML parameter file """ + +from __future__ import print_function + +import json +import logging +import os +import textwrap +from typing import Literal, Optional + +import questionary +import rich +import rich.columns + +import nf_core.list +import nf_core.utils +from nf_core.schema import PipelineSchema + +log = logging.getLogger(__name__) + +INTRO = ( + "This is an example parameter file to pass to the `-params-file` option " + "of nextflow run with the {pipeline_name} pipeline." +) + +USAGE = "Uncomment lines with a single '#' if you want to pass the parameter " "to the pipeline." + +H1_SEPERATOR = "## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +H2_SEPERATOR = "## ----------------------------------------------------------------------------" + +ModeLiteral = Literal["both", "start", "end", "none"] + + +def _print_wrapped(text, fill_char="-", mode="both", width=80, indent=0, drop_whitespace=True): + """Helper function to format text for the params-file template. + + Args: + text (str): Text to print + fill_char (str, optional): + Character to use for creating dividers. Defaults to '-'. + mode (str, optional): + Where to place dividers. Defaults to "both". + width (int, optional): + Maximum line-width of the output text. Defaults to 80. + indent (int, optional): + Number of spaces to indent the text. Defaults to 0. + drop_whitespace (bool, optional): + Whether to drop whitespace from the start and end of lines. + """ + if len(fill_char) != 1: + raise ValueError("fill_char must be a single character") + + prefix = "## " + out = "" + + if mode in ("both", "start"): + out += prefix.ljust(width, fill_char) + "\n" + + wrap_indent = f"{prefix}{' ' * indent}" + + textlines = textwrap.wrap( + text, + width=width - len(prefix), + initial_indent=wrap_indent, + subsequent_indent=wrap_indent, + drop_whitespace=drop_whitespace, + ) + + for line in textlines: + out += line + "\n" + + if mode in ("both", "end"): + out += prefix.ljust(width, fill_char) + "\n" + + return out + + +class ParamsFileBuilder: + """Class to hold config option to launch a pipeline. + + Args: + pipeline (str, optional): + Path to a local pipeline path or a remote pipeline. + revision (str, optional): + Revision of the pipeline to use. + """ + + def __init__( + self, + pipeline=None, + revision=None, + ): + """Initialise the ParamFileBuilder class + + Args: + pipeline (str, optional): Path to a local pipeline path or a remote pipeline. + revision (str, optional): Revision of the pipeline to use. + """ + self.pipeline = pipeline + self.pipeline_revision = revision + self.schema_obj: Optional[PipelineSchema] = None + + # Fetch remote workflows + self.wfs = nf_core.list.Workflows() + self.wfs.get_remote_workflows() + + def get_pipeline(self): + """ + Prompt the user for a pipeline name and get the schema + """ + # Prompt for pipeline if not supplied + if self.pipeline is None: + launch_type = questionary.select( + "Generate parameter file for local pipeline " "or remote GitHub pipeline?", + choices=["Remote pipeline", "Local path"], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + if launch_type == "Remote pipeline": + try: + self.pipeline = nf_core.utils.prompt_remote_pipeline_name(self.wfs) + except AssertionError as e: + log.error(e.args[0]) + return False + else: + self.pipeline = questionary.path( + "Path to workflow:", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + # Get the schema + self.schema_obj = nf_core.schema.PipelineSchema() + self.schema_obj.get_schema_path(self.pipeline, local_only=False, revision=self.pipeline_revision) + self.schema_obj.get_wf_params() + + def format_group(self, definition, show_hidden=False): + """Format a group of parameters of the schema as commented YAML. + + Args: + definition (dict): Definition of the group from the schema + show_hidden (bool): Whether to include hidden parameters + + Returns: + str: Formatted output for a group + """ + out = "" + title = definition.get("title", definition) + description = definition.get("description", "") + properties = definition.get("properties") + + hidden_props = set() + for param_key, param_props in properties.items(): + if param_props.get("hidden", False): + hidden_props.add(param_key) + + out += _print_wrapped(title, "=", mode="both") + if description: + out += _print_wrapped(description, mode="none") + + out += "\n" + if len(hidden_props) > 0: + out += _print_wrapped(f"({len(hidden_props)} hidden parameters are not shown)", mode="none") + out += "\n\n" + + required_props = definition.get("required", []) + + for prop_key, props in properties.items(): + param_out = self.format_param(prop_key, props, required_props, show_hidden=show_hidden) + if param_out is not None: + out += param_out + out += "\n" + + return out + + def format_param(self, name, properties, required_properties=(), show_hidden=False): + """ + Format a single parameter of the schema as commented YAML + + Args: + name (str): Name of the parameter + properties (dict): Properties of the parameter + required_properties (list): List of required properties + show_hidden (bool): Whether to include hidden parameters + + Returns: + str: Section of a params-file.yml for given parameter + None: + If the parameter is skipped because it is hidden and + show_hidden is not set + """ + out = "" + hidden = properties.get("hidden", False) + + if not show_hidden and hidden: + return None + + description = properties.get("description", "") + self.schema_obj.get_schema_defaults() + default = properties.get("default") + typ = properties.get("type") + required = name in required_properties + + out += _print_wrapped(name, "-", mode="both") + + if description: + out += _print_wrapped(description + "\n", mode="none", indent=4) + + if typ: + out += _print_wrapped(f"Type: {typ}", mode="none", indent=4) + + out += _print_wrapped("\n", mode="end") + out += f"# {name} = {json.dumps(default)}\n" + + return out + + def generate_params_file(self, show_hidden=False): + """Generate the contents of a parameter template file. + + Assumes the pipeline has been fetched (if remote) and the schema loaded. + + Args: + show_hidden (bool): Whether to include hidden parameters + + Returns: + str: Formatted output for the pipeline schema + """ + schema = self.schema_obj.schema + pipeline_name = self.schema_obj.pipeline_manifest.get("name", self.pipeline) + pipeline_version = self.schema_obj.pipeline_manifest.get("version", "0.0.0") + + # Build the header section + out = "" + + out += _print_wrapped(f"{pipeline_name} {pipeline_version}", "~", mode="both", indent=4) + out += _print_wrapped(INTRO.format(pipeline_name=pipeline_name), " ", mode="none", indent=4) + out += _print_wrapped("\n", " ", mode="none", indent=4, drop_whitespace=False) + out += _print_wrapped(USAGE, "-", mode="end", indent=4) + out += "\n" + + # Add all parameter groups + for definition in schema.get("definitions", {}).values(): + out += self.format_group(definition, show_hidden=show_hidden) + out += "\n" + + return out + + def write_params_file(self, output_fn="nf-params.yaml", show_hidden=False, force=False): + """Build a template file for the pipeline schema. + + Args: + output_fn (str, optional): Filename to write the template to. + show_hidden (bool, optional): + Include parameters marked as hidden in the output + force (bool, optional): Whether to overwrite existing output file. + + Returns: + bool: True if the template was written successfully, False otherwise + """ + + self.get_pipeline() + + try: + self.schema_obj.load_schema() + self.schema_obj.validate_schema() + except AssertionError as e: + log.error(f'Pipeline schema file is invalid ("{self.schema_obj.schema_filename}"): {e}') + log.info("Please fix this file, then try again.") + return False + + schema_out = self.generate_params_file(show_hidden=show_hidden) + + if os.path.exists(output_fn) and not force: + log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") + return False + with open(output_fn, "w") as fh: + fh.write(schema_out) + log.info(f"Parameter file written to '{output_fn}'") + + return True diff --git a/tests/test_params_file.py b/tests/test_params_file.py new file mode 100644 index 0000000000..824e8fe345 --- /dev/null +++ b/tests/test_params_file.py @@ -0,0 +1,79 @@ +import json +import os +import shutil +import tempfile +from pathlib import Path + +import nf_core.create +import nf_core.schema +from nf_core.params_file import ParamsFileBuilder + + +class TestParamsFileBuilder: + """Class for schema tests""" + + @classmethod + def setup_class(cls): + """Create a new PipelineSchema object""" + cls.schema_obj = nf_core.schema.PipelineSchema() + cls.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + # Create a test pipeline in temp directory + cls.tmp_dir = tempfile.mkdtemp() + cls.template_dir = os.path.join(cls.tmp_dir, "wf") + create_obj = nf_core.create.PipelineCreate( + "testpipeline", "", "", outdir=cls.template_dir, no_git=True, plain=True + ) + create_obj.init_pipeline() + + cls.template_schema = os.path.join(cls.template_dir, "nextflow_schema.json") + cls.params_template_builder = ParamsFileBuilder(cls.template_dir) + cls.invalid_template_schema = os.path.join(cls.template_dir, "nextflow_schema_invalid.json") + + # Remove the allOf section to make the schema invalid + with open(cls.template_schema, "r") as fh: + o = json.load(fh) + del o["allOf"] + + with open(cls.invalid_template_schema, "w") as fh: + json.dump(o, fh) + + @classmethod + def teardown_class(cls): + if os.path.exists(cls.tmp_dir): + shutil.rmtree(cls.tmp_dir) + + def test_build_template(self): + outfile = os.path.join(self.tmp_dir, "params-file.yml") + self.params_template_builder.write_params_file(outfile) + + assert os.path.exists(outfile) + + with open(outfile, "r") as fh: + out = fh.read() + + assert "nf-core/testpipeline" in out + + def test_build_template_invalid_schema(self, caplog): + """Build a schema from a template""" + outfile = os.path.join(self.tmp_dir, "params-file-invalid.yml") + builder = ParamsFileBuilder(self.invalid_template_schema) + res = builder.write_params_file(outfile) + + assert res is False + assert "Pipeline schema file is invalid" in caplog.text + + def test_build_template_file_exists(self, caplog): + """Build a schema from a template""" + + # Creates a new empty file + outfile = Path(self.tmp_dir) / "params-file.yml" + with open(outfile, "w") as fp: + pass + + res = self.params_template_builder.write_params_file(outfile) + + assert res is False + assert f"File '{outfile}' exists!" in caplog.text + + outfile.unlink()