diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 51bf2ae5f7..1741b934f2 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -17,7 +17,7 @@ runs: # Set up Nextflow - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: ${{ matrix.NXF_VER }} diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index 4835b0d0f0..471665e4b1 100644 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -79,7 +79,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]: section, section_header = _determine_change_type(pr_title) # Remove section indicator from the PR title. -pr_title = re.sub(rf"{section}[:\s]*", "", pr_title, flags=re.IGNORECASE) +pr_title = re.sub(rf"{section}:[\s]*", "", pr_title, flags=re.IGNORECASE) # Prepare the change log entry. pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))" @@ -91,7 +91,6 @@ def _determine_change_type(pr_title) -> tuple[str, str]: new_lines = [ f"- {pr_title} {pr_link}\n", ] - print(f"Adding new lines into section '{section}':\n" + "".join(new_lines)) # Finally, updating the changelog. @@ -144,10 +143,11 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st # Parse version from the line `## v2.12dev` or # `## [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]` ... - if not (m := re.match(r".*(v\d+\.\d+(dev)?).*", line)): + if not (m := re.match(r".*(v\d+\.\d+.\d*(dev)?).*", line)): print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr) sys.exit(1) version = m.group(1) + print(f"Found version: {version}") if not inside_version_dev: if not version.endswith("dev"): @@ -209,6 +209,7 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st # If the line already contains a link to the PR, don't add it again. line = _skip_existing_entry_for_this_pr(line, same_section=True) section_lines.append(line) + else: updated_lines.append(line) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e69229c051..7a72e18d22 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -7,6 +7,9 @@ on: - "docs/**" - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 188cccfd75..0d65c40b07 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -6,6 +6,9 @@ on: paths: - nf_core/pipeline-template/** pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: @@ -72,7 +75,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: latest-everything diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index cd36d4e62d..9800ac1910 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -7,6 +7,9 @@ on: - "docs/**" - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: @@ -59,7 +62,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: ${{ matrix.NXF_VER }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index eec676a746..2aed1752ff 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -12,7 +12,9 @@ on: paths-ignore: - "docs/**" - "CHANGELOG.md" + # ignore github workflows except for the current one - ".github/**" + - "!.github/workflows/pytest.yml" release: types: [published] workflow_dispatch: @@ -119,32 +121,10 @@ jobs: run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - - - name: Look if nf-test is already installed and write to env variable - id: check-nftest - run: | - if [ -f /usr/local/bin/nf-test ]; then - echo "nftest_installed=true" >> $GITHUB_ENV - else - echo "nftest_installed=false" >> $GITHUB_ENV - fi - - - name: Cache nf-test installation - if: env.nftest_installed != 'true' - id: cache-software - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest-${{ env.date }} + uses: nf-core/setup-nextflow@v2 - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' && env.nftest_installed != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + uses: nf-core/setup-nf-test@v1 - name: move coveragerc file up run: | @@ -197,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85 # v4 + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 8748f3d7b3..13fd71a883 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -15,22 +15,10 @@ jobs: cache-dependency-path: setup.py - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest-${{ env.date }} + uses: nf-core/setup-nextflow@v2 - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + uses: nf-core/setup-nf-test@v1 - name: Install nf-core/tools run: pip install git+https://github.com/nf-core/tools.git@dev diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index c998552ddd..19e895f452 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -67,7 +67,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "latest-everything" diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index f357bed03b..0fa3adaf20 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -23,11 +23,11 @@ jobs: - name: Update modules run: nf-core modules update --all - working-directory: nf-core/pipeline-template + working-directory: nf_core/pipeline-template - name: Update subworkflows run: nf-core subworkflows update --all - working-directory: nf-core/pipeline-template + working-directory: nf_core/pipeline-template # Commit the changes - name: Commit changes diff --git a/.gitpod.yml b/.gitpod.yml index 445cb35706..b2fbb73133 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03fbb7bedf..d0d5381b9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.3.2 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix @@ -9,6 +9,8 @@ repos: rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "2.7.3" @@ -17,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.8.0" + rev: "v1.9.0" hooks: - id: mypy additional_dependencies: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3189edaa..6bb9843197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # nf-core/tools: Changelog +## v2.13.2dev + +### Template + +- Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) +- Update templates to use nf-core/setup-nextflow v2 +- Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) +- Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) + +### Linting + +- Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) + +### Components + +### General + +- Update CI to use nf-core/setup-nextflow v2 +- Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) +- Fix path in component update script ([#2823](https://github.com/nf-core/tools/pull/2823)) +- Update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) +- Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) +- Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 ([#2836](https://github.com/nf-core/tools/pull/2836)) +- Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) +- Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) +- Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) +- Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) + ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template diff --git a/Dockerfile b/Dockerfile index 62431be140..60768499a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a +FROM python:3.11-slim@sha256:a2eb07f336e4f194358382611b4fea136c632b40baa6314cb27a366deeaf0144 LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt index fc5184b95f..abffe30740 100644 --- a/docs/api/requirements.txt +++ b/docs/api/requirements.txt @@ -1,4 +1,5 @@ Sphinx>=3.3.1 sphinxcontrib-napoleon +sphinx-markdown-builder sphinx_rtd_theme>=0.5.0 myst-parser diff --git a/nf_core/__init__.py b/nf_core/__init__.py index d96be73f3d..2d4fe45a0a 100644 --- a/nf_core/__init__.py +++ b/nf_core/__init__.py @@ -1,4 +1,4 @@ -""" Main nf_core module file. +"""Main nf_core module file. Shouldn't do much, as everything is under subcommands. """ diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7d2d083fa9..d6f6077be9 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -""" nf-core: Helper tools for use with nf-core Nextflow pipelines. """ +"""nf-core: Helper tools for use with nf-core Nextflow pipelines.""" + import logging import os import sys diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index f1a9e7c401..9b81f54f06 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -2,7 +2,6 @@ The ComponentsTest class handles the generation and testing of nf-test snapshots. """ - import logging import os import re @@ -91,9 +90,9 @@ def run(self) -> None: """Run build steps""" self.check_inputs() os.environ["NFT_DIFF"] = "pdiff" # set nf-test differ to pdiff to get a better diff output - os.environ[ - "NFT_DIFF_ARGS" - ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + os.environ["NFT_DIFF_ARGS"] = ( + "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + ) with nf_core.utils.set_wd(Path(self.dir)): self.check_snapshot_stability() if len(self.errors) > 0: diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c4b477a0ab..d2169e3a72 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -2,7 +2,6 @@ The ComponentCreate class handles generating of module and subworkflow templates """ - import glob import json import logging diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 3c2fb9dde3..c99934bca3 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -3,7 +3,6 @@ in nf-core pipelines """ - import logging import operator import os diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 2f73afe9d3..d9731ba7c9 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -1,6 +1,7 @@ """ The NFCoreComponent class holds information and utility functions for a single module or subworkflow """ + import logging import re from pathlib import Path diff --git a/nf_core/create.py b/nf_core/create.py index c094d33a22..b420b1c86d 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -1,6 +1,7 @@ """Creates a nf-core pipeline matching the current organization's specification based on a template. """ + import configparser import logging import os diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index f04c7dcd04..3136d02e97 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,14 +1,14 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:c15ee2f4de8902421ccbfbd77329a9e8e4abf18477e1b5ae8a1a572790254647 +FROM gitpod/workspace-base@sha256:1e133e5691add6c19443672594b9f3d7d9c3372ead4c86a4490c2701dbfa32e3 USER root # Install util tools. # software-properties-common is needed to add ppa support for Apptainer installation RUN apt-get update --quiet && \ - apt-get install --quiet --yes \ + apt-get install --quiet --yes --no-install-recommends \ apt-transport-https \ apt-utils \ sudo \ @@ -18,18 +18,17 @@ RUN apt-get update --quiet && \ curl \ tree \ graphviz \ - software-properties-common - -# Install Apptainer (Singularity) -RUN add-apt-repository -y ppa:apptainer/ppa && \ + software-properties-common && \ + add-apt-repository -y ppa:apptainer/ppa && \ apt-get update --quiet && \ - apt install -y apptainer - -# Install Conda -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + apt-get install --quiet --yes apptainer && \ + wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-latest-Linux-x86_64.sh + rm Miniconda3-latest-Linux-x86_64.sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# Set PATH for Conda ENV PATH="/opt/conda/bin:$PATH" # Add the nf-core source files to the image @@ -47,23 +46,20 @@ RUN conda config --add channels defaults && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base \ - mamba \ nextflow \ - nf-core \ nf-test \ prettier \ pre-commit \ ruff \ + mypy \ openjdk \ pytest-workflow && \ conda clean --all --force-pkgs-dirs --yes -# Update Nextflow -RUN nextflow self-update - -# Install nf-core -RUN python -m pip install . --no-cache-dir +# Update Nextflow and Install nf-core +RUN nextflow self-update && \ + python -m pip install . --no-cache-dir # Setup pdiff for nf-test diffs -RUN export NFT_DIFF="pdiff" && \ - export NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" +ENV NFT_DIFF="pdiff" +ENV NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" diff --git a/nf_core/launch.py b/nf_core/launch.py index 25bb4c150c..bc0cd58aec 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -1,5 +1,4 @@ -""" Launch a pipeline, interactively collecting params """ - +"""Launch a pipeline, interactively collecting params""" import copy import json diff --git a/nf_core/licences.py b/nf_core/licences.py index a8a35334dd..be737280f8 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -1,6 +1,5 @@ """Lists software licences for a given workflow.""" - import json import logging import os diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index d3e29d2363..47b7d78f5e 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -245,10 +245,9 @@ def nextflow_config(self): raise AssertionError() except (AssertionError, IndexError): failed.append( - "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( - manifest_homepage - ) + f"Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {manifest_homepage}" ) + else: passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") @@ -375,7 +374,6 @@ def nextflow_config(self): schema.load_schema() schema.get_schema_defaults() # Get default values from schema schema.get_schema_types() # Get types from schema - self.nf_config.keys() # Params in nextflow.config for param_name in schema.schema_defaults.keys(): param = "params." + param_name if param in ignore_defaults: diff --git a/nf_core/list.py b/nf_core/list.py index 67d1a76878..658f4dc6d2 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -1,6 +1,5 @@ """Lists available nf-core pipelines and versions.""" - import json import logging import os diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index b9003be974..9b54174d5a 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -3,7 +3,6 @@ or for a single module """ - import logging import os import re diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 866e6312aa..b2816dab6a 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -6,7 +6,6 @@ nf-core modules lint """ - import logging import os diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index ee8cabebe1..eb76f4b88b 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -1,6 +1,7 @@ """ Check whether the content of a module has changed compared to the original repository """ + import shutil import tempfile from pathlib import Path diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 520f8cf0a2..b1a611d70a 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -1,6 +1,7 @@ """ Lint the tests of a module in nf-core/modules """ + import json import logging from pathlib import Path diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 267fe7086a..78798b065e 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -1,5 +1,4 @@ -""" Create a YAML parameter file """ - +"""Create a YAML parameter file""" import json import logging diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index d04335275a..ca0687733e 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). +- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --profile +docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 84c727f60d..3880b2c4dc 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index b53a427e06..44d06c0cd5 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -14,6 +14,8 @@ on: pull_request: types: - opened + - edited + - synchronize branches: - master pull_request_target: @@ -28,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 @@ -68,8 +70,17 @@ jobs: - name: Inspect download run: tree ./${{ env.REPOTITLE_LOWERCASE }} - - name: Run the downloaded pipeline + - name: Run the downloaded pipeline (stub) + id: stub_run_pipeline + continue-on-error: true env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results{% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results + - name: Run the downloaded pipeline (stub run not supported) + id: run_pipeline + if: ${{ job.steps.stub_run_pipeline.status == failure() }} + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 59b85f95fc..612467ff6e 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index af57081f60..4dc0f1dcd7 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -3,6 +3,9 @@ repos: rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "2.7.3" hooks: diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 17e75f18a4..2f9b349916 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -18,7 +18,6 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false - fasta = null {%- endif -%} // MultiQC options diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 5bc51caab8..e02a113755 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -143,6 +143,10 @@ workflow PIPELINE_COMPLETION { imNotification(summary_params, hook_url) } } + + workflow.onError { + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + } } /* diff --git a/nf_core/schema.py b/nf_core/schema.py index df04dc5a1e..373f8bbaa1 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -1,5 +1,4 @@ -""" Code to deal with pipeline JSON Schema """ - +"""Code to deal with pipeline JSON Schema""" import copy import json @@ -84,16 +83,13 @@ def load_lint_schema(self): self.get_schema_defaults() self.validate_default_params() if len(self.invalid_nextflow_config_default_parameters) > 0: + params = "\n --".join( + [f"{param}: {msg}" for param, msg in self.invalid_nextflow_config_default_parameters.items()] + ) log.info( - "[red][✗] Invalid default parameters found:\n --{}\n\nNOTE: Use null in config for no default.".format( - "\n --".join( - [ - f"{param}: {msg}" - for param, msg in self.invalid_nextflow_config_default_parameters.items() - ] - ) - ) + f"[red][✗] Invalid default parameters found:\n {params} \n\nNOTE: Use null in config for no default." ) + else: log.info(f"[green][✓] Pipeline schema looks valid[/] [dim](found {num_params} params)") except json.decoder.JSONDecodeError as e: @@ -282,9 +278,9 @@ def validate_default_params(self): if param in self.pipeline_params: self.validate_config_default_parameter(param, group_properties[param], self.pipeline_params[param]) else: - self.invalid_nextflow_config_default_parameters[ - param - ] = "Not in pipeline parameters. Check `nextflow.config`." + self.invalid_nextflow_config_default_parameters[param] = ( + "Not in pipeline parameters. Check `nextflow.config`." + ) # Go over ungrouped params if any exist ungrouped_properties = self.schema.get("properties") @@ -297,9 +293,9 @@ def validate_default_params(self): param, ungrouped_properties[param], self.pipeline_params[param] ) else: - self.invalid_nextflow_config_default_parameters[ - param - ] = "Not in pipeline parameters. Check `nextflow.config`." + self.invalid_nextflow_config_default_parameters[param] = ( + "Not in pipeline parameters. Check `nextflow.config`." + ) def validate_config_default_parameter(self, param, schema_param, config_default): """ @@ -314,9 +310,9 @@ def validate_config_default_parameter(self, param, schema_param, config_default) ): # Check that we are not deferring the execution of this parameter in the schema default with squiggly brakcets if schema_param["type"] != "string" or "{" not in schema_param["default"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Schema default (`{schema_param['default']}`) does not match the config default (`{config_default}`)" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Schema default (`{schema_param['default']}`) does not match the config default (`{config_default}`)" + ) return # if default is null, we're good @@ -326,28 +322,28 @@ def validate_config_default_parameter(self, param, schema_param, config_default) # Check variable types in nextflow.config if schema_param["type"] == "string": if str(config_default) in ["false", "true", "''"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"String should not be set to `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"String should not be set to `{config_default}`" + ) if schema_param["type"] == "boolean": if str(config_default) not in ["false", "true"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Booleans should only be true or false, not `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Booleans should only be true or false, not `{config_default}`" + ) if schema_param["type"] == "integer": try: int(config_default) except ValueError: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Does not look like an integer: `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Does not look like an integer: `{config_default}`" + ) if schema_param["type"] == "number": try: float(config_default) except ValueError: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Does not look like a number (float): `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Does not look like a number (float): `{config_default}`" + ) def validate_schema(self, schema=None): """ @@ -647,17 +643,13 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): # Extra help for people running offline if "Could not connect" in e.args[0]: log.info( - "If you're working offline, now copy your schema ({}) and paste at https://nf-co.re/pipeline_schema_builder".format( - self.schema_filename - ) + f"If you're working offline, now copy your schema ({self.schema_filename}) and paste at https://nf-co.re/pipeline_schema_builder" ) log.info("When you're finished, you can paste the edited schema back into the same file") if self.web_schema_build_web_url: log.info( "To save your work, open {}\n" - "Click the blue 'Finished' button, copy the schema and paste into this file: {}".format( - self.web_schema_build_web_url, self.schema_filename - ) + f"Click the blue 'Finished' button, copy the schema and paste into this file: { self.web_schema_build_web_url, self.schema_filename}" ) return False diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 3a87190422..96d2e0ab92 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -6,7 +6,6 @@ nf-core subworkflows lint """ - import logging import os diff --git a/nf_core/subworkflows/lint/subworkflow_changes.py b/nf_core/subworkflows/lint/subworkflow_changes.py index b7fa13d931..a9c9616a21 100644 --- a/nf_core/subworkflows/lint/subworkflow_changes.py +++ b/nf_core/subworkflows/lint/subworkflow_changes.py @@ -1,6 +1,7 @@ """ Check whether the content of a subworkflow has changed compared to the original repository """ + from pathlib import Path import nf_core.modules.modules_repo diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index f7284320ea..796a56d018 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -1,6 +1,7 @@ """ Lint the tests of a subworkflow in nf-core/modules """ + import json import logging from pathlib import Path diff --git a/nf_core/sync.py b/nf_core/sync.py index 995baeacd2..93a79c3d50 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -1,5 +1,4 @@ -"""Synchronise a pipeline TEMPLATE branch with the template. -""" +"""Synchronise a pipeline TEMPLATE branch with the template.""" import json import logging diff --git a/nf_core/utils.py b/nf_core/utils.py index e1778b55b3..9261388a39 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,6 +1,7 @@ """ Common utility functions for the nf-core python package. """ + import concurrent.futures import datetime import errno @@ -288,7 +289,7 @@ def fetch_wf_config(wf_path, cache_config=True): main_nf = os.path.join(wf_path, "main.nf") with open(main_nf) as fh: for line in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=", line) + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line) if match: config[match.group(1)] = "null" except FileNotFoundError as e: diff --git a/pytest.ini b/pytest.ini index 652bdf8e53..cf37159478 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,4 @@ [pytest] -filterwarnings = - ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 testpaths = tests python_files = test_*.py diff --git a/requirements.txt b/requirements.txt index e4319d2352..6b5b3ab57d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ pillow pdiff pre-commit prompt_toolkit>=3.0.3 -pytest-workflow>=1.6.0 +pytest-workflow>=2.0.0 pytest>=7.0.0 pyyaml questionary>=1.8.0 diff --git a/setup.py b/setup.py index c7f3fa41c6..fc7b69ac1e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.13.1" +version = "2.13.2dev" with open("README.md") as f: readme = f.read() diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index c5067d7210..50024a8ebb 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,4 +1,5 @@ """Test generate a snapshot""" + import json from pathlib import Path from unittest.mock import MagicMock diff --git a/tests/components/snapshot_test.py b/tests/components/snapshot_test.py index d774618476..b3fc259770 100644 --- a/tests/components/snapshot_test.py +++ b/tests/components/snapshot_test.py @@ -1,4 +1,5 @@ """Test the 'modules test' or 'subworkflows test' command which runs nf-test test.""" + import shutil from pathlib import Path diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index fa85568f14..06af8c4fb8 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -83,7 +83,6 @@ def test_default_values_fail(self): with open(nf_schema_file) as f: content = f.read() fail_content = re.sub(r'"default": "128.GB"', '"default": "18.GB"', content) - print(fail_content) with open(nf_schema_file, "w") as f: f.write(fail_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -100,6 +99,36 @@ def test_default_values_fail(self): ) +def test_catch_params_assignment_in_main_nf(self): + """Test linting fails if main.nf contains an assignment to a parameter from nextflow_schema.json.""" + new_pipeline = self._make_pipeline_copy() + # Add parameter assignment in main.nf + main_nf_file = Path(new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.max_time = 42") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 1 + assert ( + result["failed"][0] + == "Config default value incorrect: `params.max_time` is set as `240.h` in `nextflow_schema.json` but is `null` in `nextflow.config`." + ) + + +def test_allow_params_reference_in_main_nf(self): + """Test linting allows for references like `params.aligner == 'bwa'` in main.nf. The test will detect if the bug mentioned in GitHub-issue #2833 reemerges.""" + new_pipeline = self._make_pipeline_copy() + # Add parameter reference in main.nf + main_nf_file = Path(new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.max_time == 42") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + + def test_default_values_ignored(self): """Test ignoring linting of default values.""" new_pipeline = self._make_pipeline_copy() diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 658a2339d4..c697d34287 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -1,5 +1,5 @@ -"""Some tests covering the bump_version code. -""" +"""Some tests covering the bump_version code.""" + import os import yaml diff --git a/tests/test_cli.py b/tests/test_cli.py index 1261e3a9e9..d1dff0fe7b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,4 @@ -""" Tests covering the command-line code. +"""Tests covering the command-line code. Most tests check the cli arguments are passed along and that some action is taken. diff --git a/tests/test_components.py b/tests/test_components.py index b7f67eb51d..eaf999c3c3 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1,5 +1,4 @@ -""" Tests covering the modules commands -""" +"""Tests covering the modules commands""" import os import shutil diff --git a/tests/test_create.py b/tests/test_create.py index 1cc073cb54..e2672499cd 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,5 +1,5 @@ -"""Some tests covering the pipeline creation sub command. -""" +"""Some tests covering the pipeline creation sub command.""" + import os import unittest from pathlib import Path diff --git a/tests/test_download.py b/tests/test_download.py index d823040247..621379842b 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,5 +1,4 @@ -"""Tests for the download subcommand of nf-core tools -""" +"""Tests for the download subcommand of nf-core tools""" import os import re diff --git a/tests/test_launch.py b/tests/test_launch.py index dc8d6b147c..79dbe3fb97 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -1,5 +1,4 @@ -""" Tests covering the pipeline launch code. -""" +"""Tests covering the pipeline launch code.""" import json import os diff --git a/tests/test_licenses.py b/tests/test_licenses.py index 4fb58a107c..8023c9e891 100644 --- a/tests/test_licenses.py +++ b/tests/test_licenses.py @@ -1,5 +1,4 @@ -"""Some tests covering the pipeline creation sub command. -""" +"""Some tests covering the pipeline creation sub command.""" # import json # import os # import tempfile diff --git a/tests/test_lint.py b/tests/test_lint.py index 9839265892..d10cef37e4 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1,5 +1,5 @@ -"""Some tests covering the linting code. -""" +"""Some tests covering the linting code.""" + import fnmatch import json import os @@ -221,6 +221,8 @@ def test_sphinx_md_files(self): test_multiqc_incorrect_export_plots, ) from .lint.nextflow_config import ( # type: ignore[misc] + test_allow_params_reference_in_main_nf, + test_catch_params_assignment_in_main_nf, test_default_values_fail, test_default_values_float, test_default_values_float_fail, diff --git a/tests/test_list.py b/tests/test_list.py index c1f51e03e0..c78276b41d 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -1,5 +1,4 @@ -""" Tests covering the workflow listing code. -""" +"""Tests covering the workflow listing code.""" import json import os diff --git a/tests/test_modules.py b/tests/test_modules.py index 539d3dcc57..944a09f670 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,5 +1,4 @@ -""" Tests covering the modules commands -""" +"""Tests covering the modules commands""" import os import shutil diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py index 5440c1c477..23cc0dd14a 100644 --- a/tests/test_refgenie.py +++ b/tests/test_refgenie.py @@ -1,5 +1,4 @@ -""" Tests covering the refgenie integration code -""" +"""Tests covering the refgenie integration code""" import os import shlex diff --git a/tests/test_schema.py b/tests/test_schema.py index 89fcc98b66..29f4921985 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,5 +1,4 @@ -""" Tests covering the pipeline schema code. -""" +"""Tests covering the pipeline schema code.""" import json import os diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index cd0bd21146..6163faa7a9 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,5 +1,4 @@ -""" Tests covering the subworkflows commands -""" +"""Tests covering the subworkflows commands""" import os import shutil diff --git a/tests/test_sync.py b/tests/test_sync.py index 51a27653ab..6f0e502e8b 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,5 +1,4 @@ -""" Tests covering the sync command -""" +"""Tests covering the sync command""" import json import os diff --git a/tests/test_utils.py b/tests/test_utils.py index 4b5ab19fce..145060450f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ -""" Tests covering for utility functions. -""" +"""Tests covering for utility functions.""" import os import shutil