Merge branch 'dev' into fix-api-doc-headers

.github/actions/create-lint-wf/action.yml

@@ -15,7 +15,6 @@ runs:
         cd create-lint-wf
         export NXF_WORK=$(pwd)
 
-    # Set up Nextflow
     - name: Install Nextflow
       uses: nf-core/setup-nextflow@v2
       with:
@@ -65,6 +64,12 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
       working-directory: create-lint-wf
 
+    # Add empty ro-crate file
+    - name: add empty ro-crate file
+      shell: bash
+      run: touch nf-core-testpipeline/ro-crate-metadata.json
+      working-directory: create-lint-wf
+
     # Run nf-core pipelines linting
     - name: nf-core pipelines lint
       shell: bash

.github/workflows/create-lint-wf.yml

@@ -27,14 +27,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-env:
-  NXF_ANSI_LOG: false
-
 jobs:
   MakeTestWorkflow:
     runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
     env:
       NXF_ANSI_LOG: false
+
     strategy:
       matrix:
         NXF_VER:

.github/workflows/create-test-lint-wf-template.yml

@@ -51,6 +51,7 @@ jobs:
     needs: prepare-matrix
     env:
       NXF_ANSI_LOG: false
+
     strategy:
       matrix:
         TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }}
@@ -147,11 +148,22 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
         working-directory: create-test-lint-wf
 
+      # Add empty ro-crate file
+      - name: add empty ro-crate file
+        run: touch my-prefix-testpipeline/ro-crate-metadata.json
+        working-directory: create-test-lint-wf
+
       # Run nf-core linting
       - name: nf-core pipelines lint
         run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned
         working-directory: create-test-lint-wf
 
+      # Run code style linting
+      - name: run pre-commit
+        shell: bash
+        run: pre-commit run --all-files
+        working-directory: create-test-lint-wf
+
       # Run bump-version
       - name: nf-core pipelines bump-version
         run: nf-core --log-file log.txt pipelines bump-version --dir my-prefix-testpipeline/ 1.1

.github/workflows/create-test-wf.yml

@@ -27,15 +27,13 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-env:
-  NXF_ANSI_LOG: false
-
 jobs:
   RunTestWorkflow:
     # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default
     runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
     env:
       NXF_ANSI_LOG: false
+
     strategy:
       matrix:
         NXF_VER:

.github/workflows/pytest.yml

@@ -132,7 +132,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$?
+          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --cov-config=.coveragerc --durations=0 && exit_code=0|| exit_code=$?
           # don't fail if no tests were collected, e.g. for test_licence.py
           if [ "${exit_code}" -eq 5 ]; then
             echo "No tests were collected"
@@ -198,7 +198,7 @@ jobs:
           coverage report
           coverage xml
 
-      - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
+      - uses: codecov/codecov-action@5c47607acb93fed5485fdbf7232e8a31425f672a # v5
         with:
           files: coverage.xml
         env:

.gitignore

@@ -113,7 +113,6 @@ ENV/
 # Jetbrains IDEs
 .idea
 pip-wheel-metadata
-.vscode
 .*.sw?
 
 # Textual

.gitpod.yml

@@ -9,11 +9,4 @@ tasks:
 
 vscode:
   extensions:
-    - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code
-    - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files
-    - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar
-    - mechatroner.rainbow-csv # Highlight columns in csv files in different colors
-    - nextflow.nextflow # Nextflow syntax highlighting
-    - oderwat.indent-rainbow # Highlight indentation level
-    - streetsidesoftware.code-spell-checker # Spelling checker for source code
-    - charliermarsh.ruff # Code linter Ruff
+    - nf-core.nf-core-extensionpack # https://github.com/nf-core/vscode-extensionpack

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.8.2
     hooks:
       - id: ruff # linter
         args: [--fix, --exit-non-zero-on-fix] # sort imports and fix

CHANGELOG.md

@@ -10,19 +10,33 @@
 - fix workflow_dispatch trigger and parse more review comments in awsfulltest ([#3235](https://github.com/nf-core/tools/pull/3235))
 - Add resource limits to Gitpod profile([#3255](https://github.com/nf-core/tools/pull/3255))
 - Fix a typo ([#3268](https://github.com/nf-core/tools/pull/3268))
+- Remove `def` from `nextflow.config` and add `trace_report_suffix` param ([#3296](https://github.com/nf-core/tools/pull/3296))
+- Move `includeConfig 'conf/modules.config'` next to `includeConfig 'conf/base.config'` to not overwrite tests profiles configurations ([#3301](https://github.com/nf-core/tools/pull/3301))
+- Use `params.monochrome_logs` in the template and update nf-core components ([#3310](https://github.com/nf-core/tools/pull/3310))
+- Fix some typos and improve writing in `usage.md` and `CONTRIBUTING.md` ([#3302](https://github.com/nf-core/tools/pull/3302))
+- Add `manifest.contributors` to `nextflow.config` ([#3311](https://github.com/nf-core/tools/pull/3311))
 
 ### Download
 
 - First steps towards fixing [#3179](https://github.com/nf-core/tools/issues/3179): Modify `prioritize_direct_download()` to retain Seqera Singularity https:// Container URIs and hardcode Seqera Containers into `gather_registries()` ([#3244](https://github.com/nf-core/tools/pull/3244)).
+- Further steps towards fixing [#3179](https://github.com/nf-core/tools/issues/3179): Enable limited support for `oras://` container paths (_only absolute URIs, no flexible registries like with Docker_) and prevent unnecessary image downloads for Seqera Container modules with `reconcile_seqera_container_uris()` ([#3293](https://github.com/nf-core/tools/pull/3293)).
+- Update dawidd6/action-download-artifact action to v7 ([#3306](https://github.com/nf-core/tools/pull/3306))
 
 ### Linting
 
+- allow mixed `str` and `dict` entries in lint config ([#3228](https://github.com/nf-core/tools/pull/3228))
+- fix meta_yml linting test failing due to module.process_name always being "" ([#3317](https://github.com/nf-core/tools/pull/3317))
+- fix module section regex matching wrong things ([#3321](https://github.com/nf-core/tools/pull/3321))
+
 ### Modules
 
 - add a panel around diff previews when updating ([#3246](https://github.com/nf-core/tools/pull/3246))
 
 ### Subworkflows
 
+- Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861))
+- Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298))
+
 ### General
 
 - Include .nf-core.yml in `nf-core pipelines bump-version` ([#3220](https://github.com/nf-core/tools/pull/3220))
@@ -36,6 +50,20 @@
 - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250))
 - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323))
+- handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276))
+- Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274))
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.3 ([#3275](https://github.com/nf-core/tools/pull/3275))
+- Add hint to solve git errors with a synced repo ([#3279](https://github.com/nf-core/tools/pull/3279))
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.4 ([#3282](https://github.com/nf-core/tools/pull/3282))
+- Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283))
+- Update python:3.12-slim Docker digest to 2a6386a ([#3284](https://github.com/nf-core/tools/pull/3284))
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.0 ([#3299](https://github.com/nf-core/tools/pull/3299))
+- Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309))
+- Run pre-commit when testing linting the template pipeline ([#3280](https://github.com/nf-core/tools/pull/3280))
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325))
+- Make CLI prompt less nf-core specific ([#3326](https://github.com/nf-core/tools/pull/3326))
+- Update gitpod vscode extensions to use nf-core extension pack ([#3327](https://github.com/nf-core/tools/pull/3327))
+
 
 ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11]
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim@sha256:032c52613401895aa3d418a4c563d2d05f993bc3ecc065c8f4e2280978acd249
+FROM python:3.12-slim@sha256:2b0079146a74e23bf4ae8f6a28e1b484c6292f6fb904cbb51825b4a19812fcd8
 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \
     description="Docker image containing requirements for nf-core/tools"
 

nf_core/__main__.py

@@ -36,6 +36,7 @@
     pipelines_launch,
     pipelines_lint,
     pipelines_list,
+    pipelines_rocrate,
     pipelines_schema_build,
     pipelines_schema_docs,
     pipelines_schema_lint,
@@ -86,7 +87,7 @@
         },
         {
             "name": "For developers",
-            "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"],
+            "commands": ["create", "lint", "bump-version", "sync", "schema", "rocrate", "create-logo"],
         },
     ],
     "nf-core modules": [
@@ -287,7 +288,7 @@ def command_pipelines_create(ctx, name, description, author, version, force, out
 @click.option(
     "--release",
     is_flag=True,
-    default=os.path.basename(os.path.dirname(os.environ.get("GITHUB_REF", "").strip(" '\""))) == "master"
+    default=Path(os.environ.get("GITHUB_REF", "").strip(" '\"")).parent.name in ["master", "main"]
     and os.environ.get("GITHUB_REPOSITORY", "").startswith("nf-core/")
     and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools",
     help="Execute additional checks for release-ready workflows.",
@@ -570,6 +571,44 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived):
     pipelines_list(ctx, keywords, sort, json, show_archived)
 
 
+# nf-core pipelines rocrate
+@pipelines.command("rocrate")
+@click.argument(
+    "pipeline_dir",
+    type=click.Path(exists=True),
+    default=Path.cwd(),
+    required=True,
+    metavar="<pipeline directory>",
+)
+@click.option(
+    "-j",
+    "--json_path",
+    default=Path.cwd(),
+    type=str,
+    help="Path to save RO Crate metadata json file to",
+)
+@click.option("-z", "--zip_path", type=str, help="Path to save RO Crate zip file to")
+@click.option(
+    "-pv",
+    "--pipeline_version",
+    type=str,
+    help="Version of pipeline to use for RO Crate",
+    default="",
+)
+@click.pass_context
+def rocrate(
+    ctx,
+    pipeline_dir: str,
+    json_path: str,
+    zip_path: str,
+    pipeline_version: str,
+):
+    """
+    Make an Research Object Crate
+    """
+    pipelines_rocrate(ctx, pipeline_dir, json_path, zip_path, pipeline_version)
+
+
 # nf-core pipelines sync
 @pipelines.command("sync")
 @click.pass_context
@@ -1024,7 +1063,7 @@ def command_modules_update(
     default=".",
     help=r"Pipeline directory. [dim]\[default: current working directory][/]",
 )
-@click.option("-r", "--remove", is_flag=True, default=False)
+@click.option("-r", "--remove", is_flag=True, default=False, help="Remove an existent patch file and regenerate it.")
 def command_modules_patch(ctx, tool, directory, remove):
     """
     Create a patch file for minor changes in a module
@@ -1567,6 +1606,43 @@ def command_subworkflows_install(ctx, subworkflow, directory, prompt, force, sha
     subworkflows_install(ctx, subworkflow, directory, prompt, force, sha)
 
 
+# nf-core subworkflows patch
+@subworkflows.command("patch")
+@click.pass_context
+@click.argument("tool", type=str, required=False, metavar="<tool> or <tool/subtool>")
+@click.option(
+    "-d",
+    "--dir",
+    type=click.Path(exists=True),
+    default=".",
+    help=r"Pipeline directory. [dim]\[default: current working directory][/]",
+)
+@click.option("-r", "--remove", is_flag=True, default=False, help="Remove an existent patch file and regenerate it.")
+def subworkflows_patch(ctx, tool, dir, remove):
+    """
+    Create a patch file for minor changes in a subworkflow
+
+    Checks if a subworkflow has been modified locally and creates a patch file
+    describing how the module has changed from the remote version
+    """
+    from nf_core.subworkflows import SubworkflowPatch
+
+    try:
+        subworkflow_patch = SubworkflowPatch(
+            dir,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+        )
+        if remove:
+            subworkflow_patch.remove(tool)
+        else:
+            subworkflow_patch.patch(tool)
+    except (UserWarning, LookupError) as e:
+        log.error(e)
+        sys.exit(1)
+
+
 # nf-core subworkflows remove
 @subworkflows.command("remove")
 @click.pass_context
@@ -1721,7 +1797,7 @@ def command_schema_validate(pipeline, params):
 @click.option(
     "--url",
     type=str,
-    default="https://nf-co.re/pipeline_schema_builder",
+    default="https://oldsite.nf-co.re/pipeline_schema_builder",
     help="Customise the builder URL (for development work)",
 )
 def command_schema_build(directory, no_prompts, web_only, url):
@@ -2164,7 +2240,7 @@ def command_download(
 @click.option(
     "--release",
     is_flag=True,
-    default=os.path.basename(os.path.dirname(os.environ.get("GITHUB_REF", "").strip(" '\""))) == "master"
+    default=Path(os.environ.get("GITHUB_REF", "").strip(" '\"")).parent.name in ["master", "main"]
     and os.environ.get("GITHUB_REPOSITORY", "").startswith("nf-core/")
     and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools",
     help="Execute additional checks for release-ready workflows.",

nf_core/commands_pipelines.py

@@ -2,6 +2,7 @@
 import os
 import sys
 from pathlib import Path
+from typing import Optional, Union
 
 import rich
 
@@ -277,6 +278,33 @@ def pipelines_list(ctx, keywords, sort, json, show_archived):
     stdout.print(list_workflows(keywords, sort, json, show_archived))
 
 
+# nf-core pipelines rocrate
+def pipelines_rocrate(
+    ctx,
+    pipeline_dir: Union[str, Path],
+    json_path: Optional[Union[str, Path]],
+    zip_path: Optional[Union[str, Path]],
+    pipeline_version: str,
+) -> None:
+    from nf_core.pipelines.rocrate import ROCrate
+
+    if json_path is None and zip_path is None:
+        log.error("Either `--json_path` or `--zip_path` must be specified.")
+        sys.exit(1)
+    else:
+        pipeline_dir = Path(pipeline_dir)
+        if json_path is not None:
+            json_path = Path(json_path)
+        if zip_path is not None:
+            zip_path = Path(zip_path)
+        try:
+            rocrate_obj = ROCrate(pipeline_dir, pipeline_version)
+            rocrate_obj.create_rocrate(json_path=json_path, zip_path=zip_path)
+        except (UserWarning, LookupError, FileNotFoundError) as e:
+            log.error(e)
+            sys.exit(1)
+
+
 # nf-core pipelines sync
 def pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr):
     """