diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d5f377a486..a3941f3ce1 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -17,7 +17,7 @@ env: jobs: MakeTestWorkflow: - runs-on: ubuntu-latest + runs-on: self-hosted env: NXF_ANSI_LOG: false strategy: @@ -27,7 +27,7 @@ jobs: - "latest-everything" steps: # Get the repo code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository # Set up nf-core/tools @@ -48,7 +48,9 @@ jobs: version: ${{ matrix.NXF_VER }} # Install the Prettier linting tools - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 5ce9b41118..4be3098629 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -33,7 +33,7 @@ jobs: - "template_skip_nf_core_configs.yml" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python 3.12 @@ -52,7 +52,9 @@ jobs: version: latest-everything # Install the Prettier linting tools - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d96518d7c6..4f66adae6b 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -26,7 +26,7 @@ jobs: - "23.04.0" - "latest-everything" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 9b46f0731b..b847df9218 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index f56cb927fc..d846151205 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -24,7 +24,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index a951230a6a..168fac653c 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -14,11 +14,13 @@ concurrency: jobs: EditorConfig: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -28,11 +30,13 @@ jobs: run: editorconfig-checker -exclude README.md $(git ls-files | grep -v 'test\|.py\|md\|json\|yml\|yaml\|html\|css\|Makefile') Prettier: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier @@ -41,9 +45,9 @@ jobs: run: prettier --check ${GITHUB_WORKSPACE} PythonBlack: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check code lints with Black uses: psf/black@stable @@ -71,10 +75,10 @@ jobs: allow-repeats: false isort: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - name: Check out source-code repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v4 @@ -87,10 +91,10 @@ jobs: requirementsFiles: "requirements.txt requirements-dev.txt" static-type-check: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 with: python-version: 3.12 cache: "pip" diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index dea28cdd35..1230bfc9d3 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -13,7 +13,7 @@ concurrency: jobs: push_dockerhub: name: Push new Docker image to Docker Hub (dev) - runs-on: ubuntu-latest + runs-on: self-hosted # Only run for the nf-core repo, for releases and merged PRs if: ${{ github.repository == 'nf-core/tools' }} env: @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 857b241022..49ce17dd84 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 03d5bc65e5..9f8172a3be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -25,7 +25,7 @@ env: jobs: setup: - runs-on: ${{ matrix.runner }} + runs-on: ["ubuntu-latest"] strategy: matrix: python-version: ["3.8", "3.12"] @@ -55,6 +55,7 @@ jobs: run-tests: ${{ env.run-tests }} test: + name: Test with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} runs-on: ${{ needs.setup.outputs.runner }} @@ -77,9 +78,9 @@ jobs: if: ${{ needs.setup.outputs.runner == 'ubuntu-20.04' && needs.setup.outputs.python-version == '3.8' }} run: | sudo apt update - sudo apt remove git git-man + sudo apt remove -y git git-man sudo add-apt-repository --remove ppa:git-core/ppa - sudo apt install git + sudo apt install -y git - name: Get current date id: date run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 54aaf240df..8368255390 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 5e7719973f..b8fb287944 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -36,10 +36,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out nf-core/tools - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 3a4725a944..f6106bd8b5 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -20,11 +20,11 @@ concurrency: jobs: api-docs: name: Build & push Sphinx API docs - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Check out source-code repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v4 diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 8a6eda1318..412784c233 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -19,7 +19,7 @@ jobs: - ${{ github.event.release.tag_name }} steps: - name: Check out source-code repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7aeeb5bc9..578d17f769 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,13 @@ repos: rev: "v2.7.1" hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.7.1" # Use the sha / tag you want to point at + hooks: + - id: mypy + additional_dependencies: + - types-PyYAML + - types-requests + - types-jsonschema + - types-Markdown + - types-setuptools diff --git a/CHANGELOG.md b/CHANGELOG.md index a68fcf810c..a467c75023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490)) - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514)) - Run tests with Python 3.12 ([#2522](https://github.com/nf-core/tools/pull/2522)). +- Add mypy to pre-commit config for the tools repo ([#2545](https://github.com/nf-core/tools/pull/2545)) # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25] diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index 4d8ae661d5..27eaf9bcb3 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -14,6 +14,7 @@ # import os import sys +from typing import Dict sys.path.insert(0, os.path.abspath("../../../nf_core")) import nf_core @@ -58,7 +59,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language: str = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -113,7 +114,7 @@ # -- Options for LaTeX output ------------------------------------------------ -latex_elements = { +latex_elements: Dict[str, str] = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', diff --git a/mypy.ini b/mypy.ini index f869fa7e47..c48aa5884b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] warn_unused_configs = True +ignore_missing_imports = true diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 6b8befb062..dfdd60f428 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -151,7 +151,7 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): # nf-core list -@nf_core_cli.command() +@nf_core_cli.command("list") @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option( "-s", @@ -162,7 +162,7 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): ) @click.option("--json", is_flag=True, default=False, help="Print full output as JSON") @click.option("--show-archived", is_flag=True, default=False, help="Print archived workflows") -def list(keywords, sort, json, show_archived): +def list_pipelines(keywords, sort, json, show_archived): """ List available nf-core pipelines with local info. @@ -553,9 +553,9 @@ def subworkflows(ctx, git_remote, branch, no_pull): # nf-core modules list subcommands -@modules.group() +@modules.group("list") @click.pass_context -def list(ctx): +def modules_list(ctx): """ List modules in a local pipeline or remote repository. """ @@ -563,11 +563,11 @@ def list(ctx): # nf-core modules list remote -@list.command() +@modules_list.command("remote") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def remote(ctx, keywords, json): +def modules_list_remote(ctx, keywords, json): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -588,7 +588,7 @@ def remote(ctx, keywords, json): # nf-core modules list local -@list.command() +@modules_list.command("local") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") @@ -599,7 +599,7 @@ def remote(ctx, keywords, json): default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin +def modules_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ @@ -620,7 +620,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin # nf-core modules install -@modules.command() +@modules.command("install") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -633,7 +633,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @click.option("-f", "--force", is_flag=True, default=False, help="Force reinstallation of module if it already exists") @click.option("-s", "--sha", type=str, metavar="", help="Install module at commit SHA") -def install(ctx, tool, dir, prompt, force, sha): +def modules_install(ctx, tool, dir, prompt, force, sha): """ Install DSL2 modules within a pipeline. @@ -660,7 +660,7 @@ def install(ctx, tool, dir, prompt, force, sha): # nf-core modules update -@modules.command() +@modules.command("update") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -699,7 +699,7 @@ def install(ctx, tool, dir, prompt, force, sha): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps): +def modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 modules within a pipeline. @@ -767,7 +767,7 @@ def patch(ctx, tool, dir, remove): # nf-core modules remove -@modules.command() +@modules.command("remove") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -777,7 +777,7 @@ def patch(ctx, tool, dir, remove): default=".", help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) -def remove(ctx, dir, tool): +def modules_remove(ctx, dir, tool): """ Remove a module from a pipeline. """ @@ -899,7 +899,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once): # nf-core modules lint -@modules.command() +@modules.command("lint") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -924,9 +924,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once): show_default=True, ) @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") -def lint( - ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version -): # pylint: disable=redefined-outer-name +def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version): """ Lint one or more modules in a directory. @@ -971,7 +969,7 @@ def lint( # nf-core modules info -@modules.command() +@modules.command("info") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -981,7 +979,7 @@ def lint( default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def info(ctx, tool, dir): +def modules_info(ctx, tool, dir): """ Show developer usage information about a given module. @@ -1077,7 +1075,6 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") -@click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") @@ -1108,9 +1105,9 @@ def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once): # nf-core subworkflows list subcommands -@subworkflows.group() +@subworkflows.group("list") @click.pass_context -def list(ctx): +def subworkflows_list(ctx): """ List subworkflows in a local pipeline or remote repository. """ @@ -1118,11 +1115,11 @@ def list(ctx): # nf-core subworkflows list remote -@list.command() +@subworkflows_list.command("remote") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def remote(ctx, keywords, json): +def subworkflows_list_remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -1144,7 +1141,7 @@ def remote(ctx, keywords, json): # nf-core subworkflows list local -@list.command() +@subworkflows_list.command("local") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") @@ -1155,7 +1152,7 @@ def remote(ctx, keywords, json): default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin +def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List subworkflows installed locally in a pipeline """ @@ -1176,7 +1173,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin # nf-core subworkflows lint -@subworkflows.command() +@subworkflows.command("lint") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -1200,9 +1197,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin help="Sort lint output by subworkflow or test name.", show_default=True, ) -def lint( - ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by -): # pylint: disable=redefined-outer-name +def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by): """ Lint one or more subworkflows in a directory. @@ -1246,7 +1241,7 @@ def lint( # nf-core subworkflows info -@subworkflows.command() +@subworkflows.command("info") @click.pass_context @click.argument("tool", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1256,7 +1251,7 @@ def lint( default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def info(ctx, tool, dir): +def subworkflows_info(ctx, tool, dir): """ Show developer usage information about a given subworkflow. @@ -1285,7 +1280,7 @@ def info(ctx, tool, dir): # nf-core subworkflows install -@subworkflows.command() +@subworkflows.command("install") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1300,7 +1295,7 @@ def info(ctx, tool, dir): "-f", "--force", is_flag=True, default=False, help="Force reinstallation of subworkflow if it already exists" ) @click.option("-s", "--sha", type=str, metavar="", help="Install subworkflow at commit SHA") -def install(ctx, subworkflow, dir, prompt, force, sha): +def subworkflows_install(ctx, subworkflow, dir, prompt, force, sha): """ Install DSL2 subworkflow within a pipeline. @@ -1327,7 +1322,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): # nf-core subworkflows remove -@subworkflows.command() +@subworkflows.command("remove") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1337,7 +1332,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): default=".", help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) -def remove(ctx, dir, subworkflow): +def subworkflows_remove(ctx, dir, subworkflow): """ Remove a subworkflow from a pipeline. """ @@ -1357,7 +1352,7 @@ def remove(ctx, dir, subworkflow): # nf-core subworkflows update -@subworkflows.command() +@subworkflows.command("update") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1395,7 +1390,7 @@ def remove(ctx, dir, subworkflow): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): +def subworkflows_update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 subworkflow within a pipeline. @@ -1409,7 +1404,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save force, prompt, sha, - all, + install_all, preview, save_diff, update_deps, @@ -1509,11 +1504,11 @@ def build(dir, no_prompts, web_only, url): # nf-core schema lint -@schema.command() +@schema.command("lint") @click.argument( "schema_path", type=click.Path(exists=True), default="nextflow_schema.json", metavar="" ) -def lint(schema_path): +def schema_lint(schema_path): """ Check that a given pipeline schema is valid. diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index c46d68ff22..efffc28e85 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -10,7 +10,9 @@ import os from pathlib import Path -import rich +import rich.box +import rich.console +import rich.panel from rich.markdown import Markdown from rich.table import Table @@ -209,7 +211,7 @@ def _print_results(self, show_passed=False, sort_by="test"): self.failed.sort(key=operator.attrgetter(*sort_order)) # Find maximum module name length - max_name_len = 40 + max_name_len = len(self.component_type[:-1] + " name") for tests in [self.passed, self.warned, self.failed]: try: for lint_result in tests: @@ -264,7 +266,7 @@ def format_result(test_results, table): table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len) table.add_column("File path") - table.add_column("Test message") + table.add_column("Test message", overflow="fold") table = format_result(self.warned, table) console.print( rich.panel.Panel( @@ -278,10 +280,15 @@ def format_result(test_results, table): # Table of failing tests if len(self.failed) > 0: - table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + table = Table( + style="red", + box=rich.box.MINIMAL, + pad_edge=False, + border_style="dim", + ) table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len) table.add_column("File path") - table.add_column("Test message") + table.add_column("Test message", overflow="fold") table = format_result(self.failed, table) console.print( rich.panel.Panel( diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 190310faae..8d9b6840b0 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -4,6 +4,7 @@ import logging import re from pathlib import Path +from typing import Union log = logging.getLogger(__name__) @@ -77,7 +78,7 @@ def __init__( self.test_yml = None self.test_main_nf = None - def _get_main_nf_tags(self, test_main_nf: str): + def _get_main_nf_tags(self, test_main_nf: Union[Path, str]): """Collect all tags from the main.nf.test file.""" tags = [] with open(test_main_nf, "r") as fh: @@ -86,13 +87,19 @@ def _get_main_nf_tags(self, test_main_nf: str): tags.append(line.strip().split()[1].strip('"')) return tags - def _get_included_components(self, main_nf: str): + def _get_included_components(self, main_nf: Union[Path, str]): """Collect all included components from the main.nf file.""" included_components = [] with open(main_nf, "r") as fh: for line in fh: if line.strip().startswith("include"): - included_components.append(line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/")) + # get tool/subtool or subworkflow name from include statement, can be in the form + #'../../../modules/nf-core/hisat2/align/main' + #'../bam_sort_stats_samtools/main' + #'../subworkflows/nf-core/bam_sort_stats_samtools/main' + component = line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/") + component = component.replace("'../", "subworkflows/") + included_components.append(component) return included_components def get_inputs_from_main_nf(self): diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 39986b95c2..5c50c53fb9 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -89,7 +89,7 @@ def __init__( self, pipeline=None, revision=None, - ): + ) -> None: """Initialise the ParamFileBuilder class Args: diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 521f3e664a..3edd49f09d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index f3dc3e50fe..31e8cd2b36 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index edce89065f..64d1851f2e 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,9 +14,9 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -27,9 +27,9 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @@ -40,7 +40,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check code lints with Black uses: psf/black@stable @@ -71,7 +71,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 6a6bee6a83..31917a3e31 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -121,6 +121,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + runOptions = '-u $(id -u):$(id -g)' } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test index b0a212f2a3..59c4fdb679 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -8,7 +8,6 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" - tag "{{ component_name }}" tag "subworkflows/{{ component_name }}" // TODO nf-core: Add tags for all modules used within this subworkflow. Example: tag "samtools" diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 44c7c21a37..ffba41f9da 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -29,12 +29,12 @@ class SubworkflowLint(ComponentLint): """ # Import lint functions - from .main_nf import main_nf - from .meta_yml import meta_yml - from .subworkflow_changes import subworkflow_changes - from .subworkflow_tests import subworkflow_tests - from .subworkflow_todos import subworkflow_todos - from .subworkflow_version import subworkflow_version + from .main_nf import main_nf # type: ignore[misc] + from .meta_yml import meta_yml # type: ignore[misc] + from .subworkflow_changes import subworkflow_changes # type: ignore[misc] + from .subworkflow_tests import subworkflow_tests # type: ignore[misc] + from .subworkflow_todos import subworkflow_todos # type: ignore[misc] + from .subworkflow_version import subworkflow_version # type: ignore[misc] def __init__( self, diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index e82d00f636..a125be152f 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -102,11 +102,11 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): "subworkflows", f"subworkflows/{subworkflow.component_name}", "subworkflows_nfcore", - subworkflow.component_name, ] included_components = [] if subworkflow.main_nf.is_file(): included_components = subworkflow._get_included_components(subworkflow.main_nf) + log.debug(f"Required tags: {required_tags}") missing_tags = [] for tag in required_tags + included_components: if tag not in main_nf_tags: diff --git a/pyproject.toml b/pyproject.toml index f0702742fd..2380073107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,3 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu profile = "black" known_first_party = ["nf_core"] multi_line_output = 3 - -[tool.mypy] -ignore_missing_imports = true -follow_imports = "skip" -disable_error_code = "no-redef" diff --git a/requirements-dev.txt b/requirements-dev.txt index 12183eb98c..3ef0593085 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,4 @@ types-PyYAML types-requests types-jsonschema types-Markdown +types-setuptools diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 5bbe746f2e..1380db2260 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -28,9 +28,8 @@ def test_subworkflows_lint_new_subworkflow(self): """lint a new subworkflow""" subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) - assert ( - len(subworkflow_lint.failed) == 1 # test snap missing after creating a subworkflow - ), f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -68,7 +67,6 @@ def test_subworkflows_lint_multiple_remotes(self): def test_subworkflows_lint_snapshot_file(self): """Test linting a subworkflow with a snapshot file""" - Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" @@ -78,8 +76,10 @@ def test_subworkflows_lint_snapshot_file(self): def test_subworkflows_lint_snapshot_file_missing_fail(self): """Test linting a subworkflow with a snapshot file missing, which should fail""" + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -96,8 +96,11 @@ def test_subworkflows_lint_snapshot_file_not_needed(self): Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test"), "w" ) as fh: fh.write(new_content) + + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 diff --git a/tests/test_lint.py b/tests/test_lint.py index 67104e3ad0..69917b279d 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -178,44 +178,44 @@ def test_sphinx_md_files(self): ####################### # SPECIFIC LINT TESTS # ####################### - from .lint.actions_awsfulltest import ( + from .lint.actions_awsfulltest import ( # type: ignore[misc] test_actions_awsfulltest_fail, test_actions_awsfulltest_pass, test_actions_awsfulltest_warn, ) - from .lint.actions_awstest import ( + from .lint.actions_awstest import ( # type: ignore[misc] test_actions_awstest_fail, test_actions_awstest_pass, ) - from .lint.actions_ci import ( + from .lint.actions_ci import ( # type: ignore[misc] test_actions_ci_fail_wrong_nf, test_actions_ci_fail_wrong_trigger, test_actions_ci_pass, ) - from .lint.actions_schema_validation import ( + from .lint.actions_schema_validation import ( # type: ignore[misc] test_actions_schema_validation_fails_for_additional_property, test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) - from .lint.files_exist import ( + from .lint.files_exist import ( # type: ignore[misc] test_files_exist_depreciated_file, test_files_exist_missing_config, test_files_exist_missing_main, test_files_exist_pass, ) - from .lint.files_unchanged import ( + from .lint.files_unchanged import ( # type: ignore[misc] test_files_unchanged_fail, test_files_unchanged_pass, ) - from .lint.merge_markers import test_merge_markers_found - from .lint.modules_json import test_modules_json_pass - from .lint.nextflow_config import ( + from .lint.merge_markers import test_merge_markers_found # type: ignore[misc] + from .lint.modules_json import test_modules_json_pass # type: ignore[misc] + from .lint.nextflow_config import ( # type: ignore[misc] test_nextflow_config_bad_name_fail, test_nextflow_config_dev_in_release_mode_failed, test_nextflow_config_example_pass, test_nextflow_config_missing_test_profile_failed, ) - from .lint.version_consistency import test_version_consistency + from .lint.version_consistency import test_version_consistency # type: ignore[misc] # TODO nf-core: Assess and strip out if no longer required for DSL2 diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 638a4bedb1..aaf4080a7b 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -4,6 +4,7 @@ import os import shutil import unittest +from pathlib import Path import nf_core.create import nf_core.modules @@ -21,22 +22,18 @@ def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" - root_dir = os.path.join(tmp_dir, "modules") - os.makedirs(os.path.join(root_dir, "modules")) - os.makedirs(os.path.join(root_dir, "subworkflows")) - os.makedirs(os.path.join(root_dir, "subworkflows", "nf-core")) - os.makedirs(os.path.join(root_dir, "tests", "modules")) - os.makedirs(os.path.join(root_dir, "tests", "subworkflows")) - os.makedirs(os.path.join(root_dir, "tests", "config")) - with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: - fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + root_dir = Path(tmp_dir, "modules") + Path(root_dir, "modules").mkdir(parents=True, exist_ok=True) + Path(root_dir, "subworkflows").mkdir(parents=True, exist_ok=True) + Path(root_dir, "subworkflows", "nf-core").mkdir(parents=True, exist_ok=True) + Path(root_dir, "tests", "config").mkdir(parents=True, exist_ok=True) + with open(Path(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # TODO Add a mock here subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) subworkflow_create.create() + Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() return root_dir