Merge pull request #1782 from nf-core/dev

Dev -> Master for v2.5

.editorconfig

@@ -8,5 +8,5 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = space
 
-[*.{md,yml,yaml,html,css,scss,js}]
+[*.{md,yml,yaml,html,css,scss,js,cff}]
 indent_size = 2

.github/CONTRIBUTING.md

@@ -33,7 +33,9 @@ Then install your local fork of nf-core/tools:
 pip install -e .
 ```
 
-## Code formatting with Black
+## Code formatting
+
+### Black
 
 All Python code in nf-core/tools must be passed through the [Black Python code formatter](https://black.readthedocs.io/en/stable/).
 This ensures a harmonised code formatting style throughout the package, from all contributors.
@@ -51,6 +53,42 @@ You can also set it up to run when you [make a commit](https://black.readthedocs
 There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
 any code does not adhere to Black formatting.
 
+### isort
+
+All Python code must also be passed through [isort](https://pycqa.github.io/isort/index.html).
+This ensures a harmonised imports throughout the package, from all contributors.
+
+To run isort on the command line recursively on the whole repository you can use:
+
+```bash
+isort .
+```
+
+isort also has [plugins for most common editors](https://github.com/pycqa/isort/wiki/isort-Plugins)
+to automatically format code when you hit save.
+Or [version control integration](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) to set it up to run when you make a commit.
+
+There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
+any code does not adhere to isort formatting.
+
+### pre-commit hooks
+
+This repository comes with [pre-commit](https://pre-commit.com/) hooks for black, isort and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
+
+You can use the pre-commit hooks if you like, but you don't have to. The CI on Github will run the same checks as the tools installed with pre-commit. If the pre-commit checks pass, then the same checks in the CI will pass, too.
+
+You can install the pre-commit hooks into the development environment by running the following command in the root directory of the repository.
+
+```bash
+pre-commit install --install-hooks
+```
+
+You can also run all pre-commit hooks without making a commit:
+
+```bash
+pre-commit run --all
+```
+
 ## API Documentation
 
 We aim to write function docstrings according to the [Google Python style-guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings). These are used to automatically generate package documentation on the nf-core website using Sphinx.

.github/workflows/create-lint-wf.yml

@@ -1,9 +1,13 @@
 name: Create a pipeline and run nf-core linting
 on: [push, pull_request]
 
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   NXF_ANSI_LOG: false
-  CAPSULE_LOG: none
 
 jobs:
   MakeTestWorkflow:
@@ -12,14 +16,9 @@ jobs:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        # Nextflow versions
-        include:
-          # Test pipeline minimum Nextflow version
-          - NXF_VER: "21.10.3"
-            NXF_EDGE: ""
-          # Test latest edge release of Nextflow
-          - NXF_VER: ""
-            NXF_EDGE: "1"
+        NXF_VER:
+          - "21.10.3"
+          - "latest-everything"
     steps:
       # Get the repo code
       - uses: actions/checkout@v2
@@ -38,14 +37,9 @@ jobs:
 
       # Set up Nextflow
       - name: Install Nextflow
-        env:
-          NXF_VER: ${{ matrix.NXF_VER }}
-          # Uncomment only if the edge release is more recent than the latest stable release
-          # See https://github.com/nextflow-io/nextflow/issues/2467
-          # NXF_EDGE: ${{ matrix.NXF_EDGE }}
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: ${{ matrix.NXF_VER }}
 
       # Install the Prettier linting tools
       - uses: actions/setup-node@v2
@@ -59,7 +53,7 @@ jobs:
 
       # Build a pipeline from the template
       - name: nf-core create
-        run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
+        run: nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
 
       # Try syncing it before we change anything
       - name: nf-core sync
@@ -80,9 +74,13 @@ jobs:
       - name: nf-core modules update
         run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview
 
+      # Remove TODO statements
+      - name: remove TODO
+        run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
+
       # Run nf-core linting
       - name: nf-core lint
-        run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored
+        run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --fail-warned
 
       # Run the other nf-core commands
       - name: nf-core list
@@ -98,11 +96,23 @@ jobs:
         run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1
 
       - name: nf-core lint in release mode
-        run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --release
+        run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release
 
       - name: nf-core modules install
         run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force
 
+      - name: nf-core modules install gitlab
+        run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git install fastqc --dir nf-core-testpipeline/
+
+      - name: nf-core modules list local
+        run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/
+
+      - name: nf-core modules list remote
+        run: nf-core --log-file log.txt modules list remote
+
+      - name: nf-core modules list remote gitlab
+        run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote
+
       - name: Upload log file artifact
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

.github/workflows/create-test-wf.yml

@@ -1,9 +1,13 @@
 name: Create a pipeline and test it
 on: [push, pull_request]
 
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   NXF_ANSI_LOG: false
-  CAPSULE_LOG: none
 
 jobs:
   RunTestWorkflow:
@@ -12,14 +16,9 @@ jobs:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        # Nextflow versions
-        include:
-          # Test pipeline minimum Nextflow version
-          - NXF_VER: "21.10.3"
-            NXF_EDGE: ""
-          # Test latest edge release of Nextflow
-          - NXF_VER: ""
-            NXF_EDGE: "1"
+        NXF_VER:
+          - "21.10.3"
+          - "latest-everything"
     steps:
       - uses: actions/checkout@v2
         name: Check out source-code repository
@@ -35,18 +34,13 @@ jobs:
           pip install .
 
       - name: Install Nextflow
-        env:
-          NXF_VER: ${{ matrix.NXF_VER }}
-          # Uncomment only if the edge release is more recent than the latest stable release
-          # See https://github.com/nextflow-io/nextflow/issues/2467
-          # NXF_EDGE: ${{ matrix.NXF_EDGE }}
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: ${{ matrix.NXF_VER }}
 
       - name: Run nf-core/tools
         run: |
-          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
+          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
           nextflow run nf-core-testpipeline -profile test,docker --outdir ./results
 
       - name: Upload log file artifact

.github/workflows/deploy-pypi.yml

@@ -3,6 +3,11 @@ on:
   release:
     types: [published]
 
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-n-publish:
     runs-on: ubuntu-latest

.github/workflows/fix-linting.yml

@@ -38,6 +38,16 @@ jobs:
           # Override to remove the default --check flag so that we make changes
           options: "--color"
 
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: python-isort
+        uses: isort/isort-action@v0.1.0
+        with:
+          isortVersion: "latest"
+          requirementsFiles: "requirements.txt requirements-dev.txt"
+
       - name: Commit & push changes
         run: |
           git config user.email "core@nf-co.re"

.github/workflows/lint-code.yml

@@ -1,6 +1,11 @@
 name: Lint tools code formatting
 on: [push, pull_request]
 
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   EditorConfig:
     runs-on: ubuntu-latest
@@ -58,3 +63,19 @@ jobs:
             Thanks again for your contribution!
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allow-repeats: false
+
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source-code repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: python-isort
+        uses: isort/isort-action@v0.1.0
+        with:
+          isortVersion: "latest"
+          requirementsFiles: "requirements.txt requirements-dev.txt"

.github/workflows/push_dockerhub_dev.yml

@@ -5,6 +5,11 @@ on:
   push:
     branches: [dev]
 
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   push_dockerhub:
     name: Push new Docker image to Docker Hub (dev)

.github/workflows/push_dockerhub_release.yml

@@ -5,6 +5,11 @@ on:
   release:
     types: [published]
 
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   push_dockerhub:
     name: Push new Docker image to Docker Hub (release)

.github/workflows/pytest.yml

@@ -5,15 +5,17 @@ on:
   push:
   pull_request:
 
-# Uncomment if we need an edge release of Nextflow again
-# env: NXF_EDGE: 1
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   pytest:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
       - uses: actions/checkout@v2
@@ -30,11 +32,9 @@ jobs:
           pip install -e .
 
       - name: Install Nextflow
-        env:
-          CAPSULE_LOG: none
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "latest-everything"
 
       - name: Test with pytest
         run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core

.github/workflows/rich-codex.yml

@@ -0,0 +1,37 @@
+name: Generate images for docs
+on:
+  workflow_dispatch:
+jobs:
+  rich_codex:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.x
+          cache: pip
+          cache-dependency-path: setup.py
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
+
+      - name: Install nf-core/tools
+        run: pip install .
+
+      - name: Generate terminal images with rich-codex
+        uses: ewels/rich-codex@v1
+        env:
+          COLUMNS: 100
+          NFCORE_LINT_HIDE_PROGRESS: true
+          NFCORE_MODULES_LINT_HIDE_PROGRESS: true
+        with:
+          commit_changes: "true"
+          clean_img_paths: docs/images/*.svg
+          terminal_width: 100
+          before_command: >
+            which nextflow &&
+            which nf-core &&
+            nextflow -version &&
+            nf-core --version