Parallelize pytest (#1)

.github/.coveragerc

@@ -1,2 +1,5 @@
 [run]
-omit = nf_core/pipeline-template/*
+omit = nf_core/*-template/*
+source = nf_core
+relative_files = True
+

.github/workflows/create-lint-wf.yml

@@ -26,6 +26,12 @@ jobs:
           - "23.04.0"
           - "latest-everything"
     steps:
+      - name: go to subdirectory and change nextflow workdir
+        run: |
+          mkdir -p create-lint-wf
+          cd create-lint-wf
+          export NXF_WORK=$(pwd)
+
       # Get the repo code
       - uses: actions/checkout@v4
         name: Check out source-code repository
@@ -136,3 +142,7 @@ jobs:
         with:
           name: nf-core-log-file
           path: create-lint-wf/log.txt
+
+      - name: Cleanup work directory
+        run: sudo rm -rf create-lint-wf
+        if: always()

.github/workflows/create-test-lint-wf-template.yml

@@ -20,19 +20,32 @@ env:
 
 jobs:
   RunTestWorkflow:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     env:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
         TEMPLATE:
-          - "template_skip_all.yml"
           - "template_skip_github_badges.yml"
           - "template_skip_igenomes.yml"
           - "template_skip_ci.yml"
-          - "template_skip_nf_core_configs.yml"
+        runner: ["self-hosted"]
+        profile: ["self_hosted_runner"]
+        include:
+          - TEMPLATE: "template_skip_all.yml"
+            runner: ubuntu-latest
+            profile: "docker"
+          - TEMPLATE: "template_skip_nf_core_configs.yml"
+            runner: ubuntu-latest
+            profile: "docker"
 
     steps:
+      - name: go to working directory
+        run: |
+          mkdir -p create-lint-wf-template
+          cd create-lint-wf-template
+          export NXF_WORK=$(pwd)
+
       - uses: actions/checkout@v4
         name: Check out source-code repository
 
@@ -83,7 +96,7 @@ jobs:
       - name: run the pipeline
         run: |
           cd create-test-lint-wf
-          nextflow run my-prefix-testpipeline -profile test,docker --outdir ./results
+          nextflow run my-prefix-testpipeline -profile test,${{matrix.profile}} --outdir ./results
 
       # Remove results folder before linting
       - name: remove results folder
@@ -134,3 +147,7 @@ jobs:
         with:
           name: nf-core-log-file
           path: create-test-lint-wf/artifact_files.tar
+
+      - name: Cleanup work directory
+        run: sudo rm -rf create-test-lint-wf
+        if: always()

.github/workflows/create-test-wf.yml

@@ -17,7 +17,7 @@ env:
 
 jobs:
   RunTestWorkflow:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     env:
       NXF_ANSI_LOG: false
     strategy:
@@ -26,6 +26,12 @@ jobs:
           - "23.04.0"
           - "latest-everything"
     steps:
+      - name: go to working directory
+        run: |
+          mkdir -p create-test-wf
+          cd create-test-wf
+          export NXF_WORK=$(pwd)
+
       - uses: actions/checkout@v4
         name: Check out source-code repository
 
@@ -49,11 +55,16 @@ jobs:
           mkdir create-test-wf && cd create-test-wf
           export NXF_WORK=$(pwd)
           nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
-          nextflow run nf-core-testpipeline -profile test,docker --outdir ./results
+          nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results
 
       - name: Upload log file artifact
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
           name: nf-core-log-file
           path: create-test-wf/log.txt
+
+      - name: Cleanup work directory
+        # cleanup work directory
+        run: sudo rm -rf create-test-wf
+        if: always()

.github/workflows/fix-linting.yml

@@ -10,7 +10,7 @@ jobs:
       contains(github.event.comment.html_url, '/pull/') &&
       contains(github.event.comment.body, '@nf-core-bot fix linting') &&
       github.repository == 'nf-core/tools'
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       # Use the @nf-core-bot token to check out so we can push later
       - uses: actions/checkout@v4

.github/workflows/lint-code.yml

@@ -14,19 +14,20 @@ concurrency:
 
 jobs:
   Pre-commit:
-    name: Pre-commit
-    runs-on: ["ubuntu-latest"]
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
           cache: "pip"
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip -r requirements-dev.txt
-          pip install -e .
+      - name: Install pre-commit
+        run: pip install pre-commit
 
       - name: Run pre-commit
         run: pre-commit run --all-files

.github/workflows/push_dockerhub_dev.yml

@@ -13,7 +13,7 @@ concurrency:
 jobs:
   push_dockerhub:
     name: Push new Docker image to Docker Hub (dev)
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     # Only run for the nf-core repo, for releases and merged PRs
     if: ${{ github.repository == 'nf-core/tools' }}
     env:

.github/workflows/pytest.yml

@@ -14,8 +14,9 @@ on:
       - "CHANGELOG.md"
   release:
     types: [published]
+  workflow_dispatch:
 
-# Cancel if a newer run is started
+# Cancel if a newer run with the same workflow name is queued
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -25,7 +26,7 @@ env:
 
 jobs:
   setup:
-    runs-on: ["ubuntu-latest"]
+    runs-on: "ubuntu-latest"
     strategy:
       matrix:
         python-version: ["3.8", "3.11"]
@@ -44,12 +45,36 @@ jobs:
       runner: ${{ matrix.runner }}
       run-tests: ${{ steps.conditions.outputs.run-tests }}
 
+  # create a test matrix based on all python files in /tests
+  list_tests:
+    name: Get test file matrix
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+        name: Check out source-code repository
+
+      - name: List tests
+        id: list_tests
+        run: |
+          echo "tests=$(find tests/test_* | tac | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
+    outputs:
+      tests: ${{ steps.list_tests.outputs.tests }}
+
   test:
-    name: Test with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
-    needs: setup
-    if: ${{ needs.setup.outputs.run-tests}}
-    runs-on: ${{ needs.setup.outputs.runner }}
+    name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
+    needs: [setup, list_tests]
+    if: ${{ needs.setup.outputs.run-tests }}
+    runs-on: self-hosted
+    strategy:
+      matrix: ${{ fromJson(needs.list_tests.outputs.tests) }}
+      fail-fast: false # run all tests even if one fails
     steps:
+      - name: go to subdirectory and change nextflow workdir
+        run: |
+          mkdir -p pytest
+          cd pytest
+          export NXF_WORK=$(pwd)
+
       - uses: actions/checkout@v4
         name: Check out source-code repository
 
@@ -58,6 +83,7 @@ jobs:
         with:
           python-version: ${{ needs.setup.outputs.python-version }}
           cache: "pip"
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install dependencies
         run: |
@@ -71,14 +97,13 @@ jobs:
           sudo apt remove -y git git-man
           sudo add-apt-repository --remove ppa:git-core/ppa
           sudo apt install -y git
+
       - name: Get current date
         id: date
         run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
-        with:
-          version: "latest-everything"
 
       - name: Cache nf-test installation
         id: cache-software
@@ -95,11 +120,65 @@ jobs:
           wget -qO- https://code.askimed.com/install/nf-test | bash
           sudo mv nf-test /usr/local/bin/
 
+      - name: move coveragerc file up
+        run: |
+          mv .github/.coveragerc .
+
       - name: Test with pytest
-        run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core
+        run: |
+          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$?
+          # don't fail if no tests were collected, e.g. for test_licence.py
+          if [ "${exit_code}" -eq 5 ]; then
+            echo "No tests were collected"
+            exit 0
+          elif [ "${exit_code}" -ne 0 ]; then
+            echo "Tests failed with exit code ${exit_code}"
+            exit 1
+          fi
+
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage_${{ matrix.test }}
+          path: .coverage
+
+  coverage:
+    needs: test
+    runs-on: self-hosted
+    steps:
+      - name: go to subdirectory
+        run: |
+          mkdir -p pytest
+          cd pytest
+
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        env:
+          AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/
+        with:
+          python-version: 3.11
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip -r requirements-dev.txt
+          pip install -e .
+
+      - name: move coveragerc file up
+        run: |
+          mv .github/.coveragerc .
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+      - name: Run coverage
+        run: |
+          coverage combine --keep coverage*/.coverage*
+          coverage report
+          coverage xml
 
-      - uses: codecov/codecov-action@v1
-        name: Upload code coverage report
+      - uses: codecov/codecov-action@v3
         with:
-          if: success()
-          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/sync.yml

@@ -16,7 +16,7 @@ concurrency:
 
 jobs:
   get-pipelines:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:

.github/workflows/tools-api-docs-dev.yml

@@ -20,7 +20,7 @@ concurrency:
 jobs:
   api-docs:
     name: Build & push Sphinx API docs
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
 
     steps:
       - name: Check out source-code repository

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Template
 
+- Use `pre-commit` to lint files in GitHub CI ([#2635](https://github.com/nf-core/tools/pull/2635))
+
 ### Download
 
 ### Linting
@@ -14,6 +16,8 @@
 
 ### General
 
+- Run CI-pytests for nf-core tools on self-hosted runners ([#2550](https://github.com/nf-core/tools/pull/2550))
+
 - Add Ruff linter and formatter replacing Black, isort and pyupgrade ([#2620](https://github.com/nf-core/tools/pull/2620))
 - Update pre-commit hook pre-commit/mirrors-mypy to v1.8.0 ([#2630](https://github.com/nf-core/tools/pull/2630))
 - Update mshick/add-pr-comment action to v2 ([#2632](https://github.com/nf-core/tools/pull/2632))
@@ -24,7 +28,6 @@
 
 - Rename `release-announcments.yml` to `release-announcements.yml` ([#2610](https://github.com/nf-core/tools/pull/2610))
 - Fix `nextflow.config` `docker.runOptions` ([#2607](https://github.com/nf-core/tools/pull/2607))
-- Use `pre-commit` to lint files in GitHub CI ([#2635](https://github.com/nf-core/tools/pull/2635))
 
 ### General
 

codecov.yml

@@ -4,3 +4,6 @@ coverage:
       default:
         threshold: 5%
     patch: off
+comment:
+  layout: "condensed_header, condensed_files, condensed_footer" # add "condensed_" to "header", "files" and "footer"
+  hide_project_coverage: TRUE # set to true

tests/utils.py

@@ -25,7 +25,7 @@
 GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path"
 GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca"
 GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991"
-GITLAB_NFTEST_BRANCH = "nf-test-tests"
+GITLAB_NFTEST_BRANCH = "nf-test-tests-self-hosted-runners"
 
 
 def with_temporary_folder(func: Callable[..., Any]) -> Callable[..., Any]: