Wave with Dockerfiles (#4940)

* ci(renovate): Add Dockerfile building

* build: Bump quarto to 24.04

* ci: Get clever with singularity flag

* build: Try a better Dockerfile

* ci(wave): Better names

* ci(wave): Properly skip environment.yml and Dockerfiles

* style(wittyer): Remove environment.yml

* ci(wave): Let Dockerfiles fail and request @edmundmiller and @maxulysse

.github/CODEOWNERS

@@ -12,3 +12,4 @@ modules/nf-core/pints/caller/** @edmundmiller
 modules/nf-core/preseq/ccurve/** @edmundmiller
 modules/nf-core/preseq/lcextrap/** @edmundmiller
 subworkflows/nf-core/homer/** @edmundmiller
+modules/nf-core/**/Dockerfile @edmundmiller @maxulysse

.github/workflows/wave.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - "**/environment.yml"
+      - "**/Dockerfile"
 
 # TODO On complete call testing CI
 # TODO Skip testing CI if any changes to environment.yml
@@ -18,38 +19,44 @@ jobs:
     runs-on: ubuntu-latest
     # Only run on Pull Requests within the same repository, and not from forks
     if: github.event.pull_request.head.repo.full_name == github.repository
-
+    outputs:
+      conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }}
+      docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
-      - name: Calculate file differences
-        id: diff
+      - name: Find conda differences
+        id: conda-diff
         uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45
         with:
           json: true
-          quotepath: false
-          # TODO Add Dockerfiles
+          escape_json: false
           files: |
             modules/**/environment.yml
-      - name: Debug
-        run: echo ${{ steps.diff.outputs.all_changed_files }}
-      - id: set-matrix
-        run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT"
 
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      - name: Find Dockerfile differences
+        id: docker-diff
+        uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45
+        with:
+          json: true
+          escape_json: false
+          files: |
+            modules/**/Dockerfile
 
-  build:
+  conda-wave:
     # NOTE This should get skipped because generate-matrix won't run
     # if: github.repository == 'nf-core/modules'
-    if: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}"
+    if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }}
     needs: generate-matrix
-    name: build
+    name: Build Conda-based ${{ matrix.profile }} Container
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
-      matrix: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}"
+      max-parallel: 4
+      matrix:
+        files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}"
+        profile: [docker, singularity]
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
@@ -59,38 +66,64 @@ jobs:
           sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
           chmod +x /usr/local/bin/wave
 
-      - name: Create a registry name
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
-        id: registry-name
-        with:
-          result-encoding: string
-          script: |
-            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_');
-
-      - name: Build container
+      - name: Build ${{ matrix.profile }} container
         # FIXME Hack while iron out the CI
         continue-on-error: true
-        if: matrix.profile == 'docker'
+        env:
+          PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }}
         run: |
           wave --conda-file "${{ matrix.files }}" \
+              $PROFILE \
               --freeze \
               --await \
               --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
               --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
 
-      - name: Build Singularity
-        # FIXME Hack while iron out the CI
-        continue-on-error: true
-        if: matrix.profile == 'singularity'
+  docker-wave:
+    # NOTE This should get skipped because generate-matrix won't run
+    # if: github.repository == 'nf-core/modules'
+    if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }}
+    needs: generate-matrix
+    name: Build Dockerfile-based ${{ matrix.profile }} Container
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}"
+        profile: [docker, singularity]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Install wave-cli
         run: |
-          wave --conda-file "${{ matrix.files }}" \
+          wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64
+          sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave
+          chmod +x /usr/local/bin/wave
+
+      - name: Create a registry name
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
+        id: registry-name
+        with:
+          result-encoding: string
+          script: |
+            return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/Dockerfile', '').replace('/', '_');
+
+      - name: Build ${{ matrix.profile }} container
+        # NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave
+        continue-on-error: false
+        env:
+          PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }}
+        run: |
+          wave -f "${{ matrix.files }}" \
+              $PROFILE \
               --freeze \
               --await \
+              --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \
+              --cache-repository quay.io/nf-core/wave-cache \
               --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
-              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \
-              --singularity
-
-    # TODO Build from Dockerfiles
+              --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}
 
   # bump-versions:
   #   needs: generate-matrix

modules/nf-core/quartonotebook/Dockerfile

@@ -1,7 +1,7 @@
 #
 # First stage: Quarto installation
 #
-FROM ubuntu:20.04 as quarto
+FROM ubuntu:24.04 as quarto
 ARG QUARTO_VERSION=1.3.433
 ARG TARGETARCH
 RUN apt-get update \

modules/nf-core/wittyer/Dockerfile

@@ -1,11 +1,13 @@
 FROM mcr.microsoft.com/dotnet/sdk:6.0@sha256:6df1177e48b55272316d08f19cb383483af82aca5cdc67a76c414bc200847624 as builder
+
 WORKDIR /src
 COPY . /src
 RUN cd Wittyer \
     && dotnet publish -f net6.0 -r linux-x64 -c Release -o /output \
     && chmod +x /output/Wittyer
 
 FROM mcr.microsoft.com/dotnet/runtime:6.0@sha256:437cda84bdce26ce074d88b63abeec567c7226d73e8b911605077468e1d5c8d5
+
 LABEL git_repository=https://git.illumina.com/DASTE/Ilmn.Das.App.Wittyer.git
 WORKDIR /opt/Wittyer
 RUN apt-get -y update && apt-get -y install tabix libunwind8 openssl procps