From 0e2d0c6d94f048eb1ba88924a0c87edc2c1bd6ef Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:25:15 -0600 Subject: [PATCH] Wave with Dockerfiles (#4940) * ci(renovate): Add Dockerfile building * build: Bump quarto to 24.04 * ci: Get clever with singularity flag * build: Try a better Dockerfile * ci(wave): Better names * ci(wave): Properly skip environment.yml and Dockerfiles * style(wittyer): Remove environment.yml * ci(wave): Let Dockerfiles fail and request @edmundmiller and @maxulysse --- .github/CODEOWNERS | 1 + .github/workflows/wave.yml | 101 ++++++++++++++-------- modules/nf-core/quartonotebook/Dockerfile | 2 +- modules/nf-core/wittyer/Dockerfile | 2 + modules/nf-core/wittyer/environment.yml | 5 -- 5 files changed, 71 insertions(+), 40 deletions(-) delete mode 100644 modules/nf-core/wittyer/environment.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3893bf75737..9552f7c99ad 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,3 +12,4 @@ modules/nf-core/pints/caller/** @edmundmiller modules/nf-core/preseq/ccurve/** @edmundmiller modules/nf-core/preseq/lcextrap/** @edmundmiller subworkflows/nf-core/homer/** @edmundmiller +modules/nf-core/**/Dockerfile @edmundmiller @maxulysse diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 4f36575b6d1..a86fd9bb10a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - "**/environment.yml" + - "**/Dockerfile" # TODO On complete call testing CI # TODO Skip testing CI if any changes to environment.yml @@ -18,38 +19,44 @@ jobs: runs-on: ubuntu-latest # Only run on Pull Requests within the same repository, and not from forks if: github.event.pull_request.head.repo.full_name == github.repository - + outputs: + conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }} + docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Calculate file differences - id: diff + - name: Find conda differences + id: conda-diff uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45 with: json: true - quotepath: false - # TODO Add Dockerfiles + escape_json: false files: | modules/**/environment.yml - - name: Debug - run: echo ${{ steps.diff.outputs.all_changed_files }} - - id: set-matrix - run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + - name: Find Dockerfile differences + id: docker-diff + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45 + with: + json: true + escape_json: false + files: | + modules/**/Dockerfile - build: + conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}" + if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }} needs: generate-matrix - name: build + name: Build Conda-based ${{ matrix.profile }} Container runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}" + max-parallel: 4 + matrix: + files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}" + profile: [docker, singularity] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -59,38 +66,64 @@ jobs: sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - - name: Create a registry name - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: registry-name - with: - result-encoding: string - script: | - return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_'); - - - name: Build container + - name: Build ${{ matrix.profile }} container # FIXME Hack while iron out the CI continue-on-error: true - if: matrix.profile == 'docker' + env: + PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} run: | wave --conda-file "${{ matrix.files }}" \ + $PROFILE \ --freeze \ --await \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} - - name: Build Singularity - # FIXME Hack while iron out the CI - continue-on-error: true - if: matrix.profile == 'singularity' + docker-wave: + # NOTE This should get skipped because generate-matrix won't run + # if: github.repository == 'nf-core/modules' + if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }} + needs: generate-matrix + name: Build Dockerfile-based ${{ matrix.profile }} Container + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}" + profile: [docker, singularity] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Install wave-cli run: | - wave --conda-file "${{ matrix.files }}" \ + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 + sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + chmod +x /usr/local/bin/wave + + - name: Create a registry name + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: registry-name + with: + result-encoding: string + script: | + return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/Dockerfile', '').replace('/', '_'); + + - name: Build ${{ matrix.profile }} container + # NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave + continue-on-error: false + env: + PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} + run: | + wave -f "${{ matrix.files }}" \ + $PROFILE \ --freeze \ --await \ + --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ + --cache-repository quay.io/nf-core/wave-cache \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ - --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ - --singularity - - # TODO Build from Dockerfiles + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} # bump-versions: # needs: generate-matrix diff --git a/modules/nf-core/quartonotebook/Dockerfile b/modules/nf-core/quartonotebook/Dockerfile index 78d2ab20c23..26624b7ea6d 100644 --- a/modules/nf-core/quartonotebook/Dockerfile +++ b/modules/nf-core/quartonotebook/Dockerfile @@ -1,7 +1,7 @@ # # First stage: Quarto installation # -FROM ubuntu:20.04 as quarto +FROM ubuntu:24.04 as quarto ARG QUARTO_VERSION=1.3.433 ARG TARGETARCH RUN apt-get update \ diff --git a/modules/nf-core/wittyer/Dockerfile b/modules/nf-core/wittyer/Dockerfile index fae697ab9dc..fbf732f32bd 100644 --- a/modules/nf-core/wittyer/Dockerfile +++ b/modules/nf-core/wittyer/Dockerfile @@ -1,4 +1,5 @@ FROM mcr.microsoft.com/dotnet/sdk:6.0@sha256:6df1177e48b55272316d08f19cb383483af82aca5cdc67a76c414bc200847624 as builder + WORKDIR /src COPY . /src RUN cd Wittyer \ @@ -6,6 +7,7 @@ RUN cd Wittyer \ && chmod +x /output/Wittyer FROM mcr.microsoft.com/dotnet/runtime:6.0@sha256:437cda84bdce26ce074d88b63abeec567c7226d73e8b911605077468e1d5c8d5 + LABEL git_repository=https://git.illumina.com/DASTE/Ilmn.Das.App.Wittyer.git WORKDIR /opt/Wittyer RUN apt-get -y update && apt-get -y install tabix libunwind8 openssl procps diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml deleted file mode 100644 index c90cb8dccbf..00000000000 --- a/modules/nf-core/wittyer/environment.yml +++ /dev/null @@ -1,5 +0,0 @@ -channels: - - conda-forge - - bioconda -dependencies: - - tabix