Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(deps): update condaforge/mambaforge docker tag to v24.9.2 #5561

Merged
merged 14 commits into from
Nov 21, 2024
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 14 additions & 11 deletions .github/workflows/wave.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
if: github.event.pull_request.head.repo.full_name == github.repository
outputs:
conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }}
docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }}
dockerfile-matrix: ${{ steps.docker-diff.outputs.all_changed_files }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

Expand All @@ -48,7 +48,7 @@ jobs:
# if: github.repository == 'nf-core/modules'
if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }}
needs: generate-matrix
name: Build Conda-based ${{ matrix.profile }} Container
name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
Expand All @@ -57,6 +57,7 @@ jobs:
matrix:
files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}"
profile: [docker, singularity]
platform: ["linux/amd64", "linux/arm64"]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

Expand All @@ -76,23 +77,27 @@ jobs:
$PROFILE \
--freeze \
--await \
--platform ${{ matrix.platform }} \
--tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
--tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}

docker-wave:
dockerfile-wave:
# NOTE This should get skipped because generate-matrix won't run
# if: github.repository == 'nf-core/modules'
if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }}
if: ${{ needs.generate-matrix.outputs.dockerfile-matrix != '[]' }}
needs: generate-matrix
name: Build Dockerfile-based ${{ matrix.profile }} Container
name: Build ${{ matrix.profile }} ${{ matrix.platform }} Container
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: 4
matrix:
files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}"
profile: [docker, singularity]
files: "${{ fromJson(needs.generate-matrix.outputs.dockerfile-matrix) }}"
# NOTE singularity build requires a Singularity definition file in place of Dockerfile
# https://nextflow.slack.com/archives/C0477AS31T5/p1731755350230149?thread_ts=1731697852.661849&cid=C0477AS31T5
profile: [docker]
platform: ["linux/amd64", "linux/arm64"]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

Expand All @@ -113,15 +118,13 @@ jobs:
- name: Build ${{ matrix.profile }} container
# NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave
continue-on-error: false
env:
PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }}
run: |
wave -f "${{ matrix.files }}" \
$PROFILE \
--freeze \
--await \
--platform ${{ matrix.platform }} \
--build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \
--cache-repository quay.io/nf-core/wave-cache \
--cache-repository quay.io/nf-core/wave-cache/${{ matrix.profile }}/${{ matrix.platform }} \
--tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \
--tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }}

Expand Down
32 changes: 10 additions & 22 deletions modules/nf-core/quartonotebook/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,25 +1,17 @@
#
# First stage: Quarto installation
#
FROM ubuntu:24.04 as quarto
FROM condaforge/mambaforge:24.9.2-0
#@sha256:e4edd160d0a10c6f577b127021d853b460e7e8c1bf1850c9aa4a52c4b8d9c8ee
ARG QUARTO_VERSION=1.3.433
ARG TARGETARCH
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
&& apt-get clean

RUN mkdir -p /opt/quarto \
&& curl -o quarto.tar.gz -L "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${TARGETARCH}.tar.gz" \
&& tar -zxvf quarto.tar.gz -C /opt/quarto/ --strip-components=1 \
&& rm quarto.tar.gz
LABEL org.opencontainers.image.authors="Erik Fasterius"
LABEL org.opencontainers.image.description="Dockerfile for the quartonotebook nf-core module"
LABEL org.opencontainers.image.version="$QUARTO_VERSION"
LABEL org.opencontainers.image.documentation="https://github.com/nf-core/modules/blob/master/modules/nf-core/quartonotebook/README.md"
LABEL org.opencontainers.image.source="https://github.com/nf-core/modules"
LABEL org.opencontainers.image.vendor="nf-core"
LABEL org.opencontainers.image.license="https://github.com/nf-core/modules/blob/master/LICENSE"

#
# Second stage: Conda environment
#
FROM condaforge/mambaforge:24.1.2-0@sha256:64c45c1a743737f61cf201f54cae974b5c853be94f9c1a84f5e82e0e854f0407
COPY --from=quarto /opt/quarto /opt/quarto
ADD https://github.com/quarto-dev/quarto-cli#v${QUARTO_VERSION} /opt/quarto
ENV PATH="${PATH}:/opt/quarto/bin"

# Install packages using Mamba; also remove static libraries, python bytecode
Expand All @@ -32,7 +24,3 @@ RUN mamba env update --name base --file environment.yml \
&& find /opt/conda -follow -type f -name '*.js.map' -delete

CMD /bin/bash

LABEL \
authors = "Erik Fasterius" \
description = "Dockerfile for the quartonotebook nf-core module"
8 changes: 7 additions & 1 deletion modules/nf-core/quartonotebook/environment.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
channels:
- conda-forge
- bioconda

dependencies:
# renovate: datasource=conda depName=conda-forge/quarto
# FIXME https://github.com/nf-core/modules/issues/7006
- conda-forge::quarto=1.5.57
# renovate: datasource=conda depName=conda-forge/jupyter
- conda-forge::jupyter=1.0.0
# renovate: datasource=conda depName=conda-forge/matplotlib
- conda-forge::matplotlib=3.4.3
# renovate: datasource=conda depName=conda-forge/papermill
- conda-forge::papermill=2.4.0
# renovate: datasource=conda depName=conda-forge/r-rmarkdown
- conda-forge::r-rmarkdown=2.25
50 changes: 30 additions & 20 deletions modules/nf-core/quartonotebook/main.nf
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
include { dumpParamsYaml; indentCodeBlock } from "./parametrize"
include {
dumpParamsYaml ;
indentCodeBlock
} from "./parametrize"

// NB: You'll likely want to override this with a container containing all
// required dependencies for your analyses. Or use wave to build the container
// for you from the environment.yml You'll at least need Quarto itself,
// Papermill and whatever language you are running your analyses on; you can see
// an example in this module's Dockerfile.
process QUARTONOTEBOOK {
tag "$meta.id"
tag "${meta.id}"
label 'process_low'

// NB: You'll likely want to override this with a container containing all
// required dependencies for your analyses. You'll at least need Quarto
// itself, Papermill and whatever language you are running your analyses on;
// you can see an example in this module's Dockerfile.
container "docker.io/erikfas/quartonotebook"
conda "${moduleDir}/environment.yml"
container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container
? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/25/25d0a9decefd5d369b8f9b8c934640bd61493df2c95c39b0c580e765e0d2a644/data'
: 'community.wave.seqera.io/library/quarto_jupyter_matplotlib_papermill_r-rmarkdown:95c6620495eabcd1'}"

input:
tuple val(meta), path(notebook)
Expand All @@ -17,29 +23,33 @@ process QUARTONOTEBOOK {
path extensions

output:
tuple val(meta), path("*.html") , emit: html
tuple val(meta), path("*.html"), emit: html
tuple val(meta), path("${notebook}"), emit: notebook
tuple val(meta), path("artifacts/*"), emit: artifacts, optional: true
tuple val(meta), path("params.yml") , emit: params_yaml, optional: true
tuple val(meta), path("params.yml"), emit: params_yaml, optional: true
tuple val(meta), path("_extensions"), emit: extensions, optional: true
path "versions.yml" , emit: versions
path "versions.yml", emit: versions

when:
task.ext.when == null || task.ext.when

script:
// Exit if running this module with -profile conda / -profile mamba
// FIXME https://github.com/nf-core/modules/issues/7006
// Exit if running this module with -profile conda / -profile mamba on ARM64
// This is because of issues with getting a homogenous environment across
// both AMD64 and ARM64 architectures; please find more information at
// https://github.com/nf-core/modules/pull/4876#discussion_r1483541037.
if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
exit 1, "The QUARTONOTEBOOK module does not support Conda/Mamba, please use Docker / Singularity / Podman instead."
arch = System.getProperty("os.arch")
if (arch == "arm64" || arch == "aarch64") {
exit(1, "The QUARTONOTEBOOK module does not support Conda/Mamba on ARM64. Please use Docker / Singularity / Podman instead.")
}
}
def args = task.ext.args ?: ''
def prefix = task.ext.prefix ?: "${meta.id}"
def parametrize = (task.ext.parametrize == null) ? true : task.ext.parametrize
def implicit_params = (task.ext.implicit_params == null) ? true : task.ext.implicit_params
def meta_params = (task.ext.meta_params == null) ? true : task.ext.meta_params
def parametrize = task.ext.parametrize == null ? true : task.ext.parametrize
def implicit_params = task.ext.implicit_params == null ? true : task.ext.implicit_params
def meta_params = task.ext.meta_params == null ? true : task.ext.meta_params

// Dump parameters to yaml file.
// Using a YAML file over using the CLI params because
Expand Down Expand Up @@ -74,10 +84,10 @@ process QUARTONOTEBOOK {
export XDG_DATA_HOME="./.xdg_data_home"

# Set parallelism for BLAS/MKL etc. to avoid over-booking of resources
export MKL_NUM_THREADS="$task.cpus"
export OPENBLAS_NUM_THREADS="$task.cpus"
export OMP_NUM_THREADS="$task.cpus"
export NUMBA_NUM_THREADS="$task.cpus"
export MKL_NUM_THREADS="${task.cpus}"
export OPENBLAS_NUM_THREADS="${task.cpus}"
export OMP_NUM_THREADS="${task.cpus}"
export NUMBA_NUM_THREADS="${task.cpus}"

# Render notebook
quarto render \\
Expand Down
8 changes: 4 additions & 4 deletions modules/nf-core/quartonotebook/tests/main.nf.test
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ nextflow_process {
assertAll(
{ assert process.success },
{ assert snapshot(
process.out.versions,
process.out.artifacts,
process.out.params_yaml,
path(process.out.versions.get(0)).yaml,
).match() },
{ assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } }
{ assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } },
)
}

Expand Down Expand Up @@ -116,11 +116,11 @@ nextflow_process {
assertAll(
{ assert process.success },
{ assert snapshot(
process.out.versions,
process.out.artifacts,
process.out.params_yaml,
path(process.out.versions.get(0)).yaml,
).match() },
{ assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } }
{ assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } },
)
}

Expand Down
Loading
Loading