From 2800d49fc27ae133c4ab04a5b8175f97aae180dd Mon Sep 17 00:00:00 2001 From: oesteban Date: Tue, 8 Oct 2019 14:11:40 -0700 Subject: [PATCH] enh: add a ``sbatch.slurm`` submission file for SLURM and update singularity --- docs/_static/sbatch.slurm | 60 +++++++++++++++++++++++++++++++++++++ docs/singularity.rst | 63 ++++++++++----------------------------- 2 files changed, 76 insertions(+), 47 deletions(-) create mode 100644 docs/_static/sbatch.slurm diff --git a/docs/_static/sbatch.slurm b/docs/_static/sbatch.slurm new file mode 100644 index 000000000..5b3c66f9e --- /dev/null +++ b/docs/_static/sbatch.slurm @@ -0,0 +1,60 @@ +#!/bin/bash +# +#SBATCH -J fmriprep +#SBATCH --time=48:00:00 +#SBATCH -n 1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem-per-cpu=4G +#SBATCH -p normal,mygroup # Queue names you can submit to +# Outputs ---------------------------------- +#SBATCH -o log/%x-%A-%a.out +#SBATCH -e log/%x-%A-%a.err +#SBATCH --mail-user=%u@domain.tld +#SBATCH --mail-type=ALL +# ------------------------------------------ + +BIDS_DIR="$STUDY/data" +DERIVS_DIR="derivatives/fmriprep-1.5.0" + +# Prepare some writeable bind-mount points. +TEMPLATEFLOW_HOST_HOME=$HOME/.cache/templateflow +FMRIPREP_HOST_CACHE=$HOME/.cache/fmriprep +mkdir -p ${TEMPLATEFLOW_HOST_HOME} +mkdir -p ${FMRIPREP_HOST_CACHE} + +# Prepare derivatives folder +mkdir -p ${BIDS_DIR}/${DERIVS_DIR} + +# This trick will help you reuse freesurfer results across pipelines and fMRIPrep versions +mkdir -p ${BIDS_DIR}/derivatives/freesurfer-6.0.1 +if [ ! -d ${BIDS_DIR}/${DERIVS_DIR}/freesurfer ]; then + ln -s ${BIDS_DIR}/derivatives/freesurfer-6.0.1 ${BIDS_DIR}/${DERIVS_DIR}/freesurfer +fi + +# Make sure FS_LICENSE is defined in the container. +export SINGULARITYENV_FS_LICENSE=$HOME/.freesurfer.txt + +# Designate a templateflow bind-mount point +export SINGULARITYENV_TEMPLATEFLOW_HOME="/templateflow" +SINGULARITY_CMD="singularity run --cleanenv -B $BIDS_DIR:/data -B ${TEMPLATEFLOW_HOST_HOME}:${SINGULARITYENV_TEMPLATEFLOW_HOME} -B $L_SCRATCH:/work $STUDY/images/poldracklab_fmriprep_1.5.0.simg" + +# Parse the participants.tsv file and extract one subject ID from the line corresponding to this SLURM task. +subject=$( sed -n -E "$((${SLURM_ARRAY_TASK_ID} + 1))s/sub-(\S*)\>.*/\1/gp" ${BIDS_DIR}/participants.tsv ) + +# Remove IsRunning files from FreeSurfer +find ${BIDS_DIR}/derivatives/freesurfer-6.0.1/sub-$subject/ -name "*IsRunning*" -type f -delete + +# Compose the command line +cmd="${SINGULARITY_CMD} /data /data/${DERIVS_DIR} participant --participant-label $subject -w /work/ -vv --omp-nthreads 8 --nthreads 12 --mem_mb 30000 --output-spaces MNI152NLin2009cAsym:res-2 anat fsnative fsaverage5 --use-aroma" + +# Setup done, run the command +echo Running task ${SLURM_ARRAY_TASK_ID} +echo Commandline: $cmd +eval $cmd +exitcode=$? + +# Output results to a table +echo "sub-$subject ${SLURM_ARRAY_TASK_ID} $exitcode" \ + >> ${SLURM_JOB_NAME}.${SLURM_ARRAY_JOB_ID}.tsv +echo Finished tasks ${SLURM_ARRAY_TASK_ID} with exit code $exitcode +exit $exitcode \ No newline at end of file diff --git a/docs/singularity.rst b/docs/singularity.rst index 3a9bbbaff..adba1e07c 100644 --- a/docs/singularity.rst +++ b/docs/singularity.rst @@ -315,53 +315,22 @@ The next step is checking the environment and access to *fMRIPrep* resources, us ... + Running Singularity on a SLURM system ------------------------------------- -An example of ``sbatch`` script to run *fMRIPrep* on a SLURM system with Singularity -available is given below: :: - - #!/bin/bash - # - #SBATCH -J fmriprep - #SBATCH --array=1-36 # Replace indices with the right number of subjects - #SBATCH --time=48:00:00 - #SBATCH -n 1 - #SBATCH --cpus-per-task=16 - #SBATCH --mem-per-cpu=4G - #SBATCH -p queues,you,can,submit # Partition names, separated by comma - # Outputs ---------------------------------- - #SBATCH -o log/%x-%A-%a.out - #SBATCH -e log/%x-%A-%a.err - #SBATCH --mail-user=%u@domain.tld - #SBATCH --mail-type=ALL - # ------------------------------------------ - - BIDS_DIR="$PROJECT/data/ds000109" - DERIVS_DIR="derivatives/fmriprep-1.5.0" - - mkdir -p $HOME/.cache/templateflow - mkdir -p ${BIDS_DIR}/${DERIVS_DIR} - mkdir -p ${BIDS_DIR}/derivatives/freesurfer-6.0.1 - ln -s ${BIDS_DIR}/derivatives/freesurfer-6.0.1 ${BIDS_DIR}/${DERIVS_DIR}/freesurfer - - - export SINGULARITYENV_FS_LICENSE=$HOME/.freesurfer.txt - export SINGULARITYENV_TEMPLATEFLOW_HOME="/templateflow" - SINGULARITY_CMD="singularity run --cleanenv -B $PROJECT:/project -B $HOME/.cache/templateflow:/templateflow -B $L_SCRATCH:/work $PROJECT/images/poldracklab_fmriprep_1.5.0-2019-09-10-6157fec3d0ea.simg" - - - subject=$( sed -n -E "$((${SLURM_ARRAY_TASK_ID} + 1))s/sub-(\S*)\>.*/\1/gp" ${BIDS_DIR}/participants.tsv ) - cmd="${SINGULARITY_CMD} /project/data/ds000109 /project/data/ds000109/${DERIVS_DIR} participant --participant-label $subject -w /work/ -vv --omp-nthreads 8 --nthreads 12 --mem_mb 30000 --output-spaces MNI152NLin2009cAsym:res-2 anat fsnative fsaverage5 --cifti-output --use-aroma" - - # Setup done, run the command - echo Running task ${SLURM_ARRAY_TASK_ID} - echo Commandline: $cmd - eval $cmd - exitcode=$? - - # Output results to a table - echo "sub-$subject ${SLURM_ARRAY_TASK_ID} $exitcode" \ - >> ${SLURM_JOB_NAME}.${SLURM_ARRAY_JOB_ID}.tsv - echo Finished tasks ${SLURM_ARRAY_TASK_ID} with exit code $exitcode - exit $exitcode +An example of ``sbatch`` script to run *fMRIPrep* on a SLURM system [#1]_ is given `below `__. +The submission script will generate one task per subject using a *job array*. +Submission is as easy as: +:: + + $ export STUDY=/path/to/some/folder + $ sbatch --array=1-$(( $( wc -l $STUDY/data/participants.tsv | cut -f1 -d' ' ) - 1 )) sbatch.slurm + + +.. literalinclude:: _static/sbatch.slurm + :language: bash + :name: sbatch.slurm + :caption: **sbatch.slurm**: + +.. [#1] assuming that *job arrays* and Singularity are available