DBG: Add debug logs to track down segfault

.circleci/config.yml

@@ -124,9 +124,9 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - data-v8-{{ .Branch }}-{{ .Revision }}
-            - data-v8-{{ .Branch }}
-            - data-v8-
+            - data-v9-{{ .Branch }}-{{ .Revision }}
+            - data-v9-{{ .Branch }}
+            - data-v9-
       - run:
           name: Get test data from ds000005
           command: |
@@ -203,7 +203,7 @@ jobs:
               echo "sMRIPrep derivatives of ds000210 were cached"
             fi
       - save_cache:
-         key: data-v8-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
+         key: data-v9-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/data
             - /tmp/ds005/freesurfer
@@ -265,7 +265,7 @@ jobs:
             - /tmp/images
       - restore_cache:
           keys:
-            - data-v8-{{ .Revision }}
+            - data-v9-{{ .Revision }}
       - run:
           name: Docker authentication
           command: |
@@ -366,7 +366,7 @@ jobs:
             - /tmp/images
       - restore_cache:
           keys:
-            - data-v8-{{ .Branch }}-{{ .Revision }}
+            - data-v9-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           keys:
             - ds005-anat-v18-{{ .Branch }}-{{ .Revision }}
@@ -605,7 +605,7 @@ jobs:
             - /tmp/images
       - restore_cache:
           keys:
-            - data-v8-{{ .Branch }}-{{ .Revision }}
+            - data-v9-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           keys:
             - ds054-anat-v14-{{ .Branch }}-{{ .Revision }}
@@ -777,7 +777,7 @@ jobs:
             - /tmp/images
       - restore_cache:
           keys:
-            - data-v8-{{ .Branch }}-{{ .Revision }}
+            - data-v9-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           keys:
             - ds210-anat-v12-{{ .Branch }}-{{ .Revision }}

Dockerfile

@@ -1,12 +1,17 @@
-# Use Ubuntu 16.04 LTS
-FROM ubuntu:xenial-20200706
+# Use Ubuntu 20.04 LTS
+FROM ubuntu:focal-20210416
+
 
 # Pre-cache neurodebian key
 COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg
+ENV DEBIAN_FRONTEND="noninteractive" \
+    LANG="en_US.UTF-8" \
+    LC_ALL="en_US.UTF-8"
 
 # Prepare environment
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+                    apt-utils \
                     curl \
                     bzip2 \
                     ca-certificates \
@@ -15,19 +20,18 @@ RUN apt-get update && \
                     autoconf \
                     libtool \
                     pkg-config \
+                    graphviz \
+                    pandoc \
+                    pandoc-citeproc \
                     git && \
-    curl -sSL https://deb.nodesource.com/setup_10.x | bash - && \
+    curl -sSL https://deb.nodesource.com/setup_14.x | bash - && \
     apt-get install -y --no-install-recommends \
                     nodejs && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Install latest pandoc
-RUN curl -o pandoc-2.2.2.1-1-amd64.deb -sSL "https://github.com/jgm/pandoc/releases/download/2.2.2.1/pandoc-2.2.2.1-1-amd64.deb" && \
-    dpkg -i pandoc-2.2.2.1-1-amd64.deb && \
-    rm pandoc-2.2.2.1-1-amd64.deb
-
 # Installing freesurfer
-RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /opt \
+RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz \
+    | tar zxv --no-same-owner -C /opt \
     --exclude='freesurfer/diffusion' \
     --exclude='freesurfer/docs' \
     --exclude='freesurfer/fsfast' \
@@ -48,7 +52,7 @@ RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/frees
     --exclude='freesurfer/trctrain'
 
 # Simulate SetUpFreeSurfer.sh
-ENV FSL_DIR="/usr/share/fsl/5.0" \
+ENV FSL_DIR="/opt/fsl-5.0.11" \
     OS="Linux" \
     FS_OVERRIDE=0 \
     FIX_VERTEX_AREA="" \
@@ -65,33 +69,108 @@ ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
     MNI_PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
     PATH="$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH"
 
-# Installing Neurodebian packages (FSL, AFNI, git-annex)
+# Installing Neurodebian packages
 RUN curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \
     apt-key add /usr/local/etc/neurodebian.gpg && \
     (apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true)
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-                    fsl-core=5.0.9-5~nd16.04+1 \
-                    fsl-mni152-templates=5.0.7-2 \
-                    afni=16.2.07~dfsg.1-5~nd16.04+1 \
-                    convert3d \
-                    connectome-workbench=1.3.2-2~nd16.04+1 \
-                    git-annex-standalone && \
+                    connectome-workbench=1.5.0-1~nd20.04+1 \
+                    git-annex-standalone=8.20210223-1~ndall+1 && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Setting FSL and AFNI envvars
-ENV FSLDIR="/usr/share/fsl/5.0" \
+# FSL 5.0.11 (neurodocker build)
+RUN apt-get update -qq \
+    && apt-get install -y -q --no-install-recommends \
+           bc \
+           dc \
+           file \
+           libfontconfig1 \
+           libfreetype6 \
+           libgl1-mesa-dev \
+           libgl1-mesa-dri \
+           libglu1-mesa-dev \
+           libgomp1 \
+           libice6 \
+           libxcursor1 \
+           libxft2 \
+           libxinerama1 \
+           libxrandr2 \
+           libxrender1 \
+           libxt6 \
+           sudo \
+           wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && echo "Downloading FSL ..." \
+    && mkdir -p /opt/fsl-5.0.11 \
+    && curl -fsSL --retry 5 https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.11-centos6_64.tar.gz \
+    | tar -xz -C /opt/fsl-5.0.11 --strip-components 1 \
+    && echo "Installing FSL conda environment ..." \
+    && bash /opt/fsl-5.0.11/etc/fslconf/fslpython_install.sh -f /opt/fsl-5.0.11
+ENV FSLDIR="/opt/fsl-5.0.11" \
+    PATH="/opt/fsl-5.0.11/bin:$PATH" \
     FSLOUTPUTTYPE="NIFTI_GZ" \
     FSLMULTIFILEQUIT="TRUE" \
-    POSSUMDIR="/usr/share/fsl/5.0" \
-    FSLTCLSH="/usr/bin/tclsh" \
-    FSLWISH="/usr/bin/wish" \
-    AFNI_MODELPATH="/usr/lib/afni/models" \
+    FSLTCLSH="/opt/fsl-5.0.11/bin/fsltclsh" \
+    FSLWISH="/opt/fsl-5.0.11/bin/fslwish" \
+    FSLLOCKDIR="" \
+    FSLMACHINELIST="" \
+    FSLREMOTECALL="" \
+    FSLGECUDAQ="cuda.q" \
+    POSSUMDIR="/opt/fsl-5.0.11" \
+    LD_LIBRARY_PATH="/opt/fsl-5.0.11:$LD_LIBRARY_PATH"
+
+# Convert3D (neurodocker build)
+RUN echo "Downloading Convert3D ..." \
+    && mkdir -p /opt/convert3d-1.0.0 \
+    && curl -fsSL --retry 5 https://sourceforge.net/projects/c3d/files/c3d/1.0.0/c3d-1.0.0-Linux-x86_64.tar.gz/download \
+    | tar -xz -C /opt/convert3d-1.0.0 --strip-components 1
+ENV C3DPATH="/opt/convert3d-1.0.0" \
+    PATH="/opt/convert3d-1.0.0/bin:$PATH"
+
+# AFNI latest (neurodocker build)
+RUN apt-get update -qq \
+    && apt-get install -y -q --no-install-recommends \
+           apt-utils \
+           ed \
+           gsl-bin \
+           libglib2.0-0 \
+           libglu1-mesa-dev \
+           libglw1-mesa \
+           libgomp1 \
+           libjpeg62 \
+           libxm4 \
+           netpbm \
+           tcsh \
+           xfonts-base \
+           xvfb \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl -sSL --retry 5 -o /tmp/multiarch.deb http://archive.ubuntu.com/ubuntu/pool/main/g/glibc/multiarch-support_2.27-3ubuntu1.2_amd64.deb \
+    && dpkg -i /tmp/multiarch.deb \
+    && rm /tmp/multiarch.deb \
+    && curl -sSL --retry 5 -o /tmp/libxp6.deb http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb \
+    && dpkg -i /tmp/libxp6.deb \
+    && rm /tmp/libxp6.deb \
+    && curl -sSL --retry 5 -o /tmp/libpng.deb http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb \
+    && dpkg -i /tmp/libpng.deb \
+    && rm /tmp/libpng.deb \
+    && apt-get install -f \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && gsl2_path="$(find / -name 'libgsl.so.19' || printf '')" \
+    && if [ -n "$gsl2_path" ]; then \
+         ln -sfv "$gsl2_path" "$(dirname $gsl2_path)/libgsl.so.0"; \
+    fi \
+    && ldconfig \
+    && echo "Downloading AFNI ..." \
+    && mkdir -p /opt/afni-latest \
+    && curl -fsSL --retry 5 https://afni.nimh.nih.gov/pub/dist/tgz/linux_openmp_64.tgz \
+    | tar -xz -C /opt/afni-latest --strip-components 1
+ENV PATH="/opt/afni-latest:$PATH" \
     AFNI_IMSAVE_WARNINGS="NO" \
-    AFNI_TTATLAS_DATASET="/usr/share/afni/atlases" \
-    AFNI_PLUGINPATH="/usr/lib/afni/plugins" \
-    PATH="/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH" \
-    LD_LIBRARY_PATH="/usr/lib/fsl/5.0:$LD_LIBRARY_PATH"
+    AFNI_PLUGINPATH="/opt/afni-latest"
 
 # Installing ANTs 2.3.4 (NeuroDocker build)
 ENV ANTSPATH="/usr/lib/ants" \
@@ -101,7 +180,7 @@ RUN curl -sSL "https://dl.dropbox.com/s/gwf51ykkk5bifyj/ants-Linux-centos6_x86_6
     | tar -xzC $ANTSPATH --strip-components 1
 
 # Installing SVGO and bids-validator
-RUN npm install -g svgo bids-validator@1.5.4 \
+RUN npm install -g svgo@^2.3 bids-validator@1.5.7 \
   && rm -rf ~/.npm ~/.empty
 
 # Installing and setting up ICA_AROMA
@@ -113,9 +192,9 @@ ENV PATH="/opt/ICA-AROMA:$PATH" \
     AROMA_VERSION="0.4.5"
 
 # Installing and setting up miniconda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-4.5.11-Linux-x86_64.sh
+RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh && \
+    bash Miniconda3-py38_4.9.2-Linux-x86_64.sh -b -p /usr/local/miniconda && \
+    rm Miniconda3-py38_4.9.2-Linux-x86_64.sh
 
 # Set CPATH for packages relying on compiled libs (e.g. indexed_gzip)
 ENV PATH="/usr/local/miniconda/bin:$PATH" \
@@ -125,20 +204,18 @@ ENV PATH="/usr/local/miniconda/bin:$PATH" \
     PYTHONNOUSERSITE=1
 
 # Installing precomputed python packages
-RUN conda install -y python=3.7.1 \
-                     pip=19.1 \
-                     mkl=2018.0.3 \
-                     mkl-service \
-                     numpy=1.15.4 \
-                     scipy=1.1.0 \
-                     scikit-learn=0.19.1 \
-                     matplotlib=2.2.2 \
-                     pandas=0.23.4 \
-                     libxml2=2.9.8 \
-                     libxslt=1.1.32 \
-                     graphviz=2.40.1 \
-                     traits=4.6.0 \
-                     zlib; sync && \
+RUN conda install -y python=3.8 \
+                     pip=21.0 \
+                     mkl=2021.2 \
+                     mkl-service=2.3 \
+                     numpy=1.20 \
+                     scipy=1.6 \
+                     scikit-learn=0.24 \
+                     matplotlib=3.3 \
+                     pandas=1.2 \
+                     libxslt=1.1 \
+                     traits=6.2 \
+                     zstd=1.4; sync && \
     chmod -R a+rX /usr/local/miniconda; sync && \
     chmod +x /usr/local/miniconda/bin/*; sync && \
     conda clean -y --all && sync && \

fmriprep/cli/run.py

@@ -150,6 +150,7 @@ def main():
         from fmriprep.reports.core import generate_reports
         from pkg_resources import resource_filename as pkgrf
 
+        config.loggers.workflow.log(25, "Generating reports...")
         # Generate reports phase
         failed_reports = generate_reports(
             config.execution.participant_label,
@@ -164,10 +165,16 @@ def main():
         write_bidsignore(config.execution.fmriprep_dir)
 
         if failed_reports and not config.execution.notrack:
+            config.loggers.workflow.log(25, "Sending Sentry error...")
             sentry_sdk.capture_message(
                 "Report generation failed for %d subjects" % failed_reports,
                 level="error",
             )
+        if not (config.execution.notrack or sentry_sdk.Hub.current.client is None):
+            config.loggers.workflow.log(25, "Closing Sentry client pre-shutdown...")
+            sentry_sdk.Hub.current.client.close(timeout=2.0)
+
+        config.loggers.workflow.log(25, "Exiting...")
         sys.exit(int((errno + failed_reports) > 0))
 
 

fmriprep/utils/bids.py

@@ -148,10 +148,10 @@ def validate_input_dir(exec_env, bids_dir, participant_label):
         if ignored_subs:
             for sub in ignored_subs:
                 validator_config_dict["ignoredFiles"].append("/sub-%s/**" % sub)
-    with tempfile.NamedTemporaryFile('w+') as temp:
+    with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as temp:
         temp.write(json.dumps(validator_config_dict))
         temp.flush()
         try:
-            subprocess.check_call(['bids-validator', bids_dir, '-c', temp.name])
+            subprocess.check_call(['bids-validator', str(bids_dir), '-c', temp.name])
         except FileNotFoundError:
             print("bids-validator does not appear to be installed", file=sys.stderr)

fmriprep/utils/sentry.py

@@ -52,7 +52,8 @@ def sentry_setup():
     sentry_sdk.init("https://d5a16b0c38d84d1584dfc93b9fb1ade6@sentry.io/1137693",
                     release=release,
                     environment=environment,
-                    before_send=before_send)
+                    before_send=before_send,
+                    shutdown_timeout=0)
     with sentry_sdk.configure_scope() as scope:
         for k, v in config.get(flat=True).items():
             scope.set_tag(k, v)

wrapper/fmriprep_docker.py

@@ -136,6 +136,14 @@ def check_memory(image):
 
 
 def merge_help(wrapper_help, target_help):
+    def _get_posargs(usage):
+        posargs = []
+        for t_arg in t_usage.split('\n')[-3:]:
+            line = t_arg.lstrip()
+            if line[0].isalnum() or line[0] == "{":
+                posargs.append(line)
+        return " ".join(posargs)
+
     # Matches all flags with up to one nested square bracket
     opt_re = re.compile(r'(\[--?[\w-]+(?:[^\[\]]+(?:\[[^\[\]]+\])?)?\])')
     # Matches flag name only
@@ -151,7 +159,7 @@ def merge_help(wrapper_help, target_help):
     t_groups = t_details.split('\n\n')
 
     w_posargs = w_usage.split('\n')[-1].lstrip()
-    t_posargs = t_usage.split('\n')[-1].lstrip()
+    t_posargs = _get_posargs(t_usage)
 
     w_options = opt_re.findall(w_usage)
     w_flags = sum(map(flag_re.findall, w_options), [])