Improve test image and build/test process

Added mpi4py to build image and removed unwanted ohpc numpy package.

Use github api to retrieve real branch name, use that along with other
information to generate a meaningful build name.  To keep this from
happening too often, we only attempt to retrieve the branch name once
for each job during the first step, then communicate that name to
subsequent steps via an environment variable.  This required that we
stop using --login option when running the job steps from circleci,
as the lmod environment setup uses the same variable (BASH_ENV) to
store the path to the profile setup file that CircleCI sources for
each "run" in the job steps.

Use github api and the robot user oauth key (from CircleCI env) to post
an extra status message on each tested commit pointing to the CDash results.
By also adding an "update" step to the default build/test job and using
CTEST_UPDATE_VERSION_ONLY=TRUE, we can generate a deep link to the CDash
results specific to the appropriate commit.

.circleci/config.yml

@@ -3,15 +3,18 @@ defaults: &defaults
   steps:
     - checkout:
         path: /home/adios2/source
+    - run:
+        name: Update
+        command: bash source/scripts/circle/runOnCircle.sh update
     - run:
         name: Configure
-        command: bash --login source/scripts/circle/runOnCircle.sh configure
+        command: bash source/scripts/circle/runOnCircle.sh configure
     - run:
         name: Build
-        command: bash --login source/scripts/circle/runOnCircle.sh build
+        command: bash source/scripts/circle/runOnCircle.sh build
     - run:
         name: Test
-        command: bash --login source/scripts/circle/runOnCircle.sh test
+        command: bash source/scripts/circle/runOnCircle.sh test
 
 version: 2
 

scripts/circle/.dockerignore

@@ -1,3 +1,4 @@
 runOnCircle.sh
 postComment.sh
 *.cmake
+findStatus.py

scripts/circle/Dockerfile

@@ -13,8 +13,7 @@ RUN yum -y install \
 
 # Install gcc7 OpenHPC packages
 RUN yum -y install \
-      lmod-ohpc gnu7-compilers-ohpc openmpi-gnu7-ohpc phdf5-gnu7-openmpi-ohpc \
-      python-numpy-gnu7-ohpc
+      lmod-ohpc gnu7-compilers-ohpc openmpi-gnu7-ohpc phdf5-gnu7-openmpi-ohpc
 
 # Cleanup headers and packages
 RUN yum clean all
@@ -63,14 +62,23 @@ RUN cd source && \
 # Cleanup
 RUN rm -rf /opt/adios1/1.12.0/source
 
+# mpi4py
+WORKDIR /opt
+RUN . /etc/profile && \
+    module purge && \
+    module load gnu7 openmpi && \
+    curl -L https://bitbucket.org/mpi4py/mpi4py/downloads/mpi4py-2.0.0.tar.gz | \
+      tar -xzv && \
+    cd mpi4py-2.0.0 && \
+    python setup.py build && \
+    python setup.py install && \
+    cd ../ && rm -rf mpi4py-2.0.0
+
 # Create a non-root user to run the builds/tests
 RUN export uid=1001 gid=1001 && \
     mkdir -p /home/adios2 && \
     echo "adios2:x:${uid}:${gid}:adios2,,,:/home/adios2:/bin/bash" >> /etc/passwd && \
     echo "adios2:x:${uid}:" >> /etc/group && \
-    mkdir -p /etc/sudoers.d && \
-    echo "adios2 ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/adios2 && \
-    chmod 0440 /etc/sudoers.d/adios2 && \
     chown ${uid}:${gid} -R /home/adios2
 
 USER adios2

scripts/circle/circle_el7-gcc48.cmake

@@ -1,14 +1,15 @@
 # Client maintainer: chuck.atkins@kitware.com
 set(CTEST_SITE "CircleCI")
-set(CTEST_BUILD_NAME "$ENV{CIRCLE_BRANCH}_$ENV{CIRCLE_JOB}")
 set(CTEST_BUILD_CONFIGURATION Release)
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j4")
 set(CTEST_TEST_ARGS PARALLEL_LEVEL 4)
 
 set(dashboard_model Experimental)
-set(dashboard_binary_name "build-$ENV{CIRCLE_JOB}")
+set(dashboard_binary_name "build_$ENV{CIRCLE_JOB}")
 
+set(CTEST_GIT_COMMAND "/usr/bin/git")
+set(CTEST_UPDATE_VERSION_ONLY TRUE)
 set(CTEST_SOURCE_DIRECTORY "$ENV{CIRCLE_WORKING_DIRECTORY}/source")
 set(CTEST_DASHBOARD_ROOT "$ENV{HOME}")
 

scripts/circle/circle_el7-gcc7-openmpi.cmake

@@ -1,6 +1,5 @@
 # Client maintainer: chuck.atkins@kitware.com
 set(CTEST_SITE "CircleCI")
-set(CTEST_BUILD_NAME "$ENV{CIRCLE_BRANCH}_$ENV{CIRCLE_JOB}")
 set(CTEST_BUILD_CONFIGURATION Release)
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j4")
@@ -9,6 +8,8 @@ set(CTEST_TEST_ARGS PARALLEL_LEVEL 4)
 set(dashboard_model Experimental)
 set(dashboard_binary_name "build_$ENV{CIRCLE_JOB}")
 
+set(CTEST_GIT_COMMAND "/usr/bin/git")
+set(CTEST_UPDATE_VERSION_ONLY TRUE)
 set(CTEST_SOURCE_DIRECTORY "$ENV{CIRCLE_WORKING_DIRECTORY}/source")
 set(CTEST_DASHBOARD_ROOT "$ENV{HOME}")
 
@@ -17,7 +18,6 @@ module(purge)
 module(load gnu7)
 module(load openmpi)
 module(load phdf5)
-module(load python)
 
 set(ENV{CC}  gcc)
 set(ENV{CXX} g++)
@@ -30,7 +30,7 @@ ADIOS2_USE_DataMan:STRING=ON
 ADIOS2_USE_Fortran:STRING=ON
 ADIOS2_USE_HDF5:STRING=ON
 ADIOS2_USE_MPI:STRING=ON
-ADIOS2_USE_Python:STRING=OFF
+ADIOS2_USE_Python:STRING=ON
 ADIOS2_USE_ZFP:STRING=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
 ZFP_ROOT_DIR:PATH=/opt/zfp/install

scripts/circle/findStatus.py

@@ -0,0 +1,25 @@
+import sys
+import json
+import argparse
+
+
+def searchForContext(context):
+    print('Searching for a status with context: %s' % context)
+    statuses = json.load(sys.stdin)
+
+    for stat in statuses:
+        if 'context' in stat and stat['context'] == context:
+            sys.exit(0)
+
+    sys.exit(1)
+
+
+# =============================================================================
+# Main
+# =============================================================================
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Parse github api status list")
+    parser.add_argument("--context", default=None, help="context of interest")
+    args = parser.parse_args()
+    searchForContext(args.context)

scripts/circle/postComment.sh

@@ -37,3 +37,4 @@ for i in "${ADDR[@]}"; do
       echo "Unable to find PR number in ${i}"
     fi
 done
+

scripts/circle/runOnCircle.sh

@@ -1,5 +1,46 @@
 #!/bin/bash
 
+API_BASE="https://api.github.com/repos/ornladios/adios2"
+USER=${STATUS_ROBOT_NAME}
+TOKEN=${STATUS_ROBOT_KEY}
+COMMIT=${CIRCLE_SHA1}
+CDASH_STATUS_CONTEXT="cdash-results-link"
+
+build_status_body() {
+  cat <<EOF
+{
+  "state": "success",
+  "target_url": "https://open.cdash.org/index.php?compare1=61&filtercount=1&field1=revision&project=ADIOS&showfilters=0&limit=100&value1=${COMMIT}&showfeed=0",
+  "description": "Build and test results can also be viewed on CDash",
+  "context": "${CDASH_STATUS_CONTEXT}"
+}
+EOF
+}
+
+check_and_post_status() {
+  PYTHON_SCRIPT="${SOURCE_DIR}/scripts/circle/findStatus.py"
+  curl -u "${STATUS_ROBOT_NAME}:${STATUS_ROBOT_KEY}" "${API_BASE}/commits/${COMMIT}/statuses" | python ${PYTHON_SCRIPT} --context ${CDASH_STATUS_CONTEXT}
+  if [ $? -ne 0 ]
+  then
+    echo "Need to post a status for context ${CDASH_STATUS_CONTEXT}"
+    postBody="$(build_status_body)"
+    postUrl="${API_BASE}/statuses/${COMMIT}"
+    curl -u "${OWNER}:${TOKEN}" "${postUrl}" -H "Content-Type: application/json" -H "Accept: application/vnd.github.v3+json" -d "${postBody}"
+  fi
+}
+
+get_real_branch_name() {
+  APIURL="${API_BASE}/pulls/${CIRCLE_PR_NUMBER}"
+  RESULT=`curl -s ${APIURL} | python -c "import sys, json; print(json.load(sys.stdin)['head']['ref'])" 2> /dev/null`
+
+  if [ $? -eq 0 ]
+  then
+    REALBRANCH=$RESULT
+  else
+    REALBRANCH=$CIRCLE_BRANCH
+  fi
+}
+
 check_var() {
   if [ -z "$1" ]
   then
@@ -11,6 +52,21 @@ check_var() {
 check_var CIRCLE_WORKING_DIRECTORY
 check_var CIRCLE_BRANCH
 check_var CIRCLE_JOB
+check_var CIRCLE_BUILD_NUM
+check_var CIRCLE_PR_NUMBER
+
+if [ ! "${CUSTOM_BUILD_NAME}" ]
+then
+  get_real_branch_name
+
+  LINETOSAVE="export CUSTOM_BUILD_NAME=${REALBRANCH}_${CIRCLE_BUILD_NUM}_${CIRCLE_JOB}"
+
+  # Set the custom build name for this step
+  eval $LINETOSAVE
+
+  # Also make sure it will get set for the following steps
+  echo "${LINETOSAVE}" >> $BASH_ENV
+fi
 
 SOURCE_DIR=${CIRCLE_WORKING_DIRECTORY}/source
 CTEST_SCRIPT="${SOURCE_DIR}/scripts/circle/circle_${CIRCLE_JOB}.cmake"
@@ -22,13 +78,20 @@ then
 fi
 
 case "$1" in
-  configure|build|test)
+  update|configure|build|test)
     STEP=$1
+    if [ "$STEP" == "update" ]
+    then
+      check_and_post_status
+    fi
     ;;
   *)
-    echo "Usage: $0 (configure|build|test)"
+    echo "Usage: $0 (update|configure|build|test)"
     exit 3
     ;;
 esac
 
-/opt/cmake/3.6.0/bin/ctest -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF -Ddashboard_do_${STEP}=TRUE
+# Manually source the bash env setup, freeing up $BASH_ENV used by circleci
+. /etc/profile >/dev/null
+
+/opt/cmake/3.6.0/bin/ctest -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF -Ddashboard_do_${STEP}=TRUE -DCTEST_BUILD_NAME=${CUSTOM_BUILD_NAME}

scripts/dashboard/adios_common.cmake

@@ -32,6 +32,7 @@
 #   dashboard_cache           = Initial CMakeCache.txt file content
 
 #   dashboard_do_checkout  = True to enable source checkout via git
+#   dashboard_do_update    = True to enable source update
 #   dashboard_do_configure = True to enable the Configure step
 #   dashboard_do_build     = True to enable the Build step
 #   dashboard_do_test      = True to enable the Test step

scripts/dashboard/common.cmake

@@ -33,6 +33,7 @@
 #   dashboard_cache       = Initial CMakeCache.txt file content
 
 #   dashboard_do_checkout  = True to enable source checkout via git
+#   dashboard_do_update    = True to enable the Update step
 #   dashboard_do_configure = True to enable the Configure step
 #   dashboard_do_build     = True to enable the Build step
 #   dashboard_do_test      = True to enable the Test step
@@ -68,6 +69,10 @@ if(NOT DEFINED dashboard_full)
 endif()
 
 # Initialize all build steps to "ON"
+if(NOT DEFINED dashboard_do_update)
+  set(dashboard_do_update ${dashboard_full})
+endif()
+
 if(NOT DEFINED dashboard_do_checkout)
   set(dashboard_do_checkout ${dashboard_full})
 endif()
@@ -94,7 +99,7 @@ if(NOT DEFINED dashboard_do_memcheck)
 endif()
 
 if(NOT DEFINED dashboard_fresh)
-  if(dashboard_full OR dashboard_do_configure)
+  if(dashboard_full OR dashboard_do_update)
     set(dashboard_fresh TRUE)
   else()
     set(dashboard_fresh FALSE)
@@ -368,6 +373,18 @@ else()
   ctest_start(${dashboard_model} APPEND)
 endif()
 
+# Look for updates.
+if(dashboard_do_update)
+  if(COMMAND dashboard_hook_update)
+    dashboard_hook_update()
+  endif()
+  message("Calling ctest_update...")
+  ctest_update(RETURN_VALUE count)
+  set(CTEST_CHECKOUT_COMMAND) # checkout on first iteration only
+  message("Found ${count} changed files")
+  ctest_submit(PARTS Update)
+endif()
+
 if(dashboard_do_configure)
   if(COMMAND dashboard_hook_configure)
     dashboard_hook_configure()