Fix build to use ScaLapack

* pass in explicitly scalapack library
* clean up checks for existence of linear algebra functions

CMakeLists.txt

@@ -297,6 +297,7 @@ endif()
 set(BML_SCALAPACK FALSE CACHE BOOL "Whether to use ScaLAPACK library")
 if(BML_SCALAPACK)
   add_definitions(-DBML_USE_SCALAPACK)
+  message(STATUS "SCALAPACK_LIBRARIES=${SCALAPACK_LIBRARIES}")
 endif()
 
 set(BML_INTERNAL_BLAS FALSE
@@ -339,32 +340,29 @@ if(NOT (LAPACK_FOUND OR NOBLAS OR MAGMA_FOUND))
   endif()
 endif()
 message(STATUS "LAPACK_LIBRARIES=${LAPACK_LIBRARIES}")
+if(MAGMA_FOUND)
+  if(BML_CUSOLVER)
+    message(STATUS "cuSOLVER: ${CUDA_cusolver_LIBRARY}")
+  endif()
+endif()
 
+#check existence of required math and linear algebra functions
+#in third party libraries
 include(CheckFunctionExists)
 include(CheckFortranFunctionExists)
+include(${PROJECT_SOURCE_DIR}/cmake/bmlCheckCFortranFunctionExists.cmake)
 
 set(CMAKE_REQUIRED_LIBRARIES ${LINK_LIBRARIES} -lm)
 check_function_exists(fabs HAVE_FABS)
 if(NOT HAVE_FABS)
   message(FATAL_ERROR "Could not find the fabs() function")
 endif()
-list(APPEND LINK_LIBRARIES -lm)
 
-include(${PROJECT_SOURCE_DIR}/cmake/bmlCheckCFortranFunctionExists.cmake)
-if(MAGMA_FOUND)
-  set(CMAKE_REQUIRED_LIBRARIES
-    ${MAGMA_LIBRARIES})
-  list(APPEND LINK_LIBRARIES ${MAGMA_LIBRARIES})
-  if(BML_CUSOLVER)
-    message(STATUS "cuSOLVER: ${CUDA_cusolver_LIBRARY}")
-    list(APPEND LINK_LIBRARIES ${CUDA_cusolver_LIBRARY})
-  endif()
-endif()
 if(BLAS_FOUND)
   add_definitions(-DHAVE_BLAS)
 
   set(CMAKE_REQUIRED_LIBRARIES
-    ${LINK_LIBRARIES} ${BLAS_LIBRARIES} ${OpenMP_C_FLAGS})
+    ${BLAS_LIBRARIES} ${OpenMP_C_FLAGS})
   bml_check_C_Fortran_function_exists(dgemm C_DGEMM REQUIRED)
   bml_check_C_Fortran_function_exists(sgemm C_SGEMM REQUIRED)
   bml_check_C_Fortran_function_exists(cgemm C_CGEMM REQUIRED)
@@ -390,15 +388,13 @@ if(BLAS_FOUND)
     -DC_DAXPY=${C_DAXPY}
     -DC_CAXPY=${C_CAXPY}
     -DC_ZAXPY=${C_ZAXPY})
-
-  list(APPEND LINK_LIBRARIES ${BLAS_LIBRARIES})
 endif()
 
 if(LAPACK_FOUND)
   add_definitions(-DHAVE_LAPACK)
 
   set(CMAKE_REQUIRED_LIBRARIES
-    ${LINK_LIBRARIES} ${LAPACK_LIBRARIES} ${OpenMP_C_FLAGS})
+    ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${OpenMP_C_FLAGS})
   bml_check_C_Fortran_function_exists(ssyev C_SSYEV REQUIRED)
   bml_check_C_Fortran_function_exists(dsyev C_DSYEV REQUIRED)
   bml_check_C_Fortran_function_exists(cheevr C_CHEEVR REQUIRED)
@@ -432,16 +428,17 @@ if(LAPACK_FOUND)
     -DC_DLACPY=${C_DLACPY}
     -DC_CLACPY=${C_CLACPY}
     -DC_ZLACPY=${C_ZLACPY})
-
-  list(APPEND LINK_LIBRARIES ${LAPACK_LIBRARIES})
 endif()
 
 if(BML_SCALAPACK)
+  set(CMAKE_REQUIRED_LIBRARIES ${SCALAPACK_LIBRARIES}
+      ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${OpenMP_C_FLAGS})
+  message(STATUS "SCALAPACK_LIBRARIES=${SCALAPACK_LIBRARIES}")
+  bml_check_C_Fortran_function_exists(numroc NUMROC REQUIRED)
   bml_check_C_Fortran_function_exists(pssyevd PSSYEVD REQUIRED)
   bml_check_C_Fortran_function_exists(pdsyevd PDSYEVD REQUIRED)
   bml_check_C_Fortran_function_exists(pcheevd PCHEEVD REQUIRED)
   bml_check_C_Fortran_function_exists(pzheevd PZHEEVD REQUIRED)
-  bml_check_C_Fortran_function_exists(numroc NUMROC REQUIRED)
   add_definitions(
     -DPSSYEVD=${PSSYEVD}
     -DPDSYEVD=${PDSYEVD}
@@ -450,6 +447,25 @@ if(BML_SCALAPACK)
     -DNUMROC=${NUMROC})
 endif()
 
+#append linear algebra libraries to list of libraries to link with
+#warning: order matters!
+if(MAGMA_FOUND)
+  list(APPEND LINK_LIBRARIES ${MAGMA_LIBRARIES})
+  if(BML_CUSOLVER)
+    list(APPEND LINK_LIBRARIES ${CUDA_cusolver_LIBRARY})
+  endif()
+endif()
+if(BML_SCALAPACK)
+  list(APPEND LINK_LIBRARIES ${SCALAPACK_LIBRARIES})
+endif()
+if(LAPACK_FOUND)
+  list(APPEND LINK_LIBRARIES ${LAPACK_LIBRARIES})
+endif()
+if(BLAS_FOUND)
+  list(APPEND LINK_LIBRARIES ${BLAS_LIBRARIES})
+endif()
+list(APPEND LINK_LIBRARIES -lm)
+
 # Check whether the compiler supports complex types. The Nvidia
 # compiler is known not to support them for example.
 

build.sh

@@ -74,7 +74,7 @@ EOF
     echo "CMAKE_CXX_FLAGS        Set C++ compiler flags      (default is '${CMAKE_CXX_FLAGS}')"
     echo "CMAKE_Fortran_FLAGS    Set Fortran compiler flags  (default is '${CMAKE_Fortran_FLAGS}')"
     echo "BLAS_LIBRARIES         Blas libraries              (default is '${BLAS_LIBRARIES}')"
-    echo "LAPACK LIBRARIES       Lapack libraries            (default is '${LAPACK_LIBRARIES}')"
+    echo "LAPACK_LIBRARIES       Lapack libraries            (default is '${LAPACK_LIBRARIES}')"
     echo "EXTRA_CFLAGS           Extra C flags               (default is '${EXTRA_CFLAGS}')"
     echo "EXTRA_FFLAGS           Extra fortran flags         (default is '${EXTRA_FFLAGS}')"
     echo "EXTRA_LINK_FLAGS       Add extra link flags        (default is '${EXTRA_LINK_FLAGS}')"
@@ -85,6 +85,7 @@ EOF
     echo "BML_CUSOLVER           Build with cuSOLVER         (default is ${BML_CUSOLVER})"
     echo "BML_XSMM               Build with XSMM             (default is ${BML_XSMM})"
     echo "BML_SCALAPACK          Build with SCALAPACK        (default is ${BML_SCALAPACK})"
+    echo "SCALAPACK_LIBRARIES    ScaLapack libraries         (default is ${SCALAPACK_LIBRARIES})"
     echo "BML_ELLBLOCK_MEMPOOL   Use ellblock memory pool    (default is ${BML_ELLBLOCK_MEMPOOL}"
     echo "CUDA_TOOLKIT_ROOT_DIR  Path to CUDA dir            (default is ${CUDA_TOOLKIT_ROOT_DIR})"
     echo "INTEL_OPT              {yes, no}                   (default is ${INTEL_OPT})"
@@ -115,6 +116,7 @@ set_defaults() {
     : ${CMAKE_Fortran_FLAGS:=}
     : ${BLAS_LIBRARIES:=}
     : ${LAPACK_LIBRARIES:=}
+    : ${SCALAPACK_LIBRARIES:=}
     : ${BML_TESTING:=yes}
     : ${BML_VALGRIND:=no}
     : ${BML_COVERAGE:=no}
@@ -187,6 +189,7 @@ configure() {
         -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
         -DBLAS_LIBRARIES="${BLAS_LIBRARIES}" \
         -DLAPACK_LIBRARIES="${LAPACK_LIBRARIES}" \
+        -DSCALAPACK_LIBRARIES="${SCALAPACK_LIBRARIES}" \
         -DBML_OPENMP="${BML_OPENMP}" \
         -DMKL_GPU="${MKL_GPU}" \
         -DBML_MPI="${BML_MPI}" \

scripts/ci-MPI-double-complex.sh

@@ -12,8 +12,8 @@ export BML_OPENMP=${BML_OPENMP:-no}
 export BML_INTERNAL_BLAS=${BML_INTERNAL_BLAS:-no}
 export BML_MPI=${BML_MPI:-yes}
 export TESTING_EXTRA_ARGS=${TESTING_EXTRA_ARGS:-"-R MPI-C-.*-double_complex"}
-export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:-"-lscalapack-openmpi"}
 export BML_SCALAPACK=${BML_SCALAPACK:-yes}
+export SCALAPACK_LIBRARIES=${SCALAPACK_LIBRARIES:=scalapack-openmpi.so}
 
 [[ -f ${basedir}/scripts/ci-defaults.sh ]] && . ${basedir}/scripts/ci-defaults.sh
 

scripts/ci-MPI-double-real.sh

@@ -12,8 +12,8 @@ export BML_OPENMP=${BML_OPENMP:-no}
 export BML_INTERNAL_BLAS=${BML_INTERNAL_BLAS:-no}
 export BML_MPI=${BML_MPI:-yes}
 export TESTING_EXTRA_ARGS=${TESTING_EXTRA_ARGS:-"-R MPI-C-.*-double_real"}
-export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:-"-lscalapack-openmpi"}
 export BML_SCALAPACK=${BML_SCALAPACK:-yes}
+export SCALAPACK_LIBRARIES=${SCALAPACK_LIBRARIES:=scalapack-openmpi.so}
 
 [[ -f ${basedir}/scripts/ci-defaults.sh ]] && . ${basedir}/scripts/ci-defaults.sh
 

scripts/ci-MPI-single-complex.sh

@@ -12,8 +12,8 @@ export BML_OPENMP=${BML_OPENMP:-no}
 export BML_INTERNAL_BLAS=${BML_INTERNAL_BLAS:-no}
 export BML_MPI=${BML_MPI:-yes}
 export TESTING_EXTRA_ARGS=${TESTING_EXTRA_ARGS:-"-R MPI-C-.*-single_complex"}
-export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:-"-lscalapack-openmpi"}
 export BML_SCALAPACK=${BML_SCALAPACK:-yes}
+export SCALAPACK_LIBRARIES=${SCALAPACK_LIBRARIES:=scalapack-openmpi.so}
 
 [[ -f ${basedir}/scripts/ci-defaults.sh ]] && . ${basedir}/scripts/ci-defaults.sh
 

scripts/ci-MPI-single-real.sh

@@ -12,8 +12,8 @@ export BML_OPENMP=${BML_OPENMP:-no}
 export BML_INTERNAL_BLAS=${BML_INTERNAL_BLAS:-no}
 export BML_MPI=${BML_MPI:-yes}
 export TESTING_EXTRA_ARGS=${TESTING_EXTRA_ARGS:-"-R MPI-C-.*-single_real"}
-export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:-"-lscalapack-openmpi"}
 export BML_SCALAPACK=${BML_SCALAPACK:-yes}
+export SCALAPACK_LIBRARIES=${SCALAPACK_LIBRARIES:=scalapack-openmpi.so}
 
 [[ -f ${basedir}/scripts/ci-defaults.sh ]] && . ${basedir}/scripts/ci-defaults.sh