Run time detection of avx2 for conda packages. (#957)

* Build conda CPU packages with run-time detection of AVX2.

* Add cudatoolkit 10.1 to conda list.

* Default to sse4 on x86_64.

acinclude/ax_check_cpu.m4

@@ -3,24 +3,22 @@
 AC_DEFUN([AX_CPU_ARCH], [
 
 AC_MSG_CHECKING([for cpu arch])
-AC_CANONICAL_TARGET
 
-case $target in
-  amd64-* | x86_64-*)
-    ARCH_CPUFLAGS="-mpopcnt"
-    ARCH_CXXFLAGS="-m64"
+  AC_CANONICAL_TARGET
 
-    AX_GCC_X86_CPU_SUPPORTS(avx2,
-        [ARCH_CPUFLAGS+=" -mavx2 -mf16c"],
-        [ARCH_CPUFLAGS+=" -msse4"])
-    ;;
-  aarch64*-*)
+  case $target in
+    amd64-* | x86_64-*)
+      ARCH_CPUFLAGS="-mpopcnt -msse4"
+      ARCH_CXXFLAGS="-m64"
+      ;;
+    aarch64*-*)
 dnl This is an arch for Nvidia Xavier a proper detection would be nice.
-    ARCH_CPUFLAGS="-march=armv8.2-a"
-    ;;
-  *) ;;
-esac
-AC_MSG_RESULT([$target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS])
+      ARCH_CPUFLAGS="-march=armv8.2-a"
+      ;;
+    *) ;;
+  esac
+
+AC_MSG_RESULT([$target CPUFLAGS+="$ARCH_CPUFLAGS" CXXFLAGS+="$ARCH_CXXFLAGS"])
 
 AC_SUBST(ARCH_CPUFLAGS)
 AC_SUBST(ARCH_CXXFLAGS)

conda/faiss-gpu/build.sh

@@ -1,5 +1,16 @@
+# Build avx2 version
+CXXFLAGS="-mavx2 -mf16c" ./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
+make -j $CPU_COUNT
+make -C python _swigfaiss_avx2.so
+make clean
+
+# Build vanilla version (no avx)
 ./configure --with-cuda=$CUDA_ROOT --with-cuda-arch="$CUDA_ARCH"
 make -j $CPU_COUNT
+make -C python _swigfaiss.so
+
+make -C python build
+
 cd python
-make
+
 $PYTHON setup.py install --single-version-externally-managed --record=record.txt

conda/faiss-gpu/conda_build_config.yaml

@@ -5,7 +5,7 @@ cudatoolkit:
   - 9.0
   - 9.2
   - 10.0
-#  - 10.1 # cudatoolkit 10.1 packages are not yet available on Anaconda.
+  - 10.1
 pin_run_as_build:
   cudatoolkit:
     max_pin: x.x

conda/faiss/build.sh

@@ -1,5 +1,16 @@
+# Build avx2 version
+CXXFLAGS="-mavx2 -mf16c" ./configure --without-cuda
+make -j $CPU_COUNT
+make -C python _swigfaiss_avx2.so
+make clean
+
+# Build vanilla version (no avx)
 ./configure --without-cuda
 make -j $CPU_COUNT
+make -C python _swigfaiss.so
+
+make -C python build
+
 cd python
-make
+
 $PYTHON setup.py install --single-version-externally-managed --record=record.txt

configure

@@ -2065,48 +2065,6 @@ $as_echo "$ac_res" >&6; }
   eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
 
 } # ac_fn_cxx_check_func
-
-# ac_fn_c_try_run LINENO
-# ----------------------
-# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes
-# that executables *can* be run.
-ac_fn_c_try_run ()
-{
-  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
-  if { { ac_try="$ac_link"
-case "(($ac_try" in
-  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
-  *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
-  (eval "$ac_link") 2>&5
-  ac_status=$?
-  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
-  test $ac_status = 0; } && { ac_try='./conftest$ac_exeext'
-  { { case "(($ac_try" in
-  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
-  *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
-$as_echo "$ac_try_echo"; } >&5
-  (eval "$ac_try") 2>&5
-  ac_status=$?
-  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
-  test $ac_status = 0; }; }; then :
-  ac_retval=0
-else
-  $as_echo "$as_me: program exited with status $ac_status" >&5
-       $as_echo "$as_me: failed program was:" >&5
-sed 's/^/| /' conftest.$ac_ext >&5
-
-       ac_retval=$ac_status
-fi
-  rm -rf conftest.dSYM conftest_ipa8_conftest.oo
-  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
-  as_fn_set_status $ac_retval
-
-} # ac_fn_c_try_run
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
@@ -6849,136 +6807,25 @@ test -n "$target_alias" &&
     NONENONEs,x,x, &&
   program_prefix=${target_alias}-
 
-ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for gcc __builtin_cpu_init function" >&5
-$as_echo_n "checking for gcc __builtin_cpu_init function... " >&6; }
-if ${ax_cv_gcc_check_x86_cpu_init+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  if test "$cross_compiling" = yes; then :
-  { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "cannot run test program while cross compiling
-See \`config.log' for more details" "$LINENO" 5; }
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-#include <stdlib.h>
-int
-main ()
-{
-__builtin_cpu_init ();
-  ;
-  return 0;
-}
-
-_ACEOF
-if ac_fn_c_try_run "$LINENO"; then :
-  ax_cv_gcc_check_x86_cpu_init=yes
-else
-  ax_cv_gcc_check_x86_cpu_init=no
-fi
-rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
-  conftest.$ac_objext conftest.beam conftest.$ac_ext
-fi
-
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_gcc_check_x86_cpu_init" >&5
-$as_echo "$ax_cv_gcc_check_x86_cpu_init" >&6; }
-  if test "X$ax_cv_gcc_check_x86_cpu_init" = "Xno"; then :
-  as_fn_error $? "Need GCC to support X86 CPU features tests" "$LINENO" 5
-fi
-
-
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for cpu arch" >&5
 $as_echo_n "checking for cpu arch... " >&6; }
 
 
-case $target in
-  amd64-* | x86_64-*)
-    ARCH_CPUFLAGS="-mpopcnt"
-    ARCH_CXXFLAGS="-m64"
-
-
-
-   ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-
-   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for x86 avx2 instruction support" >&5
-$as_echo_n "checking for x86 avx2 instruction support... " >&6; }
-if ${ax_cv_gcc_x86_cpu_supports_avx2+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  if test "$cross_compiling" = yes; then :
-  { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "cannot run test program while cross compiling
-See \`config.log' for more details" "$LINENO" 5; }
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-#include <stdlib.h>
-int
-main ()
-{
- __builtin_cpu_init ();
-         if (__builtin_cpu_supports("avx2"))
-           return 0;
-         return 1;
-
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_c_try_run "$LINENO"; then :
-  ax_cv_gcc_x86_cpu_supports_avx2=yes
-else
-  ax_cv_gcc_x86_cpu_supports_avx2=no
-
-fi
-rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
-  conftest.$ac_objext conftest.beam conftest.$ac_ext
-fi
-
-
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ax_cv_gcc_x86_cpu_supports_avx2" >&5
-$as_echo "$ax_cv_gcc_x86_cpu_supports_avx2" >&6; }
-   ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-   if test "x$ax_cv_gcc_x86_cpu_supports_avx2" = xyes; then :
-
-$as_echo "#define HAVE_AVX2_INSTRUCTIONS 1" >>confdefs.h
-
-          ARCH_CPUFLAGS+=" -mavx2 -mf16c"
-else
-  ARCH_CPUFLAGS+=" -msse4"
-
-fi
 
+  case $target in
+    amd64-* | x86_64-*)
+      ARCH_CPUFLAGS="-mpopcnt -msse4"
+      ARCH_CXXFLAGS="-m64"
+      ;;
+    aarch64*-*)
+      ARCH_CPUFLAGS="-march=armv8.2-a"
+      ;;
+    *) ;;
+  esac
 
-    ;;
-  aarch64*-*)
-    ARCH_CPUFLAGS="-march=armv8.2-a"
-    ;;
-  *) ;;
-esac
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS" >&5
-$as_echo "$target CPUFLAGS+=$ARCH_CPUFLAGS CXXFLAGS+=$ARCH_CXXFLAGS" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $target CPUFLAGS+=\"$ARCH_CPUFLAGS\" CXXFLAGS+=\"$ARCH_CXXFLAGS\"" >&5
+$as_echo "$target CPUFLAGS+=\"$ARCH_CPUFLAGS\" CXXFLAGS+=\"$ARCH_CXXFLAGS\"" >&6; }
 
 
 

makefile.inc.in

@@ -8,7 +8,7 @@ CXXCPP       = @CXXCPP@
 CPPFLAGS     = -DFINTEGER=int @CPPFLAGS@ @OPENMP_CXXFLAGS@ @NVCC_CPPFLAGS@
 CXXFLAGS     = -fPIC @ARCH_CXXFLAGS@ -Wno-sign-compare @CXXFLAGS@
 CPUFLAGS     = @ARCH_CPUFLAGS@
-LDFLAGS      = @OPENMP_CXXFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@
+LDFLAGS      = @OPENMP_LDFLAGS@ @LDFLAGS@ @NVCC_LDFLAGS@
 LIBS         = @BLAS_LIBS@ @LAPACK_LIBS@ @LIBS@ @NVCC_LIBS@
 PYTHONCFLAGS = @PYTHON_CFLAGS@ -I@NUMPY_INCLUDE@
 SWIGFLAGS    = -DSWIGWORDSIZE64