Merge Add more CI for checking lowest requirement and fix nohipfft/zi…

…p_iterator

Add CI test, adjust the compiler version to check the lowest requirement, fix the corresponding error
This PR adds more CI test for checking lowest requirement.
Moreover, it fixes no hipfft issue, zip_iterator constructor in gcc5, and relax error bound when hip with old gcc.

Related PR: #989

.gitlab-ci.yml

@@ -530,12 +530,12 @@ build/cuda114/nompi/gcc/cuda/debug/shared:
     EXTRA_CMAKE_FLAGS: "-DCMAKE_CUDA_FLAGS=-diag-suppress=177"
     CUDA_ARCH: 61
 
-# HIP AMD
-build/amd/nompi/gcc/hip/debug/shared:
+# ROCm 3.5 and friends
+build/amd/nompi/gcc/rocm35/debug/shared:
   <<: *default_build_with_test
   extends:
     - .quick_test_condition
-    - .use_gko-amd-openmpi-gnu8-llvm7
+    - .use_gko-rocm35-openmpi-gnu5-llvm50
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -544,11 +544,11 @@ build/amd/nompi/gcc/hip/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
 
-build/amd/openmpi/clang/hip/release/static:
+build/amd/openmpi/clang/rocm35/release/static:
   <<: *default_build_with_test
   extends:
-    - .quick_test_condition
-    - .use_gko-amd-openmpi-gnu8-llvm7
+    - .full_test_condition
+    - .use_gko-rocm35-openmpi-gnu5-llvm50
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -560,17 +560,73 @@ build/amd/openmpi/clang/hip/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
-# Build HIP AMD without omp
-build/amd/nompi/clang/hip_wo_omp/release/shared:
+# ROCm 4.5 and friends
+build/amd/mvapich2/gcc/rocm45/release/shared:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-rocm45-mvapich2-gnu8-llvm8
+  variables:
+    <<: *default_variables
+    BUILD_OMP: "ON"
+    BUILD_HIP: "ON"
+    RUN_EXAMPLES: "ON"
+    BUILD_TYPE: "Release"
+
+build/amd/mvapich2/clang/rocm45/debug/shared:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-rocm45-mvapich2-gnu8-llvm8
+  variables:
+    <<: *default_variables
+    C_COMPILER: "clang"
+    CXX_COMPILER: "clang++"
+    BUILD_OMP: "ON"
+    BUILD_HIP: "ON"
+    RUN_EXAMPLES: "ON"
+    BUILD_TYPE: "Debug"
+
+# ROCm 5.0.2 and friends
+build/amd/openmpi/gcc/rocm502/debug/static:
   <<: *default_build_with_test
   extends:
     - .full_test_condition
-    - .use_gko-amd-openmpi-gnu8-llvm7
+    - .use_gko-rocm502-openmpi-gnu11-llvm11
+  variables:
+    <<: *default_variables
+    BUILD_OMP: "ON"
+    BUILD_HIP: "ON"
+    RUN_EXAMPLES: "ON"
+    BUILD_TYPE: "Debug"
+    BUILD_SHARED_LIBS: "OFF"
+
+build/amd/openmpi/clang/rocm502/release/shared:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-rocm502-openmpi-gnu11-llvm11
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
+    BUILD_OMP: "ON"
+    BUILD_HIP: "ON"
+    RUN_EXAMPLES: "ON"
+    BUILD_TYPE: "Release"
+
+# without omp
+build/amd/nompi/gcc/rocm502_wo_omp/release/shared:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-rocm502-openmpi-gnu11-llvm11
+  variables:
+    <<: *default_variables
+    BUILD_OMP: "OFF"
+    BUILD_MPI: "OFF"
     BUILD_HIP: "ON"
+    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Release"
 
 # no cuda but latest gcc and clang
@@ -597,18 +653,6 @@ build/nocuda/nompi/clang/core/release/shared:
     CXX_COMPILER: "clang++"
     BUILD_TYPE: "Release"
 
-build/nocuda/nompi/intel/core/debug/shared:
-  <<: *default_build_with_test
-  extends:
-    - .quick_test_condition
-    - .use_gko-nocuda-mvapich2-gnu9-llvm8-intel
-  variables:
-    <<: *default_variables
-    C_COMPILER: "icc"
-    CXX_COMPILER: "icpc"
-    BUILD_TYPE: "Debug"
-    FAST_TESTS: "ON"
-
 build/nocuda/nompi/gcc/omp/release/shared:
   <<: *default_build_with_test
   extends:
@@ -635,19 +679,46 @@ build/nocuda/openmpi/clang/omp/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
 
-build/nocuda/nompi/intel/omp/release/static:
+# nocuda with the oldest supported compiler
+build/nocuda/nompi/intel/core/debug/shared:
   <<: *default_build_with_test
   extends:
     - .quick_test_condition
-    - .use_gko-nocuda-mvapich2-gnu9-llvm8-intel
+    - .use_gko-nocuda-mvapich2-gnu5-llvm39-intel2018
   variables:
     <<: *default_variables
+    # intel with old gcc without include path leads error: identifier "____m128d" is undefined
+    CXX_FLAGS: "-I /opt/intel/include/icc"
     C_COMPILER: "icc"
     CXX_COMPILER: "icpc"
+    BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
+
+build/nocuda/nompi/gcc/omp/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-mvapich2-gnu5-llvm39-intel2018
+  variables:
+    <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
+build/nocuda-nomixed/nompi/clang/omp/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-mvapich2-gnu5-llvm39-intel2018
+  variables:
+    <<: *default_variables
+    C_COMPILER: "clang"
+    CXX_COMPILER: "clang++"
+    BUILD_OMP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OFF"
+    MIXED_PRECISION: "OFF"
+
 build/nocuda-nomixed/openmpi/gcc/omp/release/shared:
   <<: *default_build_with_test
   extends:
@@ -675,20 +746,6 @@ build/nocuda-nomixed/nompi/clang/omp/debug/static:
     BUILD_SHARED_LIBS: "OFF"
     MIXED_PRECISION: "OFF"
 
-build/nocuda-nomixed/nompi/intel/omp/release/static:
-  <<: *default_build_with_test
-  extends:
-    - .full_test_condition
-    - .use_gko-nocuda-mvapich2-gnu9-llvm8-intel
-  variables:
-    <<: *default_variables
-    C_COMPILER: "icc"
-    CXX_COMPILER: "icpc"
-    BUILD_OMP: "ON"
-    BUILD_TYPE: "Release"
-    BUILD_SHARED_LIBS: "OFF"
-    MIXED_PRECISION: "OFF"
-
 build/dpcpp/cpu/release/static:
   <<: *default_build_with_test
   extends:
@@ -1021,7 +1078,7 @@ cudamemcheck:
 fineci-benchmark-build:
   stage: benchmark-build
   extends:
-    - .use_gko-nocuda-mvapich2-gnu9-llvm8-intel
+    - .use_gko-nocuda-mvapich2-gnu5-llvm39-intel2018
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1099,7 +1156,7 @@ fineci-benchmark-build:
 fineci-benchmark-cuda:
   stage: benchmark-cuda
   extends:
-    - .use_gko-nocuda-mvapich2-gnu9-llvm8-intel
+    - .use_gko-nocuda-mvapich2-gnu5-llvm39-intel2018
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI

.gitlab/image.yml

@@ -5,8 +5,8 @@
     - cpu
     - amdci
 
-.use_gko-nocuda-mvapich2-gnu9-llvm8-intel:
-  image: ginkgohub/cpu:mvapich2-gnu9-llvm8-intel2020
+.use_gko-nocuda-mvapich2-gnu5-llvm39-intel2018:
+  image: ginkgohub/cpu:mvapich2-gnu5-llvm39-intel2018
   tags:
     - private_ci
     - cpu
@@ -72,8 +72,22 @@
     - private_ci
     - nvidia-gpu
 
-.use_gko-amd-openmpi-gnu8-llvm7:
-  image: ginkgohub/rocm:gnu8-openmpi-llvm7
+.use_gko-rocm35-openmpi-gnu5-llvm50:
+  image: ginkgohub/rocm:35-openmpi-gnu5-llvm50
+  tags:
+    - private_ci
+    - amdci
+    - gpu
+
+.use_gko-rocm45-mvapich2-gnu8-llvm8:
+  image: ginkgohub/rocm:45-mvapich2-gnu8-llvm8
+  tags:
+    - private_ci
+    - amdci
+    - gpu
+
+.use_gko-rocm502-openmpi-gnu11-llvm11:
+  image: ginkgohub/rocm:502-openmpi-gnu11-llvm11
   tags:
     - private_ci
     - amdci

CMakeLists.txt

@@ -380,6 +380,7 @@ add_custom_target(test_install
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
     -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
+    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
     # `--config cfg` is ignored by single-configuration generator.
     # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
     # single-configuration generator.
@@ -400,6 +401,7 @@ add_custom_target(test_exportbuild
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
     -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
+    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
     # `--config cfg` is ignored by single-configuration generator.
     # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
     # single-configuration generator.

core/base/iterator_factory.hpp

@@ -84,7 +84,11 @@ class zip_iterator_reference
     template <std::size_t... idxs>
     value_type cast_impl(std::index_sequence<idxs...>) const
     {
-        return {std::get<idxs>(*this)...};
+        // gcc 5 throws error as using unintialized array
+        // std::tuple<int, char> t = { 1, '2' }; is not allowed.
+        // converting to 'std::tuple<...>' from initializer list would use
+        // explicit constructor
+        return value_type(std::get<idxs>(*this)...);
     }
 
     template <std::size_t... idxs>

hip/CMakeLists.txt

@@ -1,3 +1,7 @@
+if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
+    set(CMAKE_HIP_ARCHITECTURES OFF)
+endif()
+
 if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}"
     AND GINKGO_BUILD_CUDA AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)
     message(FATAL_ERROR "Ginkgo HIP backend requires CUDA >= 9.2.")
@@ -275,7 +279,12 @@ if(GINKGO_HIP_AMDGPU)
     endforeach()
 endif()
 
-set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS} "-std=c++14 -DGKO_COMPILING_HIP")
+if(GINKGO_STATIC_OR_SHARED MATCHES "STATIC")
+    # Debug Static: Hip requires PIC flags
+    set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS} "-std=c++14 -DGKO_COMPILING_HIP $<$<CONFIG:Debug>:-fPIC>")
+else()
+    set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS} "-std=c++14 -DGKO_COMPILING_HIP")
+endif()
 set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_ARCH} ${GINKGO_HIP_NVCC_ADDITIONAL_FLAGS})
 set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 

hip/matrix/fft_kernels_stub.hip.cpp

@@ -39,6 +39,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 namespace gko {
+
+
+std::string HipfftError::get_error(int64 error_code)
+{
+    return "Does not contain hipfft.";
+}
+
+
 namespace kernels {
 namespace hip {
 /**

test/base/device_matrix_data_kernels.cpp

@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/device_matrix_data_kernels.hpp"
+#include <ginkgo/core/base/device_matrix_data.hpp>
 
 
 #include <memory>
@@ -41,11 +41,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/device_matrix_data.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/matrix_data.hpp>
 
 
+#include "core/base/device_matrix_data_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "core/test/utils/assertions.hpp"
 #include "test/utils/executor.hpp"
@@ -365,7 +365,8 @@ TYPED_TEST(DeviceMatrixData, SumsDuplicates)
             max_error, std::abs(arrays.values.get_const_data()[i] -
                                 ref_arrays.values.get_const_data()[i]));
     }
-    ASSERT_LT(max_error, r<value_type>::value);
+    // when Hip with GNU < 7, it will give a little difference.
+    ASSERT_LT(max_error, 2 * r<value_type>::value);
 }
 
 

test/test_install/CMakeLists.txt

@@ -57,6 +57,8 @@ if(GINKGO_BUILD_HIP)
     if (GINKGO_BUILD_SHARED_LIBS)
         set (GINKGO_PIC_OPTION "-fPIC")
         set (GINKGO_CUDA_PIC_OPTION "-Xcompiler '-fPIC'")
+    else()
+        set (GINKGO_PIC_OPTION "$<$<CONFIG:Debug>:-fPIC>")
     endif()
     if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
         set(TESTINSTALL_CUDA_ARCH_FLAGS "${GINKGO_CUDA_ARCH_FLAGS}")