Merge support for different precisions in benchmark

This PR exports the precision all benchmark code is compiled in into the file `benchmark/utils/types.hpp`. The `etype` defines in which precision the benchmark code is compiled in. All benchmark code is now compiled with the following precisions: - single - double - single complex - double complex Each generating a different binary, allowing the benchmarks to be run for different precisions with the `run_all_benchmarks.sh` script variable `BENCHMARK_PRECISION` or by appending a precision suffix to the benchmark binaries. Related PR: #675
ginkgo-project · Jan 23, 2021 · c5f0e48 · c5f0e48
2 parents 7284619 + b1fa08d
commit c5f0e48
Show file tree

Hide file tree

Showing 23 changed files with 302 additions and 174 deletions.
diff --git a/BENCHMARKING.md b/BENCHMARKING.md
@@ -156,6 +156,8 @@ benchmark options). Here are the most important options:
 * `SEGMENT_ID=<I>` - used in combination with the `SEGMENTS` variable. `<I>`
     should be an integer between 1 and `<N>`, the number of `SEGMENTS`. If
     specified, only the `<I>`-th segment of the benchmark suite will be run.
+* `BENCHMARK_PRECISION` - defines the precision the benchmarks are run in.
+    Supported values are: "double" (default), "single", "dcomplex" and "scomplex"
 * `MATRIX_LIST_FILE=/path/to/matrix_list.file` - allows to list SuiteSparse
     matrix id or name to benchmark. As an example, a matrix list file containing
     the following will ensure that benchmarks are ran for only those three

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
@@ -54,6 +54,51 @@ function(ginkgo_benchmark_hipsp_linops name)
     target_link_libraries("${name}" ${HIPSPARSE_LIBRARIES})
 endfunction()
 
+
+# Generates an executable for one precision. Each executable will be linked to
+# `ginkgo`, `gflags` and `rapidjson`.
+# Note: This should only be used by `ginkgo_add_typed_benchmark_executables`
+#
+# \param name            name for the executable to create (including type suffix)
+# \param use_lib_linops  Boolean indicating if linking against hipsparse/cusparse
+#                        is necessary
+# \param macro_def       preprocessor macro name that will be defined during
+#                        building (to compile for a specific type)
+# All remaining arguments will be treated as source files
+function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def)
+    add_executable("${name}" ${ARGN})
+    target_link_libraries("${name}" ginkgo gflags rapidjson)
+    target_compile_definitions("${name}" PRIVATE "${macro_def}")
+    if("${use_lib_linops}")
+        if (GINKGO_BUILD_CUDA)
+            ginkgo_benchmark_cusp_linops("${name}")
+        endif()
+        if (GINKGO_BUILD_HIP)
+            ginkgo_benchmark_hipsp_linops("${name}")
+        endif()
+    endif()
+endfunction(ginkgo_add_single_benchmark_executable)
+
+
+# Generates an executable for each supported precision. Each executable will be
+# linked to `ginkgo`, `gflags` and `rapidjson`.
+#
+# \param name            base-name for the executable to create
+# \param use_lib_linops  Boolean indicating if linking against hipsparse/cusparse
+#                        is necessary
+# All remaining arguments will be treated as source files
+function(ginkgo_add_typed_benchmark_executables name use_lib_linops)
+    ginkgo_add_single_benchmark_executable(
+        "${name}" use_lib_linops "GKO_BENCHMARK_USE_DOUBLE_PRECISION" ${ARGN})
+    ginkgo_add_single_benchmark_executable(
+        "${name}_single" use_lib_linops "GKO_BENCHMARK_USE_SINGLE_PRECISION" ${ARGN})
+    ginkgo_add_single_benchmark_executable(
+        "${name}_dcomplex" use_lib_linops "GKO_BENCHMARK_USE_DOUBLE_COMPLEX_PRECISION" ${ARGN})
+    ginkgo_add_single_benchmark_executable(
+        "${name}_scomplex" use_lib_linops "GKO_BENCHMARK_USE_SINGLE_COMPLEX_PRECISION" ${ARGN})
+endfunction(ginkgo_add_typed_benchmark_executables)
+
+
 add_subdirectory(conversions)
 add_subdirectory(matrix_generator)
 add_subdirectory(matrix_statistics)

diff --git a/benchmark/conversions/CMakeLists.txt b/benchmark/conversions/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(conversions conversions.cpp)
-target_link_libraries(conversions ginkgo gflags rapidjson)
+ginkgo_add_typed_benchmark_executables(conversions "NO" conversions.cpp)
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
@@ -48,9 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/spmv_common.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // This function supposes that management of `FLAGS_overwrite` is done before
@@ -66,7 +64,7 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         add_or_set_member(conversion_case, conversion_name,
                           rapidjson::Value(rapidjson::kObjectType), allocator);
 
-        gko::matrix_data<> data{gko::dim<2>{1, 1}, 1};
+        gko::matrix_data<etype> data{gko::dim<2>{1, 1}, 1};
         auto matrix_to =
             share(formats::matrix_factory.at(format_to)(exec, data));
         // warm run
@@ -138,7 +136,7 @@ int main(int argc, char *argv[])
 
         std::clog << "Running test case: " << test_case << std::endl;
         std::ifstream mtx_fd(test_case["filename"].GetString());
-        gko::matrix_data<> data;
+        gko::matrix_data<etype> data;
         try {
             data = gko::read_raw<etype>(mtx_fd);
         } catch (std::exception &e) {

diff --git a/benchmark/matrix_generator/CMakeLists.txt b/benchmark/matrix_generator/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(matrix_generator matrix_generator.cpp)
-target_link_libraries(matrix_generator ginkgo gflags rapidjson)
+ginkgo_add_typed_benchmark_executables(matrix_generator "NO" matrix_generator.cpp)
diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp
@@ -40,10 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "benchmark/utils/general.hpp"
-
-
-// some Ginkgo shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 namespace {
@@ -109,8 +106,8 @@ gko::matrix_data<etype> generate_block_diagonal(rapidjson::Value &config,
     auto num_blocks = config["num_blocks"].GetUint();
     auto block_size = config["block_size"].GetUint();
     auto block = gko::matrix_data<etype>(
-        gko::dim<2>(block_size), std::uniform_real_distribution<>(-1.0, 1.0),
-        engine);
+        gko::dim<2>(block_size),
+        std::uniform_real_distribution<rc_etype>(-1.0, 1.0), engine);
     return gko::matrix_data<etype>::diag(num_blocks, block);
 }
 

diff --git a/benchmark/matrix_statistics/CMakeLists.txt b/benchmark/matrix_statistics/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(matrix_statistics matrix_statistics.cpp)
-target_link_libraries(matrix_statistics ginkgo gflags rapidjson)
+ginkgo_add_typed_benchmark_executables(matrix_statistics "NO" matrix_statistics.cpp)
diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp
@@ -42,10 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/spmv_common.hpp"
-
-
-// some Ginkgo shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // See en.wikipedia.org/wiki/Five-number_summary

diff --git a/benchmark/preconditioner/CMakeLists.txt b/benchmark/preconditioner/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(preconditioner preconditioner.cpp)
-target_link_libraries(preconditioner ginkgo gflags rapidjson)
+ginkgo_add_typed_benchmark_executables(preconditioner "NO" preconditioner.cpp)
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
@@ -48,10 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/preconditioners.hpp"
 #include "benchmark/utils/spmv_common.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-// some shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // preconditioner generation and application

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
@@ -65,6 +65,25 @@ if [ ! "${SOLVERS_RHS}" ]; then
     SOLVERS_RHS="unit"
 fi
 
+if [ ! "${BENCHMARK_PRECISION}" ]; then
+    BENCHMARK_PRECISION="double"
+    echo "BENCHMARK_PRECISION not set - assuming \"${BENCHMARK_PRECISION}\"" 1>&2
+fi
+
+if [ "${BENCHMARK_PRECISION}" == "double" ]; then
+    BENCH_SUFFIX=""
+elif [ "${BENCHMARK_PRECISION}" == "single" ]; then
+    BENCH_SUFFIX="_single"
+elif [ "${BENCHMARK_PRECISION}" == "dcomplex" ]; then
+    BENCH_SUFFIX="_dcomplex"
+elif [ "${BENCHMARK_PRECISION}" == "scomplex" ]; then
+    BENCH_SUFFIX="_scomplex"
+else
+    echo "BENCHMARK_PRECISION is set to the not supported \"${BENCHMARK_PRECISION}\"." 1>&2
+    echo "Currently supported values: \"double\", \"single\", \"dcomplex\" and \"scomplex\"" 1>&2
+    exit 1
+fi
+
 if [ "${SOLVERS_RHS}" == "random" ]; then
     SOLVERS_RHS_FLAG="--random_rhs=true"
 else
@@ -132,7 +151,7 @@ keep_latest() {
 compute_matrix_statistics() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./matrix_statistics/matrix_statistics \
+    ./matrix_statistics/matrix_statistics${BENCH_SUFFIX} \
         --backup="$1.bkp" --double_buffer="$1.bkp2" \
         <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
@@ -147,7 +166,7 @@ compute_matrix_statistics() {
 run_conversion_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./conversions/conversions --backup="$1.bkp" --double_buffer="$1.bkp2" \
+    ./conversions/conversions${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
@@ -163,7 +182,7 @@ run_conversion_benchmarks() {
 run_spmv_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./spmv/spmv --backup="$1.bkp" --double_buffer="$1.bkp2" \
+    ./spmv/spmv${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
@@ -179,7 +198,7 @@ run_spmv_benchmarks() {
 run_solver_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./solver/solver --backup="$1.bkp" --double_buffer="$1.bkp2" \
+    ./solver/solver${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                     --executor="${EXECUTOR}" --solvers="${SOLVERS}" \
                     --preconditioners="${PRECONDS}" \
                     --max_iters=${SOLVERS_MAX_ITERATIONS} --rel_res_goal=${SOLVERS_PRECISION} \
@@ -204,7 +223,7 @@ run_preconditioner_benchmarks() {
         for prec in ${PRECISIONS}; do
             echo -e "\t\t running jacobi ($prec) for block size ${bsize}" 1>&2
             cp "$1" "$1.imd" # make sure we're not loosing the original input
-            ./preconditioner/preconditioner \
+            ./preconditioner/preconditioner${BENCH_SUFFIX} \
                 --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --preconditioners="jacobi" \
                 --jacobi_max_block_size="${bsize}" \
@@ -358,7 +377,7 @@ EOT
 generate_problem() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.tmp"
-    ./matrix_generator/matrix_generator <"$1.tmp" 2>&1 >"$1"
+    ./matrix_generator/matrix_generator${BENCH_SUFFIX} <"$1.tmp" 2>&1 >"$1"
     keep_latest "$1" "$1.tmp"
 }
 

diff --git a/benchmark/solver/CMakeLists.txt b/benchmark/solver/CMakeLists.txt
@@ -1,8 +1 @@
-add_executable(solver solver.cpp)
-target_link_libraries(solver ginkgo gflags rapidjson)
-if (GINKGO_BUILD_CUDA)
-    ginkgo_benchmark_cusp_linops(solver)
-endif()
-if (GINKGO_BUILD_HIP)
-    ginkgo_benchmark_hipsp_linops(solver)
-endif()
+ginkgo_add_typed_benchmark_executables(solver "YES" solver.cpp)
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
@@ -41,6 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <limits>
+#include <sstream>
 #include <vector>
 
 
@@ -50,10 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/overhead_linop.hpp"
 #include "benchmark/utils/preconditioners.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-// some Ginkgo shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // Command-line arguments
@@ -131,13 +130,15 @@ std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
 {
     std::shared_ptr<const gko::stop::CriterionFactory> residual_stop;
     if (FLAGS_rel_residual) {
-        residual_stop = gko::share(gko::stop::RelativeResidualNorm<>::build()
-                                       .with_tolerance(FLAGS_rel_res_goal)
-                                       .on(exec));
+        residual_stop = gko::share(
+            gko::stop::RelativeResidualNorm<rc_etype>::build()
+                .with_tolerance(static_cast<rc_etype>(FLAGS_rel_res_goal))
+                .on(exec));
     } else {
         residual_stop =
-            gko::share(gko::stop::ResidualNormReduction<>::build()
-                           .with_reduction_factor(FLAGS_rel_res_goal)
+            gko::share(gko::stop::ResidualNormReduction<rc_etype>::build()
+                           .with_reduction_factor(
+                               static_cast<rc_etype>(FLAGS_rel_res_goal))
                            .on(exec));
     }
     auto iteration_stop = gko::share(
@@ -164,25 +165,25 @@ std::unique_ptr<gko::LinOpFactory> create_solver(
 const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                                 std::shared_ptr<const gko::Executor>,
                                 std::shared_ptr<const gko::LinOpFactory>)>>
-    solver_factory{{"bicgstab", create_solver<gko::solver::Bicgstab<>>},
-                   {"bicg", create_solver<gko::solver::Bicg<>>},
-                   {"cg", create_solver<gko::solver::Cg<>>},
-                   {"cgs", create_solver<gko::solver::Cgs<>>},
-                   {"fcg", create_solver<gko::solver::Fcg<>>},
+    solver_factory{{"bicgstab", create_solver<gko::solver::Bicgstab<etype>>},
+                   {"bicg", create_solver<gko::solver::Bicg<etype>>},
+                   {"cg", create_solver<gko::solver::Cg<etype>>},
+                   {"cgs", create_solver<gko::solver::Cgs<etype>>},
+                   {"fcg", create_solver<gko::solver::Fcg<etype>>},
                    {"idr",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::Idr<>::build()
+                        return gko::solver::Idr<etype>::build()
                             .with_criteria(create_criterion(exec))
                             .with_subspace_dim(FLAGS_idr_subspace_dim)
-                            .with_kappa(FLAGS_idr_kappa)
+                            .with_kappa(static_cast<rc_etype>(FLAGS_idr_kappa))
                             .with_preconditioner(give(precond))
                             .on(exec);
                     }},
                    {"gmres",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::Gmres<>::build()
+                        return gko::solver::Gmres<etype>::build()
                             .with_criteria(create_criterion(exec))
                             .with_krylov_dim(FLAGS_gmres_restart)
                             .with_preconditioner(give(precond))
@@ -191,18 +192,18 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                    {"lower_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory>) {
-                        return gko::solver::LowerTrs<>::build()
+                        return gko::solver::LowerTrs<etype>::build()
                             .with_num_rhs(FLAGS_nrhs)
                             .on(exec);
                     }},
                    {"upper_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory>) {
-                        return gko::solver::UpperTrs<>::build()
+                        return gko::solver::UpperTrs<etype>::build()
                             .with_num_rhs(FLAGS_nrhs)
                             .on(exec);
                     }},
-                   {"overhead", create_solver<gko::Overhead<>>}};
+                   {"overhead", create_solver<gko::Overhead<etype>>}};
 
 
 void write_precond_info(const gko::LinOp *precond,
@@ -415,12 +416,14 @@ int main(int argc, char *argv[])
         "format\n\n";
     initialize_argument_parsing(&argc, &argv, header, format);
 
-    std::string extra_information = "Running " + FLAGS_solvers + " with " +
-                                    std::to_string(FLAGS_max_iters) +
-                                    " iterations and residual goal of " +
-                                    std::to_string(FLAGS_rel_res_goal) +
-                                    "\nThe number of right hand sides is " +
-                                    std::to_string(FLAGS_nrhs) + "\n";
+    std::stringstream ss_rel_res_goal;
+    ss_rel_res_goal << std::scientific << FLAGS_rel_res_goal;
+
+    std::string extra_information =
+        "Running " + FLAGS_solvers + " with " +
+        std::to_string(FLAGS_max_iters) + " iterations and residual goal of " +
+        ss_rel_res_goal.str() + "\nThe number of right hand sides is " +
+        std::to_string(FLAGS_nrhs) + "\n";
     print_general_information(extra_information);
 
     auto exec = get_executor();
@@ -472,13 +475,14 @@ int main(int argc, char *argv[])
             std::clog << "Running test case: " << test_case << std::endl;
             std::ifstream mtx_fd(test_case["filename"].GetString());
 
-            using Vec = gko::matrix::Dense<>;
+            using Vec = gko::matrix::Dense<etype>;
             std::shared_ptr<gko::LinOp> system_matrix;
             std::unique_ptr<Vec> b;
             std::unique_ptr<Vec> x;
             if (FLAGS_overhead) {
                 system_matrix = gko::initialize<Vec>({1.0}, exec);
-                b = gko::initialize<Vec>({std::nan("")}, exec);
+                b = gko::initialize<Vec>(
+                    {std::numeric_limits<rc_etype>::quiet_NaN()}, exec);
                 x = gko::initialize<Vec>({0.0}, exec);
             } else {
                 auto data = gko::read_raw<etype>(mtx_fd);

diff --git a/benchmark/spmv/CMakeLists.txt b/benchmark/spmv/CMakeLists.txt
@@ -1,8 +1 @@
-add_executable(spmv spmv.cpp)
-target_link_libraries(spmv ginkgo gflags rapidjson)
-if (GINKGO_BUILD_CUDA)
-    ginkgo_benchmark_cusp_linops(spmv)
-endif()
-if (GINKGO_BUILD_HIP)
-    ginkgo_benchmark_hipsp_linops(spmv)
-endif()
+ginkgo_add_typed_benchmark_executables(spmv "YES" spmv.cpp)