From 3aff0e2be98f3712dc26394222a290ddd784fe23 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Mon, 14 Aug 2023 10:38:52 +0200 Subject: [PATCH 1/6] allow passing matrix filename as benchmark input --- benchmark/blas/blas.cpp | 2 ++ benchmark/blas/distributed/multi_vector.cpp | 2 ++ .../matrix_generator/matrix_generator.cpp | 2 ++ benchmark/solver/distributed/solver.cpp | 2 ++ benchmark/solver/solver.cpp | 2 ++ benchmark/utils/general.hpp | 19 ++++++++++++++++++- 6 files changed, 28 insertions(+), 1 deletion(-) diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp index 11228ed5818..ce3a98a9a68 100644 --- a/benchmark/blas/blas.cpp +++ b/benchmark/blas/blas.cpp @@ -131,6 +131,8 @@ Parameters for a benchmark case are: stride_C: stride for C matrix in gemm (optional, default m) )"; std::string format = example_config; + // this benchmark doesn't use input matrices + matrix_input = false; initialize_argument_parsing(&argc, &argv, header, format); std::string extra_information = diff --git a/benchmark/blas/distributed/multi_vector.cpp b/benchmark/blas/distributed/multi_vector.cpp index be326b08b96..bcf77594c4b 100644 --- a/benchmark/blas/distributed/multi_vector.cpp +++ b/benchmark/blas/distributed/multi_vector.cpp @@ -61,6 +61,8 @@ Parameters for a benchmark case are: stride_y: stride for in/out vector y (optional, default r) )"; std::string format = example_config; + // this benchmark doesn't use input matrices + matrix_input = false; initialize_argument_parsing(&argc, &argv, header, format); const auto comm = gko::experimental::mpi::communicator(MPI_COMM_WORLD); diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp index 138b5a9c2ce..94883f5f63d 100644 --- a/benchmark/matrix_generator/matrix_generator.cpp +++ b/benchmark/matrix_generator/matrix_generator.cpp @@ -127,6 +127,8 @@ int main(int argc, char* argv[]) std::string header = "A utility that generates various types of " "matrices.\n"; + // this benchmark doesn't use input matrices + matrix_input = false; initialize_argument_parsing(&argc, &argv, header, input_format); std::clog << gko::version_info::get() << std::endl; diff --git a/benchmark/solver/distributed/solver.cpp b/benchmark/solver/distributed/solver.cpp index 2db71c16ca3..4f583bcd8a8 100644 --- a/benchmark/solver/distributed/solver.cpp +++ b/benchmark/solver/distributed/solver.cpp @@ -98,6 +98,8 @@ int main(int argc, char* argv[]) "-", where both "local_format" and "non_local_format" can be any of the recognized spmv formats. )"; + // this benchmark needs an additional "optimal" object in the input + matrix_input_additional_json = ",\"optimal\":{\"spmv\":\"csr-csr\"}"; initialize_argument_parsing(&argc, &argv, header, format); const auto comm = gko::experimental::mpi::communicator(MPI_COMM_WORLD); diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp index 9190c99dad0..c5010116dea 100644 --- a/benchmark/solver/solver.cpp +++ b/benchmark/solver/solver.cpp @@ -61,6 +61,8 @@ int main(int argc, char* argv[]) std::string format = example_config + R"( "optimal":"spmv" can be one of the recognized spmv formats )"; + // this benchmark needs an additional "optimal" object in the input + matrix_input_additional_json = ",\"optimal\":{\"spmv\":\"csr\"}"; initialize_argument_parsing(&argc, &argv, header, format); std::stringstream ss_rel_res_goal; diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp index 335ed687002..d1b39263468 100644 --- a/benchmark/utils/general.hpp +++ b/benchmark/utils/general.hpp @@ -95,7 +95,9 @@ DEFINE_string(double_buffer, "", DEFINE_string( input, "", "If set, the value is used as the input for the benchmark (if set to a " - "json string ending with ]) or as input file path (otherwise)."); + "json string ending with ]), as the \"filename\" of a generated JSON input " + "(if the variable points to a MatrixMarket or Ginkgo binary matrix file) " + "or as JSON input file path (otherwise)."); DEFINE_bool(detailed, true, "If set, performs several runs to obtain more detailed results"); @@ -297,6 +299,12 @@ std::vector split(const std::string& s, char delimiter = ',') } +// allow matrix files as -input value +bool matrix_input = true; +// additional JSON to append to the input_str if the input file is a matrix file +std::string matrix_input_additional_json = ""; + + // returns the stream to be used as input of the application std::istream& get_input_stream() { @@ -308,6 +316,15 @@ std::istream& get_input_stream() if (input_str.back() == ']') { return std::make_unique(input_str); } + if (matrix_input) { + auto first_char = std::ifstream{input_str}.peek(); + // if the input looks like a MatrixMarket or Ginkgo binary file + if (first_char == '%' || first_char == 'G') { + input_str = "[{\"filename\":\"" + input_str + "\"" + + matrix_input_additional_json + "}]"; + return std::make_unique(input_str); + } + } return std::make_unique(input_str); }(); if (stream) { From faf9724935de9c9b9b3b5c147d021ae4b3359596 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Wed, 16 Aug 2023 13:33:24 +0200 Subject: [PATCH 2/6] add `-input_matrix` flag --- benchmark/blas/blas.cpp | 2 - benchmark/blas/distributed/multi_vector.cpp | 2 - benchmark/conversions/conversions.cpp | 4 +- .../matrix_generator/matrix_generator.cpp | 2 - .../matrix_statistics/matrix_statistics.cpp | 4 +- benchmark/preconditioner/preconditioner.cpp | 4 +- benchmark/solver/distributed/solver.cpp | 8 +-- benchmark/solver/solver.cpp | 8 +-- benchmark/sparse_blas/sparse_blas.cpp | 4 +- benchmark/spmv/distributed/spmv.cpp | 4 +- benchmark/spmv/spmv.cpp | 4 +- benchmark/utils/general.hpp | 44 ++++-------- benchmark/utils/general_matrix.hpp | 72 +++++++++++++++++++ 13 files changed, 106 insertions(+), 56 deletions(-) create mode 100644 benchmark/utils/general_matrix.hpp diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp index ce3a98a9a68..11228ed5818 100644 --- a/benchmark/blas/blas.cpp +++ b/benchmark/blas/blas.cpp @@ -131,8 +131,6 @@ Parameters for a benchmark case are: stride_C: stride for C matrix in gemm (optional, default m) )"; std::string format = example_config; - // this benchmark doesn't use input matrices - matrix_input = false; initialize_argument_parsing(&argc, &argv, header, format); std::string extra_information = diff --git a/benchmark/blas/distributed/multi_vector.cpp b/benchmark/blas/distributed/multi_vector.cpp index bcf77594c4b..be326b08b96 100644 --- a/benchmark/blas/distributed/multi_vector.cpp +++ b/benchmark/blas/distributed/multi_vector.cpp @@ -61,8 +61,6 @@ Parameters for a benchmark case are: stride_y: stride for in/out vector y (optional, default r) )"; std::string format = example_config; - // this benchmark doesn't use input matrices - matrix_input = false; initialize_argument_parsing(&argc, &argv, header, format); const auto comm = gko::experimental::mpi::communicator(MPI_COMM_WORLD); diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp index b249293116b..d9684321e2d 100644 --- a/benchmark/conversions/conversions.cpp +++ b/benchmark/conversions/conversions.cpp @@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/utils/formats.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" #include "benchmark/utils/timer.hpp" @@ -115,7 +115,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring performance of Ginkgo's conversions.\n"; std::string format_str = example_config; - initialize_argument_parsing(&argc, &argv, header, format_str); + initialize_argument_parsing_matrix(&argc, &argv, header, format_str); std::string extra_information = std::string() + "The formats are " + FLAGS_formats + "\n"; diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp index 94883f5f63d..138b5a9c2ce 100644 --- a/benchmark/matrix_generator/matrix_generator.cpp +++ b/benchmark/matrix_generator/matrix_generator.cpp @@ -127,8 +127,6 @@ int main(int argc, char* argv[]) std::string header = "A utility that generates various types of " "matrices.\n"; - // this benchmark doesn't use input matrices - matrix_input = false; initialize_argument_parsing(&argc, &argv, header, input_format); std::clog << gko::version_info::get() << std::endl; diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp index 09cae6a7554..fccf4391ad5 100644 --- a/benchmark/matrix_statistics/matrix_statistics.cpp +++ b/benchmark/matrix_statistics/matrix_statistics.cpp @@ -38,7 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" #include "benchmark/utils/types.hpp" @@ -173,7 +173,7 @@ int main(int argc, char* argv[]) "A utility that collects additional statistical properties of the " "matrix.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); std::clog << gko::version_info::get() << std::endl; diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp index d125b46bb34..e7859e992dc 100644 --- a/benchmark/preconditioner/preconditioner.cpp +++ b/benchmark/preconditioner/preconditioner.cpp @@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/utils/formats.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/loggers.hpp" #include "benchmark/utils/preconditioners.hpp" @@ -262,7 +262,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring preconditioner performance.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); std::string extra_information = "Running with preconditioners: " + FLAGS_preconditioners + "\n"; diff --git a/benchmark/solver/distributed/solver.cpp b/benchmark/solver/distributed/solver.cpp index 4f583bcd8a8..8b285e343ce 100644 --- a/benchmark/solver/distributed/solver.cpp +++ b/benchmark/solver/distributed/solver.cpp @@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/solver/solver_common.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" @@ -98,9 +98,9 @@ int main(int argc, char* argv[]) "-", where both "local_format" and "non_local_format" can be any of the recognized spmv formats. )"; - // this benchmark needs an additional "optimal" object in the input - matrix_input_additional_json = ",\"optimal\":{\"spmv\":\"csr-csr\"}"; - initialize_argument_parsing(&argc, &argv, header, format); + std::string additional_json = ",\"optimal\":{\"spmv\":\"csr-csr\"}"; + initialize_argument_parsing_matrix(&argc, &argv, header, format, + additional_json); const auto comm = gko::experimental::mpi::communicator(MPI_COMM_WORLD); const auto rank = comm.rank(); diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp index c5010116dea..910bb54d89a 100644 --- a/benchmark/solver/solver.cpp +++ b/benchmark/solver/solver.cpp @@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/solver/solver_common.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" @@ -61,9 +61,9 @@ int main(int argc, char* argv[]) std::string format = example_config + R"( "optimal":"spmv" can be one of the recognized spmv formats )"; - // this benchmark needs an additional "optimal" object in the input - matrix_input_additional_json = ",\"optimal\":{\"spmv\":\"csr\"}"; - initialize_argument_parsing(&argc, &argv, header, format); + std::string additional_json = ",\"optimal\":{\"spmv\":\"csr\"}"; + initialize_argument_parsing_matrix(&argc, &argv, header, format, + additional_json); std::stringstream ss_rel_res_goal; ss_rel_res_goal << std::scientific << FLAGS_rel_res_goal; diff --git a/benchmark/sparse_blas/sparse_blas.cpp b/benchmark/sparse_blas/sparse_blas.cpp index d906e9f9e12..8c054709fdf 100644 --- a/benchmark/sparse_blas/sparse_blas.cpp +++ b/benchmark/sparse_blas/sparse_blas.cpp @@ -45,7 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/sparse_blas/operations.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" #include "benchmark/utils/types.hpp" @@ -161,7 +161,7 @@ int main(int argc, char* argv[]) "A benchmark for measuring performance of Ginkgo's sparse BLAS " "operations.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer); diff --git a/benchmark/spmv/distributed/spmv.cpp b/benchmark/spmv/distributed/spmv.cpp index 3c2986846b3..9b7e4ad8c8f 100644 --- a/benchmark/spmv/distributed/spmv.cpp +++ b/benchmark/spmv/distributed/spmv.cpp @@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/spmv/spmv_common.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/timer.hpp" #include "benchmark/utils/types.hpp" @@ -102,7 +102,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring performance of Ginkgo's spmv.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); if (rank == 0) { std::string extra_information = "The formats are [" + diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp index df000cecd47..034437907c8 100644 --- a/benchmark/spmv/spmv.cpp +++ b/benchmark/spmv/spmv.cpp @@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/spmv/spmv_common.hpp" #include "benchmark/utils/formats.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" @@ -64,7 +64,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring performance of Ginkgo's spmv.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); std::string extra_information = "The formats are " + FLAGS_formats + "\nThe number of right hand sides is " + diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp index d1b39263468..b7ec0e72cf1 100644 --- a/benchmark/utils/general.hpp +++ b/benchmark/utils/general.hpp @@ -95,9 +95,7 @@ DEFINE_string(double_buffer, "", DEFINE_string( input, "", "If set, the value is used as the input for the benchmark (if set to a " - "json string ending with ]), as the \"filename\" of a generated JSON input " - "(if the variable points to a MatrixMarket or Ginkgo binary matrix file) " - "or as JSON input file path (otherwise)."); + "json string ending with ]) or as input file path (otherwise)."); DEFINE_bool(detailed, true, "If set, performs several runs to obtain more detailed results"); @@ -147,6 +145,9 @@ DEFINE_double( "is lower than or equal to 1, the timing region is always 1 repetition."); +std::unique_ptr input_stream; + + /** * Parses arguments through gflags and initialize a documentation string. * @@ -186,6 +187,14 @@ void initialize_argument_parsing(int* argc, char** argv[], std::string& header, FLAGS_profiler_hook = "auto"; } } + std::string input_str(FLAGS_input); + if (!input_str.empty()) { + if (input_str.back() == ']') { + input_stream = std::make_unique(input_str); + } else { + input_stream = std::make_unique(input_str); + } + } } /** @@ -299,36 +308,11 @@ std::vector split(const std::string& s, char delimiter = ',') } -// allow matrix files as -input value -bool matrix_input = true; -// additional JSON to append to the input_str if the input file is a matrix file -std::string matrix_input_additional_json = ""; - - // returns the stream to be used as input of the application std::istream& get_input_stream() { - static auto stream = []() -> std::unique_ptr { - std::string input_str(FLAGS_input); - if (input_str.empty()) { - return nullptr; - } - if (input_str.back() == ']') { - return std::make_unique(input_str); - } - if (matrix_input) { - auto first_char = std::ifstream{input_str}.peek(); - // if the input looks like a MatrixMarket or Ginkgo binary file - if (first_char == '%' || first_char == 'G') { - input_str = "[{\"filename\":\"" + input_str + "\"" + - matrix_input_additional_json + "}]"; - return std::make_unique(input_str); - } - } - return std::make_unique(input_str); - }(); - if (stream) { - return *stream; + if (input_stream) { + return *input_stream; } return std::cin; } diff --git a/benchmark/utils/general_matrix.hpp b/benchmark/utils/general_matrix.hpp new file mode 100644 index 00000000000..3791976e3ab --- /dev/null +++ b/benchmark/utils/general_matrix.hpp @@ -0,0 +1,72 @@ +/************************************************************* +Copyright (c) 2017-2023, the Ginkgo authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*************************************************************/ + +#ifndef GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ +#define GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ + + +#include + + +#include + + +#include "benchmark/utils/general.hpp" + + +DEFINE_string(input_matrix, "", + "Filename of a matrix to be used as the single input. Overwrites " + "the value of the -input flag"); + + +/** + * @copydoc initialize_argument_parsing + * @param additional_matrix_file_json text to be appended to the + * `{"filename":"..."}` JSON object that + * will be used as input for the benchmark + * if the `-input_matrix` flag is used. + */ +void initialize_argument_parsing_matrix( + int* argc, char** argv[], std::string& header, std::string& format, + std::string additional_matrix_file_json = "") +{ + initialize_argument_parsing(argc, argv, header, format); + std::string input_matrix_str{FLAGS_input_matrix}; + if (!input_matrix_str.empty()) { + auto input_json = "[{\"filename\":\"" + input_matrix_str + "\"" + + additional_matrix_file_json + "}]"; + input_stream = std::make_unique(input_json); + } +} + + +#endif // GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ \ No newline at end of file From e122911c28f1b4faeac89bff0f6f3282cbda4074 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Wed, 16 Aug 2023 13:33:51 +0200 Subject: [PATCH 3/6] test `-input_matrix` flag --- benchmark/test/conversion.py | 12 ++++ benchmark/test/matrix_statistics.py | 7 +++ benchmark/test/preconditioner.py | 7 +++ .../test/reference/conversion.matrix.stderr | 46 +++++++++++++++ .../test/reference/conversion.matrix.stdout | 19 +++++++ .../distributed_solver.matrix.stderr | 17 ++++++ .../distributed_solver.matrix.stdout | 57 +++++++++++++++++++ .../reference/matrix_statistics.matrix.stderr | 8 +++ .../reference/matrix_statistics.matrix.stdout | 38 +++++++++++++ .../reference/preconditioner.matrix.stderr | 42 ++++++++++++++ .../reference/preconditioner.matrix.stdout | 30 ++++++++++ benchmark/test/reference/solver.matrix.stderr | 17 ++++++ benchmark/test/reference/solver.matrix.stdout | 55 ++++++++++++++++++ .../test/reference/sparse_blas.matrix.stderr | 36 ++++++++++++ .../test/reference/sparse_blas.matrix.stdout | 25 ++++++++ benchmark/test/reference/spmv.matrix.stderr | 31 ++++++++++ benchmark/test/reference/spmv.matrix.stdout | 20 +++++++ benchmark/test/solver.py | 7 +++ benchmark/test/solver_distributed.py | 7 +++ benchmark/test/sparse_blas.py | 12 ++++ benchmark/test/spmv.py | 7 +++ benchmark/test/test_framework.py.in | 12 ++-- 22 files changed, 505 insertions(+), 7 deletions(-) create mode 100644 benchmark/test/reference/conversion.matrix.stderr create mode 100644 benchmark/test/reference/conversion.matrix.stdout create mode 100644 benchmark/test/reference/distributed_solver.matrix.stderr create mode 100644 benchmark/test/reference/distributed_solver.matrix.stdout create mode 100644 benchmark/test/reference/matrix_statistics.matrix.stderr create mode 100644 benchmark/test/reference/matrix_statistics.matrix.stdout create mode 100644 benchmark/test/reference/preconditioner.matrix.stderr create mode 100644 benchmark/test/reference/preconditioner.matrix.stdout create mode 100644 benchmark/test/reference/solver.matrix.stderr create mode 100644 benchmark/test/reference/solver.matrix.stdout create mode 100644 benchmark/test/reference/sparse_blas.matrix.stderr create mode 100644 benchmark/test/reference/sparse_blas.matrix.stdout create mode 100644 benchmark/test/reference/spmv.matrix.stderr create mode 100644 benchmark/test/reference/spmv.matrix.stdout diff --git a/benchmark/test/conversion.py b/benchmark/test/conversion.py index cf2e33983af..2eada100731 100755 --- a/benchmark/test/conversion.py +++ b/benchmark/test/conversion.py @@ -29,6 +29,18 @@ expected_stderr="conversion.simple.stderr", ) +# input matrixfile +test_framework.compare_output( + [ + "-input_matrix", + str(test_framework.matrixpath), + "-formats", + "coo,csr", + ], + expected_stdout="conversion.matrix.stdout", + expected_stderr="conversion.matrix.stderr", +) + # check that all conversions work test_framework.compare_output( [ diff --git a/benchmark/test/matrix_statistics.py b/benchmark/test/matrix_statistics.py index a29c80a0a7a..6e4d8b1d2f5 100755 --- a/benchmark/test/matrix_statistics.py +++ b/benchmark/test/matrix_statistics.py @@ -23,3 +23,10 @@ expected_stdout="matrix_statistics.simple.stdout", expected_stderr="matrix_statistics.simple.stderr", ) + +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="matrix_statistics.matrix.stdout", + expected_stderr="matrix_statistics.matrix.stderr", +) diff --git a/benchmark/test/preconditioner.py b/benchmark/test/preconditioner.py index a5a8dd3f13f..e05e5b780ac 100755 --- a/benchmark/test/preconditioner.py +++ b/benchmark/test/preconditioner.py @@ -24,6 +24,13 @@ expected_stderr="preconditioner.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="preconditioner.matrix.stdout", + expected_stderr="preconditioner.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/reference/conversion.matrix.stderr b/benchmark/test/reference/conversion.matrix.stderr new file mode 100644 index 00000000000..813e04a2100 --- /dev/null +++ b/benchmark/test/reference/conversion.matrix.stderr @@ -0,0 +1,46 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The formats are coo,csr +Benchmarking conversions. +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "conversions": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + }, + "csr-coo": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/conversion.matrix.stdout b/benchmark/test/reference/conversion.matrix.stdout new file mode 100644 index 00000000000..d3f62ec2b56 --- /dev/null +++ b/benchmark/test/reference/conversion.matrix.stdout @@ -0,0 +1,19 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + }, + "csr-coo": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/distributed_solver.matrix.stderr b/benchmark/test/reference/distributed_solver.matrix.stderr new file mode 100644 index 00000000000..7eba2fc85dd --- /dev/null +++ b/benchmark/test/reference/distributed_solver.matrix.stderr @@ -0,0 +1,17 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 1 running iterations +The random seed for right hand sides is 42 +Running cg with 1000 iterations and residual goal of 1.000000e-06 +The number of right hand sides is 1 +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr-csr" + }, + "solver": {} +} +Matrix is of size (36, 36) + Running solver: cg diff --git a/benchmark/test/reference/distributed_solver.matrix.stdout b/benchmark/test/reference/distributed_solver.matrix.stdout new file mode 100644 index 00000000000..157c40fdccf --- /dev/null +++ b/benchmark/test/reference/distributed_solver.matrix.stdout @@ -0,0 +1,57 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr-csr" + }, + "solver": { + "cg": { + "recurrent_residuals": [], + "true_residuals": [], + "implicit_residuals": [], + "iteration_timestamps": [], + "rhs_norm": 1.0, + "generate": { + "components": { + "generate()": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "time": 1.0 + }, + "apply": { + "components": { + "apply()": 1.0, + "iteration": 1.0, + "allocate": 1.0, + "dense::fill": 1.0, + "cg::initialize": 1.0, + "advanced_apply()": 1.0, + "dense::row_gather": 1.0, + "csr::advanced_spmv": 1.0, + "dense::compute_squared_norm2": 1.0, + "dense::compute_sqrt": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "dense::compute_conj_dot_dispatch": 1.0, + "check()": 1.0, + "residual_norm::residual_norm": 1.0, + "cg::step_1": 1.0, + "csr::spmv": 1.0, + "cg::step_2": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "iterations": 27, + "time": 1.0 + }, + "preconditioner": {}, + "residual_norm": 1.0, + "repetitions": 1, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/matrix_statistics.matrix.stderr b/benchmark/test/reference/matrix_statistics.matrix.stderr new file mode 100644 index 00000000000..0f7cc261c47 --- /dev/null +++ b/benchmark/test/reference/matrix_statistics.matrix.stderr @@ -0,0 +1,8 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "problem": {} +} +Matrix is of size (36, 36) diff --git a/benchmark/test/reference/matrix_statistics.matrix.stdout b/benchmark/test/reference/matrix_statistics.matrix.stdout new file mode 100644 index 00000000000..20657f7abd5 --- /dev/null +++ b/benchmark/test/reference/matrix_statistics.matrix.stdout @@ -0,0 +1,38 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "problem": { + "rows": 36, + "columns": 36, + "nonzeros": 208, + "row_distribution": { + "min": 4, + "q1": 4.5, + "median": 6.0, + "q3": 7.0, + "max": 9, + "mean": 5.777777777777778, + "variance": 2.061728395061728, + "skewness": 0.3366362745126052, + "kurtosis": 2.0507009932231366, + "hyperskewness": 1.9165991338199193, + "hyperflatness": 6.0545648993883665 + }, + "col_distribution": { + "min": 4, + "q1": 4.5, + "median": 6.0, + "q3": 7.0, + "max": 9, + "mean": 5.777777777777778, + "variance": 2.061728395061728, + "skewness": 0.3366362745126052, + "kurtosis": 2.0507009932231366, + "hyperskewness": 1.9165991338199193, + "hyperflatness": 6.0545648993883665 + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/preconditioner.matrix.stderr b/benchmark/test/reference/preconditioner.matrix.stderr new file mode 100644 index 00000000000..d54c99ac971 --- /dev/null +++ b/benchmark/test/reference/preconditioner.matrix.stderr @@ -0,0 +1,42 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +Running with preconditioners: none +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "preconditioner": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "preconditioner": { + "none": { + "generate": { + "components": { + "generate()": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "apply": { + "components": { + "apply()": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/preconditioner.matrix.stdout b/benchmark/test/reference/preconditioner.matrix.stdout new file mode 100644 index 00000000000..22c0c9cc1c1 --- /dev/null +++ b/benchmark/test/reference/preconditioner.matrix.stdout @@ -0,0 +1,30 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "preconditioner": { + "none": { + "generate": { + "components": { + "generate()": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "apply": { + "components": { + "apply()": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/solver.matrix.stderr b/benchmark/test/reference/solver.matrix.stderr new file mode 100644 index 00000000000..78b75c301f7 --- /dev/null +++ b/benchmark/test/reference/solver.matrix.stderr @@ -0,0 +1,17 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 1 running iterations +The random seed for right hand sides is 42 +Running cg with 1000 iterations and residual goal of 1.000000e-06 +The number of right hand sides is 1 +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr" + }, + "solver": {} +} +Matrix is of size (36, 36) + Running solver: cg diff --git a/benchmark/test/reference/solver.matrix.stdout b/benchmark/test/reference/solver.matrix.stdout new file mode 100644 index 00000000000..4a68d8a599b --- /dev/null +++ b/benchmark/test/reference/solver.matrix.stdout @@ -0,0 +1,55 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr" + }, + "solver": { + "cg": { + "recurrent_residuals": [], + "true_residuals": [], + "implicit_residuals": [], + "iteration_timestamps": [], + "rhs_norm": 1.0, + "generate": { + "components": { + "generate()": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "time": 1.0 + }, + "apply": { + "components": { + "apply()": 1.0, + "iteration": 1.0, + "allocate": 1.0, + "dense::fill": 1.0, + "cg::initialize": 1.0, + "advanced_apply()": 1.0, + "csr::advanced_spmv": 1.0, + "dense::compute_norm2_dispatch": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "dense::compute_conj_dot_dispatch": 1.0, + "check()": 1.0, + "residual_norm::residual_norm": 1.0, + "cg::step_1": 1.0, + "csr::spmv": 1.0, + "cg::step_2": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "iterations": 27, + "time": 1.0 + }, + "preconditioner": {}, + "residual_norm": 1.0, + "repetitions": 1, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/sparse_blas.matrix.stderr b/benchmark/test/reference/sparse_blas.matrix.stderr new file mode 100644 index 00000000000..404a761aec9 --- /dev/null +++ b/benchmark/test/reference/sparse_blas.matrix.stderr @@ -0,0 +1,36 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The operations are transposeRunning test case +{ + "filename": "../../matrices/test/ani1.mtx", + "sparse_blas": {} +} +Matrix is of size (36, 36), 208 +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "sparse_blas": { + "transpose": { + "time": 1.0, + "flops": 1.0, + "bandwidth": 1.0, + "repetitions": 10, + "components": { + "allocate": 1.0, + "components::fill_array": 1.0, + "csr::transpose": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "completed": true + } + }, + "rows": 36, + "cols": 36, + "nonzeros": 208 + } +] diff --git a/benchmark/test/reference/sparse_blas.matrix.stdout b/benchmark/test/reference/sparse_blas.matrix.stdout new file mode 100644 index 00000000000..ae983436081 --- /dev/null +++ b/benchmark/test/reference/sparse_blas.matrix.stdout @@ -0,0 +1,25 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "sparse_blas": { + "transpose": { + "time": 1.0, + "flops": 1.0, + "bandwidth": 1.0, + "repetitions": 10, + "components": { + "allocate": 1.0, + "components::fill_array": 1.0, + "csr::transpose": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "completed": true + } + }, + "rows": 36, + "cols": 36, + "nonzeros": 208 + } +] diff --git a/benchmark/test/reference/spmv.matrix.stderr b/benchmark/test/reference/spmv.matrix.stderr new file mode 100644 index 00000000000..1096d64c74e --- /dev/null +++ b/benchmark/test/reference/spmv.matrix.stderr @@ -0,0 +1,31 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The formats are coo +The number of right hand sides is 1 +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "spmv": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "spmv": { + "coo": { + "storage": 3328, + "max_relative_norm2": 1.0, + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36, + "nnz": 208, + "optimal": {} + } +] diff --git a/benchmark/test/reference/spmv.matrix.stdout b/benchmark/test/reference/spmv.matrix.stdout new file mode 100644 index 00000000000..b51f331f918 --- /dev/null +++ b/benchmark/test/reference/spmv.matrix.stdout @@ -0,0 +1,20 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "spmv": { + "coo": { + "storage": 3328, + "max_relative_norm2": 1.0, + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36, + "nnz": 208, + "optimal": { + "spmv": "coo" + } + } +] diff --git a/benchmark/test/solver.py b/benchmark/test/solver.py index e974f849276..025ee92707c 100755 --- a/benchmark/test/solver.py +++ b/benchmark/test/solver.py @@ -24,6 +24,13 @@ expected_stderr="solver.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="solver.matrix.stdout", + expected_stderr="solver.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/solver_distributed.py b/benchmark/test/solver_distributed.py index c6623723a43..54bbb030077 100644 --- a/benchmark/test/solver_distributed.py +++ b/benchmark/test/solver_distributed.py @@ -27,6 +27,13 @@ expected_stderr="distributed_solver.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="distributed_solver.matrix.stdout", + expected_stderr="distributed_solver.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/sparse_blas.py b/benchmark/test/sparse_blas.py index 7b0968a710c..724cdb866f0 100755 --- a/benchmark/test/sparse_blas.py +++ b/benchmark/test/sparse_blas.py @@ -29,6 +29,18 @@ expected_stderr="sparse_blas.simple.stderr", ) +# input matrix file +test_framework.compare_output( + [ + "-operations", + "transpose", + "-input_matrix", + str(test_framework.matrixpath), + ], + expected_stdout="sparse_blas.matrix.stdout", + expected_stderr="sparse_blas.matrix.stderr", +) + # profiler annotations (transpose has the smallest number of allocations) test_framework.compare_output( [ diff --git a/benchmark/test/spmv.py b/benchmark/test/spmv.py index 6e2d9f05d49..865f74bb6d0 100755 --- a/benchmark/test/spmv.py +++ b/benchmark/test/spmv.py @@ -24,6 +24,13 @@ expected_stderr="spmv.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="spmv.matrix.stdout", + expected_stderr="spmv.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/test_framework.py.in b/benchmark/test/test_framework.py.in index 6037f8c594e..1c762905c77 100644 --- a/benchmark/test/test_framework.py.in +++ b/benchmark/test/test_framework.py.in @@ -9,6 +9,7 @@ import sys sourcepath = pathlib.Path("@CMAKE_CURRENT_SOURCE_DIR@") binpath = pathlib.Path("@PROJECT_BINARY_DIR@") +matrixpath = pathlib.Path("../../matrices/test/ani1.mtx") generate = False if len(sys.argv) > 2 and sys.argv[2] == "--generate": generate = True @@ -99,8 +100,7 @@ def sanitize_json_in_text(lines: List[str]) -> List[str]: for begin, end, do_sanitize in combined_pairs ] reconstructed = [ - json.dumps(sanitize_json(json.loads(t)), - indent=4) if do_sanitize else t + json.dumps(sanitize_json(json.loads(t)), indent=4) if do_sanitize else t for t, do_sanitize in texts ] return "\n".join(reconstructed).split("\n") @@ -135,7 +135,7 @@ def determinize_text( break if keep: output_lines.append(line) - if output_lines[-1] != "": + if len(output_lines) == 0 or output_lines[-1] != "": output_lines.append("") try: return sanitize_json_in_text(output_lines) @@ -215,8 +215,7 @@ def compare_output_impl( print("FAIL: stdout differs") print( "\n".join( - difflib.unified_diff( - expected_stdout_processed, result_stdout_processed) + difflib.unified_diff(expected_stdout_processed, result_stdout_processed) ) ) failed = True @@ -224,8 +223,7 @@ def compare_output_impl( print("FAIL: stderr differs") print( "\n".join( - difflib.unified_diff( - expected_stderr_processed, result_stderr_processed) + difflib.unified_diff(expected_stderr_processed, result_stderr_processed) ) ) failed = True From 2ae11f986c9c8a488806c3cd755a5b66cf9ee612 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Thu, 17 Aug 2023 09:54:59 +0200 Subject: [PATCH 4/6] use absolute path --- benchmark/test/reference/conversion.matrix.stderr | 6 +++--- benchmark/test/reference/conversion.matrix.stdout | 2 +- benchmark/test/reference/distributed_solver.matrix.stderr | 2 +- benchmark/test/reference/distributed_solver.matrix.stdout | 2 +- benchmark/test/reference/matrix_statistics.matrix.stderr | 2 +- benchmark/test/reference/matrix_statistics.matrix.stdout | 2 +- benchmark/test/reference/preconditioner.matrix.stderr | 4 ++-- benchmark/test/reference/preconditioner.matrix.stdout | 2 +- benchmark/test/reference/solver.matrix.stderr | 2 +- benchmark/test/reference/solver.matrix.stdout | 2 +- benchmark/test/reference/sparse_blas.matrix.stderr | 4 ++-- benchmark/test/reference/sparse_blas.matrix.stdout | 2 +- benchmark/test/reference/spmv.matrix.stderr | 4 ++-- benchmark/test/reference/spmv.matrix.stdout | 2 +- benchmark/test/test_framework.py.in | 4 ++-- 15 files changed, 21 insertions(+), 21 deletions(-) diff --git a/benchmark/test/reference/conversion.matrix.stderr b/benchmark/test/reference/conversion.matrix.stderr index 813e04a2100..1d604175479 100644 --- a/benchmark/test/reference/conversion.matrix.stderr +++ b/benchmark/test/reference/conversion.matrix.stderr @@ -7,14 +7,14 @@ The formats are coo,csr Benchmarking conversions. Running test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "conversions": {} } Matrix is of size (36, 36) Current state: [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "conversions": { "coo-csr": { "time": 1.0, @@ -28,7 +28,7 @@ Current state: Current state: [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "conversions": { "coo-csr": { "time": 1.0, diff --git a/benchmark/test/reference/conversion.matrix.stdout b/benchmark/test/reference/conversion.matrix.stdout index d3f62ec2b56..e43edda0595 100644 --- a/benchmark/test/reference/conversion.matrix.stdout +++ b/benchmark/test/reference/conversion.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "conversions": { "coo-csr": { "time": 1.0, diff --git a/benchmark/test/reference/distributed_solver.matrix.stderr b/benchmark/test/reference/distributed_solver.matrix.stderr index 7eba2fc85dd..4f0c6b22edd 100644 --- a/benchmark/test/reference/distributed_solver.matrix.stderr +++ b/benchmark/test/reference/distributed_solver.matrix.stderr @@ -7,7 +7,7 @@ Running cg with 1000 iterations and residual goal of 1.000000e-06 The number of right hand sides is 1 Running test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "optimal": { "spmv": "csr-csr" }, diff --git a/benchmark/test/reference/distributed_solver.matrix.stdout b/benchmark/test/reference/distributed_solver.matrix.stdout index 157c40fdccf..34fdda13e55 100644 --- a/benchmark/test/reference/distributed_solver.matrix.stdout +++ b/benchmark/test/reference/distributed_solver.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "optimal": { "spmv": "csr-csr" }, diff --git a/benchmark/test/reference/matrix_statistics.matrix.stderr b/benchmark/test/reference/matrix_statistics.matrix.stderr index 0f7cc261c47..af205c778c0 100644 --- a/benchmark/test/reference/matrix_statistics.matrix.stderr +++ b/benchmark/test/reference/matrix_statistics.matrix.stderr @@ -2,7 +2,7 @@ This is Ginkgo 1.7.0 (develop) running with core module 1.7.0 (develop) Running test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "problem": {} } Matrix is of size (36, 36) diff --git a/benchmark/test/reference/matrix_statistics.matrix.stdout b/benchmark/test/reference/matrix_statistics.matrix.stdout index 20657f7abd5..a056241669b 100644 --- a/benchmark/test/reference/matrix_statistics.matrix.stdout +++ b/benchmark/test/reference/matrix_statistics.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "problem": { "rows": 36, "columns": 36, diff --git a/benchmark/test/reference/preconditioner.matrix.stderr b/benchmark/test/reference/preconditioner.matrix.stderr index d54c99ac971..c9ef583d79e 100644 --- a/benchmark/test/reference/preconditioner.matrix.stderr +++ b/benchmark/test/reference/preconditioner.matrix.stderr @@ -6,14 +6,14 @@ The random seed for right hand sides is 42 Running with preconditioners: none Running test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "preconditioner": {} } Matrix is of size (36, 36) Current state: [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "preconditioner": { "none": { "generate": { diff --git a/benchmark/test/reference/preconditioner.matrix.stdout b/benchmark/test/reference/preconditioner.matrix.stdout index 22c0c9cc1c1..77979f4c54b 100644 --- a/benchmark/test/reference/preconditioner.matrix.stdout +++ b/benchmark/test/reference/preconditioner.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "preconditioner": { "none": { "generate": { diff --git a/benchmark/test/reference/solver.matrix.stderr b/benchmark/test/reference/solver.matrix.stderr index 78b75c301f7..8a1ea117314 100644 --- a/benchmark/test/reference/solver.matrix.stderr +++ b/benchmark/test/reference/solver.matrix.stderr @@ -7,7 +7,7 @@ Running cg with 1000 iterations and residual goal of 1.000000e-06 The number of right hand sides is 1 Running test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "optimal": { "spmv": "csr" }, diff --git a/benchmark/test/reference/solver.matrix.stdout b/benchmark/test/reference/solver.matrix.stdout index 4a68d8a599b..6a1f8ceb959 100644 --- a/benchmark/test/reference/solver.matrix.stdout +++ b/benchmark/test/reference/solver.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "optimal": { "spmv": "csr" }, diff --git a/benchmark/test/reference/sparse_blas.matrix.stderr b/benchmark/test/reference/sparse_blas.matrix.stderr index 404a761aec9..5001c604e72 100644 --- a/benchmark/test/reference/sparse_blas.matrix.stderr +++ b/benchmark/test/reference/sparse_blas.matrix.stderr @@ -5,14 +5,14 @@ Running with 2 warm iterations and 10 running iterations The random seed for right hand sides is 42 The operations are transposeRunning test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "sparse_blas": {} } Matrix is of size (36, 36), 208 Current state: [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "sparse_blas": { "transpose": { "time": 1.0, diff --git a/benchmark/test/reference/sparse_blas.matrix.stdout b/benchmark/test/reference/sparse_blas.matrix.stdout index ae983436081..4a64c8ea1ce 100644 --- a/benchmark/test/reference/sparse_blas.matrix.stdout +++ b/benchmark/test/reference/sparse_blas.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "sparse_blas": { "transpose": { "time": 1.0, diff --git a/benchmark/test/reference/spmv.matrix.stderr b/benchmark/test/reference/spmv.matrix.stderr index 1096d64c74e..8d942cd0de5 100644 --- a/benchmark/test/reference/spmv.matrix.stderr +++ b/benchmark/test/reference/spmv.matrix.stderr @@ -7,14 +7,14 @@ The formats are coo The number of right hand sides is 1 Running test case { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "spmv": {} } Matrix is of size (36, 36) Current state: [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "spmv": { "coo": { "storage": 3328, diff --git a/benchmark/test/reference/spmv.matrix.stdout b/benchmark/test/reference/spmv.matrix.stdout index b51f331f918..47035c27549 100644 --- a/benchmark/test/reference/spmv.matrix.stdout +++ b/benchmark/test/reference/spmv.matrix.stdout @@ -1,7 +1,7 @@ [ { - "filename": "../../matrices/test/ani1.mtx", + "filename": "", "spmv": { "coo": { "storage": 3328, diff --git a/benchmark/test/test_framework.py.in b/benchmark/test/test_framework.py.in index 1c762905c77..da1b0bfd618 100644 --- a/benchmark/test/test_framework.py.in +++ b/benchmark/test/test_framework.py.in @@ -9,7 +9,7 @@ import sys sourcepath = pathlib.Path("@CMAKE_CURRENT_SOURCE_DIR@") binpath = pathlib.Path("@PROJECT_BINARY_DIR@") -matrixpath = pathlib.Path("../../matrices/test/ani1.mtx") +matrixpath = pathlib.Path("@PROJECT_BINARY_DIR@/matrices/test/ani1.mtx") generate = False if len(sys.argv) > 2 and sys.argv[2] == "--generate": generate = True @@ -22,7 +22,7 @@ denumberify_paths = [ "rhs_norm", "max_relative_norm2", ] -empty_string_paths = ["error"] +empty_string_paths = ["error", "filename"] empty_array_paths = [ "recurrent_residuals", "true_residuals", From 4a8ea1b113849d1366e2936e7f2a2d9d559a39ce Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Thu, 17 Aug 2023 10:01:34 +0200 Subject: [PATCH 5/6] review updates * -input and -input_matrix are incompatible * use R-strings for JSON Co-authored-by: Marcel Koch Co-authored-by: Yuhsiang M. Tsai --- benchmark/solver/distributed/solver.cpp | 2 +- benchmark/solver/solver.cpp | 2 +- benchmark/utils/general_matrix.hpp | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/benchmark/solver/distributed/solver.cpp b/benchmark/solver/distributed/solver.cpp index 8b285e343ce..a9b1f9c1c93 100644 --- a/benchmark/solver/distributed/solver.cpp +++ b/benchmark/solver/distributed/solver.cpp @@ -98,7 +98,7 @@ int main(int argc, char* argv[]) "-", where both "local_format" and "non_local_format" can be any of the recognized spmv formats. )"; - std::string additional_json = ",\"optimal\":{\"spmv\":\"csr-csr\"}"; + std::string additional_json = R"(,"optimal":{"spmv":"csr-csr"})"; initialize_argument_parsing_matrix(&argc, &argv, header, format, additional_json); diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp index 910bb54d89a..4efc5558a8e 100644 --- a/benchmark/solver/solver.cpp +++ b/benchmark/solver/solver.cpp @@ -61,7 +61,7 @@ int main(int argc, char* argv[]) std::string format = example_config + R"( "optimal":"spmv" can be one of the recognized spmv formats )"; - std::string additional_json = ",\"optimal\":{\"spmv\":\"csr\"}"; + std::string additional_json = R"(,"optimal":{"spmv":"csr"})"; initialize_argument_parsing_matrix(&argc, &argv, header, format, additional_json); diff --git a/benchmark/utils/general_matrix.hpp b/benchmark/utils/general_matrix.hpp index 3791976e3ab..e499d5d9326 100644 --- a/benchmark/utils/general_matrix.hpp +++ b/benchmark/utils/general_matrix.hpp @@ -62,11 +62,16 @@ void initialize_argument_parsing_matrix( initialize_argument_parsing(argc, argv, header, format); std::string input_matrix_str{FLAGS_input_matrix}; if (!input_matrix_str.empty()) { - auto input_json = "[{\"filename\":\"" + input_matrix_str + "\"" + + if (input_stream) { + std::cerr + << "-input and -input_matrix cannot be used simultaneously\n"; + std::exit(1); + } + auto input_json = R"([{"filename":")" + input_matrix_str + "\"" + additional_matrix_file_json + "}]"; input_stream = std::make_unique(input_json); } } -#endif // GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ \ No newline at end of file +#endif // GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ From b37b233f0acf033ac160b52ff037814476dfb2ac Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Thu, 17 Aug 2023 11:56:12 +0200 Subject: [PATCH 6/6] escape file path using rapidJSON --- benchmark/utils/general_matrix.hpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/benchmark/utils/general_matrix.hpp b/benchmark/utils/general_matrix.hpp index e499d5d9326..2049dadf45f 100644 --- a/benchmark/utils/general_matrix.hpp +++ b/benchmark/utils/general_matrix.hpp @@ -67,9 +67,17 @@ void initialize_argument_parsing_matrix( << "-input and -input_matrix cannot be used simultaneously\n"; std::exit(1); } - auto input_json = R"([{"filename":")" + input_matrix_str + "\"" + - additional_matrix_file_json + "}]"; - input_stream = std::make_unique(input_json); + // create JSON for the filename via RapidJSON to ensure the string is + // correctly escaped + rapidjson::Document d; + auto json_template = + R"([{"filename":"")" + additional_matrix_file_json + "}]"; + d.Parse(json_template.c_str()); + d[0]["filename"].SetString(input_matrix_str.c_str(), d.GetAllocator()); + rapidjson::StringBuffer sb; + rapidjson::PrettyWriter writer(sb); + d.Accept(writer); + input_stream = std::make_unique(sb.GetString()); } }