diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp index b249293116b..d9684321e2d 100644 --- a/benchmark/conversions/conversions.cpp +++ b/benchmark/conversions/conversions.cpp @@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/utils/formats.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" #include "benchmark/utils/timer.hpp" @@ -115,7 +115,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring performance of Ginkgo's conversions.\n"; std::string format_str = example_config; - initialize_argument_parsing(&argc, &argv, header, format_str); + initialize_argument_parsing_matrix(&argc, &argv, header, format_str); std::string extra_information = std::string() + "The formats are " + FLAGS_formats + "\n"; diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp index 09cae6a7554..fccf4391ad5 100644 --- a/benchmark/matrix_statistics/matrix_statistics.cpp +++ b/benchmark/matrix_statistics/matrix_statistics.cpp @@ -38,7 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" #include "benchmark/utils/types.hpp" @@ -173,7 +173,7 @@ int main(int argc, char* argv[]) "A utility that collects additional statistical properties of the " "matrix.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); std::clog << gko::version_info::get() << std::endl; diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp index d125b46bb34..e7859e992dc 100644 --- a/benchmark/preconditioner/preconditioner.cpp +++ b/benchmark/preconditioner/preconditioner.cpp @@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/utils/formats.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/loggers.hpp" #include "benchmark/utils/preconditioners.hpp" @@ -262,7 +262,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring preconditioner performance.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); std::string extra_information = "Running with preconditioners: " + FLAGS_preconditioners + "\n"; diff --git a/benchmark/solver/distributed/solver.cpp b/benchmark/solver/distributed/solver.cpp index 2db71c16ca3..a9b1f9c1c93 100644 --- a/benchmark/solver/distributed/solver.cpp +++ b/benchmark/solver/distributed/solver.cpp @@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/solver/solver_common.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" @@ -98,7 +98,9 @@ int main(int argc, char* argv[]) "-", where both "local_format" and "non_local_format" can be any of the recognized spmv formats. )"; - initialize_argument_parsing(&argc, &argv, header, format); + std::string additional_json = R"(,"optimal":{"spmv":"csr-csr"})"; + initialize_argument_parsing_matrix(&argc, &argv, header, format, + additional_json); const auto comm = gko::experimental::mpi::communicator(MPI_COMM_WORLD); const auto rank = comm.rank(); diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp index 9190c99dad0..4efc5558a8e 100644 --- a/benchmark/solver/solver.cpp +++ b/benchmark/solver/solver.cpp @@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/solver/solver_common.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" @@ -61,7 +61,9 @@ int main(int argc, char* argv[]) std::string format = example_config + R"( "optimal":"spmv" can be one of the recognized spmv formats )"; - initialize_argument_parsing(&argc, &argv, header, format); + std::string additional_json = R"(,"optimal":{"spmv":"csr"})"; + initialize_argument_parsing_matrix(&argc, &argv, header, format, + additional_json); std::stringstream ss_rel_res_goal; ss_rel_res_goal << std::scientific << FLAGS_rel_res_goal; diff --git a/benchmark/sparse_blas/sparse_blas.cpp b/benchmark/sparse_blas/sparse_blas.cpp index d906e9f9e12..8c054709fdf 100644 --- a/benchmark/sparse_blas/sparse_blas.cpp +++ b/benchmark/sparse_blas/sparse_blas.cpp @@ -45,7 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/sparse_blas/operations.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" #include "benchmark/utils/types.hpp" @@ -161,7 +161,7 @@ int main(int argc, char* argv[]) "A benchmark for measuring performance of Ginkgo's sparse BLAS " "operations.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer); diff --git a/benchmark/spmv/distributed/spmv.cpp b/benchmark/spmv/distributed/spmv.cpp index 3c2986846b3..9b7e4ad8c8f 100644 --- a/benchmark/spmv/distributed/spmv.cpp +++ b/benchmark/spmv/distributed/spmv.cpp @@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/spmv/spmv_common.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/timer.hpp" #include "benchmark/utils/types.hpp" @@ -102,7 +102,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring performance of Ginkgo's spmv.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); if (rank == 0) { std::string extra_information = "The formats are [" + diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp index df000cecd47..034437907c8 100644 --- a/benchmark/spmv/spmv.cpp +++ b/benchmark/spmv/spmv.cpp @@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "benchmark/spmv/spmv_common.hpp" #include "benchmark/utils/formats.hpp" -#include "benchmark/utils/general.hpp" +#include "benchmark/utils/general_matrix.hpp" #include "benchmark/utils/generator.hpp" #include "benchmark/utils/spmv_validation.hpp" @@ -64,7 +64,7 @@ int main(int argc, char* argv[]) std::string header = "A benchmark for measuring performance of Ginkgo's spmv.\n"; std::string format = example_config; - initialize_argument_parsing(&argc, &argv, header, format); + initialize_argument_parsing_matrix(&argc, &argv, header, format); std::string extra_information = "The formats are " + FLAGS_formats + "\nThe number of right hand sides is " + diff --git a/benchmark/test/conversion.py b/benchmark/test/conversion.py index cf2e33983af..2eada100731 100755 --- a/benchmark/test/conversion.py +++ b/benchmark/test/conversion.py @@ -29,6 +29,18 @@ expected_stderr="conversion.simple.stderr", ) +# input matrixfile +test_framework.compare_output( + [ + "-input_matrix", + str(test_framework.matrixpath), + "-formats", + "coo,csr", + ], + expected_stdout="conversion.matrix.stdout", + expected_stderr="conversion.matrix.stderr", +) + # check that all conversions work test_framework.compare_output( [ diff --git a/benchmark/test/matrix_statistics.py b/benchmark/test/matrix_statistics.py index a29c80a0a7a..6e4d8b1d2f5 100755 --- a/benchmark/test/matrix_statistics.py +++ b/benchmark/test/matrix_statistics.py @@ -23,3 +23,10 @@ expected_stdout="matrix_statistics.simple.stdout", expected_stderr="matrix_statistics.simple.stderr", ) + +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="matrix_statistics.matrix.stdout", + expected_stderr="matrix_statistics.matrix.stderr", +) diff --git a/benchmark/test/preconditioner.py b/benchmark/test/preconditioner.py index a5a8dd3f13f..e05e5b780ac 100755 --- a/benchmark/test/preconditioner.py +++ b/benchmark/test/preconditioner.py @@ -24,6 +24,13 @@ expected_stderr="preconditioner.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="preconditioner.matrix.stdout", + expected_stderr="preconditioner.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/reference/conversion.matrix.stderr b/benchmark/test/reference/conversion.matrix.stderr new file mode 100644 index 00000000000..1d604175479 --- /dev/null +++ b/benchmark/test/reference/conversion.matrix.stderr @@ -0,0 +1,46 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The formats are coo,csr +Benchmarking conversions. +Running test case +{ + "filename": "", + "conversions": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] +Current state: +[ + { + "filename": "", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + }, + "csr-coo": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/conversion.matrix.stdout b/benchmark/test/reference/conversion.matrix.stdout new file mode 100644 index 00000000000..e43edda0595 --- /dev/null +++ b/benchmark/test/reference/conversion.matrix.stdout @@ -0,0 +1,19 @@ + +[ + { + "filename": "", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + }, + "csr-coo": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/distributed_solver.matrix.stderr b/benchmark/test/reference/distributed_solver.matrix.stderr new file mode 100644 index 00000000000..4f0c6b22edd --- /dev/null +++ b/benchmark/test/reference/distributed_solver.matrix.stderr @@ -0,0 +1,17 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 1 running iterations +The random seed for right hand sides is 42 +Running cg with 1000 iterations and residual goal of 1.000000e-06 +The number of right hand sides is 1 +Running test case +{ + "filename": "", + "optimal": { + "spmv": "csr-csr" + }, + "solver": {} +} +Matrix is of size (36, 36) + Running solver: cg diff --git a/benchmark/test/reference/distributed_solver.matrix.stdout b/benchmark/test/reference/distributed_solver.matrix.stdout new file mode 100644 index 00000000000..34fdda13e55 --- /dev/null +++ b/benchmark/test/reference/distributed_solver.matrix.stdout @@ -0,0 +1,57 @@ + +[ + { + "filename": "", + "optimal": { + "spmv": "csr-csr" + }, + "solver": { + "cg": { + "recurrent_residuals": [], + "true_residuals": [], + "implicit_residuals": [], + "iteration_timestamps": [], + "rhs_norm": 1.0, + "generate": { + "components": { + "generate()": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "time": 1.0 + }, + "apply": { + "components": { + "apply()": 1.0, + "iteration": 1.0, + "allocate": 1.0, + "dense::fill": 1.0, + "cg::initialize": 1.0, + "advanced_apply()": 1.0, + "dense::row_gather": 1.0, + "csr::advanced_spmv": 1.0, + "dense::compute_squared_norm2": 1.0, + "dense::compute_sqrt": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "dense::compute_conj_dot_dispatch": 1.0, + "check()": 1.0, + "residual_norm::residual_norm": 1.0, + "cg::step_1": 1.0, + "csr::spmv": 1.0, + "cg::step_2": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "iterations": 27, + "time": 1.0 + }, + "preconditioner": {}, + "residual_norm": 1.0, + "repetitions": 1, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/matrix_statistics.matrix.stderr b/benchmark/test/reference/matrix_statistics.matrix.stderr new file mode 100644 index 00000000000..af205c778c0 --- /dev/null +++ b/benchmark/test/reference/matrix_statistics.matrix.stderr @@ -0,0 +1,8 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running test case +{ + "filename": "", + "problem": {} +} +Matrix is of size (36, 36) diff --git a/benchmark/test/reference/matrix_statistics.matrix.stdout b/benchmark/test/reference/matrix_statistics.matrix.stdout new file mode 100644 index 00000000000..a056241669b --- /dev/null +++ b/benchmark/test/reference/matrix_statistics.matrix.stdout @@ -0,0 +1,38 @@ + +[ + { + "filename": "", + "problem": { + "rows": 36, + "columns": 36, + "nonzeros": 208, + "row_distribution": { + "min": 4, + "q1": 4.5, + "median": 6.0, + "q3": 7.0, + "max": 9, + "mean": 5.777777777777778, + "variance": 2.061728395061728, + "skewness": 0.3366362745126052, + "kurtosis": 2.0507009932231366, + "hyperskewness": 1.9165991338199193, + "hyperflatness": 6.0545648993883665 + }, + "col_distribution": { + "min": 4, + "q1": 4.5, + "median": 6.0, + "q3": 7.0, + "max": 9, + "mean": 5.777777777777778, + "variance": 2.061728395061728, + "skewness": 0.3366362745126052, + "kurtosis": 2.0507009932231366, + "hyperskewness": 1.9165991338199193, + "hyperflatness": 6.0545648993883665 + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/preconditioner.matrix.stderr b/benchmark/test/reference/preconditioner.matrix.stderr new file mode 100644 index 00000000000..c9ef583d79e --- /dev/null +++ b/benchmark/test/reference/preconditioner.matrix.stderr @@ -0,0 +1,42 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +Running with preconditioners: none +Running test case +{ + "filename": "", + "preconditioner": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "", + "preconditioner": { + "none": { + "generate": { + "components": { + "generate()": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "apply": { + "components": { + "apply()": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/preconditioner.matrix.stdout b/benchmark/test/reference/preconditioner.matrix.stdout new file mode 100644 index 00000000000..77979f4c54b --- /dev/null +++ b/benchmark/test/reference/preconditioner.matrix.stdout @@ -0,0 +1,30 @@ + +[ + { + "filename": "", + "preconditioner": { + "none": { + "generate": { + "components": { + "generate()": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "apply": { + "components": { + "apply()": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/solver.matrix.stderr b/benchmark/test/reference/solver.matrix.stderr new file mode 100644 index 00000000000..8a1ea117314 --- /dev/null +++ b/benchmark/test/reference/solver.matrix.stderr @@ -0,0 +1,17 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 1 running iterations +The random seed for right hand sides is 42 +Running cg with 1000 iterations and residual goal of 1.000000e-06 +The number of right hand sides is 1 +Running test case +{ + "filename": "", + "optimal": { + "spmv": "csr" + }, + "solver": {} +} +Matrix is of size (36, 36) + Running solver: cg diff --git a/benchmark/test/reference/solver.matrix.stdout b/benchmark/test/reference/solver.matrix.stdout new file mode 100644 index 00000000000..6a1f8ceb959 --- /dev/null +++ b/benchmark/test/reference/solver.matrix.stdout @@ -0,0 +1,55 @@ + +[ + { + "filename": "", + "optimal": { + "spmv": "csr" + }, + "solver": { + "cg": { + "recurrent_residuals": [], + "true_residuals": [], + "implicit_residuals": [], + "iteration_timestamps": [], + "rhs_norm": 1.0, + "generate": { + "components": { + "generate()": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "time": 1.0 + }, + "apply": { + "components": { + "apply()": 1.0, + "iteration": 1.0, + "allocate": 1.0, + "dense::fill": 1.0, + "cg::initialize": 1.0, + "advanced_apply()": 1.0, + "csr::advanced_spmv": 1.0, + "dense::compute_norm2_dispatch": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "dense::compute_conj_dot_dispatch": 1.0, + "check()": 1.0, + "residual_norm::residual_norm": 1.0, + "cg::step_1": 1.0, + "csr::spmv": 1.0, + "cg::step_2": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "iterations": 27, + "time": 1.0 + }, + "preconditioner": {}, + "residual_norm": 1.0, + "repetitions": 1, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/sparse_blas.matrix.stderr b/benchmark/test/reference/sparse_blas.matrix.stderr new file mode 100644 index 00000000000..5001c604e72 --- /dev/null +++ b/benchmark/test/reference/sparse_blas.matrix.stderr @@ -0,0 +1,36 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The operations are transposeRunning test case +{ + "filename": "", + "sparse_blas": {} +} +Matrix is of size (36, 36), 208 +Current state: +[ + { + "filename": "", + "sparse_blas": { + "transpose": { + "time": 1.0, + "flops": 1.0, + "bandwidth": 1.0, + "repetitions": 10, + "components": { + "allocate": 1.0, + "components::fill_array": 1.0, + "csr::transpose": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "completed": true + } + }, + "rows": 36, + "cols": 36, + "nonzeros": 208 + } +] diff --git a/benchmark/test/reference/sparse_blas.matrix.stdout b/benchmark/test/reference/sparse_blas.matrix.stdout new file mode 100644 index 00000000000..4a64c8ea1ce --- /dev/null +++ b/benchmark/test/reference/sparse_blas.matrix.stdout @@ -0,0 +1,25 @@ + +[ + { + "filename": "", + "sparse_blas": { + "transpose": { + "time": 1.0, + "flops": 1.0, + "bandwidth": 1.0, + "repetitions": 10, + "components": { + "allocate": 1.0, + "components::fill_array": 1.0, + "csr::transpose": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "completed": true + } + }, + "rows": 36, + "cols": 36, + "nonzeros": 208 + } +] diff --git a/benchmark/test/reference/spmv.matrix.stderr b/benchmark/test/reference/spmv.matrix.stderr new file mode 100644 index 00000000000..8d942cd0de5 --- /dev/null +++ b/benchmark/test/reference/spmv.matrix.stderr @@ -0,0 +1,31 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The formats are coo +The number of right hand sides is 1 +Running test case +{ + "filename": "", + "spmv": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "", + "spmv": { + "coo": { + "storage": 3328, + "max_relative_norm2": 1.0, + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36, + "nnz": 208, + "optimal": {} + } +] diff --git a/benchmark/test/reference/spmv.matrix.stdout b/benchmark/test/reference/spmv.matrix.stdout new file mode 100644 index 00000000000..47035c27549 --- /dev/null +++ b/benchmark/test/reference/spmv.matrix.stdout @@ -0,0 +1,20 @@ + +[ + { + "filename": "", + "spmv": { + "coo": { + "storage": 3328, + "max_relative_norm2": 1.0, + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36, + "nnz": 208, + "optimal": { + "spmv": "coo" + } + } +] diff --git a/benchmark/test/solver.py b/benchmark/test/solver.py index e974f849276..025ee92707c 100755 --- a/benchmark/test/solver.py +++ b/benchmark/test/solver.py @@ -24,6 +24,13 @@ expected_stderr="solver.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="solver.matrix.stdout", + expected_stderr="solver.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/solver_distributed.py b/benchmark/test/solver_distributed.py index c6623723a43..54bbb030077 100644 --- a/benchmark/test/solver_distributed.py +++ b/benchmark/test/solver_distributed.py @@ -27,6 +27,13 @@ expected_stderr="distributed_solver.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="distributed_solver.matrix.stdout", + expected_stderr="distributed_solver.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/sparse_blas.py b/benchmark/test/sparse_blas.py index 7b0968a710c..724cdb866f0 100755 --- a/benchmark/test/sparse_blas.py +++ b/benchmark/test/sparse_blas.py @@ -29,6 +29,18 @@ expected_stderr="sparse_blas.simple.stderr", ) +# input matrix file +test_framework.compare_output( + [ + "-operations", + "transpose", + "-input_matrix", + str(test_framework.matrixpath), + ], + expected_stdout="sparse_blas.matrix.stdout", + expected_stderr="sparse_blas.matrix.stderr", +) + # profiler annotations (transpose has the smallest number of allocations) test_framework.compare_output( [ diff --git a/benchmark/test/spmv.py b/benchmark/test/spmv.py index 6e2d9f05d49..865f74bb6d0 100755 --- a/benchmark/test/spmv.py +++ b/benchmark/test/spmv.py @@ -24,6 +24,13 @@ expected_stderr="spmv.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="spmv.matrix.stdout", + expected_stderr="spmv.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/test_framework.py.in b/benchmark/test/test_framework.py.in index 6037f8c594e..da1b0bfd618 100644 --- a/benchmark/test/test_framework.py.in +++ b/benchmark/test/test_framework.py.in @@ -9,6 +9,7 @@ import sys sourcepath = pathlib.Path("@CMAKE_CURRENT_SOURCE_DIR@") binpath = pathlib.Path("@PROJECT_BINARY_DIR@") +matrixpath = pathlib.Path("@PROJECT_BINARY_DIR@/matrices/test/ani1.mtx") generate = False if len(sys.argv) > 2 and sys.argv[2] == "--generate": generate = True @@ -21,7 +22,7 @@ denumberify_paths = [ "rhs_norm", "max_relative_norm2", ] -empty_string_paths = ["error"] +empty_string_paths = ["error", "filename"] empty_array_paths = [ "recurrent_residuals", "true_residuals", @@ -99,8 +100,7 @@ def sanitize_json_in_text(lines: List[str]) -> List[str]: for begin, end, do_sanitize in combined_pairs ] reconstructed = [ - json.dumps(sanitize_json(json.loads(t)), - indent=4) if do_sanitize else t + json.dumps(sanitize_json(json.loads(t)), indent=4) if do_sanitize else t for t, do_sanitize in texts ] return "\n".join(reconstructed).split("\n") @@ -135,7 +135,7 @@ def determinize_text( break if keep: output_lines.append(line) - if output_lines[-1] != "": + if len(output_lines) == 0 or output_lines[-1] != "": output_lines.append("") try: return sanitize_json_in_text(output_lines) @@ -215,8 +215,7 @@ def compare_output_impl( print("FAIL: stdout differs") print( "\n".join( - difflib.unified_diff( - expected_stdout_processed, result_stdout_processed) + difflib.unified_diff(expected_stdout_processed, result_stdout_processed) ) ) failed = True @@ -224,8 +223,7 @@ def compare_output_impl( print("FAIL: stderr differs") print( "\n".join( - difflib.unified_diff( - expected_stderr_processed, result_stderr_processed) + difflib.unified_diff(expected_stderr_processed, result_stderr_processed) ) ) failed = True diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp index 335ed687002..b7ec0e72cf1 100644 --- a/benchmark/utils/general.hpp +++ b/benchmark/utils/general.hpp @@ -145,6 +145,9 @@ DEFINE_double( "is lower than or equal to 1, the timing region is always 1 repetition."); +std::unique_ptr input_stream; + + /** * Parses arguments through gflags and initialize a documentation string. * @@ -184,6 +187,14 @@ void initialize_argument_parsing(int* argc, char** argv[], std::string& header, FLAGS_profiler_hook = "auto"; } } + std::string input_str(FLAGS_input); + if (!input_str.empty()) { + if (input_str.back() == ']') { + input_stream = std::make_unique(input_str); + } else { + input_stream = std::make_unique(input_str); + } + } } /** @@ -300,18 +311,8 @@ std::vector split(const std::string& s, char delimiter = ',') // returns the stream to be used as input of the application std::istream& get_input_stream() { - static auto stream = []() -> std::unique_ptr { - std::string input_str(FLAGS_input); - if (input_str.empty()) { - return nullptr; - } - if (input_str.back() == ']') { - return std::make_unique(input_str); - } - return std::make_unique(input_str); - }(); - if (stream) { - return *stream; + if (input_stream) { + return *input_stream; } return std::cin; } diff --git a/benchmark/utils/general_matrix.hpp b/benchmark/utils/general_matrix.hpp new file mode 100644 index 00000000000..2049dadf45f --- /dev/null +++ b/benchmark/utils/general_matrix.hpp @@ -0,0 +1,85 @@ +/************************************************************* +Copyright (c) 2017-2023, the Ginkgo authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*************************************************************/ + +#ifndef GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ +#define GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_ + + +#include + + +#include + + +#include "benchmark/utils/general.hpp" + + +DEFINE_string(input_matrix, "", + "Filename of a matrix to be used as the single input. Overwrites " + "the value of the -input flag"); + + +/** + * @copydoc initialize_argument_parsing + * @param additional_matrix_file_json text to be appended to the + * `{"filename":"..."}` JSON object that + * will be used as input for the benchmark + * if the `-input_matrix` flag is used. + */ +void initialize_argument_parsing_matrix( + int* argc, char** argv[], std::string& header, std::string& format, + std::string additional_matrix_file_json = "") +{ + initialize_argument_parsing(argc, argv, header, format); + std::string input_matrix_str{FLAGS_input_matrix}; + if (!input_matrix_str.empty()) { + if (input_stream) { + std::cerr + << "-input and -input_matrix cannot be used simultaneously\n"; + std::exit(1); + } + // create JSON for the filename via RapidJSON to ensure the string is + // correctly escaped + rapidjson::Document d; + auto json_template = + R"([{"filename":"")" + additional_matrix_file_json + "}]"; + d.Parse(json_template.c_str()); + d[0]["filename"].SetString(input_matrix_str.c_str(), d.GetAllocator()); + rapidjson::StringBuffer sb; + rapidjson::PrettyWriter writer(sb); + d.Accept(writer); + input_stream = std::make_unique(sb.GetString()); + } +} + + +#endif // GKO_BENCHMARK_UTILS_GENERAL_MATRIX_HPP_