From 93f16ddd58fec16384878db10c7520ab4b2b0008 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Mon, 7 Mar 2022 13:56:20 +0100 Subject: [PATCH 1/3] add binary IO for matrix_data --- benchmark/conversions/conversions.cpp | 2 +- .../matrix_statistics/matrix_statistics.cpp | 2 +- benchmark/preconditioner/preconditioner.cpp | 2 +- benchmark/solver/solver.cpp | 2 +- benchmark/sparse_blas/sparse_blas.cpp | 2 +- benchmark/spmv/spmv.cpp | 2 +- benchmark/tools/CMakeLists.txt | 4 +- benchmark/tools/matrix.cpp | 16 +- benchmark/tools/mtx_to_binary.cpp | 105 +++++ core/base/mtx_io.cpp | 192 ++++++++ core/test/base/mtx_io.cpp | 436 ++++++++++++++++++ core/test/utils/assertions.hpp | 1 + include/ginkgo/core/base/mtx_io.hpp | 144 +++++- include/ginkgo/core/matrix/dense.hpp | 1 - 14 files changed, 895 insertions(+), 16 deletions(-) create mode 100644 benchmark/tools/mtx_to_binary.cpp diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp index 39e3af0b596..3036d1504f9 100644 --- a/benchmark/conversions/conversions.cpp +++ b/benchmark/conversions/conversions.cpp @@ -150,7 +150,7 @@ int main(int argc, char* argv[]) std::ifstream mtx_fd(test_case["filename"].GetString()); gko::matrix_data data; try { - data = gko::read_raw(mtx_fd); + data = gko::read_generic_raw(mtx_fd); } catch (std::exception& e) { std::cerr << "Error setting up matrix data, what(): " << e.what() << std::endl; diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp index 0abda9041e0..88911534539 100644 --- a/benchmark/matrix_statistics/matrix_statistics.cpp +++ b/benchmark/matrix_statistics/matrix_statistics.cpp @@ -203,7 +203,7 @@ int main(int argc, char* argv[]) std::clog << "Running test case: " << test_case << std::endl; std::ifstream ifs(test_case["filename"].GetString()); - auto matrix = gko::read_raw(ifs); + auto matrix = gko::read_generic_raw(ifs); ifs.close(); std::clog << "Matrix is of size (" << matrix.size[0] << ", " diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp index 4289fb8da88..3f760af36ca 100644 --- a/benchmark/preconditioner/preconditioner.cpp +++ b/benchmark/preconditioner/preconditioner.cpp @@ -296,7 +296,7 @@ int main(int argc, char* argv[]) std::clog << "Running test case: " << test_case << std::endl; std::ifstream mtx_fd(test_case["filename"].GetString()); - auto data = gko::read_raw(mtx_fd); + auto data = gko::read_generic_raw(mtx_fd); auto system_matrix = share(formats::matrix_factory.at(FLAGS_formats)(exec, data)); diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp index 5d0f01be68a..6e1ccecca2b 100644 --- a/benchmark/solver/solver.cpp +++ b/benchmark/solver/solver.cpp @@ -609,7 +609,7 @@ int main(int argc, char* argv[]) {std::numeric_limits::quiet_NaN()}, exec); x = gko::initialize({0.0}, exec); } else { - auto data = gko::read_raw(mtx_fd); + auto data = gko::read_generic_raw(mtx_fd); system_matrix = share(formats::matrix_factory.at( test_case["optimal"]["spmv"].GetString())(exec, data)); if (test_case.HasMember("rhs")) { diff --git a/benchmark/sparse_blas/sparse_blas.cpp b/benchmark/sparse_blas/sparse_blas.cpp index 3840ced32e7..8eb7df02b74 100644 --- a/benchmark/sparse_blas/sparse_blas.cpp +++ b/benchmark/sparse_blas/sparse_blas.cpp @@ -488,7 +488,7 @@ int main(int argc, char* argv[]) auto& sp_blas_case = test_case[benchmark_name]; std::clog << "Running test case: " << test_case << std::endl; std::ifstream mtx_fd(test_case["filename"].GetString()); - auto data = gko::read_raw(mtx_fd); + auto data = gko::read_generic_raw(mtx_fd); data.ensure_row_major_order(); std::clog << "Matrix is of size (" << data.size[0] << ", " << data.size[1] << "), " << data.nonzeros.size() diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp index e46c0e1472e..06a83b7f865 100644 --- a/benchmark/spmv/spmv.cpp +++ b/benchmark/spmv/spmv.cpp @@ -212,7 +212,7 @@ int main(int argc, char* argv[]) } std::clog << "Running test case: " << test_case << std::endl; std::ifstream mtx_fd(test_case["filename"].GetString()); - auto data = gko::read_raw(mtx_fd); + auto data = gko::read_generic_raw(mtx_fd); auto nrhs = FLAGS_nrhs; auto b = create_matrix(exec, gko::dim<2>{data.size[1], nrhs}, diff --git a/benchmark/tools/CMakeLists.txt b/benchmark/tools/CMakeLists.txt index c6a4ada0b0a..3bab475d471 100644 --- a/benchmark/tools/CMakeLists.txt +++ b/benchmark/tools/CMakeLists.txt @@ -2,4 +2,6 @@ add_executable(matrix matrix.cpp) add_executable(matrix_complex matrix.cpp) target_compile_definitions(matrix_complex PRIVATE GKO_TOOL_COMPLEX) target_link_libraries(matrix Ginkgo::ginkgo) -target_link_libraries(matrix_complex Ginkgo::ginkgo) \ No newline at end of file +target_link_libraries(matrix_complex Ginkgo::ginkgo) +add_executable(mtx_to_binary mtx_to_binary.cpp) +target_link_libraries(mtx_to_binary Ginkgo::ginkgo) \ No newline at end of file diff --git a/benchmark/tools/matrix.cpp b/benchmark/tools/matrix.cpp index ff35f9266cc..36752c9be48 100644 --- a/benchmark/tools/matrix.cpp +++ b/benchmark/tools/matrix.cpp @@ -175,9 +175,10 @@ int main(int argc, char** argv) if (argc == 1) { std::cerr << "Usage: " << argv[0] - << " [operation1] [operation2]\nApplies the given operations " + << " [-b] [operation1] [operation2]\nApplies the given operations " "to the input matrix read from stdin\nand writes it to " - "stdout.\nOperations are:\n" + "stdout.\nUses binary format if -b is set, otherwise matrix " + "market format.Operations are:\n" " lower-triangular removes nonzeros above the diagonal\n" " upper-triangular removes nonzeros below the diagonal\n" " remove-diagonal removes diagonal entries\n" @@ -194,10 +195,11 @@ int main(int argc, char** argv) << std::endl; return 1; } + bool binary = std::string{argv[1]} == "-b"; - auto data = gko::read_raw(std::cin); + auto data = gko::read_generic_raw(std::cin); data.ensure_row_major_order(); - for (int argi = 1; argi < argc; argi++) { + for (int argi = binary ? 2 : 1; argi < argc; argi++) { std::string arg{argv[argi]}; if (arg == "lower-triangular") { data = make_lower_triangular(data); @@ -232,5 +234,9 @@ int main(int argc, char** argv) return 1; } } - gko::write_raw(std::cout, data, gko::layout_type::coordinate); + if (binary) { + gko::write_binary_raw(std::cout, data); + } else { + gko::write_raw(std::cout, data, gko::layout_type::coordinate); + } } diff --git a/benchmark/tools/mtx_to_binary.cpp b/benchmark/tools/mtx_to_binary.cpp new file mode 100644 index 00000000000..8e26ee87b50 --- /dev/null +++ b/benchmark/tools/mtx_to_binary.cpp @@ -0,0 +1,105 @@ +/************************************************************* +Copyright (c) 2017-2022, the Ginkgo authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*************************************************************/ + +#include +#include +#include + + +#include +#include + + +template +void process(const char* input, const char* output, bool validate) +{ + std::ifstream is(input); + std::cerr << "Reading from " << input << '\n'; + auto data = gko::read_raw(is); + { + std::ofstream os(output, std::ios_base::out | std::ios_base::binary); + std::cerr << "Writing to " << output << '\n'; + if (data.size[0] <= std::numeric_limits::max()) { + gko::matrix_data int_data(data.size); + for (auto entry : data.nonzeros) { + int_data.nonzeros.emplace_back( + static_cast(entry.row), + static_cast(entry.column), entry.value); + } + gko::write_binary_raw(os, int_data); + } else { + gko::write_binary_raw(os, data); + } + } + if (validate) { + std::ifstream ois(output, std::ios_base::in | std::ios_base::binary); + auto data2 = gko::read_binary_raw(ois); + std::cerr << "Comparing against previously read data\n"; + if (data.size != data2.size) { + throw GKO_STREAM_ERROR("Mismatching sizes!"); + } + if (data.nonzeros != data2.nonzeros) { + throw GKO_STREAM_ERROR("Differing data!"); + } + std::cerr << "Validation successful!\n"; + } +} + + +int main(int argc, char** argv) +{ + if (argc < 3 || (std::string{argv[1]} == "-v" && argc < 4)) { + std::cerr << "Usage: " << argv[0] << " [-v] [input] [output]\n"; + return 1; + } + bool validate = std::string{argv[1]} == "-v"; + const auto input = validate ? argv[2] : argv[1]; + const auto output = validate ? argv[3] : argv[2]; + std::string header; + { + // read header, close file again + std::ifstream is(input); + std::getline(is, header); + } + try { + if (header.find("complex") != std::string::npos) { + std::cerr << "Input matrix is complex\n"; + process>(input, output, validate); + } else { + std::cerr << "Input matrix is real\n"; + process(input, output, validate); + } + } catch (gko::Error& err) { + std::cerr << err.what() << '\n'; + return 2; + } +} diff --git a/core/base/mtx_io.cpp b/core/base/mtx_io.cpp index 301b9caa2ef..a345dafc2d7 100644 --- a/core/base/mtx_io.cpp +++ b/core/base/mtx_io.cpp @@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include #include @@ -760,6 +761,187 @@ matrix_data read_raw(std::istream& is) } +template +static constexpr uint64_t binary_format_magic() +{ + constexpr auto is_int = std::is_same::value; + constexpr auto is_long = std::is_same::value; + constexpr auto is_double = std::is_same::value; + constexpr auto is_float = std::is_same::value; + constexpr auto is_complex_double = + std::is_same>::value; + constexpr auto is_complex_float = + std::is_same>::value; + static_assert(is_int || is_long, "invalid storage index type"); + static_assert( + is_double || is_float || is_complex_double || is_complex_float, + "invalid storage value type"); + constexpr auto index_bit = is_int ? 'I' : 'L'; + constexpr auto value_bit = + is_double ? 'D' : (is_float ? 'S' : (is_complex_double ? 'Z' : 'C')); + constexpr uint64 type_bits = index_bit * 256ull + value_bit; + return 'G' + + 256ull * + ('I' + + 256ull * + ('N' + + 256ull * + ('K' + + 256ull * + ('G' + 256ull * ('O' + 256ull * type_bits))))); +} + + +namespace { + + +template +struct select_helper {}; + +template <> +struct select_helper { + template + static T1 get(T1 val, T2) + { + return val; + } +}; + +template <> +struct select_helper { + template + static T2 get(T1, T2 val) + { + return val; + } +}; + + +template +matrix_data read_binary_convert(std::istream& is, + uint64 num_rows, + uint64 num_cols, + uint64 num_entries) +{ + if (num_rows > std::numeric_limits::max() || + num_cols > std::numeric_limits::max()) { + throw GKO_STREAM_ERROR( + "cannot read into this format, its index type would overflow"); + } + if (is_complex() && !is_complex()) { + throw GKO_STREAM_ERROR( + "cannot read into this format, would assign complex to real"); + } + matrix_data result(gko::dim<2>{num_rows, num_cols}); + result.nonzeros.resize(num_entries); + constexpr auto entry_binary_size = + sizeof(FileValueType) + 2 * sizeof(FileIndexType); + for (size_type i = 0; i < num_entries; i++) { + std::array block; + GKO_CHECK_STREAM(is.read(block.data(), entry_binary_size), + "failed reading entry " + std::to_string(i)); + FileValueType value{}; + FileIndexType row{}; + FileIndexType column{}; + std::memcpy(&row, &block[0], sizeof(FileIndexType)); + std::memcpy(&column, &block[sizeof(FileIndexType)], + sizeof(FileIndexType)); + std::memcpy(&value, &block[2 * sizeof(FileIndexType)], + sizeof(FileValueType)); + result.nonzeros[i].value = static_cast( + select_helper()>::get(value, real(value))); + result.nonzeros[i].row = row; + result.nonzeros[i].column = column; + } + return result; +} + + +} // namespace + + +template +matrix_data read_binary_raw(std::istream& is) +{ + std::array header{}; + GKO_CHECK_STREAM(is.read(header.data(), 32), "failed reading header"); + uint64 magic{}; + uint64 num_rows{}; + uint64 num_cols{}; + uint64 num_entries{}; + std::memcpy(&magic, &header[0], 8); + std::memcpy(&num_rows, &header[8], 8); + std::memcpy(&num_cols, &header[16], 8); + std::memcpy(&num_entries, &header[24], 8); +#define DECLARE_OVERLOAD(_vtype, _itype) \ + else if (magic == binary_format_magic<_vtype, _itype>()) \ + { \ + return read_binary_convert<_vtype, _itype, ValueType, IndexType>( \ + is, num_rows, num_cols, num_entries); \ + } + if (false) { + } + DECLARE_OVERLOAD(double, int32) + DECLARE_OVERLOAD(float, int32) + DECLARE_OVERLOAD(std::complex, int32) + DECLARE_OVERLOAD(std::complex, int32) + DECLARE_OVERLOAD(double, int64) + DECLARE_OVERLOAD(float, int64) + DECLARE_OVERLOAD(std::complex, int64) + DECLARE_OVERLOAD(std::complex, int64) +#undef DECLARE_OVERLOAD + else + { + throw GKO_STREAM_ERROR("invalid header magic number '" + + std::string(header.data(), 8) + "'"); + } +} + + +template +matrix_data read_generic_raw(std::istream& is) +{ + auto first_char = is.peek(); + GKO_CHECK_STREAM(is, "failed reading from stream"); + if (first_char == '%') { + return read_raw(is); + } else { + return read_binary_raw(is); + } +} + + +template +void write_binary_raw(std::ostream& os, + const matrix_data& mtx) +{ + uint64 magic = binary_format_magic(); + uint64 num_rows = mtx.size[0]; + uint64 num_cols = mtx.size[1]; + uint64 num_entries = mtx.nonzeros.size(); + std::array header{}; + std::memcpy(&header[0], &magic, 8); + std::memcpy(&header[8], &num_rows, 8); + std::memcpy(&header[16], &num_cols, 8); + std::memcpy(&header[24], &num_entries, 8); + GKO_CHECK_STREAM(os.write(header.data(), 32), "failed writing header"); + constexpr auto entry_binary_size = + sizeof(ValueType) + 2 * sizeof(IndexType); + for (size_type i = 0; i < num_entries; i++) { + std::array block; + std::memcpy(&block[0], &mtx.nonzeros[i].row, sizeof(IndexType)); + std::memcpy(&block[sizeof(IndexType)], &mtx.nonzeros[i].column, + sizeof(IndexType)); + std::memcpy(&block[2 * sizeof(IndexType)], &mtx.nonzeros[i].value, + sizeof(ValueType)); + GKO_CHECK_STREAM(os.write(block.data(), entry_binary_size), + "failed writing entry " + std::to_string(i)); + } + os.flush(); +} + + /** * Writes raw data to the stream. * @@ -786,8 +968,18 @@ void write_raw(std::ostream& os, const matrix_data& data, void write_raw(std::ostream& os, \ const matrix_data& data, \ layout_type layout) +#define GKO_DECLARE_READ_BINARY_RAW(ValueType, IndexType) \ + matrix_data read_binary_raw(std::istream& is) +#define GKO_DECLARE_WRITE_BINARY_RAW(ValueType, IndexType) \ + void write_binary_raw(std::ostream& os, \ + const matrix_data& data) +#define GKO_DECLARE_READ_GENERIC_RAW(ValueType, IndexType) \ + matrix_data read_generic_raw(std::istream& is) GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_READ_RAW); GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_WRITE_RAW); +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_READ_BINARY_RAW); +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_WRITE_BINARY_RAW); +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_READ_GENERIC_RAW); } // namespace gko diff --git a/core/test/base/mtx_io.cpp b/core/test/base/mtx_io.cpp index def4bc977c9..5c42225cdd2 100644 --- a/core/test/base/mtx_io.cpp +++ b/core/test/base/mtx_io.cpp @@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include #include @@ -41,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include @@ -394,6 +396,230 @@ TEST(MtxReader, ReadsSparseComplexHermitianMtx) } +std::array build_binary_complex_data() +{ + gko::uint64 int_val{}; + gko::uint64 neg_int_val{}; + double dbl_val = 2.5; + double neg_dbl_val = -2.5; + std::memcpy(&int_val, &dbl_val, sizeof(double)); + std::memcpy(&neg_int_val, &neg_dbl_val, sizeof(double)); + std::array data{ + 'G' + 256ull * + ('I' + + 256ull * + ('N' + + 256ull * + ('K' + + 256ull * + ('G' + + 256ull * + ('O' + 256ull * ('Z' + 256ull * 'L')))))), + 64, // num_rows + 32, // num_cols + 4, // num_entries + 0, // row + 1, // col + 0, // real val + int_val, // imag val + 1, // row + 1, // col + int_val, + neg_int_val, + 4, // row + 2, // col + 0, + neg_int_val, + 16, // row + 25, // col + 0, + 0}; + return data; +} + + +std::array build_binary_real_data() +{ + gko::uint64 int_val{}; + gko::uint64 neg_int_val{}; + double dbl_val = 2.5; + double neg_dbl_val = -2.5; + std::memcpy(&int_val, &dbl_val, sizeof(double)); + std::memcpy(&neg_int_val, &neg_dbl_val, sizeof(double)); + std::array data{ + 'G' + 256ull * + ('I' + + 256ull * + ('N' + + 256ull * + ('K' + + 256ull * + ('G' + + 256ull * + ('O' + 256ull * ('D' + 256ull * 'L')))))), + 64, // num_rows + 32, // num_cols + 4, // num_entries + 0, // row + 1, // col + 0, // val + 1, // row + 1, // col + int_val, + 4, // row + 2, // col + neg_int_val, + 16, // row + 25, // col + 0}; + return data; +} + + +TEST(MtxReader, ReadsBinary) +{ + auto raw_data = build_binary_real_data(); + auto test_read = [&](auto mtx_data) { + SCOPED_TRACE(gko::name_demangling::get_static_type(mtx_data)); + using value_type = + typename std::decay_t::value_type; + using index_type = + typename std::decay_t::index_type; + std::stringstream ss{ + std::string{reinterpret_cast(raw_data.data()), + raw_data.size() * sizeof(gko::uint64)}}; + + auto data = gko::read_binary_raw(ss); + + ASSERT_EQ(data.size, gko::dim<2>(64, 32)); + ASSERT_EQ(data.nonzeros.size(), 4); + ASSERT_EQ(data.nonzeros[0].row, 0); + ASSERT_EQ(data.nonzeros[1].row, 1); + ASSERT_EQ(data.nonzeros[2].row, 4); + ASSERT_EQ(data.nonzeros[3].row, 16); + ASSERT_EQ(data.nonzeros[0].column, 1); + ASSERT_EQ(data.nonzeros[1].column, 1); + ASSERT_EQ(data.nonzeros[2].column, 2); + ASSERT_EQ(data.nonzeros[3].column, 25); + ASSERT_EQ(data.nonzeros[0].value, value_type{0.0}); + ASSERT_EQ(data.nonzeros[1].value, value_type{2.5}); + ASSERT_EQ(data.nonzeros[2].value, value_type{-2.5}); + ASSERT_EQ(data.nonzeros[3].value, value_type{0.0}); + }; + + test_read(gko::matrix_data{}); + test_read(gko::matrix_data{}); + test_read(gko::matrix_data, gko::int32>{}); + test_read(gko::matrix_data, gko::int32>{}); + test_read(gko::matrix_data{}); + test_read(gko::matrix_data{}); + test_read(gko::matrix_data, gko::int64>{}); + test_read(gko::matrix_data, gko::int64>{}); +} + + +TEST(MtxReader, ReadsComplexBinary) +{ + auto raw_data = build_binary_complex_data(); + auto test_read = [&](auto mtx_data) { + SCOPED_TRACE(gko::name_demangling::get_static_type(mtx_data)); + using value_type = + typename std::decay_t::value_type; + using index_type = + typename std::decay_t::index_type; + std::stringstream ss{ + std::string{reinterpret_cast(raw_data.data()), + raw_data.size() * sizeof(gko::uint64)}}; + auto data = gko::read_binary_raw(ss); + + ASSERT_EQ(data.size, gko::dim<2>(64, 32)); + ASSERT_EQ(data.nonzeros.size(), 4); + ASSERT_EQ(data.nonzeros[0].row, 0); + ASSERT_EQ(data.nonzeros[1].row, 1); + ASSERT_EQ(data.nonzeros[2].row, 4); + ASSERT_EQ(data.nonzeros[3].row, 16); + ASSERT_EQ(data.nonzeros[0].column, 1); + ASSERT_EQ(data.nonzeros[1].column, 1); + ASSERT_EQ(data.nonzeros[2].column, 2); + ASSERT_EQ(data.nonzeros[3].column, 25); + ASSERT_EQ(data.nonzeros[0].value, value_type(0.0, 2.5)); + ASSERT_EQ(data.nonzeros[1].value, value_type(2.5, -2.5)); + ASSERT_EQ(data.nonzeros[2].value, value_type(0.0, -2.5)); + ASSERT_EQ(data.nonzeros[3].value, value_type(0.0, 0.0)); + }; + + auto test_read_fail = [&](auto mtx_data) { + SCOPED_TRACE(gko::name_demangling::get_static_type(mtx_data)); + using value_type = + typename std::decay_t::value_type; + using index_type = + typename std::decay_t::index_type; + std::stringstream ss{ + std::string{reinterpret_cast(raw_data.data()), + raw_data.size() * sizeof(gko::uint64)}}; + + ASSERT_THROW((gko::read_binary_raw(ss)), + gko::StreamError); + }; + + test_read_fail(gko::matrix_data{}); + test_read_fail(gko::matrix_data{}); + test_read(gko::matrix_data, gko::int32>{}); + test_read(gko::matrix_data, gko::int32>{}); + test_read_fail(gko::matrix_data{}); + test_read_fail(gko::matrix_data{}); + test_read(gko::matrix_data, gko::int64>{}); + test_read(gko::matrix_data, gko::int64>{}); +} + + +TEST(MtxReader, ReadsGenericBinary) +{ + auto raw_data = build_binary_real_data(); + std::stringstream ss{std::string{reinterpret_cast(raw_data.data()), + raw_data.size() * sizeof(gko::uint64)}}; + + auto data = gko::read_generic_raw(ss); + + ASSERT_EQ(data.size, gko::dim<2>(64, 32)); + ASSERT_EQ(data.nonzeros.size(), 4); + ASSERT_EQ(data.nonzeros[0].row, 0); + ASSERT_EQ(data.nonzeros[1].row, 1); + ASSERT_EQ(data.nonzeros[2].row, 4); + ASSERT_EQ(data.nonzeros[3].row, 16); + ASSERT_EQ(data.nonzeros[0].column, 1); + ASSERT_EQ(data.nonzeros[1].column, 1); + ASSERT_EQ(data.nonzeros[2].column, 2); + ASSERT_EQ(data.nonzeros[3].column, 25); + ASSERT_EQ(data.nonzeros[0].value, 0.0); + ASSERT_EQ(data.nonzeros[1].value, 2.5); + ASSERT_EQ(data.nonzeros[2].value, -2.5); + ASSERT_EQ(data.nonzeros[3].value, 0.0); +} + + +TEST(MtxReader, ReadsGenericMtx) +{ + using tpl = gko::matrix_data::nonzero_type; + std::istringstream iss( + "%%MatrixMarket matrix coordinate real general\n" + "2 3 4\n" + "1 1 1.0\n" + "2 2 5.0\n" + "1 2 3.0\n" + "1 3 2.0\n"); + + auto data = gko::read_generic_raw(iss); + + ASSERT_EQ(data.size, gko::dim<2>(2, 3)); + auto& v = data.nonzeros; + ASSERT_EQ(v[0], tpl(0, 0, 1.0)); + ASSERT_EQ(v[1], tpl(0, 1, 3.0)); + ASSERT_EQ(v[2], tpl(0, 2, 2.0)); + ASSERT_EQ(v[3], tpl(1, 1, 5.0)); +} + + TEST(MtxReader, FailsWhenReadingSparseComplexMtxToRealMtx) { using cpx = std::complex; @@ -598,6 +824,44 @@ TEST(MatrixData, WritesComplexFloatMatrixToMatrixMarketCoordinateWith64Index) } +TEST(MtxReader, WritesBinary) +{ + auto ref_data = build_binary_real_data(); + std::stringstream ss; + gko::matrix_data data; + data.size = gko::dim<2>{64, 32}; + data.nonzeros.resize(4); + data.nonzeros[0] = {0, 1, 0.0}; + data.nonzeros[1] = {1, 1, 2.5}; + data.nonzeros[2] = {4, 2, -2.5}; + data.nonzeros[3] = {16, 25, 0.0}; + + gko::write_binary_raw(ss, data); + + ASSERT_EQ(ss.str(), std::string(reinterpret_cast(ref_data.data()), + ref_data.size() * sizeof(gko::uint64))); +} + + +TEST(MtxReader, WritesComplexBinary) +{ + auto ref_data = build_binary_complex_data(); + std::stringstream ss; + gko::matrix_data, gko::int64> data; + data.size = gko::dim<2>{64, 32}; + data.nonzeros.resize(4); + data.nonzeros[0] = {0, 1, {0.0, 2.5}}; + data.nonzeros[1] = {1, 1, {2.5, -2.5}}; + data.nonzeros[2] = {4, 2, {0.0, -2.5}}; + data.nonzeros[3] = {16, 25, {0.0, 0.0}}; + + gko::write_binary_raw(ss, data); + + ASSERT_EQ(ss.str(), std::string(reinterpret_cast(ref_data.data()), + ref_data.size() * sizeof(gko::uint64))); +} + + template class DummyLinOp : public gko::EnableLinOp>, @@ -675,6 +939,65 @@ TYPED_TEST(RealDummyLinOpTest, ReadsLinOpFromStream) } +TYPED_TEST(RealDummyLinOpTest, ReadsGenericLinOpFromStream) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using tpl = typename gko::matrix_data::nonzero_type; + std::istringstream iss( + "%%MatrixMarket matrix array real general\n" + "2 3\n" + "1.0\n" + "0.0\n" + "3.0\n" + "5.0\n" + "2.0\n" + "0.0\n"); + + auto lin_op = gko::read_generic>( + iss, gko::ReferenceExecutor::create()); + + const auto& data = lin_op->data_; + ASSERT_EQ(data.size, gko::dim<2>(2, 3)); + const auto& v = data.nonzeros; + ASSERT_EQ(v[0], tpl(0, 0, 1.0)); + ASSERT_EQ(v[1], tpl(0, 1, 3.0)); + ASSERT_EQ(v[2], tpl(0, 2, 2.0)); + ASSERT_EQ(v[3], tpl(1, 0, 0.0)); + ASSERT_EQ(v[4], tpl(1, 1, 5.0)); + ASSERT_EQ(v[5], tpl(1, 2, 0.0)); +} + + +TYPED_TEST(RealDummyLinOpTest, ReadsGenericLinOpFromBinaryStream) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + auto raw_data = build_binary_real_data(); + std::istringstream iss(std::string{reinterpret_cast(raw_data.data()), + raw_data.size() * sizeof(gko::uint64)}); + + auto lin_op = gko::read_generic>( + iss, gko::ReferenceExecutor::create()); + + const auto& data = lin_op->data_; + ASSERT_EQ(data.size, gko::dim<2>(64, 32)); + ASSERT_EQ(data.nonzeros.size(), 4); + ASSERT_EQ(data.nonzeros[0].row, 0); + ASSERT_EQ(data.nonzeros[1].row, 1); + ASSERT_EQ(data.nonzeros[2].row, 4); + ASSERT_EQ(data.nonzeros[3].row, 16); + ASSERT_EQ(data.nonzeros[0].column, 1); + ASSERT_EQ(data.nonzeros[1].column, 1); + ASSERT_EQ(data.nonzeros[2].column, 2); + ASSERT_EQ(data.nonzeros[3].column, 25); + ASSERT_EQ(data.nonzeros[0].value, value_type{0.0}); + ASSERT_EQ(data.nonzeros[1].value, value_type{2.5}); + ASSERT_EQ(data.nonzeros[2].value, value_type{-2.5}); + ASSERT_EQ(data.nonzeros[3].value, value_type{0.0}); +} + + TYPED_TEST(RealDummyLinOpTest, WritesLinOpToStreamArray) { using value_type = typename TestFixture::value_type; @@ -756,6 +1079,33 @@ TYPED_TEST(RealDummyLinOpTest, WritesLinOpToStreamDefault) } +TYPED_TEST(RealDummyLinOpTest, WritesAndReadsBinaryLinOpToStreamArray) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + std::istringstream iss( + "%%MatrixMarket matrix array real general\n" + "2 3\n" + "1.0\n" + "0.0\n" + "3.0\n" + "5.0\n" + "2.0\n" + "0.0\n"); + auto ref = gko::ReferenceExecutor::create(); + auto lin_op = gko::read>(iss, ref); + std::ostringstream oss{}; + + gko::write_binary(oss, lend(lin_op)); + std::istringstream iss2{oss.str()}; + auto lin_op2 = + gko::read_binary>(iss2, ref); + + ASSERT_EQ(lin_op->data_.size, lin_op2->data_.size); + ASSERT_EQ(lin_op->data_.nonzeros, lin_op2->data_.nonzeros); +} + + template class DenseTest : public ::testing::Test { protected: @@ -841,6 +1191,65 @@ TYPED_TEST(ComplexDummyLinOpTest, ReadsLinOpFromStream) } +TYPED_TEST(ComplexDummyLinOpTest, ReadsGenericLinOpFromStream) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using tpl = typename gko::matrix_data::nonzero_type; + std::istringstream iss( + "%%MatrixMarket matrix array complex general\n" + "2 3\n" + "1.0 2.0\n" + "0.0 0.0\n" + "3.0 4.0\n" + "5.0 6.0\n" + "2.0 3.0\n" + "0.0 0.0\n"); + + auto lin_op = gko::read_generic>( + iss, gko::ReferenceExecutor::create()); + + const auto& data = lin_op->data_; + ASSERT_EQ(data.size, gko::dim<2>(2, 3)); + const auto& v = data.nonzeros; + ASSERT_EQ(v[0], tpl(0, 0, value_type{1.0, 2.0})); + ASSERT_EQ(v[1], tpl(0, 1, value_type{3.0, 4.0})); + ASSERT_EQ(v[2], tpl(0, 2, value_type{2.0, 3.0})); + ASSERT_EQ(v[3], tpl(1, 0, value_type{0.0, 0.0})); + ASSERT_EQ(v[4], tpl(1, 1, value_type{5.0, 6.0})); + ASSERT_EQ(v[5], tpl(1, 2, value_type{0.0, 0.0})); +} + + +TYPED_TEST(ComplexDummyLinOpTest, ReadsGenericLinOpFromBinaryStream) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + auto raw_data = build_binary_real_data(); + std::istringstream iss(std::string{reinterpret_cast(raw_data.data()), + raw_data.size() * sizeof(gko::uint64)}); + + auto lin_op = gko::read_generic>( + iss, gko::ReferenceExecutor::create()); + + const auto& data = lin_op->data_; + ASSERT_EQ(data.size, gko::dim<2>(64, 32)); + ASSERT_EQ(data.nonzeros.size(), 4); + ASSERT_EQ(data.nonzeros[0].row, 0); + ASSERT_EQ(data.nonzeros[1].row, 1); + ASSERT_EQ(data.nonzeros[2].row, 4); + ASSERT_EQ(data.nonzeros[3].row, 16); + ASSERT_EQ(data.nonzeros[0].column, 1); + ASSERT_EQ(data.nonzeros[1].column, 1); + ASSERT_EQ(data.nonzeros[2].column, 2); + ASSERT_EQ(data.nonzeros[3].column, 25); + ASSERT_EQ(data.nonzeros[0].value, value_type{0.0}); + ASSERT_EQ(data.nonzeros[1].value, value_type{2.5}); + ASSERT_EQ(data.nonzeros[2].value, value_type{-2.5}); + ASSERT_EQ(data.nonzeros[3].value, value_type{0.0}); +} + + TYPED_TEST(ComplexDummyLinOpTest, WritesLinOpToStreamArray) { using value_type = typename TestFixture::value_type; @@ -872,4 +1281,31 @@ TYPED_TEST(ComplexDummyLinOpTest, WritesLinOpToStreamArray) } +TYPED_TEST(ComplexDummyLinOpTest, WritesAndReadsBinaryLinOpToStreamArray) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + std::istringstream iss( + "%%MatrixMarket matrix array complex general\n" + "2 3\n" + "1.0 2.0\n" + "0.0 0.0\n" + "3.0 4.0\n" + "5.0 6.0\n" + "2.0 3.0\n" + "0.0 0.0\n"); + auto ref = gko::ReferenceExecutor::create(); + auto lin_op = gko::read>(iss, ref); + std::ostringstream oss{}; + + gko::write_binary(oss, lend(lin_op)); + std::istringstream iss2{oss.str()}; + auto lin_op2 = + gko::read_binary>(iss2, ref); + + ASSERT_EQ(lin_op->data_.size, lin_op2->data_.size); + ASSERT_EQ(lin_op->data_.nonzeros, lin_op2->data_.nonzeros); +} + + } // namespace diff --git a/core/test/utils/assertions.hpp b/core/test/utils/assertions.hpp index bd3fe7529f3..b9b9a091bb8 100644 --- a/core/test/utils/assertions.hpp +++ b/core/test/utils/assertions.hpp @@ -50,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include diff --git a/include/ginkgo/core/base/mtx_io.hpp b/include/ginkgo/core/base/mtx_io.hpp index bc955a72eb3..2bf4d42cfb1 100644 --- a/include/ginkgo/core/base/mtx_io.hpp +++ b/include/ginkgo/core/base/mtx_io.hpp @@ -61,6 +61,46 @@ template matrix_data read_raw(std::istream& is); +/** + * Reads a matrix stored in Ginkgo's binary matrix format from an input stream. + * Note that this format depends on the processor's endianness, + * so files from a big endian processor can't be read from a little endian + * processor and vice-versa. + * + * @tparam ValueType type of matrix values + * @tparam IndexType type of matrix indexes + * + * @param is input stream from which to read the data + * + * @return A matrix_data structure containing the matrix. The nonzero elements + * are sorted in lexicographic order of their (row, colum) indexes. + * + * @note This is an advanced routine that will return the raw matrix data + * structure. Consider using gko::read_binary instead. + */ +template +matrix_data read_binary_raw(std::istream& is); + + +/** + * Reads a matrix stored in either binary or matrix market format from an input + * stream. + * + * @tparam ValueType type of matrix values + * @tparam IndexType type of matrix indexes + * + * @param is input stream from which to read the data + * + * @return A matrix_data structure containing the matrix. The nonzero elements + * are sorted in lexicographic order of their (row, colum) indexes. + * + * @note This is an advanced routine that will return the raw matrix data + * structure. Consider using gko::read_generic instead. + */ +template +matrix_data read_generic_raw(std::istream& is); + + /** * Specifies the layout type when writing data in matrix market format. */ @@ -95,6 +135,28 @@ void write_raw(std::ostream& os, const matrix_data& data, layout_type layout = layout_type::array); +/** + * Writes a matrix_data structure to a stream in binary format. + * Note that this format depends on the processor's endianness, + * so files from a big endian processor can't be read from a little endian + * processor and vice-versa. + * + * @tparam ValueType type of matrix values + * @tparam IndexType type of matrix indexes + * + * @param os output stream where the data is to be written + * @param data the matrix data to write + * @param layout the layout used in the output + * + * @note This is an advanced routine that writes the raw matrix data structure. + * If you are trying to write an existing matrix, consider using + * gko::write_binary instead. + */ +template +void write_binary_raw(std::ostream& os, + const matrix_data& data); + + /** * Reads a matrix stored in matrix market format from an input stream. * @@ -119,6 +181,57 @@ inline std::unique_ptr read(StreamType&& is, MatrixArgs&&... args) } +/** + * Reads a matrix stored in binary format from an input stream. + * + * @tparam MatrixType a ReadableFromMatrixData LinOp type used to store the + * matrix once it's been read from disk. + * @tparam StreamType type of stream used to write the data to + * @tparam MatrixArgs additional argument types passed to MatrixType + * constructor + * + * @param is input stream from which to read the data + * @param args additional arguments passed to MatrixType constructor + * + * @return A MatrixType LinOp filled with data from filename + */ +template +inline std::unique_ptr read_binary(StreamType&& is, + MatrixArgs&&... args) +{ + auto mtx = MatrixType::create(std::forward(args)...); + mtx->read(read_binary_raw(is)); + return mtx; +} + + +/** + * Reads a matrix stored either in binary or matrix market format from an input + * stream. + * + * @tparam MatrixType a ReadableFromMatrixData LinOp type used to store the + * matrix once it's been read from disk. + * @tparam StreamType type of stream used to write the data to + * @tparam MatrixArgs additional argument types passed to MatrixType + * constructor + * + * @param is input stream from which to read the data + * @param args additional arguments passed to MatrixType constructor + * + * @return A MatrixType LinOp filled with data from filename + */ +template +inline std::unique_ptr read_generic(StreamType&& is, + MatrixArgs&&... args) +{ + auto mtx = MatrixType::create(std::forward(args)...); + mtx->read(read_generic_raw(is)); + return mtx; +} + + namespace matrix { @@ -184,10 +297,10 @@ struct mtx_io_traits { /** - * Reads a matrix stored in matrix market format from an input stream. + * Writes a matrix into an output stream in matrix market format. * - * @tparam MatrixType a ReadableFromMatrixData LinOp type used to store the - * matrix once it's been read from disk. + * @tparam MatrixType a WritableToMatrixData object providing data to be + * written. * @tparam StreamType type of stream used to write the data to * * @param os output stream where the data is to be written @@ -208,6 +321,31 @@ inline void write( } +/** + * Writes a matrix into an output stream in binary format. + * Note that this format depends on the processor's endianness, + * so files from a big endian processor can't be read from a little endian + * processor and vice-versa. + * + * @tparam MatrixType a WritableToMatrixData object providing data to be + * written. + * @tparam StreamType type of stream used to write the data to + * + * @param os output stream where the data is to be written + * @param matrix the matrix to write + * @param layout the layout used in the output + */ +template +inline void write_binary(StreamType&& os, MatrixType* matrix) +{ + matrix_data + data{}; + matrix->write(data); + write_binary_raw(os, data); +} + + } // namespace gko diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp index bfd08d2e948..2b3d17fee13 100644 --- a/include/ginkgo/core/matrix/dense.hpp +++ b/include/ginkgo/core/matrix/dense.hpp @@ -41,7 +41,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include #include #include #include From 27f45e6dbbfbf0b7a3922f57e9fe71052dedc2d1 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Mon, 7 Mar 2022 18:18:09 +0100 Subject: [PATCH 2/3] fix hypersparse symmetric MatrixMarket read --- core/base/mtx_io.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/base/mtx_io.cpp b/core/base/mtx_io.cpp index a345dafc2d7..be338a602e4 100644 --- a/core/base/mtx_io.cpp +++ b/core/base/mtx_io.cpp @@ -342,7 +342,8 @@ class mtx_io { size_type get_reservation_size(size_type num_rows, size_type num_cols, size_type num_nonzeros) const override { - return 2 * num_nonzeros - max(num_rows, num_cols); + return 2 * num_nonzeros - + min(2 * num_nonzeros, max(num_rows, num_cols)); } /** @@ -432,7 +433,8 @@ class mtx_io { size_type get_reservation_size(size_type num_rows, size_type num_cols, size_type num_nonzeros) const override { - return 2 * num_nonzeros - max(num_rows, num_cols); + return 2 * num_nonzeros - + min(2 * num_nonzeros, max(num_rows, num_cols)); } /** From 092c57a2db5e19990c8c57bae8d0c52a6a1d4b67 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Mon, 7 Mar 2022 18:32:02 +0100 Subject: [PATCH 3/3] review updates * Simplify binary magic setup * Improve documentation * Sort output of binary read Co-authored-by: Yuhsiang Tsai Co-authored-by: Marcel Koch Co-authored-by: Pratik Nayak --- benchmark/tools/CMakeLists.txt | 2 +- benchmark/tools/mtx_to_binary.cpp | 11 ++++- core/base/mtx_io.cpp | 24 +++++++--- core/test/base/mtx_io.cpp | 71 ++++++++++++++--------------- include/ginkgo/core/base/mtx_io.hpp | 22 +++++++-- 5 files changed, 78 insertions(+), 52 deletions(-) diff --git a/benchmark/tools/CMakeLists.txt b/benchmark/tools/CMakeLists.txt index 3bab475d471..0f3671b4779 100644 --- a/benchmark/tools/CMakeLists.txt +++ b/benchmark/tools/CMakeLists.txt @@ -4,4 +4,4 @@ target_compile_definitions(matrix_complex PRIVATE GKO_TOOL_COMPLEX) target_link_libraries(matrix Ginkgo::ginkgo) target_link_libraries(matrix_complex Ginkgo::ginkgo) add_executable(mtx_to_binary mtx_to_binary.cpp) -target_link_libraries(mtx_to_binary Ginkgo::ginkgo) \ No newline at end of file +target_link_libraries(mtx_to_binary Ginkgo::ginkgo) diff --git a/benchmark/tools/mtx_to_binary.cpp b/benchmark/tools/mtx_to_binary.cpp index 8e26ee87b50..4d0421a5cf8 100644 --- a/benchmark/tools/mtx_to_binary.cpp +++ b/benchmark/tools/mtx_to_binary.cpp @@ -78,7 +78,16 @@ void process(const char* input, const char* output, bool validate) int main(int argc, char** argv) { if (argc < 3 || (std::string{argv[1]} == "-v" && argc < 4)) { - std::cerr << "Usage: " << argv[0] << " [-v] [input] [output]\n"; + std::cerr + << "Usage: " << argv[0] + << " [-v] [input] [output]\n" + "Reads the input file in MatrixMarket format and converts it" + "to Ginkgo's binary format.\nWith the optional -v flag, reads " + "the written binary output again and compares it with the " + "original input to validate the conversion.\n" + "The conversion uses a complex value type if necessary, " + "the highest possible value precision and the smallest " + "possible index type.\n"; return 1; } bool validate = std::string{argv[1]} == "-v"; diff --git a/core/base/mtx_io.cpp b/core/base/mtx_io.cpp index be338a602e4..f0ad3ef9ae0 100644 --- a/core/base/mtx_io.cpp +++ b/core/base/mtx_io.cpp @@ -763,8 +763,15 @@ matrix_data read_raw(std::istream& is) } +/** + * Returns the magic number at the beginning of the binary format header for the + * given type parameters. + * + * @tparam ValueType the value type to be used for the binary storage + * @tparam IndexType the index type to be used for the binary storage + */ template -static constexpr uint64_t binary_format_magic() +static constexpr uint64 binary_format_magic() { constexpr auto is_int = std::is_same::value; constexpr auto is_long = std::is_same::value; @@ -781,16 +788,16 @@ static constexpr uint64_t binary_format_magic() constexpr auto index_bit = is_int ? 'I' : 'L'; constexpr auto value_bit = is_double ? 'D' : (is_float ? 'S' : (is_complex_double ? 'Z' : 'C')); - constexpr uint64 type_bits = index_bit * 256ull + value_bit; + constexpr uint64 shift = 256; + constexpr uint64 type_bits = index_bit * shift + value_bit; return 'G' + - 256ull * + shift * ('I' + - 256ull * + shift * ('N' + - 256ull * + shift * ('K' + - 256ull * - ('G' + 256ull * ('O' + 256ull * type_bits))))); + shift * ('G' + shift * ('O' + shift * type_bits))))); } @@ -856,6 +863,8 @@ matrix_data read_binary_convert(std::istream& is, result.nonzeros[i].row = row; result.nonzeros[i].column = column; } + // sort the entries + result.ensure_row_major_order(); return result; } @@ -866,6 +875,7 @@ matrix_data read_binary_convert(std::istream& is, template matrix_data read_binary_raw(std::istream& is) { + static_assert(sizeof(uint64) == 8, "c++ is broken"); // just to be sure std::array header{}; GKO_CHECK_STREAM(is.read(header.data(), 32), "failed reading header"); uint64 magic{}; diff --git a/core/test/base/mtx_io.cpp b/core/test/base/mtx_io.cpp index 5c42225cdd2..7f492cdc991 100644 --- a/core/test/base/mtx_io.cpp +++ b/core/test/base/mtx_io.cpp @@ -404,17 +404,18 @@ std::array build_binary_complex_data() double neg_dbl_val = -2.5; std::memcpy(&int_val, &dbl_val, sizeof(double)); std::memcpy(&neg_int_val, &neg_dbl_val, sizeof(double)); + constexpr gko::uint64 shift = 256; + // note: the following data is not sorted! std::array data{ - 'G' + 256ull * + 'G' + shift * ('I' + - 256ull * + shift * ('N' + - 256ull * + shift * ('K' + - 256ull * - ('G' + - 256ull * - ('O' + 256ull * ('Z' + 256ull * 'L')))))), + shift * ('G' + + shift * ('O' + + shift * ('Z' + shift * 'L')))))), 64, // num_rows 32, // num_cols 4, // num_entries @@ -426,14 +427,14 @@ std::array build_binary_complex_data() 1, // col int_val, neg_int_val, - 4, // row - 2, // col - 0, - neg_int_val, 16, // row 25, // col 0, - 0}; + 0, + 4, // row + 2, // col + 0, + neg_int_val}; return data; } @@ -446,26 +447,27 @@ std::array build_binary_real_data() double neg_dbl_val = -2.5; std::memcpy(&int_val, &dbl_val, sizeof(double)); std::memcpy(&neg_int_val, &neg_dbl_val, sizeof(double)); + constexpr gko::uint64 shift = 256; + // note: the following data is not sorted! std::array data{ - 'G' + 256ull * + 'G' + shift * ('I' + - 256ull * + shift * ('N' + - 256ull * + shift * ('K' + - 256ull * - ('G' + - 256ull * - ('O' + 256ull * ('D' + 256ull * 'L')))))), + shift * ('G' + + shift * ('O' + + shift * ('D' + shift * 'L')))))), 64, // num_rows 32, // num_cols 4, // num_entries - 0, // row - 1, // col - 0, // val 1, // row 1, // col int_val, + 0, // row + 1, // col + 0, // val 4, // row 2, // col neg_int_val, @@ -831,8 +833,8 @@ TEST(MtxReader, WritesBinary) gko::matrix_data data; data.size = gko::dim<2>{64, 32}; data.nonzeros.resize(4); - data.nonzeros[0] = {0, 1, 0.0}; - data.nonzeros[1] = {1, 1, 2.5}; + data.nonzeros[0] = {1, 1, 2.5}; + data.nonzeros[1] = {0, 1, 0.0}; data.nonzeros[2] = {4, 2, -2.5}; data.nonzeros[3] = {16, 25, 0.0}; @@ -852,8 +854,8 @@ TEST(MtxReader, WritesComplexBinary) data.nonzeros.resize(4); data.nonzeros[0] = {0, 1, {0.0, 2.5}}; data.nonzeros[1] = {1, 1, {2.5, -2.5}}; - data.nonzeros[2] = {4, 2, {0.0, -2.5}}; - data.nonzeros[3] = {16, 25, {0.0, 0.0}}; + data.nonzeros[2] = {16, 25, {0.0, 0.0}}; + data.nonzeros[3] = {4, 2, {0.0, -2.5}}; gko::write_binary_raw(ss, data); @@ -973,6 +975,7 @@ TYPED_TEST(RealDummyLinOpTest, ReadsGenericLinOpFromBinaryStream) { using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; + using tpl = typename gko::matrix_data::nonzero_type; auto raw_data = build_binary_real_data(); std::istringstream iss(std::string{reinterpret_cast(raw_data.data()), raw_data.size() * sizeof(gko::uint64)}); @@ -983,18 +986,10 @@ TYPED_TEST(RealDummyLinOpTest, ReadsGenericLinOpFromBinaryStream) const auto& data = lin_op->data_; ASSERT_EQ(data.size, gko::dim<2>(64, 32)); ASSERT_EQ(data.nonzeros.size(), 4); - ASSERT_EQ(data.nonzeros[0].row, 0); - ASSERT_EQ(data.nonzeros[1].row, 1); - ASSERT_EQ(data.nonzeros[2].row, 4); - ASSERT_EQ(data.nonzeros[3].row, 16); - ASSERT_EQ(data.nonzeros[0].column, 1); - ASSERT_EQ(data.nonzeros[1].column, 1); - ASSERT_EQ(data.nonzeros[2].column, 2); - ASSERT_EQ(data.nonzeros[3].column, 25); - ASSERT_EQ(data.nonzeros[0].value, value_type{0.0}); - ASSERT_EQ(data.nonzeros[1].value, value_type{2.5}); - ASSERT_EQ(data.nonzeros[2].value, value_type{-2.5}); - ASSERT_EQ(data.nonzeros[3].value, value_type{0.0}); + ASSERT_EQ(data.nonzeros[0], tpl(0, 1, value_type{0.0})); + ASSERT_EQ(data.nonzeros[1], tpl(1, 1, value_type{2.5})); + ASSERT_EQ(data.nonzeros[2], tpl(4, 2, value_type{-2.5})); + ASSERT_EQ(data.nonzeros[3], tpl(16, 25, value_type{0.0})); } diff --git a/include/ginkgo/core/base/mtx_io.hpp b/include/ginkgo/core/base/mtx_io.hpp index 2bf4d42cfb1..5575ba3f4ae 100644 --- a/include/ginkgo/core/base/mtx_io.hpp +++ b/include/ginkgo/core/base/mtx_io.hpp @@ -52,7 +52,7 @@ namespace gko { * @param is input stream from which to read the data * * @return A matrix_data structure containing the matrix. The nonzero elements - * are sorted in lexicographic order of their (row, colum) indexes. + * are sorted in lexicographic order of their (row, column) indexes. * * @note This is an advanced routine that will return the raw matrix data * structure. Consider using gko::read instead. @@ -67,13 +67,27 @@ matrix_data read_raw(std::istream& is); * so files from a big endian processor can't be read from a little endian * processor and vice-versa. * + * The binary format has the following structure (in system endianness): + * 1. A 32 byte header consisting of 4 uint64_t values: + * magic = GINKGO__: The highest two bytes stand for value and index type. + * value type: S (float), D (double), + * C (complex), Z(complex) + * index type: I (int32), L (int64) + * num_rows: Number of rows + * num_cols: Number of columns + * num_entries: Number of (row, column, value) tuples to follow + * 2. Following are num_entries blocks of size + * sizeof(IndexType) * 2 + sizeof(ValueType). + * Each consists of a row index stored as IndexType, followed by + * a column index stored as IndexType and a value stored as ValueType. + * * @tparam ValueType type of matrix values * @tparam IndexType type of matrix indexes * * @param is input stream from which to read the data * * @return A matrix_data structure containing the matrix. The nonzero elements - * are sorted in lexicographic order of their (row, colum) indexes. + * are sorted in lexicographic order of their (row, column) indexes. * * @note This is an advanced routine that will return the raw matrix data * structure. Consider using gko::read_binary instead. @@ -92,7 +106,7 @@ matrix_data read_binary_raw(std::istream& is); * @param is input stream from which to read the data * * @return A matrix_data structure containing the matrix. The nonzero elements - * are sorted in lexicographic order of their (row, colum) indexes. + * are sorted in lexicographic order of their (row, column) indexes. * * @note This is an advanced routine that will return the raw matrix data * structure. Consider using gko::read_generic instead. @@ -146,7 +160,6 @@ void write_raw(std::ostream& os, const matrix_data& data, * * @param os output stream where the data is to be written * @param data the matrix data to write - * @param layout the layout used in the output * * @note This is an advanced routine that writes the raw matrix data structure. * If you are trying to write an existing matrix, consider using @@ -333,7 +346,6 @@ inline void write( * * @param os output stream where the data is to be written * @param matrix the matrix to write - * @param layout the layout used in the output */ template inline void write_binary(StreamType&& os, MatrixType* matrix)