From e01ce501e7027569c99c6425fee82e50c041d4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Widera?= Date: Wed, 20 Jan 2021 13:10:06 +0100 Subject: [PATCH 1/2] improve Blosc compression operator The current implementation of the Blosc ADIOS2 operator is due to limitations of Blosc not supporting variables larger than 2GiB. - Add option `threshold` to set a threshold under which data will only be copied instead of compressed (equivalent to Blosc in ADIOS1). - Use chunk format to avoid 2GiB limit for variables (datasets). - Add support to decompress blosc data written with ADIOS before this PR. --- source/adios2/common/ADIOSTypes.h | 1 + .../operator/compress/CompressBlosc.cpp | 236 ++++++++++++++++-- .../adios2/operator/compress/CompressBlosc.h | 50 +++- 3 files changed, 267 insertions(+), 20 deletions(-) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index a36b923e7a..e0c0f0e35f 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -387,6 +387,7 @@ constexpr char compressor[] = "compressor"; constexpr char clevel[] = "clevel"; constexpr char doshuffle[] = "doshuffle"; constexpr char blocksize[] = "blocksize"; +constexpr char threshold[] = "threshold"; } namespace value diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 6104c226b0..f74d112c3a 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -6,6 +6,7 @@ * * Created on: Jun 18, 2019 * Author: William F Godoy godoywf@ornl.gov + * Rene Widera r.widera@hzdr.de */ #include "CompressBlosc.h" @@ -16,6 +17,10 @@ extern "C" { #include "adios2/helper/adiosFunctions.h" +#include +#include +#include + namespace adios2 { namespace core @@ -36,6 +41,25 @@ CompressBlosc::CompressBlosc(const Params ¶meters) { } +size_t CompressBlosc::BufferMaxSize(const size_t sizeIn) const +{ + const size_t maxInputPerChunk = BLOSC_MAX_BUFFERSIZE; + const size_t numFullChunks = sizeIn / maxInputPerChunk; + const size_t sizeLastChunk = sizeIn % maxInputPerChunk; + + const size_t maxOutputPerChunk = maxInputPerChunk + BLOSC_MAX_OVERHEAD; + const size_t maxOutputLastChunk = sizeLastChunk + BLOSC_MAX_OVERHEAD; + + /* DataHeader is used to detect of old format which can only handle + * BLOSC_MAX_BUFFERSIZE (<2GiB) or the new adios2 chunked blosc format is + * used. + */ + const size_t maxRquiredDataMem = maxOutputPerChunk * numFullChunks + + maxOutputLastChunk + sizeof(DataHeader); + + return maxRquiredDataMem; +} + size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, const size_t elementSize, DataType type, void *bufferOut, const Params ¶meters, @@ -44,6 +68,10 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, const size_t sizeIn = static_cast(helper::GetTotalSize(dimensions) * elementSize); + bool useMemcpy = false; + /* input size under this bound would not compressed */ + size_t thresholdSize = 128; + blosc_init(); size_t threads = 1; // defaults @@ -95,8 +123,9 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, { throw std::invalid_argument( "ERROR: invalid compressor " + compressor + - " valid values: blosclz (default), lz4, lz4hc, snappy, " - "zlib, or ztsd, in call to ADIOS2 Blosc Compression\n"); + " valid values: blosclz (default), lz4, lz4hc, " + "snappy, " + "zlib, or zstd, in call to ADIOS2 Blosc Compression\n"); } } else if (key == "blocksize") @@ -104,39 +133,208 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, blockSize = static_cast(helper::StringTo( value, "when setting Blosc blocksize parameter\n")); } + else if (key == "threshold") + { + thresholdSize = static_cast(helper::StringTo( + value, "when setting Blosc threshold parameter\n")); + if (thresholdSize < 128u) + thresholdSize = 128u; + } } - const int result = blosc_set_compressor(compressor.c_str()); - if (result == -1) + // write header to detect new compression format (set first 8 byte to zero) + DataHeader *headerPtr = reinterpret_cast(bufferOut); + + // set default header + *headerPtr = DataHeader{}; + + const uint8_t *inputDataBuff = reinterpret_cast(dataIn); + + int32_t typesize = elementSize; + if (typesize > BLOSC_MAX_TYPESIZE) + typesize = 1; + + uint8_t *outputBuff = reinterpret_cast(bufferOut); + outputBuff += sizeof(DataHeader); + + size_t currentOutputSize = 0u; + size_t inputOffset = 0u; + + if (sizeIn < thresholdSize) { - throw std::invalid_argument("ERROR: invalid compressor " + compressor + - " check if supported by blosc build, in " - "call to ADIOS2 Blosc Compression\n"); + /* disable compression */ + useMemcpy = true; } - blosc_set_nthreads(threads); - blosc_set_blocksize(blockSize); + if (!useMemcpy) + { + const int result = blosc_set_compressor(compressor.c_str()); + if (result == -1) + { + throw std::invalid_argument( + "ERROR: invalid compressor " + compressor + + " check if supported by blosc build, in " + "call to ADIOS2 Blosc Compression\n"); + } + blosc_set_nthreads(threads); + blosc_set_blocksize(blockSize); + + uint32_t chunk = 0; + for (; inputOffset < sizeIn; ++chunk) + { + size_t inputChunkSize = + std::min(sizeIn - inputOffset, + static_cast(BLOSC_MAX_BUFFERSIZE)); + bloscSize_t maxIntputSize = + static_cast(inputChunkSize); + + bloscSize_t maxChunkSize = maxIntputSize + BLOSC_MAX_OVERHEAD; + + const uint8_t *in_ptr = inputDataBuff + inputOffset; + uint8_t *out_ptr = outputBuff + currentOutputSize; - const int compressedSize = - blosc_compress(compressionLevel, doShuffle, elementSize, sizeIn, dataIn, - bufferOut, sizeIn); + bloscSize_t compressedChunkSize = + blosc_compress(compressionLevel, doShuffle, typesize, + maxIntputSize, in_ptr, out_ptr, maxChunkSize); + + if (compressedChunkSize > 0) + currentOutputSize += static_cast(compressedChunkSize); + else + { + // something went wrong with the compression switch to memcopy + useMemcpy = true; + break; + } + /* add size to written output data */ + inputOffset += static_cast(maxIntputSize); + } + + if (!useMemcpy) + { + // validate that all bytes are compressed + assert(inputOffset == sizeIn); + headerPtr->SetNumChunks(chunk); + } + } - if (compressedSize <= 0) + if (useMemcpy) { - throw std::invalid_argument( - "ERROR: from blosc_compress return size: " + - std::to_string(compressedSize) + - ", check operator parameters, " - " compression failed in ADIOS2 Blosc Compression\n"); + std::memcpy(outputBuff, inputDataBuff, sizeIn); + currentOutputSize = sizeIn; + headerPtr->SetNumChunks(0u); } blosc_destroy(); - return static_cast(compressedSize); + return currentOutputSize + sizeof(DataHeader); } size_t CompressBlosc::Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, const size_t sizeOut, Params &info) const +{ + assert(sizeIn >= sizeof(DataHeader)); + const bool isChunked = + reinterpret_cast(bufferIn)->IsChunked(); + + size_t decompressedSize = 0u; + if (isChunked) + decompressedSize = + DecompressChunkedFormat(bufferIn, sizeIn, dataOut, sizeOut, info); + else + decompressedSize = + DecompressOldFormat(bufferIn, sizeIn, dataOut, sizeOut, info); + + return decompressedSize; +} + +size_t CompressBlosc::DecompressChunkedFormat(const void *bufferIn, + const size_t sizeIn, + void *dataOut, + const size_t sizeOut, + Params &info) const +{ + const DataHeader *dataPtr = reinterpret_cast(bufferIn); + uint32_t num_chunks = dataPtr->GetNumChunks(); + size_t inputDataSize = sizeIn - sizeof(DataHeader); + + bool isCompressed = true; + if (num_chunks == 0) + isCompressed = false; + + size_t inputOffset = 0u; + size_t currentOutputSize = 0u; + + const uint8_t *inputDataBuff = + reinterpret_cast(bufferIn) + sizeof(DataHeader); + + size_t uncompressedSize = sizeOut; + + if (isCompressed) + { + blosc_init(); + uint8_t *outputBuff = reinterpret_cast(dataOut); + + while (inputOffset < inputDataSize) + { + /* move over the size of the compressed data */ + const uint8_t *in_ptr = inputDataBuff + inputOffset; + + /** read the size of the compress block from the blosc meta data + * + * blosc meta data format (all little endian): + * - 1 byte blosc format version + * - 1 byte blosclz format version + * - 1 byte flags + * - 1 byte typesize + * - 4 byte uncompressed data size + * - 4 byte block size + * - 4 byte compressed data size + * + * we need only the compressed size ( source address + 12 byte) + */ + bloscSize_t max_inputDataSize = + *reinterpret_cast(in_ptr + 12u); + + uint8_t *out_ptr = outputBuff + currentOutputSize; + + size_t outputChunkSize = + std::min(uncompressedSize - currentOutputSize, + static_cast(BLOSC_MAX_BUFFERSIZE)); + bloscSize_t max_output_size = + static_cast(outputChunkSize); + + bloscSize_t decompressdSize = + blosc_decompress(in_ptr, out_ptr, max_output_size); + + if (decompressdSize > 0) + currentOutputSize += static_cast(decompressdSize); + else + { + throw std::runtime_error( + "ERROR: ADIOS2 Blosc Decompress failed. Decompressed chunk " + "results in zero decompressed bytes.\n"); + } + inputOffset += static_cast(max_inputDataSize); + } + blosc_destroy(); + } + else + { + std::memcpy(dataOut, inputDataBuff, inputDataSize); + currentOutputSize = inputDataSize; + inputOffset += inputDataSize; + } + + assert(currentOutputSize == uncompressedSize); + assert(inputOffset == inputDataSize); + + return currentOutputSize; +} + +size_t CompressBlosc::DecompressOldFormat(const void *bufferIn, + const size_t sizeIn, void *dataOut, + const size_t sizeOut, + Params &info) const { blosc_init(); const int decompressedSize = blosc_decompress(bufferIn, dataOut, sizeOut); diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 7189e9c8ec..9251b91d1f 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -6,6 +6,7 @@ * * Created on: Jun 18, 2019 * Author: William F Godoy godoywf@ornl.gov + * Rene Widera r.widera@hzdr.de */ #ifndef ADIOS2_OPERATOR_COMPRESS_COMPRESSBLOSC_H_ @@ -34,12 +35,15 @@ class CompressBlosc : public Operator ~CompressBlosc() = default; + size_t BufferMaxSize(const size_t sizeIn) const final; + /** * Compression signature for legacy libraries that use void* * @param dataIn * @param dimensions * @param type - * @param bufferOut + * @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk | + * UncompressedData), [ BloscCompressedChunk, ...]' * @param parameters * @return size of compressed buffer in bytes */ @@ -60,6 +64,50 @@ class CompressBlosc : public Operator const size_t sizeOut, Params &info) const final; private: + using bloscSize_t = int32_t; + + /** Decompress chunked data */ + size_t DecompressChunkedFormat(const void *bufferIn, const size_t sizeIn, + void *dataOut, const size_t sizeOut, + Params &info) const; + + /** Decompress data written before ADIOS2 supported large variables larger + * 2GiB. */ + size_t DecompressOldFormat(const void *bufferIn, const size_t sizeIn, + void *dataOut, const size_t sizeOut, + Params &info) const; + + class __attribute__((packed)) DataHeader + { + /** compatible to the first 4 byte of blosc header + * + * blosc meta data format (all little endian): + * - 1 byte blosc format version + * - 1 byte blosclz format version + * - 1 byte flags + * - 1 byte typesize + * + * If zero we writing the new adios blosc format which can handle more + * than 2GiB data chunks. + */ + uint32_t format = 0u; + /** number of blosc chunks within the data blob + * + * If zero the data is not compressed and must be decompressed by using + * 'memcpy' + */ + uint32_t numberOfChunks = 0u; + + public: + void SetNumChunks(const uint32_t numChunks) + { + numberOfChunks = numChunks; + } + uint32_t GetNumChunks() const { return numberOfChunks; } + + bool IsChunked() const { return format == 0; } + }; + static const std::map m_Shuffles; static const std::set m_Compressors; }; From c83fcb741528ab40a7df9b87cfe0bc0ef1e43b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Widera?= Date: Wed, 20 Jan 2021 16:50:48 +0100 Subject: [PATCH 2/2] update blosc tests - add test combination for new option `threshold` - add test combination for shuffling --- .../bp/operations/TestBPWriteReadBlosc.cpp | 133 ++++++++++++------ 1 file changed, 93 insertions(+), 40 deletions(-) diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp index 91804d859e..7d16a81109 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp @@ -8,6 +8,7 @@ #include #include //std::iota #include +#include #include @@ -15,11 +16,13 @@ std::string engineName; // comes from command line -void BloscAccuracy1D(const std::string accuracy) +void BloscAccuracy1D(const std::string accuracy, const std::string threshold, + const std::string doshuffle) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWR_Blosc_1D_" + accuracy + ".bp"); + const std::string fname("BPWR_Blosc_1D_" + accuracy + "_" + threshold + + "_" + doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -72,9 +75,13 @@ void BloscAccuracy1D(const std::string accuracy) adios.DefineOperator("BloscCompressor", adios2::ops::LosslessBlosc); var_r32.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); var_r64.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -150,11 +157,13 @@ void BloscAccuracy1D(const std::string accuracy) } } -void BloscAccuracy2D(const std::string accuracy) +void BloscAccuracy2D(const std::string accuracy, const std::string threshold, + const std::string doshuffle) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc2D_" + accuracy + ".bp"); + const std::string fname("BPWRBlosc2D_" + accuracy + "_" + threshold + + threshold + "_" + doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -208,9 +217,13 @@ void BloscAccuracy2D(const std::string accuracy) adios.DefineOperator("BloscCompressor", adios2::ops::LosslessBlosc); var_r32.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); var_r64.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -288,11 +301,13 @@ void BloscAccuracy2D(const std::string accuracy) } } -void BloscAccuracy3D(const std::string accuracy) +void BloscAccuracy3D(const std::string accuracy, const std::string threshold, + const std::string doshuffle) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc3D_" + accuracy + ".bp"); + const std::string fname("BPWRBlosc3D_" + accuracy + "_" + threshold + + threshold + "_" + doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -347,9 +362,13 @@ void BloscAccuracy3D(const std::string accuracy) adios.DefineOperator("BloscCompressor", adios2::ops::LosslessBlosc); var_r32.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); var_r64.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -429,11 +448,13 @@ void BloscAccuracy3D(const std::string accuracy) } } -void BloscAccuracy1DSel(const std::string accuracy) +void BloscAccuracy1DSel(const std::string accuracy, const std::string threshold, + const std::string doshuffle) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc1DSel_" + accuracy + ".bp"); + const std::string fname("BPWRBlosc1DSel_" + accuracy + "_" + threshold + + threshold + "_" + doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -486,9 +507,13 @@ void BloscAccuracy1DSel(const std::string accuracy) adios.DefineOperator("BloscCompressor", adios2::ops::LosslessBlosc); var_r32.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); var_r64.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -564,11 +589,13 @@ void BloscAccuracy1DSel(const std::string accuracy) } } -void BloscAccuracy2DSel(const std::string accuracy) +void BloscAccuracy2DSel(const std::string accuracy, const std::string threshold, + const std::string doshuffle) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc2DSel_" + accuracy + ".bp"); + const std::string fname("BPWRBlosc2DSel_" + accuracy + "_" + threshold + + threshold + "_" + doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -622,9 +649,13 @@ void BloscAccuracy2DSel(const std::string accuracy) adios.DefineOperator("BloscCompressor", adios2::ops::LosslessBlosc); var_r32.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); var_r64.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -702,11 +733,13 @@ void BloscAccuracy2DSel(const std::string accuracy) } } -void BloscAccuracy3DSel(const std::string accuracy) +void BloscAccuracy3DSel(const std::string accuracy, const std::string threshold, + const std::string doshuffle) { // Each process would write a 1x8 array and all processes would // form a mpiSize * Nx 1D array - const std::string fname("BPWRBlosc3DSel_" + accuracy + ".bp"); + const std::string fname("BPWRBlosc3DSel_" + accuracy + "_" + threshold + + threshold + "_" + doshuffle + ".bp"); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -761,9 +794,13 @@ void BloscAccuracy3DSel(const std::string accuracy) adios.DefineOperator("BloscCompressor", adios2::ops::LosslessBlosc); var_r32.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::doshuffle, doshuffle}, + {adios2::ops::blosc::key::threshold, threshold}}); var_r64.AddOperation(BloscOp, - {{adios2::ops::blosc::key::clevel, accuracy}}); + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -845,7 +882,8 @@ void BloscAccuracy3DSel(const std::string accuracy) } } -class BPWriteReadBlosc : public ::testing::TestWithParam +class BPWriteReadBlosc : public ::testing::TestWithParam< + std::tuple> { public: BPWriteReadBlosc() = default; @@ -855,40 +893,55 @@ class BPWriteReadBlosc : public ::testing::TestWithParam TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc1D) { - BloscAccuracy1D(GetParam()); + BloscAccuracy1D(std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam())); } TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc2D) { - BloscAccuracy2D(GetParam()); + BloscAccuracy2D(std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam())); } TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc3D) { - BloscAccuracy3D(GetParam()); + BloscAccuracy3D(std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam())); } TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc1DSel) { - BloscAccuracy1DSel(GetParam()); + BloscAccuracy1DSel(std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam())); } TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc2DSel) { - BloscAccuracy2DSel(GetParam()); + BloscAccuracy2DSel(std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam())); } TEST_P(BPWriteReadBlosc, ADIOS2BPWriteReadBlosc3DSel) { - BloscAccuracy3DSel(GetParam()); + BloscAccuracy3DSel(std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam())); } INSTANTIATE_TEST_SUITE_P( BloscAccuracy, BPWriteReadBlosc, - ::testing::Values(adios2::ops::blosc::value::clevel_1, - adios2::ops::blosc::value::clevel_2, - adios2::ops::blosc::value::clevel_3, - adios2::ops::blosc::value::clevel_4, - adios2::ops::blosc::value::clevel_5, - adios2::ops::blosc::value::clevel_6, - adios2::ops::blosc::value::clevel_7, - adios2::ops::blosc::value::clevel_8, - adios2::ops::blosc::value::clevel_9)); + ::testing::Combine( + // test compression level + ::testing::Values(adios2::ops::blosc::value::clevel_1, + adios2::ops::blosc::value::clevel_2, + adios2::ops::blosc::value::clevel_3, + adios2::ops::blosc::value::clevel_4, + adios2::ops::blosc::value::clevel_5, + adios2::ops::blosc::value::clevel_6, + adios2::ops::blosc::value::clevel_7, + adios2::ops::blosc::value::clevel_8, + adios2::ops::blosc::value::clevel_9), + // test threshold: 128 is equal to the default, 1 GiB is used to disable + // compression + ::testing::Values("128", "1073741824"), + // test shuffling + ::testing::Values(adios2::ops::blosc::value::doshuffle_shuffle, + adios2::ops::blosc::value::doshuffle_noshuffle, + adios2::ops::blosc::value::doshuffle_bitshuffle))); int main(int argc, char **argv) {