From d273454fa32b38b0a479b56e498a8529894dd9a8 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 9 Sep 2024 15:12:40 -0400 Subject: [PATCH] Compress the lossless part of MGARD data with Zstd. It was uncompressed until now. $ ctest -R MGARD.*/.*BP5.Serial $ find testing/adios2/engine/bp/operations/bp5 -name "*bp" -exec du -sh {} \; 48K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.00001.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.00001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.01.bp 36K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.00001.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.0001.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.00001.bp 36K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.0001.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.0001.bp 32K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.01.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.00001.bp 32K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.001.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.01.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.001.bp 32K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.001.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.001.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.0001.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.01.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.001.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.01.bp 32K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.0001.bp 32K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.00001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.0001.bp 32K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.01.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.00001.bp 40K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.01.bp 44K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.0001.bp Before this commit: $ find testing/adios2/engine/bp/operations/bp5 -name "*bp" -exec du -sh {} \; 352K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.00001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.00001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.01.bp 96K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.00001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.0001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.00001.bp 96K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.0001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.0001.bp 100K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.01.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.00001.bp 100K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.01.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.001.bp 96K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.0001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1D_0.01.bp 352K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.001.bp 344K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.01.bp 100K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.0001.bp 100K testing/adios2/engine/bp/operations/bp5/BPWRMGARDNull_0.00001.bp 60K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2D_0.0001.bp 96K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3D_0.01.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD2DSel_0.00001.bp 176K testing/adios2/engine/bp/operations/bp5/BPWRMGARD1DSel_0.01.bp 352K testing/adios2/engine/bp/operations/bp5/BPWRMGARD3DSel_0.0001.bp --- source/adios2/operator/compress/CompressMGARD.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 276813d08a..dc67c4d4f3 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -151,10 +151,13 @@ size_t CompressMGARD::Operate(const char *dataIn, const Dims &blockStart, const return 0; } + mgard_x::Config config; + config.lossless = mgard_x::lossless_type::Huffman_Zstd; + PutParameter(bufferOut, bufferOutOffset, true); void *compressedData = bufferOut + bufferOutOffset; mgard_x::compress(mgardDim, mgardType, mgardCount, tolerance, s, errorBoundType, dataIn, - compressedData, sizeOut, true); + compressedData, sizeOut, config, true); bufferOutOffset += sizeOut; return bufferOutOffset;