From 2fcde453494147960ed475ca872867df4d872f06 Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 28 Feb 2024 12:57:58 -0800 Subject: [PATCH 1/3] added support to read back from H5T_STRING VARIABLES it turns out strings written out through h5py are all variable strings --- .../toolkit/interop/hdf5/HDF5Common.cpp | 71 +++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 1b8c8c409c..8efb6f6f6b 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -929,13 +929,28 @@ void HDF5Common::ReadStringScalarDataset(hid_t dataSetId, std::string &result) hid_t h5Type = H5Dget_type(dataSetId); // get actual type; size_t typesize = H5Tget_size(h5Type); - char *val = (char *)(calloc(typesize, sizeof(char))); - hid_t ret2 = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); - CHECK_H5_RETURN(ret2, "ReadStringScalarDataset"); + if (H5Tis_variable_str(h5Type)) + { + hid_t d_space = H5Dget_space(dataSetId); + std::vector vc(typesize); // byte buffer to vlen strings + auto status = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vc.data()); + result.assign(*((char **)vc.data())); - result.assign(val, typesize); - free(val); + // free dynamically allocated vlen memory from H5Dread + // recent versions shift to use H5Treclaim() + H5Dvlen_reclaim(h5Type, d_space, H5P_DEFAULT, vc.data()); + + // H5Treclaim(attr_type, attr_space, H5P_DEFAULT, vc.data()); + } + else + { + char *val = (char *)(calloc(typesize, sizeof(char))); + hid_t ret2 = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); + CHECK_H5_RETURN(ret2, "ReadStringScalarDataset"); + result.assign(val, typesize); + free(val); + } H5Tclose(h5Type); } @@ -1213,6 +1228,52 @@ void HDF5Common::ReadInStringAttr(core::IO &io, const std::string &attrName, hid hsize_t typeSize = H5Tget_size(h5Type); H5S_class_t stype = H5Sget_simple_extent_type(sid); + if (H5Tis_variable_str(h5Type)) + { + hid_t attr_space = H5Aget_space(attrId); + if (H5S_SCALAR == stype) + { + std::vector vc(typeSize); // byte buffer to vlen strings + auto status = H5Aread(attrId, h5Type, vc.data()); + CHECK_H5_RETURN(status, "ReadInStringAttr_scalar") + + std::string c_str(*((char **)vc.data())); + + // free dynamically allocated vlen memory from H5Aread + // later versions use H5Treclaim() instead. + H5Dvlen_reclaim(h5Type, attr_space, H5P_DEFAULT, vc.data()); + + io.DefineAttribute(attrName, c_str); + } + else + { + hsize_t ndims = H5Sget_simple_extent_ndims(sid); + if (ndims != 1) + CHECK_H5_RETURN(-1, "Only handles 1-D string array"); + + // ndims must be 1 + hsize_t dims[1]; + hid_t ret = H5Sget_simple_extent_dims(sid, dims, NULL); + CHECK_H5_RETURN(ret, "ReadInStringAttr"); + + std::vector vc(dims[0]); + auto status = H5Aread(attrId, h5Type, vc.data()); + CHECK_H5_RETURN(status, "ReadInStringAttr"); + + std::vector stringArray; + for (auto const &val : vc) + // stringArray.push_back(auxiliary::strip(std::string(val), {'\0'})); + stringArray.push_back(std::string(val)); + + status = H5Dvlen_reclaim(h5Type, attr_space, H5P_DEFAULT, vc.data()); + io.DefineAttribute(attrName, stringArray.data(), dims[0]); + } + + return; + } + // + // regular string, not variables + // if (H5S_SCALAR == stype) { auto val = std::unique_ptr(new char[typeSize]); From c50529951198157e2833656556a150c99c758f84 Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 28 Feb 2024 13:35:09 -0800 Subject: [PATCH 2/3] fixed warning --- source/adios2/toolkit/interop/hdf5/HDF5Common.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 8efb6f6f6b..c87786996a 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -934,6 +934,7 @@ void HDF5Common::ReadStringScalarDataset(hid_t dataSetId, std::string &result) hid_t d_space = H5Dget_space(dataSetId); std::vector vc(typesize); // byte buffer to vlen strings auto status = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vc.data()); + CHECK_H5_RETURN(status, "ReadStringScalar_variable_str"); result.assign(*((char **)vc.data())); // free dynamically allocated vlen memory from H5Dread From 6ba041bc761d4c80d955f5d64ec5de2075971a44 Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 28 Feb 2024 13:41:30 -0800 Subject: [PATCH 3/3] clang-format fix --- source/adios2/toolkit/interop/hdf5/HDF5Common.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index c87786996a..2e97a13ad9 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -934,7 +934,7 @@ void HDF5Common::ReadStringScalarDataset(hid_t dataSetId, std::string &result) hid_t d_space = H5Dget_space(dataSetId); std::vector vc(typesize); // byte buffer to vlen strings auto status = H5Dread(dataSetId, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, vc.data()); - CHECK_H5_RETURN(status, "ReadStringScalar_variable_str"); + CHECK_H5_RETURN(status, "ReadStringScalar_variable_str"); result.assign(*((char **)vc.data())); // free dynamically allocated vlen memory from H5Dread