diff --git a/docs/user_guide/source/ecosystem/h5vol/vol.rst b/docs/user_guide/source/ecosystem/h5vol/vol.rst
index 065a96b5b1..c05471b287 100644
--- a/docs/user_guide/source/ecosystem/h5vol/vol.rst
+++ b/docs/user_guide/source/ecosystem/h5vol/vol.rst
@@ -5,7 +5,7 @@ Disclaimer
The Virtual Object Layer (VOL) is a feature introduced in recent release of HDF5 1.12 (https://hdf5.wiki/index.php/New_Features_in_HDF5_Release_1.12).
-So please do make sure your HDF5 version supports VOL.
+So please do make sure your HDF5 version supports the latest VOL.
Once the ADIOS VOL is compiled, There are two ways to apply it:
@@ -70,4 +70,4 @@ Internal
-
\ No newline at end of file
+
diff --git a/docs/user_guide/source/engines/hdf5.rst b/docs/user_guide/source/engines/hdf5.rst
index d1dbb85a79..c785c32e8f 100644
--- a/docs/user_guide/source/engines/hdf5.rst
+++ b/docs/user_guide/source/engines/hdf5.rst
@@ -28,3 +28,15 @@ We can pass options to HDF5 API from ADIOS xml configuration. Currently we sup
We suggest to read HDF5 documentation before appling these options.
+
+After the subfile feature is introduced in HDF5 version 1.14, the ADIOS2 HDF5 engine will use subfiles as the default h5 format as it improves I/O in general (for example, see https://escholarship.org/uc/item/6fs7s3jb)
+
+To use the subfile feature, client needs to support MPI_Init_thread with MPI_THREAD_MULTIPLE.
+
+Useful parameters from the HDF lirbary to tune subfiles are:
+.. code-block:: xml
+
+H5FD_SUBFILING_IOC_PER_NODE (num of subfiles per node)
+ set H5FD_SUBFILING_IOC_PER_NODE to 0 if the regular h5 file is prefered, before using ADIOS2 HDF5 engine.
+H5FD_SUBFILING_STRIPE_SIZE
+H5FD_IOC_THREAD_POOL_SIZE
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 12e7b32a3f..e14d76bb05 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -22,3 +22,10 @@ endif()
if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA)
add_subdirectory(cuda)
endif()
+
+
+if(ADIOS2_HAVE_MPI AND ADIOS2_HAVE_HDF5)
+ if(HDF5_VERSION VERSION_GREATER_EQUAL 1.14)
+ add_subdirectory(h5subfile)
+ endif()
+endif()
diff --git a/examples/h5subfile/CMakeLists.txt b/examples/h5subfile/CMakeLists.txt
new file mode 100644
index 0000000000..0d94ae241b
--- /dev/null
+++ b/examples/h5subfile/CMakeLists.txt
@@ -0,0 +1,10 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0. See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+if(ADIOS2_HAVE_MPI)
+ add_executable(H5EngineSubfileTest h5_subfile.cpp)
+ target_link_libraries(H5EngineSubfileTest adios2::cxx11_mpi MPI::MPI_C)
+endif()
+
diff --git a/examples/h5subfile/h5_subfile.cpp b/examples/h5subfile/h5_subfile.cpp
new file mode 100644
index 0000000000..a980f0dbc8
--- /dev/null
+++ b/examples/h5subfile/h5_subfile.cpp
@@ -0,0 +1,300 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0. See
+ * accompanying file Copyright.txt for details.
+ *
+ * helloHDF5Writer.cpp: Simple self-descriptive example of how to write a
+ * variable to a parallel HDF5 File using MPI processes.
+ *
+ * Created on: March 6, 2023
+ * Author: Junmin
+ */
+
+#include
+#include //std::ios_base::failure
+#include //std::cout
+#include
+#include //std::invalid_argument std::exception
+#include
+#include
+
+void writeMe(adios2::IO &hdf5IO, int rank, int size, const char *testFileName)
+{
+ /** Application variable */
+ int scale = 1;
+
+ const char *temp = std::getenv("TEST_SCALE");
+ if (NULL != temp)
+ {
+ int itemp = -1;
+ sscanf(temp, "%d", &itemp);
+ if (itemp > 1)
+ scale = itemp;
+ }
+
+ const std::size_t Nx = 1024;
+ const std::size_t Ny = 1024 * scale;
+
+ std::vector myFloats(Nx * Ny, 0.1 * rank);
+ std::vector myInts(Nx * Ny, 1 + rank);
+
+ hdf5IO.SetParameter("IdleH5Writer",
+ "true"); // set this if not all ranks are writting
+
+ adios2::Variable h5Floats = hdf5IO.DefineVariable(
+ "h5Floats", {size * Nx, Ny}, {rank * Nx, 0}, {Nx, Ny},
+ adios2::ConstantDims);
+
+ adios2::Variable h5Ints =
+ hdf5IO.DefineVariable("h5Ints", {size * Nx, Ny}, {rank * Nx, 0},
+ {Nx, Ny}, adios2::ConstantDims);
+
+ /** Engine derived class, spawned to start IO operations */
+ adios2::Engine hdf5Writer = hdf5IO.Open(testFileName, adios2::Mode::Write);
+
+ int nsteps = 5;
+
+ if (size % 2 == 0)
+ {
+ // all Ranks must call Put
+ /** Write variable for buffering */
+ for (int i = 0; i < nsteps; i++)
+ {
+ hdf5Writer.BeginStep();
+ hdf5Writer.Put(h5Floats, myFloats.data());
+ hdf5Writer.Put(h5Ints, myInts.data());
+ hdf5Writer.EndStep();
+ }
+ }
+ else
+ {
+ // using collective Begin/EndStep() to run the
+ // collective HDF5 calls. Now Ranks can skip writting if no data
+ // presented
+ for (int i = 0; i < nsteps; i++)
+ {
+ hdf5Writer.BeginStep();
+ if (rank == 0)
+ {
+ hdf5Writer.Put(h5Floats, myFloats.data());
+ hdf5Writer.Put(h5Ints, myInts.data());
+ }
+ hdf5Writer.EndStep();
+ }
+ }
+ std::vector m_globalDims = {10, 20, 30, 40};
+ hdf5IO.DefineAttribute("adios2_schema/version_major",
+ std::to_string(ADIOS2_VERSION_MAJOR));
+ hdf5IO.DefineAttribute("adios2_schema/version_minor",
+ std::to_string(ADIOS2_VERSION_MINOR));
+ hdf5IO.DefineAttribute("/adios2_schema/mesh/type", "explicit");
+ hdf5IO.DefineAttribute("adios2_schema/mesh/dimension0",
+ m_globalDims[0]);
+ hdf5IO.DefineAttribute("adios2_schema/mesh/dimension1",
+ m_globalDims[1]);
+ hdf5IO.DefineAttribute("adios2_schema/mesh/dimension2",
+ m_globalDims[2]);
+ hdf5IO.DefineAttribute("adios2_schema/mesh/dimension3",
+ m_globalDims[3]);
+ hdf5IO.DefineAttribute("adios2_schema/mesh/dimension-num",
+ m_globalDims.size());
+
+ hdf5Writer.Close();
+}
+
+template
+void ReadVarData(adios2::IO h5IO, adios2::Engine &h5Reader,
+ const std::string &name)
+{
+ adios2::Variable var = h5IO.InquireVariable(name);
+
+ if (var)
+ {
+ int nDims = var.Shape().size();
+ size_t totalSize = 1;
+ for (int i = 0; i < nDims; i++)
+ {
+ totalSize *= var.Shape()[i];
+ }
+ std::vector myValues(totalSize);
+ // myFloats.data is pre-allocated
+ h5Reader.Get(var, myValues.data(), adios2::Mode::Sync);
+
+ // std::cout << "\tValues of "< totalSize - 5))
+ {
+ std::cout << number << " ";
+ }
+ else if (counter == 5)
+ {
+ std::cout << " ...... ";
+ }
+ counter++;
+ }
+ }
+ std::cout << "\n";
+ }
+}
+
+void readMe(adios2::IO &h5IO, int rank, int size, const char *fileName)
+{
+ /** Engine derived class, spawned to start IO operations */
+ adios2::Engine h5Reader = h5IO.Open(fileName, adios2::Mode::Read);
+
+ const std::map variables =
+ h5IO.AvailableVariables();
+
+ if (0 == rank)
+ std::cout << " Num Vars: " << variables.size() << std::endl;
+
+ for (const auto &variablePair : variables)
+ {
+ std::cout << "Name: " << variablePair.first;
+ std::cout << std::endl;
+
+ for (const auto ¶meter : variablePair.second)
+ {
+ std::cout << "\t" << parameter.first << ": " << parameter.second
+ << "\n";
+ if (parameter.second == "double")
+ {
+ ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ else if (parameter.second == "float")
+ {
+ ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ else if (parameter.second == "unsigned int")
+ {
+ ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ else if (parameter.second == "int")
+ {
+ ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ }
+ } // variables
+
+ const std::map attributes =
+ h5IO.AvailableAttributes();
+
+ if (0 == rank)
+ std::cout << "Num Attrs:" << attributes.size() << std::endl;
+
+ for (const auto &attrPair : attributes)
+ {
+ std::cout << "AttrName: " << attrPair.first;
+ std::cout << std::endl;
+
+ for (const auto ¶meter : attrPair.second)
+ {
+ std::cout << "\t" << parameter.first << ": " << parameter.second
+ << "\n";
+
+ if (parameter.second == "double")
+ {
+ // ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ else if (parameter.second == "float")
+ {
+ // ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ else if (parameter.second == "unsigned int")
+ {
+ // ReadVarData(h5IO, h5Reader,
+ // variablePair.first);
+ }
+ else if (parameter.second == "int")
+ {
+ // ReadVarData(h5IO, h5Reader, variablePair.first);
+ }
+ //... add more types if needed
+ }
+ }
+
+ h5Reader.Close();
+}
+
+int main(int argc, char *argv[])
+{
+ int provided;
+
+ // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+
+ if (provided < MPI_THREAD_MULTIPLE)
+ {
+ std::cout << "MPI_THREAD_MULTIPLE is not supported, not able to use "
+ "the subfile feature in HDF5. Aborting. \n"
+ << std::endl;
+ MPI_Abort(MPI_COMM_WORLD, -1);
+ }
+
+ int rank, size;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ try
+ {
+ /** ADIOS class factory of IO class objects */
+ adios2::ADIOS adios(MPI_COMM_WORLD);
+ adios2::IO writerIO = adios.DeclareIO("HDFFileIOWriter");
+
+ std::string testName = "test.h5";
+ if (argc > 1)
+ testName = argv[1];
+
+ if (1 == testName.size())
+ {
+ writerIO.SetEngine("NullCore");
+ writeMe(writerIO, rank, size, "null.bp");
+ }
+ else
+ {
+ writeMe(writerIO, rank, size, testName.c_str());
+ }
+
+ // read back if required
+ if (argc > 2)
+ {
+ MPI_Barrier(MPI_COMM_WORLD);
+ adios2::IO readerIO = adios.DeclareIO("HDFFileIOReader");
+ readMe(readerIO, rank, size, testName.c_str());
+ }
+ }
+ catch (std::invalid_argument &e)
+ {
+ std::cout << "Invalid argument exception, STOPPING PROGRAM from rank "
+ << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::ios_base::failure &e)
+ {
+ std::cout
+ << "IO System base failure exception, STOPPING PROGRAM from rank "
+ << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+ catch (std::exception &e)
+ {
+ std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n";
+ std::cout << e.what() << "\n";
+ }
+
+ MPI_Finalize();
+
+ return 0;
+}
diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt
index d304f47548..a79c4d8527 100644
--- a/source/CMakeLists.txt
+++ b/source/CMakeLists.txt
@@ -8,14 +8,16 @@ add_subdirectory(utils)
# HDF5 VOL requires 1.13+
if(ADIOS2_HAVE_HDF5)
- if(HDF5_VERSION VERSION_LESS 1.13)
+ if(HDF5_VERSION VERSION_LESS 1.14)
set(ADIOS2_HAVE_HDF5_VOL OFF CACHE INTERNAL "")
+ message(STATUS "[ADIOS2 WARNING] To enable ADIOS VOL for HDF5, please use the version 1.14+ ")
else()
set(ADIOS2_HAVE_HDF5_VOL ON CACHE INTERNAL "")
endif()
else()
set(ADIOS2_HAVE_HDF5_VOL OFF CACHE INTERNAL "")
endif()
+
if(ADIOS2_HAVE_HDF5_VOL)
add_subdirectory(h5vol)
endif()
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
index 4c59d68210..ff7327b2b1 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
@@ -222,6 +222,21 @@ void HDF5Common::Init(const std::string &name, helper::Comm const &comm,
std::string ts0;
StaticGetAdiosStepString(ts0, 0);
+#ifdef H5_HAVE_SUBFILING_VFD
+ bool useMPI = false;
+ const char *temp = getenv("H5FD_SUBFILING_IOC_PER_NODE");
+
+ if (NULL != temp)
+ {
+ int itemp = -1;
+ sscanf(temp, "%d", &itemp);
+ if (0 == itemp)
+ useMPI = true;
+ }
+
+ if (!useMPI)
+ H5Pset_fapl_subfiling(m_PropertyListId, NULL);
+#endif
if (toWrite)
{
/*
diff --git a/source/h5vol/H5Vol_dataset.c b/source/h5vol/H5Vol_dataset.c
index bf78e4d6a3..1a969db85e 100644
--- a/source/h5vol/H5Vol_dataset.c
+++ b/source/h5vol/H5Vol_dataset.c
@@ -85,20 +85,29 @@ void *H5VL_adios2_dataset_open(void *obj, const H5VL_loc_params_t *loc_params,
return result;
}
-herr_t H5VL_adios2_dataset_read(void *dset, hid_t mem_type_id,
- hid_t mem_space_id, hid_t file_space_id,
- hid_t plist_id, void *buf, void **req)
+herr_t H5VL_adios2_dataset_read(size_t count, void *dset_array[],
+ hid_t mem_type_id_array[],
+ hid_t mem_space_id_array[],
+ hid_t file_space_id_array[], hid_t dxpl_id,
+ void *buf_array[],
+ void **req) // last parameter is unused as in h5
{
- REQUIRE_NOT_NULL_ERR(dset, -1);
- H5VL_ObjDef_t *vol = (H5VL_ObjDef_t *)dset;
+ herr_t returnValue = 0;
+ for (size_t i = 0; i < count; i++)
+ {
+ REQUIRE_NOT_NULL_ERR(dset_array[i], -1);
+ H5VL_ObjDef_t *vol = (H5VL_ObjDef_t *)(dset_array[i]);
- H5VL_VarDef_t *var = (H5VL_VarDef_t *)(vol->m_ObjPtr);
+ H5VL_VarDef_t *var = (H5VL_VarDef_t *)(vol->m_ObjPtr);
- var->m_HyperSlabID = file_space_id;
- var->m_MemSpaceID = mem_space_id;
+ var->m_HyperSlabID = file_space_id_array[i];
+ var->m_MemSpaceID = mem_space_id_array[i];
- var->m_Data = buf;
- return gADIOS2ReadVar(var);
+ var->m_Data = buf_array[i];
+ if (gADIOS2ReadVar(var) < 0)
+ returnValue = -1;
+ }
+ return returnValue;
}
herr_t H5VL_adios2_dataset_get(void *dset, H5VL_dataset_get_args_t *args,
@@ -130,30 +139,34 @@ herr_t H5VL_adios2_dataset_get(void *dset, H5VL_dataset_get_args_t *args,
return 0;
}
-herr_t H5VL_adios2_dataset_write(void *dset, hid_t mem_type_id,
- hid_t mem_space_id, hid_t file_space_id,
- hid_t plist_id, const void *buf, void **req)
+herr_t H5VL_adios2_dataset_write(size_t count, void *dset_array[],
+ hid_t mem_type_id_array[],
+ hid_t mem_space_id_array[],
+ hid_t file_space_id_array[], hid_t dxpl_id,
+ const void *buf_array[], void **req)
{
- REQUIRE_NOT_NULL_ERR(dset, -1);
- H5VL_ObjDef_t *vol = (H5VL_ObjDef_t *)dset;
- H5VL_VarDef_t *varDef = (H5VL_VarDef_t *)(vol->m_ObjPtr);
+ for (size_t i = 0; i < count; i++)
+ {
+ REQUIRE_NOT_NULL_ERR(dset_array[0], -1);
+ H5VL_ObjDef_t *vol = (H5VL_ObjDef_t *)(dset_array[0]);
+ H5VL_VarDef_t *varDef = (H5VL_VarDef_t *)(vol->m_ObjPtr);
- // H5VL_VarDef_t *varDef = (H5VL_VarDef_t *)dset;
- varDef->m_Data = (void *)buf;
+ varDef->m_Data = (void *)(buf_array[i]);
- if (file_space_id > 0)
- varDef->m_HyperSlabID = file_space_id;
- else
- varDef->m_HyperSlabID = varDef->m_ShapeID;
+ if (file_space_id_array[i] > 0)
+ varDef->m_HyperSlabID = file_space_id_array[i];
+ else
+ varDef->m_HyperSlabID = varDef->m_ShapeID;
- if (mem_space_id > 0)
- varDef->m_MemSpaceID = mem_space_id;
- else
- varDef->m_MemSpaceID = varDef->m_ShapeID;
+ if (mem_space_id_array[i] > 0)
+ varDef->m_MemSpaceID = mem_space_id_array[i];
+ else
+ varDef->m_MemSpaceID = varDef->m_ShapeID;
- varDef->m_PropertyID = plist_id;
+ varDef->m_PropertyID = dxpl_id; // plist_id;
- gADIOS2CreateVar(vol->m_FileIO, varDef);
+ gADIOS2CreateVar(vol->m_FileIO, varDef);
+ }
return 0;
}
diff --git a/source/h5vol/H5Vol_def.h b/source/h5vol/H5Vol_def.h
index e84056cee6..72e789a0cb 100644
--- a/source/h5vol/H5Vol_def.h
+++ b/source/h5vol/H5Vol_def.h
@@ -167,17 +167,20 @@ extern void *H5VL_adios2_dataset_open(void *obj,
const char *name, hid_t dapl_id,
hid_t dxpl_id, void **req);
-extern herr_t H5VL_adios2_dataset_read(void *dset, hid_t mem_type_id,
- hid_t mem_space_id, hid_t file_space_id,
- hid_t plist_id, void *buf, void **req);
+extern herr_t H5VL_adios2_dataset_read(size_t count, void *dset[],
+ hid_t mem_type_id[],
+ hid_t mem_space_id[],
+ hid_t file_space_id[], hid_t dxpl_id,
+ void *buf[], void **req);
extern herr_t H5VL_adios2_dataset_get(void *dset, H5VL_dataset_get_args_t *args,
hid_t dxpl_id, void **req);
-extern herr_t H5VL_adios2_dataset_write(void *dset, hid_t mem_type_id,
- hid_t mem_space_id, hid_t file_space_id,
- hid_t plist_id, const void *buf,
- void **req);
+extern herr_t H5VL_adios2_dataset_write(size_t count, void *dset[],
+ hid_t mem_type_id[],
+ hid_t mem_space_id[],
+ hid_t file_space_id[], hid_t dxpl_id,
+ const void *buf[], void **req);
extern herr_t H5VL_adios2_dataset_close(void *dset, hid_t dxpl_id, void **req);