From 479392e8b4ab269d5e4a6811c58109c4dce76ee7 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Wed, 3 Jun 2020 11:05:39 -0700 Subject: [PATCH 1/7] Update HDF5 related code for better write performance --- Src/Base/AMReX_PlotFileUtil.cpp | 124 ++++++++++++++++++------------ Src/Particle/AMReX_ParticleHDF5.H | 51 +++++++++--- Tests/HDF5Benchmark/main.cpp | 66 ++++++++++++++-- 3 files changed, 177 insertions(+), 64 deletions(-) diff --git a/Src/Base/AMReX_PlotFileUtil.cpp b/Src/Base/AMReX_PlotFileUtil.cpp index 5da89d4158e..969628e7360 100644 --- a/Src/Base/AMReX_PlotFileUtil.cpp +++ b/Src/Base/AMReX_PlotFileUtil.cpp @@ -12,6 +12,11 @@ #ifdef AMREX_USE_HDF5 #include "hdf5.h" + +#ifdef AMREX_USE_HDF5_ASYNC +#include "h5_vol_external_async_native.h" +#endif + #endif namespace amrex { @@ -453,7 +458,6 @@ EB_WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels, #endif - #ifdef AMREX_USE_HDF5 static int CreateWriteHDF5AttrDouble(hid_t loc, const char *name, hsize_t n, const double *data) { @@ -753,12 +757,14 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, std::string filename(plotfilename + ".h5"); // Write out root level metadata - hid_t fapl, fid, grp; + hid_t fapl, dxpl, fid, grp; if(ParallelDescriptor::IOProcessor()) { // Have only one rank to create and write metadata (header) fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, MPI_COMM_SELF, MPI_INFO_NULL); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); // Create the HDF5 file fid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl); @@ -778,16 +784,62 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, ParallelDescriptor::Barrier(); + hid_t babox_id; + babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); + if (1 == AMREX_SPACEDIM) { + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT); + } + else if (2 == AMREX_SPACEDIM) { + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT); + } + else if (3 == AMREX_SPACEDIM) { + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT); + } + + hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int)); + if (1 == AMREX_SPACEDIM) { + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + } + else if (2 == AMREX_SPACEDIM) { + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); + } + else if (3 == AMREX_SPACEDIM) { + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT); + } + fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, ParallelDescriptor::Communicator(), MPI_INFO_NULL); + int alignment = 16 * 1024 * 1024; + H5Pset_alignment(fapl, alignment, alignment); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); + + dxpl = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); + + // Only use async for writing actual data + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_vol_async(fapl); + H5Pset_dxpl_async(dxpl, true); + #endif // All process open the file fid = H5Fopen(filename.c_str(), H5F_ACC_RDWR, fapl); if (fid < 0) FileOpenFailed(filename.c_str()); - H5Pclose(fapl); - // Write data for each level char level_name[32]; for (int level = 0; level <= finest_level; ++level) { @@ -798,6 +850,20 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, continue; } + /* const MultiFab* data; */ + /* std::unique_ptr mf_tmp; */ + /* if (mf[level]->nGrow() > 0) { */ + /* mf_tmp.reset(new MultiFab(mf[level]->boxArray(), */ + /* mf[level]->DistributionMap(), */ + /* mf[level]->nComp(), 0, MFInfo(), */ + /* mf[level]->Factory())); */ + /* MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0); */ + /* data = mf_tmp.get(); */ + /* } else { */ + /* data = mf[level]; */ + /* } */ + /* VisMF::Write(*data, MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix)); */ + // Get the boxes assigned to all ranks and calculate their offsets and sizes Vector procMap = mf[level]->DistributionMap().ProcessorMap(); const BoxArray& grids = mf[level]->boxArray(); @@ -814,27 +880,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, flatdims[0] = grids.size(); boxdataspace = H5Screate_simple(1, flatdims, NULL); - hid_t babox_id; - babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); - if (1 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT); - } - else if (2 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT); - } - else if (3 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT); - } - + boxdataset = H5Dcreate(grp, bdsname.c_str(), babox_id, boxdataspace, H5P_DEFAULT, H5P_DEFAULT,H5P_DEFAULT); // Create a boxarray sorted by rank @@ -862,20 +908,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, offsetdataspace = H5Screate_simple(1, oflatdims, NULL); offsetdataset = H5Dcreate(grp, odsname.c_str(), H5T_NATIVE_LLONG, offsetdataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int)); - if (1 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - } - else if (2 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); - } - else if (3 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT); - } - hsize_t centerdims[1]; centerdims[0] = sortedGrids.size() ; centerdataspace = H5Screate_simple(1, centerdims, NULL); @@ -906,9 +938,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, totalOffset += procBufferSize[proc]; } - hid_t dxpl = H5Pcreate(H5P_DATASET_XFER); - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); - if(ParallelDescriptor::IOProcessor()) { int vbCount(0); Vector vbox(sortedGrids.size() * 2 * AMREX_SPACEDIM); @@ -949,7 +978,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL); Vector a_buffer(procBufferSize[myProc], -1.0); - Long dataCount(0); + long dataCount(0); for(MFIter mfi(*mf[level]); mfi.isValid(); ++mfi) { const Box &vbox = mfi.validbox(); const Real *dataPtr = (*mf[level])[mfi].dataPtr(); @@ -969,9 +998,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, BL_PROFILE_VAR_STOP(h5dwg); - H5Tclose(center_id); - H5Tclose(babox_id); - H5Pclose(dxpl); H5Sclose(memdataspace); H5Sclose(dataspace); H5Dclose(dataset); @@ -987,6 +1013,10 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, H5Gclose(grp); } // For group + H5Tclose(center_id); + H5Tclose(babox_id); + H5Pclose(fapl); + H5Pclose(dxpl); H5Fclose(fid); /* double total_write_end_time(ParallelDescriptor::second()); */ @@ -1017,7 +1047,5 @@ WriteSingleLevelPlotfileHDF5 (const std::string& plotfilename, level_steps, ref_ratio, versionName, levelPrefix, mfPrefix, extra_dirs); } - - #endif } diff --git a/Src/Particle/AMReX_ParticleHDF5.H b/Src/Particle/AMReX_ParticleHDF5.H index 630b0b5bf24..abaddf337b4 100644 --- a/Src/Particle/AMReX_ParticleHDF5.H +++ b/Src/Particle/AMReX_ParticleHDF5.H @@ -4,6 +4,10 @@ #ifdef AMREX_USE_HDF5 #include +#ifdef AMREX_USE_HDF5_ASYNC +#include "h5_vol_external_async_native.h" +#endif + template void ParticleContainer @@ -302,7 +306,7 @@ ParticleContainer int set_stripe = 0; char setstripe[1024]; int stripe_count = 128; - int stripe_size = 32; + int stripe_size = 1; char *stripe_count_str = getenv("HDF5_STRIPE_COUNT"); char *stripe_size_str = getenv("HDF5_STRIPE_SIZE"); if (stripe_count_str) { @@ -354,7 +358,7 @@ ParticleContainer ParallelDescriptor::ReduceIntMax(maxnextid, IOProcNumber); } - hid_t fapl, fid, grp; + hid_t fapl, dxpl, fid, grp; int status; hid_t comp_dtype = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); @@ -392,6 +396,10 @@ ParticleContainer // Have only one rank to create and write metadata (header) fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, MPI_COMM_SELF, MPI_INFO_NULL); + int alignment = 1 * 1024 * 1024; + H5Pset_alignment(fapl, alignment, alignment); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); // Create the HDF5 file fid = H5Fcreate(HDF5FileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl); @@ -517,6 +525,14 @@ ParticleContainer fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, ParallelDescriptor::Communicator(), MPI_INFO_NULL); + int alignment = 1 * 1024 * 1024; + H5Pset_alignment(fapl, alignment, alignment); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); + + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_vol_async(fapl); + #endif // All process open the file fid = H5Fopen(HDF5FileName.c_str(), H5F_ACC_RDWR, fapl); @@ -590,6 +606,13 @@ ParticleContainer std::map > tile_map; + hid_t dxpl; + dxpl = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_dxpl_async(dxpl, true); + #endif + for (const auto& kv : m_particles[lev]) { const int grid = kv.first.first; @@ -649,8 +672,8 @@ ParticleContainer total_mfi += all_mfi_cnt[i]; // Create the int data - MPI_Allgather(&my_mfi_int_total_size, 1, Mpi_typemap::type(), - &(all_mfi_int_total_size[0]), 1, Mpi_typemap::type(), ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_int_total_size, 1, MPI_UNSIGNED_LONG_LONG, + &(all_mfi_int_total_size[0]), 1, MPI_UNSIGNED_LONG_LONG, ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_int_size += all_mfi_int_total_size[i]; @@ -667,8 +690,8 @@ ParticleContainer // Create the real data - MPI_Allgather(&my_mfi_real_total_size, 1, Mpi_typemap::type(), - &(all_mfi_real_total_size[0]), 1, Mpi_typemap::type(), ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_real_total_size, 1, MPI_UNSIGNED_LONG_LONG, + &(all_mfi_real_total_size[0]), 1, MPI_UNSIGNED_LONG_LONG, ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_real_size += all_mfi_real_total_size[i]; @@ -727,7 +750,7 @@ ParticleContainer /* my_int_offset << ", my_int_count = " << my_int_count << ", total_int_size = " << total_int_size << '\n'; */ int_dset_space = H5Screate_simple(1, &total_int_size, NULL); H5Sselect_hyperslab (int_dset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL); - H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, H5P_DEFAULT, istuff.dataPtr()); + H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl, istuff.dataPtr()); H5Sclose(int_mem_space); H5Sclose(int_dset_space); @@ -767,7 +790,7 @@ ParticleContainer /* my_real_offset << ", my_real_count = " << my_real_count << ", total_real_size = " << total_real_size << '\n'; */ real_dset_space = H5Screate_simple(1, &total_real_size, NULL); H5Sselect_hyperslab (real_dset_space, H5S_SELECT_SET, &my_real_offset, NULL, &my_real_count, NULL); - H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, H5P_DEFAULT, rstuff.dataPtr()); + H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl, rstuff.dataPtr()); H5Sclose(real_mem_space); H5Sclose(real_dset_space); @@ -792,7 +815,15 @@ ParticleContainer /* my_int_offset << ", my_int_count = " << my_int_count << ", total_mfi = " << total_mfi << '\n'; */ H5Sselect_hyperslab (offset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL); - H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, H5P_DEFAULT, &(my_nparticles[0])); + hid_t dxpl_col = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE); + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_dxpl_async(dxpl_col, true); + #endif + H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0])); + + H5Pclose(dxpl); + H5Pclose(dxpl_col); H5Sclose(int_mem_space); H5Dclose(offset_id); @@ -1314,6 +1345,8 @@ ParticleContainer Gpu::streamSynchronize(); } + + #endif #endif diff --git a/Tests/HDF5Benchmark/main.cpp b/Tests/HDF5Benchmark/main.cpp index 189ea997aa1..621c9b39a26 100644 --- a/Tests/HDF5Benchmark/main.cpp +++ b/Tests/HDF5Benchmark/main.cpp @@ -3,6 +3,11 @@ #include #include +#include +#ifdef AMREX_USE_HDF5_ASYNC +#include "h5_vol_external_async_native.h" +#endif + using namespace amrex; int main(int argc, char* argv[]) @@ -11,7 +16,7 @@ int main(int argc, char* argv[]) { const int nghost = 0; int ncells, max_grid_size, ncomp, nlevs, nppc; - int restart_check = 0; + int restart_check = 0, nplotfile = 1, nparticlefile = 1, sleeptime = 0; ParmParse pp; pp.get("ncells", ncells); @@ -19,6 +24,9 @@ int main(int argc, char* argv[]) pp.get("ncomp", ncomp); pp.get("nlevs", nlevs); pp.get("nppc", nppc); + pp.query("nplotfile", nplotfile); + pp.query("nparticlefile", nparticlefile); + pp.query("sleeptime", sleeptime); pp.query("restart_check", restart_check); AMREX_ALWAYS_ASSERT(nlevs < 2); // relax this later @@ -110,12 +118,36 @@ int main(int argc, char* argv[]) Vector level_steps(nlevs, 0); - WriteMultiLevelPlotfile("plt00000", nlevs, amrex::GetVecOfConstPtrs(mf), - varnames, geom, time, level_steps, ref_ratio); + char fname[128]; + for (int ts = 0; ts < nplotfile; ts++) { + sprintf(fname, "plt%05d", ts); + + // Fake computation + if (ts > 0 && sleeptime > 0) { + if (ParallelDescriptor::IOProcessor()) { + std::cout << "Sleep for " << sleeptime << " seconds." << std::endl; + fflush(stdout); + } + sleep(sleeptime); + } + + if (ParallelDescriptor::IOProcessor()) + std::cout << "Writing plot file [" << fname << "]" << std::endl; #ifdef AMREX_USE_HDF5 - WriteMultiLevelPlotfileHDF5("plt00000", nlevs, amrex::GetVecOfConstPtrs(mf), + WriteMultiLevelPlotfileHDF5(fname, nlevs, amrex::GetVecOfConstPtrs(mf), + varnames, geom, time, level_steps, ref_ratio); +#else + WriteMultiLevelPlotfile(fname, nlevs, amrex::GetVecOfConstPtrs(mf), varnames, geom, time, level_steps, ref_ratio); #endif + } + +#ifdef AMREX_USE_HDF5_ASYNC + // Complete all previous async writes + H5VLasync_waitall(); +#endif + + /* ParallelDescriptor::Barrier(); */ Vector particle_realnames; for (int i = 0; i < NStructReal + NArrayReal; ++i) @@ -129,11 +161,30 @@ int main(int argc, char* argv[]) particle_intnames.push_back("particle_int_component_" + std::to_string(i)); } + for (int ts = 0; ts < nparticlefile; ts++) { + sprintf(fname, "plt%05d", ts); + + // Fake computation + if (ts > 0 && sleeptime > 0) { + if (ParallelDescriptor::IOProcessor()) { + std::cout << "Sleep for " << sleeptime << " seconds." << std::endl; + fflush(stdout); + } + sleep(sleeptime); + } + #ifdef AMREX_USE_HDF5 - myPC.CheckpointHDF5("plt00000", "particle0", false, particle_realnames, particle_intnames); + myPC.CheckpointHDF5(fname, "particle0", false, particle_realnames, particle_intnames); #else - myPC.Checkpoint("plt00000", "particle0", false, particle_realnames, particle_intnames); - /* myPC.WriteAsciiFile("particle0_ascii"); */ + myPC.Checkpoint(fname, "particle0", false, particle_realnames, particle_intnames); + /* myPC.WriteAsciiFile("particle0_ascii"); */ +#endif + } + +#ifdef AMREX_USE_HDF5_ASYNC + // Complete all previous async writes + H5VLasync_waitall(); + /* ParallelDescriptor::Barrier(); */ #endif if (restart_check) @@ -169,5 +220,6 @@ int main(int argc, char* argv[]) } } + amrex::Finalize(); } From d4862b449dd0b96c73a1ee85bb82606964b71fa8 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Wed, 3 Jun 2020 11:32:09 -0700 Subject: [PATCH 2/7] Update Makefile for HDF5Benchmark --- Tests/HDF5Benchmark/GNUmakefile | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Tests/HDF5Benchmark/GNUmakefile b/Tests/HDF5Benchmark/GNUmakefile index 0524489fe45..4db55a6bb1f 100644 --- a/Tests/HDF5Benchmark/GNUmakefile +++ b/Tests/HDF5Benchmark/GNUmakefile @@ -18,14 +18,6 @@ TINY_PROFILE = TRUE EBASE = main -# HDF5_HOME = /home/khl7265/.local -USE_HDF5 = TRUE -ifeq ($(USE_HDF5), TRUE) -DEFINES += -DAMREX_USE_HDF5 -INCLUDE_LOCATIONS += $(HDF5_HOME)/include -LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib -endif - include $(AMREX_HOME)/Tools/GNUMake/Make.defs include ./Make.package @@ -34,3 +26,20 @@ include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Tools/GNUMake/Make.rules +USE_HDF5 = TRUE +HDF5_HOME = $(HOME)/hdf5/develop_build/hdf5 +ifeq ($(USE_HDF5), TRUE) +DEFINES += -DAMREX_USE_HDF5 +INCLUDE_LOCATIONS += $(HDF5_HOME)/include +LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib +endif + +# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://bitbucket.hdfgroup.org/projects/HDF5VOL/repos/async/browse +USE_HDF5_ASYNC = FALSE +ABT_HOME = $(HOME)/cori/argobots/install +ASYNC_HOME = $(HOME)/hdf5vol/async/src +ifeq ($(USE_HDF5_ASYNC), TRUE) +DEFINES += -DAMREX_USE_HDF5_ASYNC +INCLUDE_LOCATIONS += $(ABT_HOME)/include $(ASYNC_HOME) +LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME) -lh5async -labt +endif From cb3abcb64c5b8d464e8cd0feab8d7d56b9c51043 Mon Sep 17 00:00:00 2001 From: Houjun Date: Wed, 17 Jun 2020 15:21:43 -0700 Subject: [PATCH 3/7] Update GNUmakefile --- Tests/HDF5Benchmark/GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/HDF5Benchmark/GNUmakefile b/Tests/HDF5Benchmark/GNUmakefile index 4db55a6bb1f..509351d8214 100644 --- a/Tests/HDF5Benchmark/GNUmakefile +++ b/Tests/HDF5Benchmark/GNUmakefile @@ -39,7 +39,7 @@ USE_HDF5_ASYNC = FALSE ABT_HOME = $(HOME)/cori/argobots/install ASYNC_HOME = $(HOME)/hdf5vol/async/src ifeq ($(USE_HDF5_ASYNC), TRUE) -DEFINES += -DAMREX_USE_HDF5_ASYNC +DEFINES += -DAMREX_USE_HDF5_ASYNC -DAMREX_MPI_THREAD_MULTIPLE INCLUDE_LOCATIONS += $(ABT_HOME)/include $(ASYNC_HOME) LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME) -lh5async -labt endif From 11a95c425fa061d51e40f680b3a896d20ff77814 Mon Sep 17 00:00:00 2001 From: Houjun Date: Wed, 17 Jun 2020 15:31:16 -0700 Subject: [PATCH 4/7] Update inputs --- Tests/HDF5Benchmark/inputs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Tests/HDF5Benchmark/inputs b/Tests/HDF5Benchmark/inputs index c83791eb79f..1bbead63d1c 100644 --- a/Tests/HDF5Benchmark/inputs +++ b/Tests/HDF5Benchmark/inputs @@ -14,5 +14,14 @@ ncomp = 6 # Number of particles per cell nppc = 2 +# Number of plot files to write +nplotfile = 5 + +# Number of plot files to write +nparticlefile = 5 + +# Time to sleep before each write +sleeptime = 2 + # Whether to check the correctness of Checkpoint / Restart -restart_check = 1 \ No newline at end of file +restart_check = 1 From 26d538a7d4ede2a4bb743935e60d90ceb4ba6139 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Thu, 18 Jun 2020 16:23:20 -0400 Subject: [PATCH 5/7] Use correct MPI datatype from AMReX --- Src/Particle/AMReX_ParticleHDF5.H | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Src/Particle/AMReX_ParticleHDF5.H b/Src/Particle/AMReX_ParticleHDF5.H index abaddf337b4..528f4fc35a6 100644 --- a/Src/Particle/AMReX_ParticleHDF5.H +++ b/Src/Particle/AMReX_ParticleHDF5.H @@ -667,13 +667,13 @@ ParticleContainer } // Collect the number of mf and total size of mf from each rank - MPI_Allgather(&my_mfi_cnt, 1, MPI_INT, &(all_mfi_cnt[0]), 1, MPI_INT, ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_cnt, 1, ParallelDescriptor::Mpi_typemap::type(), &(all_mfi_cnt[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_mfi += all_mfi_cnt[i]; // Create the int data - MPI_Allgather(&my_mfi_int_total_size, 1, MPI_UNSIGNED_LONG_LONG, - &(all_mfi_int_total_size[0]), 1, MPI_UNSIGNED_LONG_LONG, ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_int_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), + &(all_mfi_int_total_size[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_int_size += all_mfi_int_total_size[i]; @@ -690,8 +690,8 @@ ParticleContainer // Create the real data - MPI_Allgather(&my_mfi_real_total_size, 1, MPI_UNSIGNED_LONG_LONG, - &(all_mfi_real_total_size[0]), 1, MPI_UNSIGNED_LONG_LONG, ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), + &(all_mfi_real_total_size[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_real_size += all_mfi_real_total_size[i]; From 7dbab500db3c0631b07442e8cbbf5cc53b281695 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 28 Jul 2020 19:45:18 -0400 Subject: [PATCH 6/7] Fix a bug that causes wrong value assignment for the boxcenter dataset --- Src/Base/AMReX_PlotFileUtil.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Base/AMReX_PlotFileUtil.cpp b/Src/Base/AMReX_PlotFileUtil.cpp index 969628e7360..46d10d624ae 100644 --- a/Src/Base/AMReX_PlotFileUtil.cpp +++ b/Src/Base/AMReX_PlotFileUtil.cpp @@ -947,7 +947,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, for(int i(0); i < AMREX_SPACEDIM; ++i) { vbox[(vbCount * 2 * AMREX_SPACEDIM) + i] = sortedGrids[b].smallEnd(i); vbox[(vbCount * 2 * AMREX_SPACEDIM) + i + AMREX_SPACEDIM] = sortedGrids[b].bigEnd(i); - centering[vbCount + i] = sortedGrids[b].ixType().test(i) ? 1 : 0; + centering[vbCount * AMREX_SPACEDIM + i] = sortedGrids[b].ixType().test(i) ? 1 : 0; } ++vbCount; } From d2726e514502083933d4654e9c68a7a30386c338 Mon Sep 17 00:00:00 2001 From: Houjun Tang Date: Tue, 28 Jul 2020 19:53:49 -0400 Subject: [PATCH 7/7] Revert unnecessary makefile change --- Tests/HDF5Benchmark/GNUmakefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Tests/HDF5Benchmark/GNUmakefile b/Tests/HDF5Benchmark/GNUmakefile index 962a1dadf10..fd7bbe5fe59 100644 --- a/Tests/HDF5Benchmark/GNUmakefile +++ b/Tests/HDF5Benchmark/GNUmakefile @@ -28,11 +28,6 @@ include $(AMREX_HOME)/Tools/GNUMake/Make.rules USE_HDF5 = TRUE HDF5_HOME = $(HOME)/hdf5/develop_build/hdf5 -ifeq ($(USE_HDF5), TRUE) -DEFINES += -DAMREX_USE_HDF5 -INCLUDE_LOCATIONS += $(HDF5_HOME)/include -LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib -endif # To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://bitbucket.hdfgroup.org/projects/HDF5VOL/repos/async/browse USE_HDF5_ASYNC = FALSE