Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Async IO for Particles #1058

Merged
merged 29 commits into from
Jul 1, 2020
Merged

Async IO for Particles #1058

Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
2fbfd91
add ability to specify allocator for a particle tile
atmyers Jun 16, 2020
88adadd
fix ifdef
atmyers Jun 16, 2020
bc9add0
missed a few places where the default allocator was used
atmyers Jun 16, 2020
d729659
add stub for particle async io
atmyers Jun 17, 2020
9c7315c
stub for Async IO Particle test
atmyers Jun 17, 2020
c1f0eb8
formatting changes
atmyers Jun 18, 2020
24e48fc
work on test for particle async io
atmyers Jun 18, 2020
e6a0d81
generalize this function template
atmyers Jun 18, 2020
9747766
formatting changes
atmyers Jun 18, 2020
b6f4624
make temporary particle tiles in pinned memory to write
atmyers Jun 18, 2020
3a957b9
add rest of the header
atmyers Jun 18, 2020
8cda358
write the complete plotfile
atmyers Jun 18, 2020
a2bc873
some work on async particle io
atmyers Jun 21, 2020
1946c8f
working async io code for particles
atmyers Jun 21, 2020
dfb948b
set up test to more steps by default
atmyers Jun 22, 2020
1c52663
tweak test
atmyers Jun 23, 2020
175c834
removed unused
atmyers Jun 23, 2020
e46c33e
add id check here
atmyers Jun 23, 2020
2c1afc9
more reorganization
atmyers Jun 23, 2020
cfa27cd
forgot to add file
atmyers Jun 23, 2020
2f29c05
tweak test
atmyers Jun 25, 2020
42d8e4e
merging
atmyers Jun 25, 2020
4544009
fix OK
atmyers Jun 25, 2020
47bc70c
some mutable
atmyers Jun 25, 2020
d9ea27d
don't need tmp copies of these
atmyers Jun 25, 2020
418c224
merge making this directories so as to call Barrier fewer times
atmyers Jun 29, 2020
1856296
can also merge these loops
atmyers Jun 29, 2020
858ba3c
generalize the filter particles function template bit
atmyers Jun 30, 2020
706b02e
some cleaning / optimization for async particle io
atmyers Jun 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Src/Base/AMReX_GpuAllocators.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ namespace amrex {

};


#ifdef AMREX_USE_GPU
template <class T>
using DefaultAllocator = amrex::ArenaAllocator<T>;
Expand Down
13 changes: 6 additions & 7 deletions Src/Particle/AMReX_ParticleContainerI.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
void
ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> :: Initialize ()
{
levelDirectoriesCreated = false;
usePrePost = false;
doUnlink = true;

Expand All @@ -43,7 +42,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> :: Initialize
{
static_assert(sizeof(ParticleType)%sizeof(RealType) == 0,
"sizeof ParticleType is not a multiple of sizeof RealType");

ParmParse pp("particles");
pp.query("do_tiling", do_tiling);
Vector<int> tilesize(AMREX_SPACEDIM);
Expand All @@ -54,7 +53,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> :: Initialize
static_assert(std::is_standard_layout<ParticleType>::value
&& std::is_trivial<ParticleType>::value,
"Particle type must be standard layout and trivial.");

pp.query("use_prepost", usePrePost);
pp.query("do_unlink", doUnlink);

Expand Down Expand Up @@ -909,7 +908,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>::
copyParticles (const ParticleContainerType& other, bool local)
{
BL_PROFILE("ParticleContainer::copyParticles");
clearParticles();
clearParticles();
addParticles(other, local);
}

Expand All @@ -926,7 +925,7 @@ addParticles (const ParticleContainerType& other, bool local)
auto& plevel = GetParticles(lev);
for(MFIter mfi = other.MakeMFIter(lev); mfi.isValid(); ++mfi)
{
auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex());
auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex());
if(plevel_other.find(index) == plevel_other.end()) continue;

auto& ptile = plevel[index];
Expand All @@ -937,10 +936,10 @@ addParticles (const ParticleContainerType& other, bool local)
auto dst_index = ptile.numParticles();
ptile.resize(dst_index + np);

amrex::copyParticles(ptile, ptile_other, 0, dst_index, np);
amrex::copyParticles(ptile, ptile_other, 0, dst_index, np);
}
}

if (not local) Redistribute();
}

Expand Down
50 changes: 23 additions & 27 deletions Src/Particle/AMReX_ParticleHDF5.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,35 +296,31 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>

std::string pdir = dir;
if ( not pdir.empty() and pdir[pdir.size()-1] != '/') pdir += '/';

if ( ! levelDirectoriesCreated)
{
if (ParallelDescriptor::IOProcessor()) {
if ( ! amrex::UtilCreateDirectory(pdir, 0755))
amrex::CreateDirectoryFailed(pdir);

int set_stripe = 0;
char setstripe[1024];
int stripe_count = 128;
int stripe_size = 1;
char *stripe_count_str = getenv("HDF5_STRIPE_COUNT");
char *stripe_size_str = getenv("HDF5_STRIPE_SIZE");
if (stripe_count_str) {
stripe_count = atoi(stripe_count_str);
set_stripe = 1;
}
if (stripe_size_str) {
stripe_size = atoi(stripe_size_str);
set_stripe = 1;
}
if (set_stripe == 1) {
sprintf(setstripe, "lfs setstripe -c %d -S %dm %s", stripe_count, stripe_size, pdir.c_str());
std::cout << "Setting stripe parameters for HDF5 output: " << setstripe << std::endl;
amrex::ignore_unused(std::system(setstripe));
}
if (ParallelDescriptor::IOProcessor()) {
if ( ! amrex::UtilCreateDirectory(pdir, 0755))
amrex::CreateDirectoryFailed(pdir);

int set_stripe = 0;
char setstripe[1024];
int stripe_count = 128;
int stripe_size = 1;
char *stripe_count_str = getenv("HDF5_STRIPE_COUNT");
char *stripe_size_str = getenv("HDF5_STRIPE_SIZE");
if (stripe_count_str) {
stripe_count = atoi(stripe_count_str);
set_stripe = 1;
}
if (stripe_size_str) {
stripe_size = atoi(stripe_size_str);
set_stripe = 1;
}
if (set_stripe == 1) {
sprintf(setstripe, "lfs setstripe -c %d -S %dm %s", stripe_count, stripe_size, pdir.c_str());
std::cout << "Setting stripe parameters for HDF5 output: " << setstripe << std::endl;
amrex::ignore_unused(std::system(setstripe));
}
ParallelDescriptor::Barrier();
}
ParallelDescriptor::Barrier();

Long nparticles = 0;
int maxnextid;
Expand Down
Loading