Skip to content

Commit

Permalink
Byte spread fixes (AMReX-Codes#2949)
Browse files Browse the repository at this point in the history
  • Loading branch information
atmyers authored Sep 15, 2022
1 parent 17c94cc commit 2cdb9df
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 15 deletions.
4 changes: 2 additions & 2 deletions Src/Particle/AMReX_ParticleContainer.H
Original file line number Diff line number Diff line change
Expand Up @@ -524,9 +524,9 @@ public:
*/
bool OK (int lev_min = 0, int lev_max = -1, int nGrow = 0) const;

void ByteSpread () const;
std::array<Long, 3> ByteSpread () const;

void PrintCapacity () const;
std::array<Long, 3> PrintCapacity () const;

void ShrinkToFit ();

Expand Down
34 changes: 21 additions & 13 deletions Src/Particle/AMReX_ParticleContainerI.H
Original file line number Diff line number Diff line change
Expand Up @@ -480,8 +480,9 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>::Nu

template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt,
template<class> class Allocator>
void
ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>::ByteSpread () const
std::array<Long, 3>
ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>
::ByteSpread () const
{
Long cnt = 0;

Expand All @@ -496,7 +497,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>::By
Long mn = cnt, mx = mn;

const int IOProc = ParallelContext::IOProcessorNumberSub();
const std::size_t sz = sizeof(ParticleType)+NumRealComps()*sizeof(Real)+NumIntComps()*sizeof(int);
const Long sz = sizeof(ParticleType)+NumRealComps()*sizeof(ParticleReal)+NumIntComps()*sizeof(int);

#ifdef AMREX_LAZY
Lazy::QueueReduction( [=] () mutable {
Expand All @@ -505,22 +506,27 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>::By
ParallelReduce::Max(mx, IOProc, ParallelContext::CommunicatorSub());
ParallelReduce::Sum(cnt, IOProc, ParallelContext::CommunicatorSub());

amrex::Print() << "ParticleContainer byte spread across MPI nodes: ["
amrex::Print() << "ParticleContainer spread across MPI nodes - bytes (num particles): [Min: "
<< mn*sz
<< " (" << mn << ")"
<< " ... "
<< ", Max: "
<< mx*sz
<< " (" << mx << ")"
<< "] total particles: (" << cnt << ")\n";
<< ", Total: "
<< cnt*sz
<< " (" << cnt << ")]\n";
#ifdef AMREX_LAZY
});
#endif

return {mn*sz, mx*sz, cnt*sz};
}

template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt,
template<class> class Allocator>
void
ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>::PrintCapacity () const
std::array<Long, 3>
ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>
::PrintCapacity () const
{
Long cnt = 0;

Expand All @@ -543,16 +549,18 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt, Allocator>::Pr
ParallelReduce::Max(mx, IOProc, ParallelContext::CommunicatorSub());
ParallelReduce::Sum(cnt, IOProc, ParallelContext::CommunicatorSub());

amrex::Print() << "ParticleContainer byte spread across MPI nodes: ["
amrex::Print() << "ParticleContainer spread across MPI nodes - bytes: [Min: "
<< mn
<< " (" << mn << ")"
<< " ... "
<< ", Max: "
<< mx
<< " (" << mx << ")"
<< "] total memory: (" << cnt << ")\n";
<< ", Total: "
<< cnt
<< "]\n";
#ifdef AMREX_LAZY
});
#endif

return {mn, mx, cnt};
}

template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt,
Expand Down

0 comments on commit 2cdb9df

Please sign in to comment.