Async IO for Particles (AMReX-Codes#1058)

dwillcox · Oct 3, 2020 · b8de549 · b8de549
1 parent 1d54028
commit b8de549
Show file tree

Hide file tree

Showing 14 changed files with 1,326 additions and 511 deletions.
diff --git a/Src/Base/AMReX_GpuAllocators.H b/Src/Base/AMReX_GpuAllocators.H
@@ -166,7 +166,6 @@ namespace amrex {
 
     };
 
-
 #ifdef AMREX_USE_GPU
     template <class T>
     using DefaultAllocator = amrex::ArenaAllocator<T>;

diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H
@@ -32,7 +32,6 @@ template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
 void
 ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> :: Initialize ()
 {
-    levelDirectoriesCreated = false;
     usePrePost = false;
     doUnlink = true;
 
@@ -43,7 +42,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> :: Initialize
     {
         static_assert(sizeof(ParticleType)%sizeof(RealType) == 0,
                       "sizeof ParticleType is not a multiple of sizeof RealType");
-        
+
         ParmParse pp("particles");
         pp.query("do_tiling", do_tiling);
         Vector<int> tilesize(AMREX_SPACEDIM);
@@ -54,7 +53,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt> :: Initialize
         static_assert(std::is_standard_layout<ParticleType>::value
                    && std::is_trivial<ParticleType>::value,
                       "Particle type must be standard layout and trivial.");
-        
+
         pp.query("use_prepost", usePrePost);
         pp.query("do_unlink", doUnlink);
 
@@ -909,7 +908,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>::
 copyParticles (const ParticleContainerType& other, bool local)
 {
     BL_PROFILE("ParticleContainer::copyParticles");
-    clearParticles();   
+    clearParticles();
     addParticles(other, local);
 }
 
@@ -926,7 +925,7 @@ addParticles (const ParticleContainerType& other, bool local)
         auto& plevel = GetParticles(lev);
         for(MFIter mfi = other.MakeMFIter(lev); mfi.isValid(); ++mfi)
         {
-            auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex());            
+            auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex());
             if(plevel_other.find(index) == plevel_other.end()) continue;
 
             auto& ptile = plevel[index];
@@ -937,10 +936,10 @@ addParticles (const ParticleContainerType& other, bool local)
             auto dst_index = ptile.numParticles();
             ptile.resize(dst_index + np);
 
-            amrex::copyParticles(ptile, ptile_other, 0, dst_index, np);            
+            amrex::copyParticles(ptile, ptile_other, 0, dst_index, np);
         }
     }
-    
+
     if (not local) Redistribute();
 }
 

diff --git a/Src/Particle/AMReX_ParticleHDF5.H b/Src/Particle/AMReX_ParticleHDF5.H
@@ -296,35 +296,31 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
 
     std::string pdir = dir;
     if ( not pdir.empty() and pdir[pdir.size()-1] != '/') pdir += '/';
-
-    if ( ! levelDirectoriesCreated)
-    {
-        if (ParallelDescriptor::IOProcessor()) {
-            if ( ! amrex::UtilCreateDirectory(pdir, 0755)) 
-                amrex::CreateDirectoryFailed(pdir);
-
-            int set_stripe = 0;
-            char setstripe[1024];
-            int stripe_count = 128;
-            int stripe_size = 1;
-            char *stripe_count_str = getenv("HDF5_STRIPE_COUNT");
-            char *stripe_size_str  = getenv("HDF5_STRIPE_SIZE");
-            if (stripe_count_str) { 
-                stripe_count = atoi(stripe_count_str);
-                set_stripe = 1;
-            }
-            if (stripe_size_str) {
-                stripe_size = atoi(stripe_size_str);
-                set_stripe = 1;
-            }
-            if (set_stripe == 1) {
-                sprintf(setstripe, "lfs setstripe -c %d -S %dm %s", stripe_count, stripe_size, pdir.c_str());
-                std::cout << "Setting stripe parameters for HDF5 output: " << setstripe << std::endl; 
-                amrex::ignore_unused(std::system(setstripe));
-            }
+    if (ParallelDescriptor::IOProcessor()) {
+        if ( ! amrex::UtilCreateDirectory(pdir, 0755)) 
+            amrex::CreateDirectoryFailed(pdir);
+
+        int set_stripe = 0;
+        char setstripe[1024];
+        int stripe_count = 128;
+        int stripe_size = 1;
+        char *stripe_count_str = getenv("HDF5_STRIPE_COUNT");
+        char *stripe_size_str  = getenv("HDF5_STRIPE_SIZE");
+        if (stripe_count_str) { 
+            stripe_count = atoi(stripe_count_str);
+            set_stripe = 1;
+        }
+        if (stripe_size_str) {
+            stripe_size = atoi(stripe_size_str);
+            set_stripe = 1;
+        }
+        if (set_stripe == 1) {
+            sprintf(setstripe, "lfs setstripe -c %d -S %dm %s", stripe_count, stripe_size, pdir.c_str());
+            std::cout << "Setting stripe parameters for HDF5 output: " << setstripe << std::endl; 
+            amrex::ignore_unused(std::system(setstripe));
         }
-        ParallelDescriptor::Barrier();
     }
+    ParallelDescriptor::Barrier();
 
     Long nparticles = 0;
     int maxnextid;
-Original file line number
+Diff line change
@@ Expand Up / @@ -166,7 +166,6 @@ namespace amrex { @@
         };
     #ifdef AMREX_USE_GPU
         template <class T>
         using DefaultAllocator = amrex::ArenaAllocator<T>;
@@ Expand Down @@