Cache the neighbor comm tags for the CPU implementation of fillNeighb…

…ors. (AMReX-Codes#2862) * Cache the neighbor comm tags for the CPU implementation of fillNeighbors. * fix areMasksValid function
Thierry992 · Jul 2, 2022 · dc8b734 · dc8b734
1 parent 2b42fb5
commit dc8b734
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 20 deletions.
diff --git a/Src/Particle/AMReX_NeighborParticles.H b/Src/Particle/AMReX_NeighborParticles.H
@@ -348,6 +348,11 @@ protected:
     ///
     void BuildMasks ();
 
+    ///
+    /// Are the masks computed by the above function still valid?
+    ///
+    bool areMasksValid ();
+
     void GetNeighborCommTags ();
 
     void GetCommTagsBox (Vector<NeighborCommTag>& tags, const int lev, const Box& in_box);

diff --git a/Src/Particle/AMReX_NeighborParticlesCPUImpl.H b/Src/Particle/AMReX_NeighborParticlesCPUImpl.H
@@ -7,8 +7,10 @@ void
 NeighborParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
 ::fillNeighborsCPU () {
     BL_PROFILE("NeighborParticleContainer::fillNeighborsCPU");
-    BuildMasks();
-    GetNeighborCommTags();
+    if (!areMasksValid()) {
+        BuildMasks();
+        GetNeighborCommTags();
+    }
     cacheNeighborInfo();
     updateNeighborsCPU(false);
 }

diff --git a/Src/Particle/AMReX_NeighborParticlesI.H b/Src/Particle/AMReX_NeighborParticlesI.H
@@ -119,6 +119,30 @@ NeighborParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
     this->Redistribute();
 }
 
+template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
+bool
+NeighborParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
+::areMasksValid () {
+
+    BL_PROFILE("NeighborParticleContainer::areMasksValid");
+
+    resizeContainers(this->numLevels());
+
+    for (int lev = 0; lev < this->numLevels(); ++lev)
+    {
+        BoxArray ba = this->ParticleBoxArray(lev);
+        const DistributionMapping& dmap = this->ParticleDistributionMap(lev);
+
+        if (mask_ptr[lev] == nullptr ||
+            ! BoxArray::SameRefs(mask_ptr[lev]->boxArray(), ba) ||
+            ! DistributionMapping::SameRefs(mask_ptr[lev]->DistributionMap(), dmap))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
 template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
 void
 NeighborParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
@@ -136,30 +160,25 @@ NeighborParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
         BoxArray ba = this->ParticleBoxArray(lev);
         const DistributionMapping& dmap = this->ParticleDistributionMap(lev);
 
-        if (mask_ptr[lev] == nullptr ||
-            ! BoxArray::SameRefs(mask_ptr[lev]->boxArray(), ba) ||
-            ! DistributionMapping::SameRefs(mask_ptr[lev]->DistributionMap(), dmap))
-        {
-            const Geometry& geom = this->Geom(lev);
+        const Geometry& geom = this->Geom(lev);
 
-            mask_ptr[lev] = std::make_unique<iMultiFab>(ba, dmap, int(num_mask_comps), m_num_neighbor_cells);
-            mask_ptr[lev]->setVal(-1, m_num_neighbor_cells);
+        mask_ptr[lev] = std::make_unique<iMultiFab>(ba, dmap, int(num_mask_comps), m_num_neighbor_cells);
+        mask_ptr[lev]->setVal(-1, m_num_neighbor_cells);
 
 #ifdef AMREX_USE_OMP
 #pragma omp parallel
 #endif
-            for (MFIter mfi(*mask_ptr[lev],this->do_tiling ? this->tile_size : IntVect::TheZeroVector());
-                 mfi.isValid(); ++mfi) {
-                const Box& box = mfi.tilebox();
-                const int grid_id = mfi.index();
-                const int tile_id = mfi.LocalTileIndex();
-                (*mask_ptr[lev])[mfi].template setVal<RunOn::Host>(grid_id, box, MaskComps::grid,  1);
-                (*mask_ptr[lev])[mfi].template setVal<RunOn::Host>(tile_id, box, MaskComps::tile,  1);
-                (*mask_ptr[lev])[mfi].template setVal<RunOn::Host>(lev    , box, MaskComps::level, 1);
-            }
-
-            mask_ptr[lev]->FillBoundary(geom.periodicity());
+        for (MFIter mfi(*mask_ptr[lev],this->do_tiling ? this->tile_size : IntVect::TheZeroVector());
+             mfi.isValid(); ++mfi) {
+            const Box& box = mfi.tilebox();
+            const int grid_id = mfi.index();
+            const int tile_id = mfi.LocalTileIndex();
+            (*mask_ptr[lev])[mfi].template setVal<RunOn::Host>(grid_id, box, MaskComps::grid,  1);
+            (*mask_ptr[lev])[mfi].template setVal<RunOn::Host>(tile_id, box, MaskComps::tile,  1);
+            (*mask_ptr[lev])[mfi].template setVal<RunOn::Host>(lev    , box, MaskComps::level, 1);
         }
+
+        mask_ptr[lev]->FillBoundary(geom.periodicity());
     }
 }