Remove some syncs from MLMG

AMReX-Codes · Feb 19, 2025 · 05d0f54 · 05d0f54
1 parent 28a1e19
commit 05d0f54
Show file tree

Hide file tree

Showing 7 changed files with 2 additions and 22 deletions.
diff --git a/Src/Base/AMReX_GpuControl.cpp b/Src/Base/AMReX_GpuControl.cpp
@@ -6,8 +6,8 @@ namespace amrex::Gpu {
 #if defined(AMREX_USE_GPU)
 bool in_launch_region = true;
 bool in_graph_region = false;
-bool in_single_stream_region = false;
-bool in_nosync_region = false;
+bool in_single_stream_region = true;
+bool in_nosync_region = true;
 #endif
 
 }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.H
@@ -841,7 +841,6 @@ MLABecLaplacianT<MF>::Fapply (int amrlev, int mglev, MF& out, const MF& in) cons
                                 dxinv, ascalar, bscalar);
             });
         }
-        Gpu::streamSynchronize();
     } else
 #endif
     {
@@ -1031,7 +1030,6 @@ MLABecLaplacianT<MF>::Fsmooth (int amrlev, int mglev, MF& sol, const MF& rhs, in
                 });
             }
         }
-        Gpu::streamSynchronize();
     } else
 #endif
     {
@@ -1271,7 +1269,6 @@ MLABecLaplacianT<MF>::normalize (int amrlev, int mglev, MF& mf) const
                                 AMREX_D_DECL(bxma[box_no],byma[box_no],bzma[box_no]),
                                 dxinv, ascalar, bscalar);
         });
-        Gpu::streamSynchronize();
     } else
 #endif
     {
@@ -1383,7 +1380,6 @@ MLABecLaplacianT<MF>::makeNLinOp (int /*grid_size*/) const
                 ama[box_no](i,j,k,n) = huge_alpha;
             }
         });
-        Gpu::streamSynchronize();
     } else
 #endif
     {
@@ -1434,7 +1430,6 @@ MLABecLaplacianT<MF>::copyNSolveSolution (MF& dst, MF const& src) const
                 dstma[box_no](i,j,k,n) = RT(0.0);
             }
         });
-        Gpu::streamSynchronize();
     } else
 #endif
     {

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.cpp
@@ -302,8 +302,6 @@ MLEBNodeFDLaplacian::prepareForSolve ()
 
     AMREX_ASSERT(!isBottomSingular());
 
-    Gpu::streamSynchronize();
-
 #if (AMREX_SPACEDIM == 2)
     if (m_rz) {
         if (m_geom[0][0].ProbLo(0) == 0._rt) {

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.cpp
@@ -109,7 +109,6 @@ MLNodeABecLaplacian::Fapply (int amrlev, int mglev, MultiFab& out, const MultiFa
         yarr_ma[box_no](i,j,k) = (dmskarr_ma[box_no](i,j,k)) ? Real(0.0)
             : alpha*acoef_ma[box_no](i,j,k)*xarr_ma[box_no](i,j,k) - beta*lap;
     });
-    Gpu::streamSynchronize();
 }
 
 void
@@ -145,7 +144,6 @@ MLNodeABecLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiF
                                   acoef_ma[box_no], bcoef_ma[box_no],
                                   dmskarr_ma[box_no], dxinvarr);
         });
-        Gpu::streamSynchronize();
         if (m_smooth_num_sweeps > 1) { nodalSync(amrlev, mglev, sol); }
     }
 #else
@@ -193,7 +191,6 @@ MLNodeABecLaplacian::restriction (int amrlev, int cmglev, MultiFab& crse, MultiF
     {
         mlndlap_restriction(i,j,k,pcrse_ma[box_no],fine_ma[box_no],msk_ma[box_no]);
     });
-    Gpu::streamSynchronize();
 
     if (need_parallel_copy) {
         crse.ParallelCopy(cfine);
@@ -225,7 +222,6 @@ MLNodeABecLaplacian::interpolation (int amrlev, int fmglev, MultiFab& fine, cons
         mlndlap_interpadd_aa(i, j, k, fine_ma[box_no], crse_ma[box_no],
                              sig_ma[box_no], msk_ma[box_no]);
     });
-    Gpu::streamSynchronize();
 }
 
 void
@@ -280,7 +276,6 @@ MLNodeABecLaplacian::fixUpResidualMask (int amrlev, iMultiFab& resmsk)
     {
         if (fmsk[bno](i,j,k) == nodelap_detail::crse_fine_node) { rmsk[bno](i,j,k) = 1; }
     });
-    Gpu::streamSynchronize();
 }
 
 void

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_misc.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_misc.cpp
@@ -265,7 +265,6 @@ MLNodeLaplacian::Fapply (int amrlev, int mglev, MultiFab& out, const MultiFab& i
 #endif
             });
         }
-        Gpu::streamSynchronize();
     } else
 #endif
     {
@@ -558,7 +557,6 @@ MLNodeLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab&
             }
         }
 
-        Gpu::streamSynchronize();
         nodalSync(amrlev, mglev, sol);
     }
     else
@@ -700,8 +698,6 @@ MLNodeLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab&
                 }
             }
         }
-
-        Gpu::streamSynchronize();
     }
 }
 

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeTensorLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeTensorLaplacian.cpp
@@ -219,7 +219,6 @@ MLNodeTensorLaplacian::Fapply (int amrlev, int mglev, MultiFab& out, const Multi
     {
         mlndtslap_adotx(i,j,k, out_a[box_no], in_a[box_no], dmsk_a[box_no], s);
     });
-    Gpu::streamSynchronize();
 #endif
 }
 
@@ -261,7 +260,6 @@ MLNodeTensorLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const Mult
             mlndtslap_gauss_seidel(i, j, k, sol_a[box_no], rhs_a[box_no], dmsk_a[box_no], s);
         }
     });
-    Gpu::streamSynchronize();
 #endif
 }
 

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLPoisson.H b/Src/LinearSolvers/MLMG/AMReX_MLPoisson.H
@@ -235,7 +235,6 @@ MLPoissonT<MF>::Fapply (int amrlev, int mglev, MF& out, const MF& in) const
                 });
             }
         }
-        Gpu::streamSynchronize();
     } else
 #endif
     {
@@ -333,7 +332,6 @@ MLPoissonT<MF>::normalize (int amrlev, int mglev, MF& mf) const
         {
             mlpoisson_normalize(i,j,k, ma[box_no], AMREX_D_DECL(dhx,dhy,dhz), dx, probxlo);
         });
-        Gpu::streamSynchronize();
     } else
 #endif
     {
-Original file line number
+Diff line change
@@ Expand Up @@
     #endif
                 });
             }
-            Gpu::streamSynchronize();
         } else
     #endif
         {
@@ Expand Down Expand Up @@
                 }
             }
-            Gpu::streamSynchronize();
             nodalSync(amrlev, mglev, sol);
         }
         else
@@ Expand Down Expand Up @@
                     }
                 }
             }
-            Gpu::streamSynchronize();
         }
     }
@@ Expand Down @@