Replace amrcore tutorial

Docs/sphinx_tutorials/source/AMR_Tutorial.rst

@@ -17,16 +17,19 @@ or with VisIt.
 
 Advection_AmrCore: This tutorial contains an AMR advection code that advects
 a single scalar field with a velocity field that is specified on faces.
+It is written entirely in C++, can be built in 2D or 3D and run with the same
+inputs file,
 
-It is an AMReX based code designed to run in parallel using MPI/OMP.
+It is an AMReX-based code designed to run in parallel using MPI+X, where X
+may be OMP for multicore machines and CUDA for hybrid CPU/GPU architectures.
 
 This example uses source code from the amrex/Src/Base, Boundary, and AmrCore
 directories.
 
 Notably, this example does not use source code from amrex/Src/Amr
 (see the tutorial Advection_AmrLevel).
 
-The directory Exec/SingleVortex includes a makefile and a sample inputs file.  
+The directory Exec includes a makefile and a sample inputs file.  
 
 **Advection_AmrLevel**
 ----------------------

Src/Base/AMReX_Box.H

@@ -473,6 +473,9 @@ public:
     AMREX_GPU_HOST_DEVICE
     Box& surroundingNodes (int dir) noexcept;
 
+    AMREX_GPU_HOST_DEVICE
+    Box& surroundingNodes (Direction d) noexcept { return surroundingNodes(static_cast<int>(d)); }
+
     //! Convert to CELL type in all directions.
     AMREX_GPU_HOST_DEVICE
     Box& enclosedCells () noexcept;
@@ -481,6 +484,9 @@ public:
     AMREX_GPU_HOST_DEVICE
     Box& enclosedCells (int dir) noexcept;
 
+    AMREX_GPU_HOST_DEVICE
+    Box& enclosedCells (Direction d) noexcept { return enclosedCells(static_cast<int>(d)); }
+
     /**
     * \brief Return Box that is intersection of this Box
     * and argument.  The Boxes MUST be of same type.
@@ -561,13 +567,19 @@ public:
     AMREX_GPU_HOST_DEVICE
     Box& grow (int idir, int n_cell) noexcept { smallend.shift(idir, -n_cell); bigend.shift(idir, n_cell); return *this; }
 
+    AMREX_GPU_HOST_DEVICE
+    Box& grow (Direction d, int n_cell) noexcept { return grow(static_cast<int>(d), n_cell); }
+
     /**
     * \brief Grow the Box on the low end by n_cell cells in direction idir.
     * NOTE: n_cell negative shrinks the Box by that number of cells.
     */
     AMREX_GPU_HOST_DEVICE
     Box& growLo (int idir, int n_cell = 1) noexcept { smallend.shift(idir, -n_cell); return *this; }
 
+    AMREX_GPU_HOST_DEVICE
+    Box& growLo (Direction d, int n_cell = 1) noexcept { return growLo(static_cast<int>(d), n_cell); }
+
     /**
     * \brief Grow the Box on the high end by n_cell cells in
     * direction idir.  NOTE: n_cell negative shrinks the Box by that
@@ -576,6 +588,9 @@ public:
     AMREX_GPU_HOST_DEVICE
     Box& growHi (int idir, int n_cell = 1) noexcept { bigend.shift(idir,n_cell); return *this; }
 
+    AMREX_GPU_HOST_DEVICE
+    Box& growHi (Direction d, int n_cell = 1) noexcept { return growHi(static_cast<int>(d), n_cell); }
+
     //! Grow in the direction of the given face.
     AMREX_GPU_HOST_DEVICE
     Box& grow (Orientation face, int n_cell = 1) noexcept {
@@ -1068,6 +1083,13 @@ Box grow (const Box& b, int idir, int n_cell) noexcept
     return result;
 }
 
+AMREX_GPU_HOST_DEVICE
+AMREX_FORCE_INLINE
+Box grow (const Box& b, Direction d, int n_cell) noexcept
+{
+    return grow(b, static_cast<int>(d), n_cell);
+}
+
 AMREX_GPU_HOST_DEVICE
 AMREX_FORCE_INLINE
 Box growLo (const Box& b, int idir, int n_cell) noexcept
@@ -1077,6 +1099,13 @@ Box growLo (const Box& b, int idir, int n_cell) noexcept
     return result;
 }
 
+AMREX_GPU_HOST_DEVICE
+AMREX_FORCE_INLINE
+Box growLo (const Box& b, Direction d, int n_cell) noexcept
+{
+    return growLo(b, static_cast<int>(d), n_cell);
+}
+
 AMREX_GPU_HOST_DEVICE
 AMREX_FORCE_INLINE
 Box growHi (const Box& b, int idir, int n_cell) noexcept
@@ -1086,6 +1115,13 @@ Box growHi (const Box& b, int idir, int n_cell) noexcept
     return result;
 }
 
+AMREX_GPU_HOST_DEVICE
+AMREX_FORCE_INLINE
+Box growHi (const Box& b, Direction d, int n_cell) noexcept
+{
+    return growHi(b, static_cast<int>(d), n_cell);
+}
+
     /**
     * \brief Coarsen Box by given (positive) refinement ratio.
     * NOTE: if type(dir) = CELL centered: lo <- lo/ratio and
@@ -1187,6 +1223,13 @@ Box surroundingNodes (const Box& b, int dir) noexcept
     return bx;
 }
 
+AMREX_GPU_HOST_DEVICE
+AMREX_FORCE_INLINE
+Box surroundingNodes (const Box& b, Direction d) noexcept
+{
+    return surroundingNodes(b, static_cast<int>(d));
+}
+
     /**
     * \brief Returns a Box with NODE based coordinates in all
     * directions that encloses Box b.
@@ -1234,6 +1277,13 @@ Box enclosedCells (const Box& b, int dir) noexcept
     return bx;
 }
 
+AMREX_GPU_HOST_DEVICE
+AMREX_FORCE_INLINE
+Box enclosedCells (const Box& b, Direction d) noexcept
+{
+    return enclosedCells(b, static_cast<int>(d));
+}
+
     /**
     * \brief Returns a Box with CELL based coordinates in all
     * directions that is enclosed by b.

Src/Base/AMReX_CoordSys.cpp

@@ -504,13 +504,15 @@ CoordSys::AreaLo (const IntVect& point, int dir) const noexcept
         case 0: return dx[1];
         case 1: return dx[0];
         }
+        return 0.; // to silent compiler warning
     case RZ:
         LoNode(point,xlo);
         switch (dir)
         {
         case 0: return TWOPI*dx[1]*xlo[0];
         case 1: return ((xlo[0]+dx[0])*(xlo[0]+dx[0])-xlo[0]*xlo[0])*(0.5*TWOPI);
         }
+        return 0.; // to silent compiler warning
     default:
         AMREX_ASSERT(0);
     }
@@ -540,13 +542,15 @@ CoordSys::AreaHi (const IntVect& point, int dir) const noexcept
         case 0: return dx[1];
         case 1: return dx[0];
         }
+        return 0.; // to silent compiler warning
     case RZ:
         HiNode(point,xhi);
         switch (dir)
         {
         case 0: return TWOPI*dx[1]*xhi[0];
         case 1: return (xhi[0]*xhi[0]-(xhi[0]-dx[0])*(xhi[0]-dx[0]))*(TWOPI*0.5);
         }
+        return 0.; // to silent compiler warning
     default:
         AMREX_ASSERT(0);
     }

Src/Base/AMReX_DistributionMapping.cpp

@@ -978,7 +978,7 @@ namespace {
 
 #if (AMREX_SPACEDIM == 3)
 
-        constexpr int imin = -static_cast<int>(1 << 29);
+        constexpr int imin = -(1 << 29);
         AMREX_ASSERT_WITH_MESSAGE(AMREX_D_TERM(iv[0] >= imin && iv[0] < -imin,
                                             && iv[1] >= imin && iv[1] < -imin,
                                             && iv[2] >= imin && iv[2] < -imin),
@@ -1005,7 +1005,7 @@ namespace {
 
 #elif (AMREX_SPACEDIM == 2)
 
-        constexpr uint32_t offset = 1 << 31;
+        constexpr uint32_t offset = 1u << 31;
         static_assert(static_cast<uint32_t>(std::numeric_limits<int>::max())+1 == offset,
                       "INT_MAX != (1<<31)-1");
         uint32_t x = (iv[0] >= 0) ? static_cast<uint32_t>(iv[0]) + offset
@@ -1021,7 +1021,7 @@ namespace {
 
 #elif (AMREX_SPACEDIM == 1)
 
-        constexpr uint32_t offset = 1 << 31;
+        constexpr uint32_t offset = 1u << 31;
         static_assert(static_cast<uint32_t>(std::numeric_limits<int>::max())+1 == offset,
                       "INT_MAX != (1<<31)-1");
         token.m_morton[0] = (iv[0] >= 0) ? static_cast<uint32_t>(iv[0]) + offset

Tutorials/Amr/Advection_AmrCore/Exec/SingleVortex/GNUmakefile → Tutorials/Amr/Advection_AmrCore/Exec/GNUmakefile

@@ -1,4 +1,3 @@
-AMREX_HOME = ../../../../..
 
 PRECISION  = DOUBLE
 PROFILE    = FALSE
@@ -12,9 +11,10 @@ DIM        = 2
 COMP	   = gnu
 
 USE_MPI    = TRUE
-USE_OMP    = FALSE
+USE_OMP    = FALSE 
+USE_CUDA   = FALSE
 
 Bpack   := ./Make.package 
 Blocs   := . 
 
-include ../Make.Adv
+include Make.Adv

Tutorials/Amr/Advection_AmrCore/Exec/GNUmakefile_movie

@@ -0,0 +1,11 @@
+PARAVIEW_PATH ?= /path/containing/paraview/executable
+
+movie2D:
+	@echo "Making a movie from 2D simulation, this will probably take <30 seconds ..."
+	@$(PARAVIEW_PATH)/pvpython paraview_amr101.py -d 2 > /dev/null 2>&1
+	@echo "Done! Generated amr101_2D.avi and amr101_2D.gif"
+
+movie3D:
+	@echo "Making a movie from 3D simulation, this will probably take <30 seconds ..."
+	@$(PARAVIEW_PATH)/pvpython paraview_amr101.py > /dev/null 2>&1
+	@echo "Done! Generated amr101_3D.avi and amr101_3D.gif"

Tutorials/Amr/Advection_AmrCore/Exec/Make.Adv

@@ -1,13 +1,15 @@
-AMREX_HOME ?= ../../../..
-ADR_DIR  ?= $(AMREX_HOME)/Tutorials/Amr/Advection_AmrCore
+AMREX_HOME = ../../../..
+ADV_DIR    = ../
 
-TOP := $(ADR_DIR)
+TOP := $(ADV_DIR)
 
 EBASE := main
 
+BL_NO_FORT = TRUE
+
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
-Bdirs 	:= Source Source/Src_nd Source/Src_$(DIM)d
+Bdirs 	:= Source Source/Src_K
 Bpack	+= $(foreach dir, $(Bdirs), $(TOP)/$(dir)/Make.package)
 Blocs   += $(foreach dir, $(Bdirs), $(TOP)/$(dir))
 
@@ -17,9 +19,6 @@ INCLUDE_LOCATIONS += $(Blocs)
 VPATH_LOCATIONS   += $(Blocs)
 
 Pdirs 	:= Base Boundary AmrCore
-ifeq ($(USE_SENSEI_INSITU),TRUE)
-	Pdirs += Amr Extern/SENSEI
-endif
 Ppack	+= $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
 
 include $(Ppack)

Tutorials/Amr/Advection_AmrCore/Exec/Make.package

@@ -0,0 +1 @@
+CEXE_headers += Prob.H

Tutorials/Amr/Advection_AmrCore/Exec/Prob.H

@@ -0,0 +1,32 @@
+#ifndef PROB_H_
+#define PROB_H_
+
+#include <AMReX_Box.H>
+#include <AMReX_FArrayBox.H>
+#include <AMReX_Geometry.H>
+
+AMREX_GPU_DEVICE
+AMREX_FORCE_INLINE
+void
+initdata(amrex::Box const& bx, amrex::Array4<amrex::Real> const& phi,
+         amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& prob_lo,
+         amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& dx)
+{
+    using namespace amrex;
+
+    const auto lo = lbound(bx);
+    const auto hi = ubound(bx);    
+
+    for         (int k = lo.z; k <= hi.z; ++k) {
+        for     (int j = lo.y; j <= hi.y; ++j) {
+            Real y = prob_lo[1] + (0.5+j) * dx[1];
+            for (int i = lo.x; i <= hi.x; ++i) {
+                Real x = prob_lo[0] + (0.5+i) * dx[0]; 
+                Real r2 = (std::pow(x-0.5, 2) + std::pow((y-0.75),2)) / 0.01;
+                phi(i,j,k) = 1.0 + std::exp(-r2);
+            }
+        }
+    }
+}
+
+#endif