From b94e1d2fca2bffd2b0e826eebfca7d63290efd40 Mon Sep 17 00:00:00 2001
From: Weiqun Zhang <WeiqunZhang@lbl.gov>
Date: Mon, 9 Nov 2020 10:30:06 -0800
Subject: [PATCH] Assert no nested MFIters (#1530)

## Summary
Add assertion to catch nested MFIters (e.g., MultiFab functions are called inside MFIter).

## Checklist

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX users
- [ ] are described in the proposed changes to the AMReX documentation, if appropriate
---
 Src/Base/AMReX_MFIter.H   |  1 +
 Src/Base/AMReX_MFIter.cpp | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/Src/Base/AMReX_MFIter.H b/Src/Base/AMReX_MFIter.H
index 8cae1464acc..976e2900563 100644
--- a/Src/Base/AMReX_MFIter.H
+++ b/Src/Base/AMReX_MFIter.H
@@ -227,6 +227,7 @@ protected:
 #endif
 
     static int nextDynamicIndex;
+    static int depth;
 
     void Initialize ();
 };
diff --git a/Src/Base/AMReX_MFIter.cpp b/Src/Base/AMReX_MFIter.cpp
index 375fd3e578c..d3eec50e7b9 100644
--- a/Src/Base/AMReX_MFIter.cpp
+++ b/Src/Base/AMReX_MFIter.cpp
@@ -7,6 +7,7 @@
 namespace amrex {
 
 int MFIter::nextDynamicIndex = std::numeric_limits<int>::min();
+int MFIter::depth = 0;
 
 MFIter::MFIter (const FabArrayBase& fabarray_, 
 		unsigned char       flags_)
@@ -200,6 +201,13 @@ MFIter::MFIter (const FabArrayBase& fabarray_, const MFItInfo& info)
 
 MFIter::~MFIter ()
 {
+#ifdef _OPENMP
+#pragma omp master
+#endif
+    {
+        depth = 0;
+    }
+
 #ifdef BL_USE_TEAM
     if ( ! (flags & NoTeamBarrier) )
 	ParallelDescriptor::MyTeam().MemoryBarrier();
@@ -242,6 +250,14 @@ MFIter::~MFIter ()
 void
 MFIter::Initialize ()
 {
+#ifdef _OPENMP
+#pragma omp master
+#endif
+    {
+        ++depth;
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(depth == 1, "Nested MFIter is not supported");
+    }
+
     if (flags & SkipInit) {
 	return;
     }