Merge pull request #1219 from parthenon-hpc-lab/jmm/serial-pool

Provide a serial "thread pool" for use in the tasking

CHANGELOG.md

@@ -34,6 +34,7 @@
 - [[PR 1173]](https://github.com/parthenon-hpc-lab/parthenon/pull/1173) Make debugging easier by making parthenon throw an error if ParameterInput is different on multiple MPI ranks.
 
 ### Infrastructure (changes irrelevant to downstream codes)
+- [[PR1218]](https://github.com/parthenon-hpc-lab/parthenon/pull/1219) Add ability to run the tasking without threading
 - [[PR 1176]](https://github.com/parthenon-hpc-lab/parthenon/pull/1176) Move some code from header to implementation files
 
 ### Removed (removing behavior/API/varaibles/...)

CMakeLists.txt

@@ -1,9 +1,9 @@
 #=========================================================================================
 # Parthenon performance portable AMR framework
-# Copyright(C) 2020-2024 The Parthenon collaboration
+# Copyright(C) 2020-2025 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #=========================================================================================
-# (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
+# (C) (or copyright) 2020-2025. Triad National Security, LLC. All rights reserved.
 #
 # This program was produced under U.S. Government contract 89233218CNA000001 for Los
 # Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -59,6 +59,7 @@ option(TEST_ERROR_CHECKING "Enables the error checking unit test. This test will
 option(CODE_COVERAGE "Enable code coverage reporting" OFF)
 option(ENABLE_ASAN "Turn on ASAN" OFF)
 option(ENABLE_HWASAN "Turn on HWASAN (currently ARM-only)" OFF)
+option(PARTHENON_USE_SERIAL_POOL "Disable threaded tasking" ON)
 
 option(PARTHENON_USE_SYSTEM_PACKAGES "Enables search for system packages when available" OFF)
 if (PARTHENON_USE_SYSTEM_PACKAGES)

doc/sphinx/src/building.rst

@@ -21,6 +21,7 @@ General list of cmake options:
 || PARTHENON\_ENABLE\_ASCENT                || OFF                           || Option || Enable Ascent for in situ visualization and analysis                                                                                                        |
 || PARTHENON\_DISABLE\_MPI                  || OFF                           || Option || MPI is enabled by default if found, set this to True to disable MPI                                                                                         |
 || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS   || OFF                           || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host.       |
+|| PARTHENON\_USE\_SERIAL\_POOL             || ON                            || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks.    |
 || PARTHENON\_DISABLE\_SPARSE               || OFF                           || Option || Disable sparse allocation of sparse variables, i.e., sparse variable still work but are always allocated. See also :ref:`sparse doc <sparse compile-time>`. |
 || ENABLE\_COMPILER\_WARNINGS               || OFF                           || Option || Enable compiler warnings                                                                                                                                    |
 || TEST\_ERROR\_CHECKING                    || OFF                           || Option || Enables the error checking unit test. This test will FAIL                                                                                                   |

src/config.hpp.in

@@ -54,6 +54,9 @@
 // define PARTHENON_ENABLE_ASCENT or not at all
 #cmakedefine PARTHENON_ENABLE_ASCENT
 
+// define PARTHENON_USE_SERIAL_POOL or not at all
+#cmakedefine PARTHENON_USE_SERIAL_POOL
+
 // Default loop patterns for MeshBlock par_for() wrappers,
 // see kokkos_abstraction.hpp for available tags.
 // Kokkos tight loop layout

src/tasks/tasks.cpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2023-2025. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -116,10 +116,10 @@ std::ostream &WriteTaskGraph(std::ostream &stream,
   return stream;
 }
 
-TaskListStatus TaskRegion::Execute(ThreadPool &pool) {
+TaskListStatus TaskRegion::Execute(Pool_t &pool) {
   // for now, require a pool with one thread
   PARTHENON_REQUIRE_THROWS(pool.size() == 1,
-                           "ThreadPool size != 1 is not currently supported.")
+                           "Pool_t size != 1 is not currently supported.")
 
   // first, if needed, finish building the graph
   if (!graph_built) BuildGraph();

src/tasks/tasks.hpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2023-2025. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -466,7 +466,7 @@ class TaskRegion {
       task_lists[i].SetID(i);
   }
 
-  TaskListStatus Execute(ThreadPool &pool);
+  TaskListStatus Execute(Pool_t &pool);
   TaskList &operator[](const int i) { return task_lists[i]; }
   size_t size() const { return task_lists.size(); }
 
@@ -494,10 +494,10 @@ class TaskCollection {
     return regions.back();
   }
   TaskListStatus Execute() {
-    static ThreadPool pool(1);
+    static Pool_t pool(1);
     return Execute(pool);
   }
-  TaskListStatus Execute(ThreadPool &pool) {
+  TaskListStatus Execute(Pool_t &pool) {
     TaskListStatus status;
     for (auto &region : regions) {
       status = region.Execute(pool);

src/tasks/thread_pool.hpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2023-2025. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -183,6 +183,45 @@ class ThreadPool {
   ThreadVector<std::shared_ptr<std::packaged_task<TaskStatus()>>> run_tasks;
 };
 
+template <typename return_t = TaskStatus>
+class SerialPool {
+ public:
+  explicit SerialPool([[maybe_unused]] const int numthreads = 1) {}
+
+  template <typename F, class... Args>
+  void enqueue(F &&f, Args &&...args) {
+    auto task = [=, func = std::forward<F>(f)] {
+      return func(std::forward<Args>(args)...);
+    };
+    queue.push(task);
+  }
+
+  int size() const { return 1; }
+  void wait() {}
+
+  TaskStatus check_task_returns() {
+    TaskStatus overall = TaskStatus::complete;
+    while (!queue.empty()) {
+      auto f = queue.front();
+      auto ret = f();
+      if constexpr (std::is_same<return_t, TaskStatus>::value) {
+        if (ret == TaskStatus::fail) overall = TaskStatus::fail;
+      }
+      queue.pop();
+    }
+    return overall;
+  }
+
+ private:
+  std::queue<std::function<return_t()>> queue;
+};
+
+#ifdef PARTHENON_USE_SERIAL_POOL
+using Pool_t = SerialPool<TaskStatus>;
+#else
+using Pool_t = ThreadPool;
+#endif
+
 } // namespace parthenon
 
 #endif // TASKS_THREAD_POOL_HPP_