From 0af302ed359e93ef41e5febd7ea8e0f4454a9469 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Jan 2025 16:31:54 -0700 Subject: [PATCH 1/7] attempt --- CMakeLists.txt | 1 + src/config.hpp.in | 3 +++ src/tasks/tasks.cpp | 4 ++-- src/tasks/tasks.hpp | 6 +++--- src/tasks/thread_pool.hpp | 38 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 590f277ee8d5..46511ce58878 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ option(TEST_ERROR_CHECKING "Enables the error checking unit test. This test will option(CODE_COVERAGE "Enable code coverage reporting" OFF) option(ENABLE_ASAN "Turn on ASAN" OFF) option(ENABLE_HWASAN "Turn on HWASAN (currently ARM-only)" OFF) +option(PARTHENON_ENABLE_SERIAL_POOL "Disable threaded tasking" OFF) option(PARTHENON_USE_SYSTEM_PACKAGES "Enables search for system packages when available" OFF) if (PARTHENON_USE_SYSTEM_PACKAGES) diff --git a/src/config.hpp.in b/src/config.hpp.in index 299fe0936978..588aff2672e8 100644 --- a/src/config.hpp.in +++ b/src/config.hpp.in @@ -54,6 +54,9 @@ // define PARTHENON_ENABLE_ASCENT or not at all #cmakedefine PARTHENON_ENABLE_ASCENT +// define PARTHENON_USE_SERIAL_POOL or not at all +#cmakedefine PARTHENON_USE_SERIAL_POOL + // Default loop patterns for MeshBlock par_for() wrappers, // see kokkos_abstraction.hpp for available tags. // Kokkos tight loop layout diff --git a/src/tasks/tasks.cpp b/src/tasks/tasks.cpp index a231c43bf380..03ef99061e4b 100644 --- a/src/tasks/tasks.cpp +++ b/src/tasks/tasks.cpp @@ -116,10 +116,10 @@ inline std::ostream &WriteTaskGraph(std::ostream &stream, return stream; } -TaskListStatus TaskRegion::Execute(ThreadPool &pool) { +TaskListStatus TaskRegion::Execute(Pool_t &pool) { // for now, require a pool with one thread PARTHENON_REQUIRE_THROWS(pool.size() == 1, - "ThreadPool size != 1 is not currently supported.") + "Pool_t size != 1 is not currently supported.") // first, if needed, finish building the graph if (!graph_built) BuildGraph(); diff --git a/src/tasks/tasks.hpp b/src/tasks/tasks.hpp index a0090c391a83..c69ff7e6c1c5 100644 --- a/src/tasks/tasks.hpp +++ b/src/tasks/tasks.hpp @@ -466,7 +466,7 @@ class TaskRegion { task_lists[i].SetID(i); } - TaskListStatus Execute(ThreadPool &pool); + TaskListStatus Execute(Pool_t &pool); TaskList &operator[](const int i) { return task_lists[i]; } size_t size() const { return task_lists.size(); } @@ -494,10 +494,10 @@ class TaskCollection { return regions.back(); } TaskListStatus Execute() { - static ThreadPool pool(1); + static Pool_t pool(1); return Execute(pool); } - TaskListStatus Execute(ThreadPool &pool) { + TaskListStatus Execute(Pool_t &pool) { TaskListStatus status; for (auto ®ion : regions) { status = region.Execute(pool); diff --git a/src/tasks/thread_pool.hpp b/src/tasks/thread_pool.hpp index 6d27d8458d32..59b9acb316e9 100644 --- a/src/tasks/thread_pool.hpp +++ b/src/tasks/thread_pool.hpp @@ -183,6 +183,44 @@ class ThreadPool { ThreadVector>> run_tasks; }; +template +class SerialPool { + public: + explicit SerialPool([[maybe_unused]] const int numthreads = 1) {} + + template + void enqueue(F &&f, Args &&...args) { + auto task = [=, func = std::forward(f)] { + return func(std::forward(args)...); + }; + queue.push(task); + } + + int size() const { return 1; } + + TaskStatus check_task_returns() { + TaskStatus overall = TaskStatus::complete; + while (!queue.empty()) { + auto f = queue.front(); + auto ret = f(); + if constexpr (std::is_same::value) { + if (ret == TaskStatus::fail) overall = TaskStatus::fail; + } + queue.pop(); + } + return overall; + } + + private: + std::queue> queue; +}; + +#ifdef PARTHENON_USE_SERIAL_POOL +using Pool_t = SerialPool; +#else +using Pool_t = ThreadPool; +#endif + } // namespace parthenon #endif // TASKS_THREAD_POOL_HPP_ From 69c71bcd3ac35df7d5ceec6a8855d93c9f069fb6 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Jan 2025 16:42:59 -0700 Subject: [PATCH 2/7] formatting --- src/tasks/thread_pool.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tasks/thread_pool.hpp b/src/tasks/thread_pool.hpp index 59b9acb316e9..e8af40cea4dd 100644 --- a/src/tasks/thread_pool.hpp +++ b/src/tasks/thread_pool.hpp @@ -197,6 +197,7 @@ class SerialPool { } int size() const { return 1; } + void wait() {} TaskStatus check_task_returns() { TaskStatus overall = TaskStatus::complete; @@ -216,7 +217,7 @@ class SerialPool { }; #ifdef PARTHENON_USE_SERIAL_POOL -using Pool_t = SerialPool; +using Pool_t = SerialPool; #else using Pool_t = ThreadPool; #endif From fcb386d9db095438021ec140e50356b1608f805c Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Jan 2025 16:46:58 -0700 Subject: [PATCH 3/7] CC --- CMakeLists.txt | 4 ++-- src/tasks/tasks.cpp | 2 +- src/tasks/tasks.hpp | 2 +- src/tasks/thread_pool.hpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46511ce58878..2ca2e45168fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,9 @@ #========================================================================================= # Parthenon performance portable AMR framework -# Copyright(C) 2020-2024 The Parthenon collaboration +# Copyright(C) 2020-2025 The Parthenon collaboration # Licensed under the 3-clause BSD License, see LICENSE file for details #========================================================================================= -# (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +# (C) (or copyright) 2020-2025. Triad National Security, LLC. All rights reserved. # # This program was produced under U.S. Government contract 89233218CNA000001 for Los # Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC diff --git a/src/tasks/tasks.cpp b/src/tasks/tasks.cpp index 462c995632d8..10fd4131c5f7 100644 --- a/src/tasks/tasks.cpp +++ b/src/tasks/tasks.cpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2023-2025. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC diff --git a/src/tasks/tasks.hpp b/src/tasks/tasks.hpp index c69ff7e6c1c5..c89cd33d3778 100644 --- a/src/tasks/tasks.hpp +++ b/src/tasks/tasks.hpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2023-2025. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC diff --git a/src/tasks/thread_pool.hpp b/src/tasks/thread_pool.hpp index e8af40cea4dd..fadc36d7a1d6 100644 --- a/src/tasks/thread_pool.hpp +++ b/src/tasks/thread_pool.hpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2023-2024. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2023-2025. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC From 8b803670e3259821674facc945acd80477a101a4 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Jan 2025 16:47:52 -0700 Subject: [PATCH 4/7] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b35447f459..770aa5022209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - [[PR 1173]](https://github.com/parthenon-hpc-lab/parthenon/pull/1173) Make debugging easier by making parthenon throw an error if ParameterInput is different on multiple MPI ranks. ### Infrastructure (changes irrelevant to downstream codes) +- [[PR1218]](https://github.com/parthenon-hpc-lab/parthenon/pull/1219) Add ability to run the tasking without threading - [[PR 1176]](https://github.com/parthenon-hpc-lab/parthenon/pull/1176) Move some code from header to implementation files ### Removed (removing behavior/API/varaibles/...) From 8baec6264e664d876636381890d05082b4b58d27 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Jan 2025 16:51:31 -0700 Subject: [PATCH 5/7] update docs --- doc/sphinx/src/building.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst index b3a42bd4caec..dad3d5f86806 100644 --- a/doc/sphinx/src/building.rst +++ b/doc/sphinx/src/building.rst @@ -21,6 +21,7 @@ General list of cmake options: || PARTHENON\_ENABLE\_ASCENT || OFF || Option || Enable Ascent for in situ visualization and analysis | || PARTHENON\_DISABLE\_MPI || OFF || Option || MPI is enabled by default if found, set this to True to disable MPI | || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS || OFF || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host. | +|| PARTHENON\_ENABLE\_SERIAL\_POOL || OFF || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks. | || PARTHENON\_DISABLE\_SPARSE || OFF || Option || Disable sparse allocation of sparse variables, i.e., sparse variable still work but are always allocated. See also :ref:`sparse doc `. | || ENABLE\_COMPILER\_WARNINGS || OFF || Option || Enable compiler warnings | || TEST\_ERROR\_CHECKING || OFF || Option || Enables the error checking unit test. This test will FAIL | From 663ae862abf842ad4159a8b41f0456a823d8a493 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 6 Jan 2025 16:56:45 -0700 Subject: [PATCH 6/7] ENABLE -> USE --- doc/sphinx/src/building.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst index dad3d5f86806..8d46fd07d972 100644 --- a/doc/sphinx/src/building.rst +++ b/doc/sphinx/src/building.rst @@ -21,7 +21,7 @@ General list of cmake options: || PARTHENON\_ENABLE\_ASCENT || OFF || Option || Enable Ascent for in situ visualization and analysis | || PARTHENON\_DISABLE\_MPI || OFF || Option || MPI is enabled by default if found, set this to True to disable MPI | || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS || OFF || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host. | -|| PARTHENON\_ENABLE\_SERIAL\_POOL || OFF || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks. | +|| PARTHENON\_USE\_SERIAL\_POOL || OFF || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks. | || PARTHENON\_DISABLE\_SPARSE || OFF || Option || Disable sparse allocation of sparse variables, i.e., sparse variable still work but are always allocated. See also :ref:`sparse doc `. | || ENABLE\_COMPILER\_WARNINGS || OFF || Option || Enable compiler warnings | || TEST\_ERROR\_CHECKING || OFF || Option || Enables the error checking unit test. This test will FAIL | From f6bab09f24f11083b1cce99268e9d0993b0c0988 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 7 Jan 2025 12:53:22 -0700 Subject: [PATCH 7/7] options default to OFF --- CMakeLists.txt | 2 +- doc/sphinx/src/building.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ca2e45168fa..f3b7ad5d7d08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,7 +59,7 @@ option(TEST_ERROR_CHECKING "Enables the error checking unit test. This test will option(CODE_COVERAGE "Enable code coverage reporting" OFF) option(ENABLE_ASAN "Turn on ASAN" OFF) option(ENABLE_HWASAN "Turn on HWASAN (currently ARM-only)" OFF) -option(PARTHENON_ENABLE_SERIAL_POOL "Disable threaded tasking" OFF) +option(PARTHENON_USE_SERIAL_POOL "Disable threaded tasking" ON) option(PARTHENON_USE_SYSTEM_PACKAGES "Enables search for system packages when available" OFF) if (PARTHENON_USE_SYSTEM_PACKAGES) diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst index 8d46fd07d972..18da60cc447a 100644 --- a/doc/sphinx/src/building.rst +++ b/doc/sphinx/src/building.rst @@ -21,7 +21,7 @@ General list of cmake options: || PARTHENON\_ENABLE\_ASCENT || OFF || Option || Enable Ascent for in situ visualization and analysis | || PARTHENON\_DISABLE\_MPI || OFF || Option || MPI is enabled by default if found, set this to True to disable MPI | || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS || OFF || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host. | -|| PARTHENON\_USE\_SERIAL\_POOL || OFF || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks. | +|| PARTHENON\_USE\_SERIAL\_POOL || ON || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks. | || PARTHENON\_DISABLE\_SPARSE || OFF || Option || Disable sparse allocation of sparse variables, i.e., sparse variable still work but are always allocated. See also :ref:`sparse doc `. | || ENABLE\_COMPILER\_WARNINGS || OFF || Option || Enable compiler warnings | || TEST\_ERROR\_CHECKING || OFF || Option || Enables the error checking unit test. This test will FAIL |