From 8ca8074b8cd9a0a40d8495d042f955c9e7577529 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 28 Oct 2023 00:00:41 -0700 Subject: [PATCH] Set `amrex.omp_threads = "nosmt"` Improve the performance on Intel and AMD CPUs by default: Avoid oversubscribing the physical number of CPU threads, as is default in OpenMP. --- Docs/source/usage/parameters.rst | 6 ++++++ Source/Initialization/WarpXAMReXInit.cpp | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a52a4c1d5e8..118fada45d9 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -167,6 +167,12 @@ Overall simulation parameters For all regular WarpX operations, we therefore do explicit memory transfers without the need for managed memory and thus changed the AMReX default to false. `Please also see the documentation in AMReX `__. +* ``amrex.omp_threads`` (``system``, ``nosmt`` or positive integer; default is ``nosmt``) + An integer number can be set in lieu of the ``OMP_NUM_THREADS`` environment variable to control the number of OpenMP threads to use for the ``OMP`` compute backend on CPUs. + By default, we use the ``nosmt`` option, which overwrites the OpenMP default of spawning one thread per logical CPU core, and instead only spawns a number of threads equal to the number of physical CPU cores on the machine. + If set, the environment variable ``OMP_NUM_THREADS`` takes precedence over ``system`` and ``nosmt``, but not over integer numbers set in this option. + + Signal Handling ^^^^^^^^^^^^^^^ diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 8502bdc4ae6..c153595604f 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -29,6 +29,10 @@ namespace { bool the_arena_is_managed = false; // AMReX' default: true pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed); + // https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html + std::string omp_threads = "nosmt"; // AMReX' default: system + pp_amrex.queryAdd("omp_threads", omp_threads); + // Work-around: // If warpx.numprocs is used for the domain decomposition, we will not use blocking factor // to generate grids. Nonetheless, AMReX has asserts in place that validate that the