Skip to content

Commit

Permalink
Simplify overwrite_amrex_parser_defaults WarpXAMReXInit.cpp (ECP-Warp…
Browse files Browse the repository at this point in the history
…X#4965)

* simplify WarpXAMReXInit.cpp

* correct some typos

* remove space
  • Loading branch information
lucafedeli88 authored Aug 28, 2024
1 parent 33fa181 commit 3bb280c
Showing 1 changed file with 64 additions and 37 deletions.
101 changes: 64 additions & 37 deletions Source/Initialization/WarpXAMReXInit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,66 +17,93 @@
#include <string>

namespace {
/** Overwrite defaults in AMReX Inputs
*
* This overwrites defaults in amrex::ParmParse for inputs.
*/
void
overwrite_amrex_parser_defaults ()
{
amrex::ParmParse pp_amrex("amrex");

#ifdef AMREX_USE_GPU
constexpr auto amrex_use_gpu = true;
#else
constexpr auto amrex_use_gpu = false;
#endif

void override_default_abort_on_out_of_gpu_memory ()
{
// https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters
bool abort_on_out_of_gpu_memory = true; // AMReX' default: false
auto pp_amrex = amrex::ParmParse{"amrex"};
bool abort_on_out_of_gpu_memory = true; // AMReX's default: false
pp_amrex.queryAdd("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory);
}

bool the_arena_is_managed = false; // AMReX' default: true
void override_default_the_arena_is_managed ()
{
auto pp_amrex = amrex::ParmParse{"amrex"};
bool the_arena_is_managed = false; // AMReX's default: true
pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed);
}

void override_default_omp_threads ()
{
// https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html
std::string omp_threads = "nosmt"; // AMReX' default: system
auto pp_amrex = amrex::ParmParse{"amrex"};
std::string omp_threads = "nosmt"; // AMReX's default: system
pp_amrex.queryAdd("omp_threads", omp_threads);
}

void set_device_synchronization ()
{
//See https://github.com/AMReX-Codes/amrex/pull/3763
auto warpx_do_device_synchronize = amrex_use_gpu;

auto pp_warpx = amrex::ParmParse{"warpx"};
pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize);
bool do_device_synchronize = warpx_do_device_synchronize;

auto pp_tiny_profiler = amrex::ParmParse{"tiny_profiler"};
if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) )
{
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
do_device_synchronize == warpx_do_device_synchronize,
"tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize.");
}

}

void apply_workaround_for_warpx_numprocs ()
{
// Work-around:
// If warpx.numprocs is used for the domain decomposition, we will not use blocking factor
// to generate grids. Nonetheless, AMReX has asserts in place that validate that the
// number of cells is a multiple of blocking factor. We set the blocking factor to 1 so those
// AMReX asserts will always pass.
const amrex::ParmParse pp_warpx("warpx");
const auto pp_warpx = amrex::ParmParse{"warpx"};
if (pp_warpx.contains("numprocs"))
{
amrex::ParmParse pp_amr("amr");
pp_amr.add("blocking_factor", 1);
}
}

//See https://github.com/AMReX-Codes/amrex/pull/3763
#ifdef AMREX_USE_GPU
bool warpx_do_device_synchronize = true;
#else
bool warpx_do_device_synchronize = false;
#endif
pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize);
bool do_device_synchronize = warpx_do_device_synchronize;
amrex::ParmParse pp_tiny_profiler("tiny_profiler");
if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) )
{
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
do_device_synchronize == warpx_do_device_synchronize,
"tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize.");
}

void override_default_tiling_option_for_particles ()
{
// Here we override the default tiling option for particles, which is always
// "false" in AMReX, to "false" if compiling for GPU execution and "true"
// if compiling for CPU.
{
amrex::ParmParse pp_particles("particles");
#ifdef AMREX_USE_GPU
bool do_tiling = false; // By default, tiling is off on GPU
#else
bool do_tiling = true;
#endif
pp_particles.queryAdd("do_tiling", do_tiling);
}
auto pp_particles = amrex::ParmParse{"particles"};
auto do_tiling = !amrex_use_gpu; // By default, tiling is off on GPU
pp_particles.queryAdd("do_tiling", do_tiling);
}

/** Overwrite defaults in AMReX Inputs
*
* This overwrites defaults in amrex::ParmParse for inputs.
*/
void
overwrite_amrex_parser_defaults ()
{
override_default_abort_on_out_of_gpu_memory();
override_default_the_arena_is_managed();
override_default_omp_threads();
apply_workaround_for_warpx_numprocs();
set_device_synchronization();
override_default_tiling_option_for_particles();
}
}

Expand Down

0 comments on commit 3bb280c

Please sign in to comment.