diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f7068a010e7..24b13c43e97 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -8,8 +8,6 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v2 - with: - path: 'warpx_directory/WarpX' - name: install dependencies run: | set +e @@ -24,10 +22,12 @@ jobs: brew install openpmd-api - name: build WarpX run: | - cd warpx_directory - git clone --depth 1 https://bitbucket.org/berkeleylab/picsar.git - git clone --depth 1 --branch development https://github.com/AMReX-Codes/amrex.git - cd WarpX - export LDFLAGS="-lomp" - make -j 2 COMP=llvm USE_OMP=FALSE USE_OPENPMD=TRUE XTRA_CXXFLAGS="-Xpreprocessor -fopenmp" - make -j 2 COMP=llvm USE_OMP=FALSE USE_OPENPMD=TRUE XTRA_CXXFLAGS="-Xpreprocessor -fopenmp" PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE + mkdir build_dp && cd build_dp + cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF + make -j 2 + + cd .. + + mkdir build_sp && cd build_sp + cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single + make -j 2 diff --git a/.github/workflows/source/hasEOLwhiteSpace b/.github/workflows/source/hasEOLwhiteSpace index 593ec9721f6..27cf1958d61 100755 --- a/.github/workflows/source/hasEOLwhiteSpace +++ b/.github/workflows/source/hasEOLwhiteSpace @@ -35,18 +35,18 @@ do if [ "$fileHasEOLws" != "-FILE CLEAN-" ] then files+=($i) - echo "$i contains EOL white spaces!" + echo "# $i contains EOL white spaces!" ok=1 fi done if [ $ok -ne 0 ] then - echo "" - echo "SUMMARY" - echo "-------" - echo "Run the following command(s) on the above files to remove your" - echo "end-of-line (EOL) white spaces:" + echo "#" + echo "# SUMMARY" + echo "# -------" + echo "# Run the following command(s) on the above files to remove your" + echo "# end-of-line (EOL) white spaces:" echo "" echo "GNU_SED=\$(sed --help >/dev/null 2>&1 && { echo 1; } || { echo 0; })" echo "[[ \${GNU_SED} -eq 1 ]] && REPLACE=\"sed -i 's/[[:blank:]]\+$//'\" || REPLACE=\"sed -i '' -E 's/[[:blank:]]+$//'\"" diff --git a/.github/workflows/source/hasTabs b/.github/workflows/source/hasTabs index baf64de1ab4..d903f5079ed 100755 --- a/.github/workflows/source/hasTabs +++ b/.github/workflows/source/hasTabs @@ -32,18 +32,18 @@ do if [ "$fileHasTabs" != "-FILE CLEAN-" ] then files+=($i) - echo "$i contains TABs instead of spaces!" + echo "# $i contains TABs instead of spaces!" ok=1 fi done if [ $ok -ne 0 ] then - echo "" - echo "SUMMARY" - echo "-------" - echo "Run the following command(s) on the above files to replace your TABs" - echo "with four white spaces:" + echo "#" + echo "# SUMMARY" + echo "# -------" + echo "# Run the following command(s) on the above files to replace your TABs" + echo "# with four white spaces:" echo "" echo "GNU_SED=\$(sed --help >/dev/null 2>&1 && { echo 1; } || { echo 0; })" echo "[[ \${GNU_SED} -eq 1 ]] && REPLACE=\"sed -i 's/\t/\ \ \ \ /g'\" || REPLACE=\"sed -i '' -E 's/\$(printf '\t')/\ \ \ \ /g'\"" diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index ce89ac3fa25..d21f0fd851a 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -10,8 +10,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - with: - path: 'warpx_directory/WarpX' - name: install dependencies run: | export DEBIAN_FRONTEND=noninteractive @@ -29,23 +27,25 @@ jobs: CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF - name: build WarpX run: | - cd warpx_directory - git clone --depth 1 https://bitbucket.org/berkeleylab/picsar.git - git clone --depth 1 --branch development https://github.com/AMReX-Codes/amrex.git - cd WarpX source /opt/intel/inteloneapi/setvars.sh export CXX=$(which icpc) export CC=$(which icc) - make -j 2 COMP=intel USE_MPI=FALSE USE_OPENPMD=TRUE - make -j 2 COMP=intel USE_MPI=FALSE USE_OPENPMD=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE + + mkdir build_dp && cd build_dp + cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF + make -j 2 + + cd .. + + mkdir build_sp && cd build_sp + cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single + make -j 2 build_dpcc: name: oneAPI DPC++ SP [Linux] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - with: - path: 'warpx_directory/WarpX' - name: install dependencies shell: bash run: | @@ -59,25 +59,25 @@ jobs: sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update - sudo apt-get install -y intel-oneapi-dpcpp-compiler intel-oneapi-mkl gfortran + sudo apt-get install -y intel-oneapi-dpcpp-compiler intel-oneapi-mkl set +e source /opt/intel/inteloneapi/setvars.sh set -e git clone https://github.com/openPMD/openPMD-api.git mkdir openPMD-api/build cd openPMD-api/build - CXX=$(which dpcpp) CC=$(which clang) cmake .. -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF + CXX=$(which dpcpp) CC=$(which clang) cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF make -j 2 sudo make install - name: build WarpX shell: bash run: | - set -ex - cd warpx_directory - git clone --depth 1 https://bitbucket.org/berkeleylab/picsar.git - git clone --depth 1 --branch development https://github.com/AMReX-Codes/amrex.git - cd WarpX set +e source /opt/intel/inteloneapi/setvars.sh set -e - make -j 2 USE_DPCPP=TRUE USE_MPI=FALSE USE_OPENPMD=TRUE USE_OMP=FALSE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE + export CXX=$(which dpcpp) + export CC=$(which clang) + + mkdir build_sp && cd build_sp + cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_COMPUTE=DPCPP -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single + make -j 2 diff --git a/.lgtm.yml b/.lgtm.yml index a4fdc14f76b..71d4b613397 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -17,10 +17,10 @@ extraction: - unzip development.zip - mv amrex-development third-party/amrex - rm development.zip - - wget https://bitbucket.org/berkeleylab/picsar/get/master.zip - - unzip master.zip - - mv berkeleylab-picsar-*/ third-party/picsar - - rm master.zip + - wget https://github.com/ECP-WarpX/picsar/archive/development.zip + - unzip development.zip + - mv picsar-development third-party/picsar + - rm development.zip configure: command: - echo "Yeah" diff --git a/.travis.yml b/.travis.yml index bb0958e9ed2..b54e029507f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,4 +46,4 @@ script: # Run the script that prepares the test environment and runs the tests - export OMP_NUM_THREADS=1 - - ./run_test.sh + - travis_wait 50 ./run_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..5a1c21484d1 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,282 @@ +# Preamble #################################################################### +# +cmake_minimum_required(VERSION 3.14.0) +project(WarpX VERSION 0.20.5) + +include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) + +# In-source tree builds are messy and can screw up the build system. +# Avoid building at least in the same dir as the root dir: +if(CMAKE_BINARY_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + message(FATAL_ERROR "Building in-source is not supported! " + "Create a build directory and remove " + "${CMAKE_SOURCE_DIR}/CMakeCache.txt ${CMAKE_SOURCE_DIR}/CMakeFiles/") +endif() + + +# CCache Support ############################################################## +# +# this is an optional tool that stores compiled object files; allows fast +# re-builds even with "make clean" in between. Mainly used to store AMReX +# objects +set_ccache() + + +# Output Directories ########################################################## +# +# temporary build directories +set_default_build_dirs() + +# install directories +set_default_install_dirs() + + +# Options and Variants ######################################################## +# +option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) +option(WarpX_MPI "Multi-node support (message-passing)" ON) +option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" OFF) +option(WarpX_PSATD "spectral solver support" OFF) +option(WarpX_QED "PICSAR QED (requires Boost and PICSAR)" OFF) +# TODO: python, sensei, legacy hdf5? + +set(WarpX_DIMS_VALUES 2 3 RZ) +set(WarpX_DIMS 3 CACHE STRING "Simulation dimensionality (2/3/RZ)") +set_property(CACHE WarpX_DIMS PROPERTY STRINGS ${WarpX_DIMS_VALUES}) +if(NOT WarpX_DIMS IN_LIST WarpX_DIMS_VALUES) + message(FATAL_ERROR "WarpX_DIMS (${WarpX_DIMS}) must be one of ${WarpX_DIMS_VALUES}") +endif() + +set(WarpX_PRECISION_VALUES double single) +set(WarpX_PRECISION double CACHE STRING "Floating point precision (single/double)") +set_property(CACHE WarpX_PRECISION PROPERTY STRINGS ${WarpX_PRECISION_VALUES}) +if(NOT WarpX_PRECISION IN_LIST WarpX_PRECISION_VALUES) + message(FATAL_ERROR "WarpX_PRECISION (${WarpX_PRECISION}) must be one of ${WarpX_PRECISION_VALUES}") +endif() + +set(WarpX_COMPUTE_VALUES NOACC OMP CUDA DPCPP) # HIP +set(WarpX_COMPUTE OMP CACHE STRING "On-node, accelerated computing backend (NOACC/OMP/CUDA/DPCPP)") +set_property(CACHE WarpX_COMPUTE PROPERTY STRINGS ${WarpX_COMPUTE_VALUES}) +if(NOT WarpX_COMPUTE IN_LIST WarpX_COMPUTE_VALUES) + message(FATAL_ERROR "WarpX_PRECISION (${WarpX_COMPUTE}) must be one of ${WarpX_COMPUTE_VALUES}") +endif() + +option(WarpX_amrex_internal "Download & build AMReX" ON) + +# change the default build type to RelWithDebInfo (or Release) instead of Debug +set_default_build_type("RelWithDebInfo") + +# this defined the variable BUILD_TESTING which is ON by default +#include(CTest) + + +# Dependencies ################################################################ +# + +# AMReX +# builds AMReX from source (default) or finds an existing install +include(${WarpX_SOURCE_DIR}/cmake/dependencies/AMReX.cmake) +# suppress warnings in AMReX headers (use -isystem instead of -I) +make_third_party_includes_system(AMReX::amrex AMReX) + +# PICSAR +# builds PICSAR from source +include(${WarpX_SOURCE_DIR}/cmake/dependencies/PICSAR.cmake) + +# openPMD +# builds openPMD-api from source (default) or finds an existing install +include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake) + +# PSATD +if(WarpX_PSATD) + # FFTW (non-GPU) and cuFFT (GPU) + if(NOT ENABLE_CUDA) + find_package(PkgConfig REQUIRED QUIET) + if(WarpX_PRECISION STREQUAL "double") + pkg_check_modules(fftw3 REQUIRED IMPORTED_TARGET fftw3) + else() + pkg_check_modules(fftw3f REQUIRED IMPORTED_TARGET fftw3f) + endif() + endif() + # BLASPP and LAPACKPP + if(WarpX_DIMS STREQUAL RZ) + # FIXME send a couple of upstream PRs to those repos... + find_package(blaspp REQUIRED) + find_package(lapackpp REQUIRED) + find_package(OpenMP REQUIRED) # pulled by the two above + endif() +endif() + + +# Targets ##################################################################### +# +# executable +add_executable(WarpX) +add_executable(WarpX::WarpX ALIAS WarpX) + +# own headers +target_include_directories(WarpX PRIVATE + $ +) + +# if we include we will need to call: +include(AMReXBuildInfo) +generate_buildinfo(WarpX "${WarpX_SOURCE_DIR}") + +# add sources +target_sources(WarpX + PRIVATE + Source/main.cpp + Source/WarpX.cpp +) +add_subdirectory(Source/BoundaryConditions) +add_subdirectory(Source/Diagnostics) +add_subdirectory(Source/Evolve) +add_subdirectory(Source/FieldSolver) +add_subdirectory(Source/Filter) +add_subdirectory(Source/Initialization) +add_subdirectory(Source/Laser) +add_subdirectory(Source/Parallelization) +add_subdirectory(Source/Parser) +add_subdirectory(Source/Particles) +#add_subdirectory(Source/Python) +add_subdirectory(Source/Utils) + +# C++ properties: at least a C++14 capable compiler is needed +target_compile_features(WarpX PUBLIC cxx_std_14) +set_target_properties(WarpX PROPERTIES + CXX_EXTENSIONS OFF + CXX_STANDARD_REQUIRED ON +) + +# link dependencies +target_link_libraries(WarpX PUBLIC WarpX::thirdparty::AMReX) + +if(WarpX_PSATD) + if(ENABLE_CUDA) + # CUDA_ADD_CUFFT_TO_TARGET(WarpX) + target_link_libraries(WarpX PUBLIC cufft) + else() + if(WarpX_PRECISION STREQUAL "double") + target_link_libraries(WarpX PUBLIC PkgConfig::fftw3) + else() + target_link_libraries(WarpX PUBLIC PkgConfig::fftw3f) + endif() + endif() + if(WarpX_DIMS STREQUAL RZ) + # FIXME send a couple of upstream PRs to those repos... + target_link_libraries(WarpX PUBLIC blaspp) + target_link_libraries(WarpX PUBLIC lapackpp) + endif() +endif() + +if(WarpX_OPENPMD) + target_compile_definitions(WarpX PUBLIC WARPX_USE_OPENPMD) + target_link_libraries(WarpX PUBLIC openPMD::openPMD) +endif() + +if(WarpX_QED) + target_compile_definitions(WarpX PUBLIC WARPX_QED) + # FIXME WARPX_QED_TABLE_GEN + target_link_libraries(WarpX PUBLIC pxrmp_qed) +endif() + +# AMReX helper function: propagate CUDA specific target & source properties +if(ENABLE_CUDA) + setup_target_for_cuda_compilation(WarpX) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) + target_compile_features(WarpX PUBLIC cuda_std_14) + set_target_properties(WarpX PROPERTIES + CUDA_EXTENSIONS OFF + CUDA_STANDARD_REQUIRED ON + ) + endif() +endif() + +# fancy binary name for build variants +set_warpx_binary_name() + + +# Defines ##################################################################### +# +if(WarpX_DIMS STREQUAL 3) + target_compile_definitions(WarpX PRIVATE WARPX_DIM_3D) +elseif(WarpX_DIMS STREQUAL 2) + target_compile_definitions(WarpX PRIVATE WARPX_DIM_XZ) +elseif(WarpX_DIMS STREQUAL RZ) + target_compile_definitions(WarpX PRIVATE WARPX_DIM_RZ) +endif() + +if(WarpX_OPENPMD) + target_compile_definitions(WarpX PRIVATE WARPX_USE_OPENPMD) +endif() + +if(WarpX_QED) + target_compile_definitions(WarpX PRIVATE WARPX_QED) + if(WarpX_QED_TABLE_GEN) + target_compile_definitions(WarpX PRIVATE WarpX_QED_TABLE_GEN) + endif() +endif() + +if(WarpX_PSATD) + target_compile_definitions(WarpX PRIVATE WARPX_USE_PSATD) +endif() + + +# Warnings #################################################################### +# +set_cxx_warnings() + + +# Generate Configuration and .pc Files ######################################## +# +# these files are used if WarpX is installed and picked up by a downstream +# project (not needed yet) + +#include(CMakePackageConfigHelpers) +#write_basic_package_version_file("WarpXConfigVersion.cmake" +# VERSION ${WarpX_VERSION} +# COMPATIBILITY SameMajorVersion +#) + + +# Installs #################################################################### +# +# headers, libraries and executables +set(WarpX_INSTALL_TARGET_NAMES WarpX) + +install(TARGETS ${WarpX_INSTALL_TARGET_NAMES} + EXPORT WarpXTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +# CMake package file for find_package(WarpX::WarpX) in depending projects +#install(EXPORT WarpXTargets +# FILE WarpXTargets.cmake +# NAMESPACE WarpX:: +# DESTINATION ${CMAKE_INSTALL_CMAKEDIR} +#) +#install( +# FILES +# ${WarpX_BINARY_DIR}/WarpXConfig.cmake +# ${WarpX_BINARY_DIR}/WarpXConfigVersion.cmake +# DESTINATION ${CMAKE_INSTALL_CMAKEDIR} +#) + + +# Tests ####################################################################### +# + +#if(BUILD_TESTING) +# enable_testing() +# +# add_test(...) +#endif() + + +# Status Summary for Build Options ############################################ +# +warpx_print_summary() diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1db18215cd8..5b945583f0f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ The basic WarpX workflow is: 1. Fork the main repo (or update it if you already created it). 2. Implement your changes and push them on a new branch ```` on your fork. -3. Create a Pull Request from branch ```` on your fork to branch `master` on the main WarpX repo. +3. Create a Pull Request from branch ```` on your fork to branch ``development`` on the main WarpX repo. First, let us setup your local git repo. Make your own fork of the main (``upstream``) WarpX repo: on the `WarpX Github page `_, press the fork button. @@ -33,7 +33,7 @@ Then, you can execute: # These 4 first lines are the same as for a standard WarpX install mkdir warpx_directory cd warpx_directory - git clone --branch master https://bitbucket.org/berkeleylab/picsar.git + git clone --branch development https://github.com/ECP-WarpX/picsar.git git clone --branch development https://github.com/AMReX-Codes/amrex.git # Clone your fork on your local computer. You can get this address on your fork's Github page. @@ -48,18 +48,18 @@ Now you are free to play with your fork (for additional information, you can vis .. note:: You do not have to re-do the setup above every time. - Instead, in the future, all you need is to update the ``master`` branch on your fork with: + Instead, in the future, all you need is to update the ``development`` branch on your fork with: .. code-block:: sh - git checkout master - git pull upstream master + git checkout development + git pull upstream development -Make sure you are on WarpX ``master`` branch with +Make sure you are on WarpX ``development`` branch with .. code-block:: sh - git checkout master + git checkout development in the WarpX directory. @@ -95,11 +95,11 @@ You can push them to your fork with git push -u origin -If you want to synchronize your branch with the ``master`` branch (this is useful when the ``master`` branch is being modified while you are working on ````), you can use +If you want to synchronize your branch with the ``development`` branch (this is useful when the ``development`` branch is being modified while you are working on ````), you can use .. code-block:: sh - git pull upstream master + git pull upstream development and fix any conflict that may occur. diff --git a/Docs/source/building/building.rst b/Docs/source/building/building.rst index 2f7cc7f9820..ec3528333cb 100644 --- a/Docs/source/building/building.rst +++ b/Docs/source/building/building.rst @@ -20,7 +20,7 @@ single directory (e.g. ``warpx_directory``): mkdir warpx_directory cd warpx_directory git clone https://github.com/ECP-WarpX/WarpX.git - git clone https://bitbucket.org/berkeleylab/picsar.git + git clone https://github.com/ECP-WarpX/picsar.git git clone --branch development https://github.com/AMReX-Codes/amrex.git Basic compilation @@ -51,6 +51,8 @@ options are: * ``COMP=gcc`` or ``intel``: Compiler. * ``USE_MPI=TRUE`` or ``FALSE``: Whether to compile with MPI support. * ``USE_OMP=TRUE`` or ``FALSE``: Whether to compile with OpenMP support. + * ``USE_GPU=TRUE`` or ``FALSE``: Whether to compile for Nvidia GPUs (requires CUDA). + * ``USE_OPENPMD=TRUE`` or ``FALSE``: Whether to support openPMD for I/O (requires openPMD-api). For a description of these different options, see the `corresponding page `__ in the AMReX documentation. @@ -68,6 +70,17 @@ In order to clean a previously compiled version (typically useful for troublesho before re-attempting compilation. +Preview: CMake Build System +--------------------------- + +We are currently transitioning to support CMake as our primary build system. +Until we have transitioned our documentation and functionality completely, please read the following preview section for details. + +.. toctree:: + :maxdepth: 1 + + cmake + Advanced building instructions ------------------------------ @@ -89,3 +102,4 @@ Building for specific platforms cori summit + juwels diff --git a/Docs/source/building/cmake.rst b/Docs/source/building/cmake.rst new file mode 100644 index 00000000000..577c3525eb5 --- /dev/null +++ b/Docs/source/building/cmake.rst @@ -0,0 +1,140 @@ +.. _building-cmake: + +CMake Transition +================ + +We are currently transitioning to support `CMake `_ as our primary build system. +Until we have transitioned our documentation and functionality completely, please read the following preview section for details. + +Progress status: `see on GitHub `_ + +Dependencies +============ + +WarpX depends on the following popular third party software. +Please see installation instructions below. + +- a mature `C++14 `_ compiler: e.g. GCC 5, Clang 3.6 or newer +- `CMake 3.14.0+ `_ +- `AMReX (development) `_: we automatically download and compile a copy of AMReX + +Optional dependencies include: + +- `MPI 3.0+ `_: for multi-node and/or multi-GPU execution +- `CUDA Toolkit 9.0+ `_: for Nvidia GPU support (see `matching host-compilers `_) +- `OpenMP 3.1+ `_: for threaded CPU execution (currently not fully accelerated) +- `FFTW3 `_: for spectral solver (PSATD) support +- `Boost 1.66.0+ `_: for QED support +- `openPMD-api 0.11.1+ `_: we automatically download and compile a copy of openPMD-api for openPMD I/O support + + - see `optional I/O backends `_ +- `CCache `_: to speed up rebuilds (needs 3.7.9+ for CUDA) + +Install Dependencies +==================== + +macOS/Linux: + +.. code-block:: bash + + spack env create warpx-dev + spack env activate warpx-dev + spack add ccache + spack add cmake + spack add fftw + spack add mpi + spack add openpmd-api + spack add pkgconfig # for fftw + # optional: + # spack add cuda + spack install + +(in new terminals, re-activate the environment with ``spack env activate warpx-dev`` again) + +or macOS/Linux: + +.. code-block:: bash + + brew update + brew install ccache + brew install cmake + brew install fftw + brew install libomp + brew install pkg-config # for fftw + brew install open-mpi + brew install openpmd-api + +Now, ``cmake --version`` should be at version 3.14.0 or newer. + +Configure your compiler +======================= + +For example, using a GCC on macOS: + +.. code-block:: bash + + export CC=$(which gcc) + export CXX=$(which g++) + +If you also want to select a CUDA compiler: + +.. code-block:: bash + + export CUDACXX=$(which nvcc) + export CUDAHOSTCXX=$(which g++) + +Build & Test +============ + +From the base of the WarpX source directory, execute: + +.. code-block:: bash + + mkdir -p build + cd build + + # find dependencies & configure + cmake .. + + # build using up to four threads + make -j 4 + + # run tests (todo) + +You can inspect and modify build options after running ``cmake ..`` with either + +.. code-block:: bash + + ccmake . + +or by providing arguments to the CMake call: ``cmake .. -D= -D=`` + +=========================== ============================================ ======================================================= +CMake Option Default & Values Description +=========================== ============================================ ======================================================= +``CMAKE_BUILD_TYPE`` **RelWithDebInfo**/Release/Debug Type of build, symbols & optimizations +``WarpX_ASCENT`` ON/**OFF** Ascent in situ visualization +``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/DPCPP On-node, accelerated computing backend +``WarpX_DIMS`` **3**/2/RZ Simulation dimensionality +``WarpX_MPI`` **ON**/OFF Multi-node support (message-passing) +``WarpX_OPENPMD`` ON/**OFF** openPMD I/O (HDF5, ADIOS) +``WarpX_PRECISION`` **double**/single Floating point precision (single/double) +``WarpX_PSATD`` ON/**OFF** Spectral solver +``WarpX_QED`` ON/**OFF** PICSAR QED (requires Boost and PICSAR) +``WarpX_amrex_repo`` ``https://github.com/AMReX-Codes/amrex.git`` Repository URI to pull and build AMReX from +``WarpX_amrex_branch`` ``development`` Repository branch for ``WarpX_amrex_repo`` +``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` +``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD library if set to ``OFF`` +=========================== ============================================ ======================================================= + +For example, one can also build against a local AMReX git repo. +Assuming AMReX' source is located in ``$HOME/src/amrex`` and changes are committed into a branch such as ``my-amrex-branch`` then pass to ``cmake`` the arguments: ``-DWarpX_amrex_repo=file://$HOME/src/amrex -DWarpX_amrex_branch=my-amrex-branch``. + +For developers, WarpX can be configured in further detail with options from AMReX, which are `documented in the AMReX manual `_. + +Run +=== + +An executable WarpX binary with the current compile-time options encoded in its file name will be created in ``bin/``. + +Additionally, a `symbolic link `_ named ``warpx`` can be found in that directory, which points to the last built WarpX executable. diff --git a/Docs/source/building/cori.rst b/Docs/source/building/cori.rst index 06015df966d..e4e9934895c 100644 --- a/Docs/source/building/cori.rst +++ b/Docs/source/building/cori.rst @@ -1,7 +1,7 @@ .. _building-cori: -Building WarpX for Cori (NERSC) -=============================== +Cori (NERSC) +============ Standard build -------------- diff --git a/Docs/source/building/juwels.rst b/Docs/source/building/juwels.rst new file mode 100644 index 00000000000..639540ae925 --- /dev/null +++ b/Docs/source/building/juwels.rst @@ -0,0 +1,74 @@ +.. _building-juwels: + +Juwels (JSC) +============ + +The `Juwels supercomputer `_ is located at JSC. + +See `this page `_ for a quick introduction. + +* Batch system: `Slurm `_ +* `Production directories `_: + + * ``$SCRATCH``: Scratch filesystem for temporary data (90 day purge) + * ``$FASTDATA/``: Storage location for large data (backuped) + * Note that the ``$HOME`` directory is not designed for simulation runs and producing output there will impact performance. + +Installation +------------ + +Use the following commands to download the WarpX source code and switch to the correct branch: + +.. code-block:: bash + + mkdir ~/src + cd ~/src + + git clone https://github.com/ECP-WarpX/WarpX.git warpx + git clone --branch QED https://github.com/ECP-WarpX/picsar.git + git clone --branch development https://github.com/AMReX-Codes/amrex.git + +We use the following modules and environments on the system. + +.. code-block:: bash + + # please set your project account + export proj= + + # required dependencies + module load GCC + module load OpenMPI + module load CUDA + + # JEWELS' job scheduler may not map ranks to GPUs, + # so we give a hint to AMReX about the node layout. + # This is usually done in Make. files in AMReX + # but there is no such file for JSC yet. + export GPUS_PER_SOCKET=2 + export GPUS_PER_NODE=4 + +Note that for now WarpX must rely on OpenMPI instead of the recommended MPI implementation on this platform MVAPICH2. + +We recommend to store the above lines in a file, such as ``$HOME/warpx.profile``, and load it into your shell after a login: + +.. code-block:: bash + + source $HOME/warpx.profile + +Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: + +.. code-block:: bash + + make -j 16 COMP=gcc USE_GPU=TRUE + +The other :ref:`general compile-time options ` apply as usual. + +Running +------- + +An example submission script reads + +.. literalinclude:: ../../../Tools/BatchScripts/batch_juwels.sh + :language: bash + +See :doc:`../visualization/yt` for more information on how to visualize the simulation results. diff --git a/Docs/source/building/spack.rst b/Docs/source/building/spack.rst index 3ee2641c947..5d9cdc87e53 100644 --- a/Docs/source/building/spack.rst +++ b/Docs/source/building/spack.rst @@ -25,7 +25,7 @@ WarpX is built with the single command spack install warpx -This will build the 3-D version of WarpX using the ``master`` branch. +This will build the 3-D version of WarpX using the ``development`` branch. At the very end of the output from build sequence, Spack tells you where the WarpX executable has been placed. Alternatively, ``spack load -r warpx`` can be called, which will put the executable in your ``PATH`` environment variable. diff --git a/Docs/source/building/summit.rst b/Docs/source/building/summit.rst index e2f5d3b178a..509ae51d248 100644 --- a/Docs/source/building/summit.rst +++ b/Docs/source/building/summit.rst @@ -1,89 +1,143 @@ .. _building-summit: -Building WarpX on Summit (OLCF) -================================ +Summit (OLCF) +============= -For the `Summit cluster -`__ at OLCF, -use the following commands to download the source code, and switch to the -correct branch: +The `Summit cluster `_ is located at OLCF. -:: +If you are new to this system, please see the following resources: - mkdir warpx_directory - cd warpx_directory +* `Summit user guide `_ +* Batch system: `LSF `_ +* `Production directories `_: - git clone https://github.com/ECP-WarpX/WarpX.git - git clone https://bitbucket.org/berkeleylab/picsar.git - git clone --branch development https://github.com/AMReX-Codes/amrex.git + * ``$PROJWORK/$proj/``: shared with all members of a project (recommended) + * ``$MEMBERWORK/$proj/``: single user (usually smaller quota) + * ``$WORLDWORK/$proj/``: shared with all users + * Note that the ``$HOME`` directory is mounted as read-only on compute nodes. + That means you cannot run in your ``$HOME``. -Then, ``cd`` into the directory ``WarpX`` and use the following set of commands to compile: -:: +Installation +------------ - module load gcc - module load cuda - make -j 16 COMP=gcc USE_GPU=TRUE +Use the following commands to download the WarpX source code and switch to the correct branch: -See :doc:`../running_cpp/platforms` for more information on how to run WarpX on Summit. +.. code-block:: bash -See :doc:`../visualization/yt` for more information on how to visualize the simulation results. + mkdir ~/src + cd ~/src -.. note:: + git clone https://github.com/ECP-WarpX/WarpX.git warpx + git clone --branch QED https://github.com/ECP-WarpX/picsar.git + git clone --branch development https://github.com/AMReX-Codes/amrex.git - To compile and run WarpX on Summit CPUs, the cuda module is not necessary. +We use the following modules and environments on the system. - But to build with the spectral solver, PSATD, you need to use an MPI-enabled version of FFTW like the `fftw` module (because the `cuFFT` tool is not available when cuda is not loaded). +.. code-block:: bash - :: + # please set your project account + export proj= - module load gcc - module load fftw - make -j 16 COMP=gcc USE_PSATD=TRUE + # required dependencies + module load gcc/6.4.0 + module load cuda + # optional: faster re-builds + module load ccache + # optional: for PSATD support + module load fftw + # optional: for QED support + module load boost/1.66.0 -.. _building-summit-openPMD: + # optional: for openPMD support + module load cmake + module load hdf5/1.10.4 + module load adios2/2.5.0 + export PKG_CONFIG_PATH=$HOME/sw/openPMD-api-install/lib64/pkgconfig:$PKG_CONFIG_PATH + export CMAKE_PREFIX_PATH=$HOME/sw/openPMD-api-install:$CMAKE_PREFIX_PATH -Building WarpX with openPMD support ------------------------------------ + # optional: Ascent in situ support + # note: build WarpX with CMake + export Alpine=/gpfs/alpine/world-shared/csc340/software/ascent/0.5.3-pre/summit/cuda/gnu + export Ascent_DIR=$Alpine/ascent-install + export Conduit_DIR=$Alpine/conduit-install -First, load the appropriate modules: + # optional: for Python bindings or libEnsemble + module load python/3.7.0 + + # optional: for libEnsemble + module load openblas/0.3.9-omp + module load netlib-lapack/3.8.0 + if [ -d "$HOME/sw/venvs/warpx-libE" ] + then + source $HOME/sw/venvs/warpx-libE/bin/activate + fi + + # optional: just an additional text editor + module load nano + + # optional: an alias to request an interactive node for two hours + alias getNode="bsub -P $proj -W 2:00 -nnodes 1 -Is /bin/bash" + + # fix system defaults: do not escape $ with a \ on tab completion + shopt -s direxpand + + # compiler environment hints + export CC=$(which gcc) + export CXX=$(which g++) + export FC=$(which gfortran) + export CUDACXX=$(which nvcc) + export CUDAHOSTCXX=$(which g++) -.. code-block:: bash - module load gcc - module load cuda - module load cmake - module load hdf5/1.10.4 - module load adios2/2.5.0 +We recommend to store the above lines in a file, such as ``$HOME/warpx.profile``, and load it into your shell after a login: - export CC=$(which gcc) - export CXX=$(which g++) - export CUDACXX=$(which nvcc) - export CUDAHOSTCXX=$(which g++) +.. code-block:: bash + + source $HOME/warpx.profile -Then, in the ``warpx_directory``, download and build openPMD-api: +Optionally, download and build openPMD-api for I/O: .. code-block:: bash git clone https://github.com/openPMD/openPMD-api.git mkdir openPMD-api-build cd openPMD-api-build - cmake ../openPMD-api -DopenPMD_USE_PYTHON=OFF -DCMAKE_INSTALL_PREFIX=../openPMD-install/ -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_RPATH='$ORIGIN' -DMPIEXEC_EXECUTABLE=$(which jsrun) + cmake ../openPMD-api -DopenPMD_USE_PYTHON=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/openPMD-api-install/ -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_RPATH='$ORIGIN' -DMPIEXEC_EXECUTABLE=$(which jsrun) cmake --build . --target install --parallel 16 -.. note: +Optionally, download and install :ref:`libEnsemble ` for dynamic ensemble optimizations: - On Summit, only compute nodes provide the infiniband hardware that Summit's MPI module expects, ``jsrun`` must be used on Summit instead of ``mpiexec``, and ``$HOME`` directories are read-only when computing. - In order to run openPMD-api unit tests, run on a compute node inside ``$PROJWORK``, e.g. via ``bsub -P -W 2:00 -nnodes 1 -Is /bin/bash``, and add ``-DMPIEXEC_EXECUTABLE=$(which jsrun)`` to the CMake options. +.. code-block:: bash -Finally, compile WarpX: + export BLAS=$OLCF_OPENBLAS_ROOT/lib/libopenblas.so + export LAPACK=$OLCF_NETLIB_LAPACK_ROOT/lib64/liblapack.so + python3 -m pip install --user --upgrade pip + python3 -m pip install --user virtualenv + python3 -m venv $HOME/sw/venvs/warpx-libE + source $HOME/sw/venvs/warpx-libE/bin/activate + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade cython + python3 -m pip install --upgrade numpy + python3 -m pip install --upgrade scipy + python3 -m pip install --upgrade mpi4py --no-binary mpi4py + python3 -m pip install --upgrade -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt + +Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: .. code-block:: bash - cd ../WarpX - export PKG_CONFIG_PATH=$PWD/../openPMD-install/lib64/pkgconfig:$PKG_CONFIG_PATH - export CMAKE_PREFIX_PATH=$PWD/../openPMD-install:$CMAKE_PREFIX_PATH make -j 16 COMP=gcc USE_GPU=TRUE USE_OPENPMD=TRUE + +The other :ref:`general compile-time options ` apply as usual. + + +Running +------- + +Please see :ref:`our example job scripts ` on how to run WarpX on Summit. + +See :doc:`../visualization/yt` for more information on how to visualize the simulation results. diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 0cf29e00eff..a4129a3044c 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -66,7 +66,7 @@ # built documents. # # The short X.Y version. -version = '20.06' +version = '20.07' # The full version, including alpha/beta/rc tags. release = '' diff --git a/Docs/source/developers/developers.rst b/Docs/source/developers/developers.rst index 1eaf4c50720..dd329b4a5d0 100644 --- a/Docs/source/developers/developers.rst +++ b/Docs/source/developers/developers.rst @@ -3,7 +3,7 @@ Developers documentation ======================== -For general information on how to contribute to WarpX, including our ``git`` workflow and code practices, have a look at our `CONTRIBUTING.md `__! +For general information on how to contribute to WarpX, including our ``git`` workflow and code practices, have a look at our `CONTRIBUTING.md `__! Our Doxygen API documentation in classic formatting `is located here <../_static/doxyhtml/index.html>`_. diff --git a/Docs/source/developers/documentation.rst b/Docs/source/developers/documentation.rst index c3f19a51ac5..6beea5b0f9d 100644 --- a/Docs/source/developers/documentation.rst +++ b/Docs/source/developers/documentation.rst @@ -36,7 +36,9 @@ A HTML rendered version of the Doxygen documentation `is located here <../_stati Breathe documentation --------------------- -Your Doxygen documentation is not only useful for people looking into the code, it is also part of the `WarpX online documentation `__ based on `Sphinx `__! This is done using the Python module `Breathe `__, that allows you to read Doxygen documentation dorectly in the source and include it in your Sphinx documentation, by calling Breathe functions. For instance, the following line will get the Doxygen documentation for ``WarpXParticleContainer`` in ``Source/Particles/WarpXParticleContainer.H`` and include it to the html page generated by Sphinx: +Your Doxygen documentation is not only useful for people looking into the code, it is also part of the `WarpX online documentation `_ based on `Sphinx `_! +This is done using the Python module `Breathe `_, that allows you to read Doxygen documentation dorectly in the source and include it in your Sphinx documentation, by calling Breathe functions. +For instance, the following line will get the Doxygen documentation for ``WarpXParticleContainer`` in ``Source/Particles/WarpXParticleContainer.H`` and include it to the html page generated by Sphinx: .. doxygenclass:: WarpXParticleContainer diff --git a/Docs/source/developers/workflow.rst b/Docs/source/developers/workflow.rst index f8a9632962c..226cf77f3c2 100644 --- a/Docs/source/developers/workflow.rst +++ b/Docs/source/developers/workflow.rst @@ -3,12 +3,13 @@ Workflow ======== -Make a new Github release -------------------------- +Create a new Github release +--------------------------- -WarpX has one release per month. To make the release, you need to: +WarpX has one release per month. +In order to create a release, you need to: - 1. Create a new branch from ``master`` and update the version number in all source files. + 1. Create a new branch from ``development`` and update the version number in all source files. There is a script for that, so you can do: .. code-block:: sh @@ -18,7 +19,7 @@ WarpX has one release per month. To make the release, you need to: Then open a PR, as usual. NOTE: do not merge this PR before step 2 is completed. - 2. Click the `Draft a new release` button at https://github.com/ECP-WarpX/WarpX/releases and follow instructions. + 2. Click the ``Draft a new release`` button at https://github.com/ECP-WarpX/WarpX/releases and follow instructions. Please specify the compatible versions of dependencies (see previous releases), and provide info on the content of the release. In order to get a list of PRs merged since last release, you may run diff --git a/Docs/source/index.rst b/Docs/source/index.rst index d718686d9c0..67d529601d6 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -1,4 +1,4 @@ -.. WarpX documentation master file, created by +.. WarpX documentation index file, created by sphinx-quickstart on Tue Mar 7 22:08:26 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. diff --git a/Docs/source/libensemble/libensemble.rst b/Docs/source/libensemble/libensemble.rst index 86d25bc669f..586dc3eef62 100644 --- a/Docs/source/libensemble/libensemble.rst +++ b/Docs/source/libensemble/libensemble.rst @@ -1,9 +1,11 @@ +.. _libensemble: + Run LibEnsemble on WarpX ======================== `LibEnsemble `__ is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. -If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., LibEnsemble provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platform (LibEnsemble targets modern HPC platforms like Summit). +If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., LibEnsemble provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (LibEnsemble targets modern HPC platforms like Summit). Scripts to run LibEnsemble on WarpX simulations can be found in ``WarpX/Tools/LibEnsemble/``. This documentation does not aim at giving a training on LibEnsemble, so please refer to the `LibEnsemble documentation `__ for technical details. @@ -47,7 +49,7 @@ You can either install all packages via `conda` (recommended), conda install -c conda-forge libensemble matplotlib numpy scipy yt -or try to install the same dependencies via `pip` (pick one *or* the other): +or try to install the same dependencies via `pip` (pick one *or* the other; note our :ref:`installation details on Summit `): .. literalinclude:: ../../../Tools/LibEnsemble/requirements.txt @@ -118,7 +120,7 @@ Adjust the ``local_specs`` dictionary in ``all_machine_specs.py`` to fix the pat .. code-block:: sh - python run_libE_on_warpx.py --comms local --nworkers 3 + python run_libensemble_on_warpx.py --comms local --nworkers 3 This is adapted to a 4-core machine, as it will use: @@ -131,6 +133,12 @@ This is adapted to a 4-core machine, as it will use: Run on Summit at OLCF ^^^^^^^^^^^^^^^^^^^^^ +- ``cp -r $HOME/warpx/Tools/LibEnsemble/* sim_directory`` +- modify ``run_libensemble_on_warpx.py`` to have ``machine = 'summit'`` +- modify ``all_machine_specs.py`` to put the right path to the WarpX executable +- modify ``summit_submit_mproc.sh`` to set ``LIBE_PLOTS`` to ``false`` and set the project ID +- ``bsub summit_submit_mproc.sh``: + .. code-block:: sh bsub summit_submit_mproc.sh diff --git a/Docs/source/running_cpp/parallelization.rst b/Docs/source/running_cpp/parallelization.rst index 86d86ffc764..b62307b5067 100644 --- a/Docs/source/running_cpp/parallelization.rst +++ b/Docs/source/running_cpp/parallelization.rst @@ -1,3 +1,5 @@ +.. _parallelization_warpx: + Parallelization in WarpX ========================= @@ -17,12 +19,22 @@ the parallelization. The main user-defined parameters are AMReX ``max_grid_size`` and ``blocking_factor`` ----------------------------------------------- -* ``amr.max_grid_size`` is the maximum number of points per **grid** along each +* ``amr.max_grid_size`` is the maximum number of cells per **grid** along each direction (default ``amr.max_grid_size=32`` in 3D). -* ``amr.blocking_factor``: The size of each **grid** must be divisible by the - `blocking_factor` along all dimensions (default ``amr.blocking_factor=8``). - Note that the ``max_grid_size`` also has to be divisible by ``blocking_factor``. +* ``amr.blocking_factor``: is the minimum number of cells per **grid** along each + direction (default ``amr.blocking_factor=8``). + Note that both the domain (at each level) and ``max_grid_size`` must be divisible by ``blocking_factor``. + + .. note:: + + You can use the parameters above if you want the same number of cells in all directions. + Or you can set ``amr.max_grid_size_x``, ``amr.max_grid_size_y`` and ``amr.max_grid_size_z``; +  ``amr.blocking_factor_x``, ``amr.blocking_factor_x`` and ``amr.blocking_factor_x`` to different numbers of cells. + +The total number of **grids** is determined using those two restrictions and the number of +ranks used to run the simulation. You can visit `AMReX `_ +documentation for more information on the two parameters. These parameters can have a dramatic impact on the code performance. Each **grid** in the decomposition is surrounded by guard cells, thus increasing the @@ -66,3 +78,10 @@ simulation, it can be cumbersome to calculate the number of cells and the physical size of the computational domain for a given resolution. This :download:`Python script<../../../Tools/DevUtils/compute_domain.py>` does it automatically. + +When using the RZ spectral solver, the values of ``amr.max_grid_size`` and ``amr.blocking_factor`` are constrained since the solver +requires that the full radial extent be within a each block. +For the radial values, any input is ignored and the max grid size and blocking factor are both set equal to the number of radial cells. +For the longitudinal values, the blocking factor has a mminimum size of 8, allowing the computational domain of each block to be large enough relative to the guard cells for reasonable performance, but the max grid size and blocking factor must also be small enough so that there will be at least one block per processor. +If max grid size and/or blocking factor are too large, they will be silently reduced as needed. +If there are too many processors so that there is not enough blocks for the number processors, WarpX will abort. diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 1d03dc7a811..5e06cac0411 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -8,6 +8,8 @@ Input parameters This section is currently in development. +.. _running-cpp-parameters-overall: + Overall simulation parameters ----------------------------- @@ -67,6 +69,8 @@ Overall simulation parameters electromagnetic effects (e.g. propagation of radiation, lasers, etc.) are not captured. +.. _running-cpp-parameters-box: + Setting up the field mesh ------------------------- @@ -132,6 +136,8 @@ Setting up the field mesh * ``warpx.n_rz_azimuthal_modes`` (`integer`; 1 by default) When using the RZ version, this is the number of azimuthal modes. +.. _running-cpp-parameters-parallelization: + Distribution across MPI ranks and parallelization ------------------------------------------------- @@ -220,6 +226,8 @@ Distribution across MPI ranks and parallelization * ``warpx.safe_guard_cells`` (`0` or `1`) optional (default `0`) For developers: run in safe mode, exchanging more guard cells, and more often in the PIC loop (for debugging). +.. _running-cpp-parameters-parser: + Math parser and user-defined constants -------------------------------------- @@ -242,6 +250,8 @@ For example, parameters ``a0`` and ``z_plateau`` can be specified with: * ``my_constants.a0 = 3.0`` * ``my_constants.z_plateau = 150.e-6`` +.. _running-cpp-parameters-particle: + Particle initialization ----------------------- @@ -278,6 +288,13 @@ Particle initialization The mass of one `physical` particle of this species. If ``species_type`` is specified, the mass will be set to the physical value and ``mass`` is optional. +* ``.xmin,ymin,zmin`` (`float`) optional (default unlimited) + When ``.xmin`` and ``.xmax`` (see below) are set, they delimit the region within which particles are injected. + The same is applicable in the other directions. + If periodic boundary conditions are used in direction ``i``, then the default (i.e. if the range is not specified) range will be the simulation box, ``[geometry.prob_hi[i], geometry.prob_lo[i]]``. + +* ``.xmax,ymax,zmax`` (`float`) optional (default unlimited) + * ``.injection_style`` (`string`) Determines how the particles will be injected in the simulation. The options are: @@ -306,14 +323,15 @@ Particle initialization symmetrize the beam in the x and y directions). * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. - It requires the additional arguments: + With it users can specify the additional arguments: ``.injection_file`` (`string`) openPMD file name and ``.q_tot`` (`double`) optional (default is ``q_tot=0`` and no re-scaling is done, ``weight=q_p``) when specified it is used to re-scale the weight of externally loaded ``N`` physical particles, each of charge ``q_p``, to inject macroparticles of ``weight=.q_tot/q_p/N``. ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. The external file must include the species ``openPMD::Record``s labeled ``position`` and ``momentum`` (`double` arrays), with dimensionality and units set via ``openPMD::setUnitDimension`` and ``setUnitSI``. If the external file also contains ``openPMD::Records``s for ``mass`` and ``charge`` (constant `double` scalars) then the species will use these, unless overwritten in the input file (see ``.mass``, ```.charge`` or ```.species_type``). - The ``external_file`` option is currently implemented for 2D and 3D geometries, with record components ``x``, ``z`` and ``y`` for 3D. + The ``external_file`` option is currently implemented for 2D, 3D and RZ geometries, with record components in the cartesian coordinates ``(x,y,z)`` for 3D and RZ, and ``(x,z)`` for 2D. For more information on the `openPMD format `__ and how to build WarpX with it, please visit :doc:`../building/openpmd`. * ``.num_particles_per_cell_each_dim`` (`3 integers in 3D and RZ, 2 integers in 2D`) @@ -503,12 +521,12 @@ Particle initialization Whether or not to use OpenMP threading for particle initialization. * ``.do_field_ionization`` (`0` or `1`) optional (default `0`) - Do field ionization for this species (using the ADK theory). Currently, - this is slow on GPU. + Do field ionization for this species (using the ADK theory). * ``.physical_element`` (`string`) Only read if `do_field_ionization = 1`. Symbol of chemical element for this species. Example: for Helium, use ``physical_element = He``. + Elements up to atomic number Z=86 (Radon) are supported, let us know if you need higher Z. * ``.ionization_product_species`` (`string`) Only read if `do_field_ionization = 1`. Name of species in which ionized @@ -559,6 +577,7 @@ Particle initialization (the name of an existing positron species must be provided). **Implementation of this feature is in progress. It requires `picsar` on the `QED` branch and to compile with QED=TRUE** +.. _running-cpp-parameters-laser: Laser initialization -------------------- @@ -880,6 +899,8 @@ Laser initialization B-field to each particle which is then added to the field values gathered from the grid in the PIC cycle. +.. _running-cpp-parameters-collision: + Collision initialization ------------------------ @@ -908,6 +929,8 @@ following the algorithm given by `Perez et al. (Phys. Plasmas 19, 083104, 2012) If this is not provided, or if a non-positive value is provided, a Coulomb logarithm will be computed automatically according to the algorithm. +.. _running-cpp-parameters-numerics: + Numerics and algorithms ----------------------- @@ -1050,10 +1073,52 @@ Numerics and algorithms See `this section of the FFTW documentation `__ for more information. -* ``psatd.do_current_correction`` (`0` or `1`; default: `0`) - If true, the current correction defined by equation (19) of - `(Vay et al, JCP 243, 2013) `_ is applied. - Only used when compiled and running with the PSATD solver. +* ``psatd.current_correction`` (`0` or `1`; default: `0`) + If true, the current correction `(Vay et al, JCP 243, 2013) `_ + + .. math:: + \widetilde{\boldsymbol{J}}^{\,n+1/2}_{\mathrm{correct}} = \widetilde{\boldsymbol{J}}^{\,n+1/2} + -\bigg[\boldsymbol{k}\cdot\widetilde{\boldsymbol{J}}^{\,n+1/2} + -i\frac{\widetilde{\rho}^{n+1}-\widetilde{\rho}^{n}}{\Delta t}\bigg] + \frac{\boldsymbol{k}}{k^2} + + is applied. This option guarantees charge conservation only when used in combination + with ``psatd.periodic_single_box_fft=1``, that is, only for periodic single-box + simulations with global FFTs without guard cells. The implementation for domain + decomposition with local FFTs over guard cells is planned but not yet completed. + +* ``psatd.update_with_rho`` (`0` or `1`; default: `0`) + If false, the update equation for the electric field reads + + .. math:: + \begin{split} + \widetilde{\boldsymbol{E}}^{\,n+1}= & \: + C\widetilde{\boldsymbol{E}}^{\,n}+i\frac{S}{c\,k}\,c^2\,\boldsymbol{k} + \times\widetilde{\boldsymbol{B}}^{\,n}-\frac{1}{\epsilon_0}\,\frac{S}{c\,k}\, + \,\widetilde{\boldsymbol{J}}^{\,n+1/2} \\ + & +\frac{1-C}{k^2}\,(\boldsymbol{k}\cdot\widetilde{\boldsymbol{E}}^{\,n})\,\boldsymbol{k} + +\frac{1}{\epsilon_0}\,\frac{1}{k^2}\,\left(\frac{S}{c\,k}-\Delta t\right)\, + (\boldsymbol{k}\cdot\widetilde{\boldsymbol{J}}^{\,n+1/2})\,\boldsymbol{k} + \end{split} + + where :math:`C=\cos(k\,c\,\Delta t)` and :math:`S=\sin(k\,c\,\Delta t)`, respectively. + + If true, the update equation for the electric field reads instead + + .. math:: + \begin{split} + \widetilde{\boldsymbol{E}}^{\,n+1}= & \: + C\widetilde{\boldsymbol{E}}^{\,n}+i\frac{S}{c\,k}\,c^2\,\boldsymbol{k} + \times\widetilde{\boldsymbol{B}}^{\,n}-\frac{1}{\epsilon_0}\,\frac{S}{c\,k} + \,\widetilde{\boldsymbol{J}}^{\,n+1/2} \\ + & -i\frac{1}{\epsilon_0}\,\frac{1}{k^2}\,\bigg[ + \left(1-\frac{S}{c\,k}\frac{1}{\Delta t}\right)\widetilde{\rho}^{n+1} + -\left(C-\frac{S}{c\,k}\frac{1}{\Delta t}\right)\widetilde{\rho}^{n}\bigg] + \,\boldsymbol{k} + \end{split} + + See `(Vay et al, JCP 243, 2013) `_ + for more details about the derivation of these equations. * ``pstad.v_galilean`` (`3 floats`, in units of the speed of light; default `0. 0. 0.`) Defines the galilean velocity. @@ -1064,6 +1129,9 @@ Numerics and algorithms It also requires the use of the `direct` current deposition option `algo.current_deposition = direct` (does not work with Esirkepov algorithm). +* ``psatd.do_time_averaging`` (`0` or `1`; default: 0) + Whether to use an averaged Galilean PSATD algorithm or standard Galilean PSATD. + * ``warpx.override_sync_int`` (`string`) optional (default `1`) Using the `Intervals parser`_ syntax, this string defines the timesteps at which synchronization of sources (`rho` and `J`) on grid nodes at box boundaries is performed. @@ -1097,6 +1165,8 @@ Numerics and algorithms If ``sort_int`` is activated particles are sorted in bins of ``sort_bin_size`` cells. In 2D, only the first two elements are read. +.. _running-cpp-parameters-boundary: + Boundary conditions ------------------- @@ -1130,6 +1200,8 @@ Boundary conditions * ``warpx.do_pml_Hi`` (`2 floats in 2D`, `3 floats in 3D`; default: `1 1 1`) The directions along which one wants a pml boundary condition for upper boundaries on mother grid. +.. _running-cpp-parameters-diagnostics: + Diagnostics and output ---------------------- @@ -1232,7 +1304,7 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``.coarsening_ratio`` (list of `int`) optional (default `1 1 1`) Reduce size of the field output by this ratio in each dimension. (This is done by averaging the field over 1 or 2 points along each direction, depending on the staggering). - ``plot_coarsening_ratio`` should be an integer divisor of ``blocking_factor``. + ``plot_coarsening_ratio`` should be an integer divisor of ``blocking_factor``, defined in the :ref:`parallelization ` section. * ``.file_prefix`` (`string`) optional (default `diags/plotfiles/plt`) Root for output file names. Supports sub-directories. @@ -1254,10 +1326,6 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``ux`` ``uy`` ``uz`` for the particle momentum, - * ``Ex`` ``Ey`` ``Ez`` for the electric field on particles, - - * ``Bx`` ``By`` ``Bz`` for the magnetic field on particles. - The particle positions are always included. Use ``.variables = none`` to plot no particle data, except particle position. diff --git a/Docs/source/running_cpp/platforms.rst b/Docs/source/running_cpp/platforms.rst index c869b05c71a..1d709332545 100644 --- a/Docs/source/running_cpp/platforms.rst +++ b/Docs/source/running_cpp/platforms.rst @@ -1,6 +1,10 @@ +.. _running-cpp: + Running on specific platforms ============================= +.. _running-cpp-cori: + Running on Cori KNL at NERSC ---------------------------- @@ -33,12 +37,20 @@ regime), the following set of parameters provided good performance: * **2 grids per MPI**, *i.e.*, 16 grids per KNL node. + +.. _running-cpp-summit: + Running on Summit at OLCF ------------------------- +.. _running-cpp-summit-V100-GPUs: + +V100 GPUs +^^^^^^^^^ + The batch script below can be used to run a WarpX simulation on 2 nodes on the supercomputer Summit at OLCF. Replace descriptions between chevrons ``<>`` -by relevalt values, for instance ```` could be +by relevant values, for instance ```` could be ``plasma_mirror_inputs``. Note that the only option so far is to run with one MPI rank per GPU. @@ -67,3 +79,33 @@ regime), the following set of parameters provided good performance: A batch script with more options regarding profiling on Summit can be found at :download:`Summit batch script<../../../Tools/BatchScripts/script_profiling_summit.sh>` + +.. _running-cpp-summit-Power9-CPUs: + +Power9 CPUs +^^^^^^^^^^^ + +Similar to above, the batch script below can be used to run a WarpX simulation on +1 node on the supercomputer Summit at OLCF, on Power9 CPUs (i.e., the GPUs are +ignored). + +.. literalinclude:: ../../../Tools/BatchScripts/batch_summit_power9.sh + :language: bash + +For a 3D simulation with a few (1-4) particles per cell using FDTD Maxwell +solver on Summit for a well load-balanced problem, the following set of +parameters provided good performance: + +* ``amr.max_grid_size=64`` and ``amr.blocking_factor=64`` + +* **Two MPI ranks per node** (i.e. 2 resource sets per node; equivalently, 1 + resource set per socket) + +* **21 physical CPU cores per MPI rank** + +* **21 OpenMP threads per MPI rank** (i.e. 1 OpenMP thread per physical core) + +* **SMT 1 (Simultaneous Multithreading level 1)** + +* **Sixteen `64x64x64` grids per MPI rank** (with default tiling in WarpX, this + results in ~49 tiles per OpenMP thread) diff --git a/Docs/source/running_cpp/pwfa.rst b/Docs/source/running_cpp/pwfa.rst index 5c8f0af0593..c62c72585a8 100644 --- a/Docs/source/running_cpp/pwfa.rst +++ b/Docs/source/running_cpp/pwfa.rst @@ -64,7 +64,9 @@ Lorentz boosted frame The boosted frame still results in a substantial reduction to the simulation computational cost. .. note:: - Regardless of the frame that is chosen for the simulation, the input file parameters are defined in respect to the laboratory frame. + Even if the simulations uses the boosted frame, most of its input file parameters are defined in respect to the laboratory frame. + + We recommend that you design your numerical setup so that the width of the box is not significantly narrower than the distance from 0 to its right edge (done, for example, by setting the right edge equal to 0). Moving window diff --git a/Docs/source/visualization/advanced.rst b/Docs/source/visualization/advanced.rst index 95b5bfc3c7c..804002a4a84 100644 --- a/Docs/source/visualization/advanced.rst +++ b/Docs/source/visualization/advanced.rst @@ -17,7 +17,7 @@ Dump additional data -------------------- In order to dump additional data in WarpX (mostly for debugging purpose), run the simulation -with parameters +with parameters (``warpx.plot_rho`` only when using back-transformed diagnostics) :: diff --git a/Docs/source/visualization/ascent.rst b/Docs/source/visualization/ascent.rst index 801b0788ef8..5f38c57bf07 100644 --- a/Docs/source/visualization/ascent.rst +++ b/Docs/source/visualization/ascent.rst @@ -1,64 +1,46 @@ +.. _visualization-ascent: + In situ Visualization with Ascent ================================= -Ascent is a system designed to meet the in-situ visualization and analysis -needs of simulation code teams running multi-physics calculations on many-core -HPC architectures. It provides rendering runtimes that can leverage many-core -CPUs and GPUs to render images of simulation meshes. +Ascent is a system designed to meet the in-situ visualization and analysis needs of simulation code teams running multi-physics calculations on many-core HPC architectures. +It provides rendering runtimes that can leverage many-core CPUs and GPUs to render images of simulation meshes. Compiling with GNU Make ----------------------- -After building and installing Ascent according to the instructions at -`Building Ascent `_, -you can enable it in WarpX by changing the line +After building and installing Ascent according to the instructions at `Building Ascent `_, you can enable support for it in WarpX by changing the line .. code-block:: bash - USE_ASCENT_INSITU = FALSE + USE_ASCENT_INSITU=FALSE -in GNUmakefile to +in ``GNUmakefile`` to .. code-block:: bash - USE_ASCENT_INSITU = TRUE + USE_ASCENT_INSITU=TRUE -Furthermore, you must ensure that either the :code:`ASCENT_HOME` shell -environment variable contains the directory where Ascent is installed -or you must specify this location when invoking make, i.e., +Furthermore, you must ensure that either the :code:`ASCENT_DIR` shell environment variable contains the directory where Ascent is installed or you must specify this location when invoking make, i.e., .. code-block:: bash - make -j 8 ASCENT_HOME = /path/to/ascent/install + make -j 8 USE_ASCENT_INSITU=TRUE ASCENT_DIR=/path/to/ascent/install + + +Inputs File Configuration +------------------------- + +Once WarpX has been compiled with Ascent support, it will need to be enabled and configured at runtime. +This is done using our usual inputs file (read with ``amrex::ParmParse``). +The supported parameters are part of the :ref:`FullDiagnostics ` with ``.format`` parameter set to ``ascent``. -ParmParse Configuration ------------------------ -Once an AMReX code has been compiled with Ascent enabled, it will need -to be enabled and configured at runtime. This is done using ParmParse input file. -The supported parameters are described in the following table. - -+-------------------------+------------------------------------------------------+---------+ -| parameter | description | default | -+=========================+======================================================+=========+ -| :code:`insitu.int` | turns in situ processing on or off and controls how | 0 | -| | often data is processed. | | -+-------------------------+------------------------------------------------------+---------+ -| :code:`insitu.start` | controls when in situ processing starts. | 0 | -+-------------------------+------------------------------------------------------+---------+ - -A typical use case is setting :code:`insitu.int` to a value of one or greater and -:code:`insitu.start` to the first time step where in situ analysis should be -performed. Visualization/Analysis Pipeline Configuration --------------------------------------------- -Ascent uses the file :code:`ascent_actions.yaml` to configure analysis and -visualization pipelines. Ascent looks for the :code:`ascent_actions.yaml` file -in the current working directory. +Ascent uses the file :code:`ascent_actions.yaml` to configure analysis and visualization pipelines. +Ascent looks for the :code:`ascent_actions.yaml` file in the current working directory. -For example, the following :code:`ascent_actions.yaml` -file extracts an isosurface of the field Ex for 15 levels and saves the -resulting images to :code:`levels_.png`. `Ascent Actions -`_ provides an -overview over all available analysis and visualization actions. +For example, the following :code:`ascent_actions.yaml` file extracts an isosurface of the field ``Ex`` for ``15`` levels and saves the resulting images to :code:`levels_.png`. +`Ascent Actions `_ provides an overview over all available analysis and visualization actions. .. code-block:: json @@ -82,8 +64,7 @@ overview over all available analysis and visualization actions. pipeline: "p1" field: "Ex" -Here is another :code:`ascent_actions.yaml` example that renders isosurfaces -and particles: +Here is another :code:`ascent_actions.yaml` example that renders isosurfaces and particles: .. code-block:: json @@ -118,9 +99,7 @@ and particles: image_prefix: "out_render_3d_%06d" -Finally, here is a more complex :code:`ascent_actions.yaml` example that -creates the same images as the prior example, but adds a trigger that -creates a Cinema Database at cycle 300: +Finally, here is a more complex :code:`ascent_actions.yaml` example that creates the same images as the prior example, but adds a trigger that creates a Cinema Database at cycle ``300``: .. code-block:: json @@ -162,8 +141,7 @@ creates a Cinema Database at cycle 300: image_prefix: "out_render_jy_part_w_3d_%06d" -When the trigger condition is meet, `cycle() == 300`, the actions in -:code:`trigger.yaml` are also executed: +When the trigger condition is meet, ``cycle() == 300``, the actions in :code:`trigger.yaml` are also executed: .. code-block:: json @@ -197,5 +175,4 @@ When the trigger condition is meet, `cycle() == 300`, the actions in theta: 10 db_name: "cinema_out" -You can view the Cinema Database result by opening -:code:`cinema_databases/cinema_out/index.html`. +You can view the Cinema Database result by opening :code:`cinema_databases/cinema_out/index.html`. diff --git a/Docs/source/visualization/inputs.2d b/Docs/source/visualization/inputs.2d index f392ac3c9c2..d21d7c77a9a 100644 --- a/Docs/source/visualization/inputs.2d +++ b/Docs/source/visualization/inputs.2d @@ -9,7 +9,6 @@ amr.blocking_factor = 32 amr.max_level = 0 amr.plot_file = "plotfiles/plt" amr.plot_int = 0 -warpx.plot_rho = 1 insitu.int = 2 insitu.start = 0 insitu.config = ez2d.xml diff --git a/Docs/source/visualization/sensei.rst b/Docs/source/visualization/sensei.rst index 5b3d31cd9c1..86da3c4387a 100644 --- a/Docs/source/visualization/sensei.rst +++ b/Docs/source/visualization/sensei.rst @@ -174,7 +174,7 @@ file that ParaView can open will be generated. Obtaining SENSEI ----------------- SENSEI is hosted on Kitware's Gitlab site at https://gitlab.kitware.com/sensei/sensei -It's best to checkout the latest release rather than working on the master branch. +It's best to checkout the latest release rather than working on the ``develop`` branch. To ease the burden of wrangling back end installs SENSEI provides two platforms with all dependencies pre-installed, a VirtualBox VM, and a NERSC Cori @@ -252,7 +252,7 @@ First, log into cori and clone the git repo's. cd warpx/ git clone https://github.com/ECP-WarpX/WarpX.git WarpX-libsim git clone --branch development https://github.com/AMReX-Codes/amrex - git clone https://bitbucket.org/berkeleylab/picsar.git + git clone https://github.com/ECP-WarpX/picsar.git cd WarpX-libsim vim GNUmakefile @@ -303,7 +303,7 @@ First, log into cori and clone the git repo's. cd warpx/ git clone https://github.com/ECP-WarpX/WarpX.git WarpX-catalyst git clone --branch development https://github.com/AMReX-Codes/amrex - git clone https://bitbucket.org/berkeleylab/picsar.git + git clone https://github.com/ECP-WarpX/picsar.git cd WarpX-catalyst vim GNUmakefile diff --git a/Docs/source/visualization/visualization.rst b/Docs/source/visualization/visualization.rst index f0810053b5a..e2253822410 100644 --- a/Docs/source/visualization/visualization.rst +++ b/Docs/source/visualization/visualization.rst @@ -56,7 +56,7 @@ quantity staggering E n n B n n j n-1/2 n-1/2 -rho n-1 n +rho n n position n n momentum n-1/2 n-1/2 E part n n diff --git a/Docs/source/visualization/yt.rst b/Docs/source/visualization/yt.rst index cffd35773a3..b939b3b85ce 100644 --- a/Docs/source/visualization/yt.rst +++ b/Docs/source/visualization/yt.rst @@ -43,7 +43,7 @@ The list of field data and particle data stored can be seen with: For a quick start-up, the most useful commands for post-processing can be found in our Jupyter notebook -:download:`Visualization.ipynb<../../../Tools/Visualization.ipynb>` +:download:`Visualization.ipynb<../../../Tools/PostProcessing/Visualization.ipynb>` Field data ~~~~~~~~~~ diff --git a/Examples/Modules/dive_cleaning/analysis.py b/Examples/Modules/dive_cleaning/analysis.py new file mode 100755 index 00000000000..5353a0f6f35 --- /dev/null +++ b/Examples/Modules/dive_cleaning/analysis.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +# Copyright 2019-2020 Axel Huebl, Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script checks the dive cleaning and PML routines, by initializing a +Gaussian beam in the center of the simulation box, and let the error in +space-charge field propagate away and be absorbed in the PML. + +This script verifies that the field at the end of the simulation corresponds +to the theoretical field of a Gaussian beam. +""" +import sys +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt +import numpy as np +import scipy.constants as scc +from scipy.special import gammainc +yt.funcs.mylog.setLevel(0) + +# Parameters from the Simulation +Qtot = -1.e-20 +r0 = 2.e-6 + +# Open data file +filename = sys.argv[1] +ds = yt.load( filename ) +# Extract data +ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array = ad0['Ex'].to_ndarray().squeeze() +if ds.dimensionality == 2: + # Rename the z dimension as y, so as to make this script work for 2d and 3d + Ey_array = ad0['Ez'].to_ndarray().squeeze() + E_array = ( Ex_array**2 + Ey_array**2 )**.5 + relative_tolerance = 0.1 +elif ds.dimensionality == 3: + Ey_array = ad0['Ey'].to_ndarray() + Ez_array = ad0['Ez'].to_ndarray() + E_array = ( Ex_array**2 + Ey_array**2 + Ez_array**2 )**.5 + relative_tolerance = 0.15 + +# Extract grid coordinates +Nx, Ny, Nz = ds.domain_dimensions +xmin, ymin, zmin = ds.domain_left_edge.v +Lx, Ly, Lz = ds.domain_width.v +x = xmin + Lx/Nx*(0.5+np.arange(Nx)) +y = ymin + Ly/Ny*(0.5+np.arange(Ny)) +z = zmin + Lz/Nz*(0.5+np.arange(Nz)) + +# Compute theoretical field +if ds.dimensionality == 2: + x_2d, y_2d = np.meshgrid(x, y, indexing='ij') + r2 = x_2d**2 + y_2d**2 + factor = (Qtot/r0)/(2*np.pi*scc.epsilon_0*r2) * (1-np.exp(-r2/(2*r0**2))) + Ex_th = x_2d * factor + Ey_th = y_2d * factor + E_th = ( Ex_th**2 + Ey_th**2 )**.5 +elif ds.dimensionality == 3: + x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing='ij') + r2 = x_2d**2 + y_2d**2 + z_2d**2 + factor = Qtot/(4*np.pi*scc.epsilon_0*r2**1.5) * gammainc(3./2, r2/(2.*r0**2)) + Ex_th = factor*x_2d + Ey_th = factor*y_2d + Ez_th = factor*z_2d + E_th = ( Ex_th**2 + Ey_th**2 + Ez_th**2 )**.5 + +# Plot theory and data +def make_2d(arr): + if arr.ndim == 3: + return arr[:,:,Nz//2] + else: + return arr +plt.figure(figsize=(10,10)) +plt.subplot(221) +plt.title('E: Theory') +plt.imshow(make_2d(E_th)) +plt.colorbar() +plt.subplot(222) +plt.title('E: Simulation') +plt.imshow(make_2d(E_array)) +plt.colorbar() +plt.subplot(223) +plt.title('E: Diff') +plt.imshow(make_2d(E_th-E_array)) +plt.colorbar() +plt.subplot(224) +plt.title('E: Relative diff') +plt.imshow(make_2d((E_th-E_array)/E_th)) +plt.colorbar() +plt.savefig('Comparison.png') + +# Automatically check the results +def check(E, E_th, label): + print( 'Relative error in %s: %.3f'%( + label, abs(E-E_th).max()/E_th.max())) + assert np.allclose( E, E_th, atol=relative_tolerance*E_th.max() ) + +check( Ex_array, Ex_th, 'Ex' ) +check( Ey_array, Ey_th, 'Ey' ) +if ds.dimensionality == 3: + check( Ez_array, Ez_th, 'Ez' ) diff --git a/Examples/Modules/dive_cleaning/inputs_3d b/Examples/Modules/dive_cleaning/inputs_3d new file mode 100644 index 00000000000..97e7b4331ac --- /dev/null +++ b/Examples/Modules/dive_cleaning/inputs_3d @@ -0,0 +1,34 @@ +max_step = 128 +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +amr.max_level = 0 + +geometry.coord_sys = 0 # 0: Cartesian +geometry.is_periodic = 0 0 0 # Is periodic? +geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 50.e-6 + +warpx.do_dive_cleaning = 1 +warpx.do_pml = 1 + +particles.nspecies = 1 +particles.species_names = beam +beam.charge = -q_e +beam.mass = 1.e30 +beam.injection_style = "gaussian_beam" +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 0.0 + +diagnostics.diags_names = diag1 +diag1.period = 8 +diag1.diag_type = Full diff --git a/Examples/Modules/gaussian_beam/PICMI_inputs_gaussian_beam.py b/Examples/Modules/gaussian_beam/PICMI_inputs_gaussian_beam.py index 737c5e94e79..1e8851dd4df 100644 --- a/Examples/Modules/gaussian_beam/PICMI_inputs_gaussian_beam.py +++ b/Examples/Modules/gaussian_beam/PICMI_inputs_gaussian_beam.py @@ -53,7 +53,7 @@ part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', period = 10, species = [electrons, protons], - data_list = ['weighting', 'momentum', 'fields']) + data_list = ['weighting', 'momentum']) sim = picmi.Simulation(solver = solver, max_steps = 10, diff --git a/Examples/Physics_applications/laser_acceleration/inputs_2d_rz b/Examples/Physics_applications/laser_acceleration/inputs_2d_rz index 6045a38ad14..e8f8eee4dfb 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_2d_rz +++ b/Examples/Physics_applications/laser_acceleration/inputs_2d_rz @@ -85,3 +85,4 @@ laser1.wavelength = 0.8e-6 # The wavelength of the laser (in m) diagnostics.diags_names = diag1 diag1.period = 200 diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py index 7fad60f19d2..07662ce3a8d 100644 --- a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py +++ b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py @@ -61,7 +61,7 @@ part_diag = picmi.ParticleDiagnostic(name = 'diag1', period = max_steps, species = [beam, plasma], - data_list = ['ux', 'uy', 'uz', 'weighting', 'Ex', 'Ey', 'Ez']) + data_list = ['ux', 'uy', 'uz', 'weighting']) sim.add_diagnostic(field_diag) sim.add_diagnostic(part_diag) diff --git a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py index bb0377e1a86..665c78bc741 100644 --- a/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py +++ b/Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py @@ -67,7 +67,7 @@ part_diag = picmi.ParticleDiagnostic(name = 'diag1', period = 2, species = [beam, plasma], - data_list = ['ux', 'uy', 'uz', 'weighting', 'Ex', 'Ey', 'Ez']) + data_list = ['ux', 'uy', 'uz', 'weighting']) sim.add_diagnostic(field_diag) sim.add_diagnostic(part_diag) diff --git a/Examples/Physics_applications/uniform_plasma/inputs_3d b/Examples/Physics_applications/uniform_plasma/inputs_3d index 587e710f9b0..a8e3f14ec84 100644 --- a/Examples/Physics_applications/uniform_plasma/inputs_3d +++ b/Examples/Physics_applications/uniform_plasma/inputs_3d @@ -38,7 +38,7 @@ electrons.uz_th = 0.01 # uth the std of the (unitless) momentum diagnostics.diags_names = diag1 chk diag1.period = 4 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz Ey Ez w +diag1.electrons.variables = ux uy uz w chk.period = 6 chk.diag_type = Full diff --git a/Examples/Tests/Langmuir/PICMI_inputs_langmuir2d.py b/Examples/Tests/Langmuir/PICMI_inputs_langmuir2d.py index 9d3dcc1db27..3b67e990d8b 100644 --- a/Examples/Tests/Langmuir/PICMI_inputs_langmuir2d.py +++ b/Examples/Tests/Langmuir/PICMI_inputs_langmuir2d.py @@ -70,7 +70,7 @@ part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', period = diagnostic_interval, species = [electrons], - data_list = ['weighting', 'ux', 'Ex']) + data_list = ['weighting', 'ux']) ########################## # simulation setup diff --git a/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rt.py b/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rt.py index 8396f06a51c..e6fd321cd0e 100644 --- a/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rt.py +++ b/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rt.py @@ -72,7 +72,7 @@ part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', period = diagnostic_interval, species = [electrons], - data_list = ['weighting', 'ux', 'Ex']) + data_list = ['weighting', 'ux']) ########################## # simulation setup diff --git a/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py b/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py index 37aad730864..e820a87db26 100644 --- a/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py +++ b/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py @@ -105,7 +105,7 @@ part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', period = diagnostic_interval, species = [electrons], - data_list = ['weighting', 'momentum', 'fields']) + data_list = ['weighting', 'momentum']) ########################## # simulation setup diff --git a/Examples/Tests/Langmuir/analysis_langmuir_multi.py b/Examples/Tests/Langmuir/analysis_langmuir_multi.py index f06aa4877fd..66fb84fa412 100755 --- a/Examples/Tests/Langmuir/analysis_langmuir_multi.py +++ b/Examples/Tests/Langmuir/analysis_langmuir_multi.py @@ -29,7 +29,7 @@ # this will be the name of the plot file fn = sys.argv[1] -# Parse test name and check if current correction (psatd.do_current_correction=1) is applied +# Parse test name and check if current correction (psatd.current_correction=1) is applied current_correction = True if re.search( 'current_correction', fn ) else False # Parameters (these parameters must match the parameters in `inputs.multi.rt`) @@ -88,11 +88,8 @@ def get_theoretical_field( field, t ): print('assert that this is NOT in ds.field_list', (species, field)) assert (species, field) not in ds.field_list species = 'positrons' -for field in ['particle_Ey']: - print('assert that this is in ds.field_list', (species, field)) - assert (species, field) in ds.field_list -for field in ['particle_momentum_y', - 'particle_momentum_z']: +for field in ['particle_momentum_x', + 'particle_momentum_y']: print('assert that this is NOT in ds.field_list', (species, field)) assert (species, field) not in ds.field_list @@ -130,7 +127,7 @@ def get_theoretical_field( field, t ): assert( error_rel < tolerance_rel ) # Check relative L-infinity spatial norm of rho/epsilon_0 - div(E) when -# current correction (psatd.do_current_correction=1) is applied +# current correction (psatd.current_correction=1) is applied if current_correction: rho = data['rho' ].to_ndarray() divE = data['divE'].to_ndarray() diff --git a/Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py b/Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py index 801b6ab38ac..222f4f625be 100755 --- a/Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py +++ b/Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py @@ -29,7 +29,7 @@ # this will be the name of the plot file fn = sys.argv[1] -# Parse test name and check if current correction (psatd.do_current_correction=1) is applied +# Parse test name and check if current correction (psatd.current_correction=1) is applied current_correction = True if re.search( 'current_correction', fn ) else False # Parameters (these parameters must match the parameters in `inputs.multi.rt`) @@ -103,7 +103,7 @@ def get_theoretical_field( field, t ): assert( error_rel < tolerance_rel ) # Check relative L-infinity spatial norm of rho/epsilon_0 - div(E) when -# current correction (psatd.do_current_correction=1) is applied +# current correction (psatd.current_correction=1) is applied if current_correction: rho = data['rho' ].to_ndarray() divE = data['divE'].to_ndarray() diff --git a/Examples/Tests/Langmuir/inputs_3d_multi_rt b/Examples/Tests/Langmuir/inputs_3d_multi_rt index c9ddad495f9..708ee2a3e0a 100644 --- a/Examples/Tests/Langmuir/inputs_3d_multi_rt +++ b/Examples/Tests/Langmuir/inputs_3d_multi_rt @@ -88,4 +88,4 @@ diag1.period = 40 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho diag1.electrons.variables = w ux -diag1.positrons.variables = Ey +diag1.positrons.variables = uz diff --git a/Examples/Tests/averaged_galilean/analysis_avg_2d.py b/Examples/Tests/averaged_galilean/analysis_avg_2d.py new file mode 100755 index 00000000000..4dae6fcaee8 --- /dev/null +++ b/Examples/Tests/averaged_galilean/analysis_avg_2d.py @@ -0,0 +1,48 @@ +#! /usr/bin/env python +""" +This script tests the result of the averaged Galilean PSATD with large +timestep: dz/dx = 4. and c*dt = dz in WarpX. +It compares the energy of the electric field of the uniform plasma calculated +using the averaged Galilean PSATD versus standard Galilean PSATD +with v_galilean = (0.,0.,0.99498743710662): + + * if standard Galilean PSATD is used (psatd.do_time_averaging == 0'): + simulation is unstable because of the arosen NCI. + + * if averaged Galilean PSATD is used ('psatd.do_time_averaging == 1) : + NCI is suppresed => simulation is stable. + +Accepted relative tolerance: 1e-5 +""" +import sys +import yt ; yt.funcs.mylog.setLevel(0) +import numpy as np +import scipy.constants as scc +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +filename = sys.argv[1] + + +ds = yt.load( filename ) + +Ex= ds.index.grids[0]['boxlib', 'Ex'].squeeze().v +Ey= ds.index.grids[0]['boxlib', 'Ey'].squeeze().v +Ez= ds.index.grids[0]['boxlib', 'Ez'].squeeze().v + +#E field energy calculated with averaged Galilean PSATD method (v_galilean = (0,0,0.99498743710662)) +energyE_averaged_psatd = np.sum(scc.epsilon_0/2*(Ex**2+Ey**2+Ez**2)) + +#E field energy precalculated with standard Galilean PSATD (v_galilean = (0,0,0.99498743710662)) +energyE_galilean_psatd = 26913.546573259937 + +error_rel = energyE_averaged_psatd / energyE_galilean_psatd +tolerance_rel = 1e-5 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = filename[:-9] # Could also be os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/averaged_galilean/analysis_avg_3d.py b/Examples/Tests/averaged_galilean/analysis_avg_3d.py new file mode 100755 index 00000000000..8bed173c6e9 --- /dev/null +++ b/Examples/Tests/averaged_galilean/analysis_avg_3d.py @@ -0,0 +1,48 @@ +#! /usr/bin/env python +""" +This script tests the result of the averaged Galilean PSATD with large +timestep: dz/dx = 2. and c*dt = dz in WarpX. +It compares the energy of the electric field of the uniform plasma calculated +using the averaged Galilean PSATD versus standard Galilean PSATD +with v_galilean = (0.,0.,0.99498743710662): + + * if standard Galilean PSATD is used (psatd.do_time_averaging == 0'): + simulation is unstable because of the arosen NCI. + + * if averaged Galilean PSATD is used ('psatd.do_time_averaging == 1) : + NCI is suppresed => simulation is stable. + +Accepted relative tolerance: 1e-4 +""" +import sys +import yt ; yt.funcs.mylog.setLevel(0) +import numpy as np +import scipy.constants as scc +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +filename = sys.argv[1] + + +ds = yt.load( filename ) + +Ex= ds.index.grids[0]['boxlib', 'Ex'].squeeze().v +Ey= ds.index.grids[0]['boxlib', 'Ey'].squeeze().v +Ez= ds.index.grids[0]['boxlib', 'Ez'].squeeze().v + +#E field energy calculated with averaged Galilean PSATD method (v_galilean = (0,0,0.99498743710662)) +energyE_averaged_psatd = np.sum(scc.epsilon_0/2*(Ex**2+Ey**2+Ez**2)) + +#E field energy precalculated with standard Galilean PSATD (v_galilean = (0,0,0.99498743710662)) +energyE_galilean_psatd = 460315.9845556079 + +error_rel = energyE_averaged_psatd / energyE_galilean_psatd +tolerance_rel = 1e-4 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = filename[:-9] # Could also be os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/averaged_galilean/inputs_avg_2d b/Examples/Tests/averaged_galilean/inputs_avg_2d new file mode 100644 index 00000000000..d03e439634a --- /dev/null +++ b/Examples/Tests/averaged_galilean/inputs_avg_2d @@ -0,0 +1,83 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 400 + +amr.n_cell = 128 128 +amr.max_grid_size = 128 +amr.blocking_factor = 128 +amr.max_level = 0 +psatd.v_galilean = 0 0 0.99498743710662 + +psatd.do_time_averaging = 1 + +geometry.coord_sys = 0 +geometry.is_periodic = 1 1 + +geometry.prob_lo = -12.3776 -49.5104 +geometry.prob_hi = 12.3776 49.5104 + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 + +algo.maxwell_fdtd_solver = ckc +algo.current_deposition = direct +algo.particle_pusher = vay + +warpx.cfl = 4. +interpolation.nox = 3 +interpolation.noy = 3 +interpolation.noz = 3 + + +################################# +############ PLASMA ############# +################################# +particles.nspecies = 2 +particles.species_names = electrons ions + +warpx.do_nodal = 1 +warpx.use_filter = 1 + +psatd.nox = 16 +psatd.noy = 16 +psatd.noz = 16 + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 2 2 +electrons.profile = constant +electrons.density = 282197938148984.7 +electrons.momentum_distribution_type = "gaussian" +electrons.uz_m = 9.9498743710661994 +electrons.xmin = -12.3776 +electrons.xmax = 12.3776 +electrons.zmin = -49.5104 +electrons.zmax = 49.5104 +electrons.ux_th = 1e-4 +electrons.uy_th = 1e-4 +electrons.uz_th = 1e-4 + +ions.charge = q_e +ions.mass = m_p +ions.injection_style = "NUniformPerCell" +ions.num_particles_per_cell_each_dim = 2 2 +ions.profile = constant +ions.density = 282197938148984.7 +ions.momentum_distribution_type = "gaussian" +ions.uz_m = 9.9498743710661994 +ions.xmin = -12.3776 +ions.xmax = 12.3776 +ions.zmin = -49.5104 +ions.zmax = 49.5104 +ions.ux_th = 1e-4 +ions.uy_th = 1e-4 +ions.uz_th = 1e-4 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.period = 100 +diag1.diag_type = Full diff --git a/Examples/Tests/averaged_galilean/inputs_avg_3d b/Examples/Tests/averaged_galilean/inputs_avg_3d new file mode 100644 index 00000000000..914bfa7a25a --- /dev/null +++ b/Examples/Tests/averaged_galilean/inputs_avg_3d @@ -0,0 +1,85 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 400 + +amr.n_cell = 32 32 32 +amr.max_grid_size = 32 +amr.blocking_factor = 32 +amr.max_level = 0 +psatd.v_galilean = 0. 0. 0.99498743710662 +psatd.do_time_averaging = 1 + +geometry.coord_sys = 0 +geometry.is_periodic = 1 1 1 +geometry.prob_lo = -9.67 -9.67 -19.34 +geometry.prob_hi = 9.67 9.67 19.34 + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 + +algo.maxwell_fdtd_solver = ckc +algo.current_deposition = direct +algo.particle_pusher = vay + +warpx.cfl = 2. +interpolation.nox = 3 +interpolation.noy = 3 +interpolation.noz = 3 + + +################################# +############ PLASMA ############# +################################# +particles.nspecies = 2 +particles.species_names = electrons ions + +warpx.do_nodal = 1 +warpx.use_filter = 1 + +psatd.nox = 16 +psatd.noy = 16 +psatd.noz = 16 + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 1 1 1 +electrons.profile = constant +electrons.density = 282197938148984.7 +electrons.momentum_distribution_type = "gaussian" +electrons.uz_m = 9.9498743710661994 +electrons.xmin = -9.67 +electrons.xmax = 9.67 +electrons.ymin = -9.67 +electrons.ymax = 9.67 +electrons.zmin = -19.34 +electrons.zmax = 19.34 +electrons.ux_th = 0.0001 +electrons.uy_th = 0.0001 +electrons.uz_th = 0.0001 + +ions.charge = q_e +ions.mass = m_p +ions.injection_style = "NUniformPerCell" +ions.num_particles_per_cell_each_dim = 1 1 1 +ions.profile = constant +ions.density = 282197938148984.7 +ions.momentum_distribution_type = "gaussian" +ions.uz_m = 9.9498743710661994 +ions.xmin = -9.67 +ions.xmax = 9.67 +ions.ymin = -9.67 +ions.ymax = 9.67 +ions.zmin = -19.34 +ions.zmax = 19.34 +ions.ux_th = 0.0001 +ions.uy_th = 0.0001 +ions.uz_th = 0.0001 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.period = 100 +diag1.diag_type = Full diff --git a/Examples/Tests/initial_plasma_profile/analysis.py b/Examples/Tests/initial_plasma_profile/analysis.py new file mode 100755 index 00000000000..3ba67f9f5af --- /dev/null +++ b/Examples/Tests/initial_plasma_profile/analysis.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python + +# Copyright 2020 Michael Rowan +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import sys +import yt +yt.funcs.mylog.setLevel(50) + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Name of the plotfile +fn = sys.argv[1] + +test_name = fn[:-9] # Could also be os.path.split(os.getcwd())[1] + +checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-4, do_particles=False) diff --git a/Examples/Tests/initial_plasma_profile/inputs b/Examples/Tests/initial_plasma_profile/inputs new file mode 100644 index 00000000000..ec9640ea1a5 --- /dev/null +++ b/Examples/Tests/initial_plasma_profile/inputs @@ -0,0 +1,50 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 1 +amr.n_cell = 128 128 +amr.max_grid_size = 64 +amr.blocking_factor = 64 +amr.max_level = 0 +geometry.coord_sys = 0 +geometry.is_periodic = 0 0 +geometry.prob_lo = -0.00024190484157981564 -0.00016126989438654374 +geometry.prob_hi = 0.00024190484157981564 1.e-6 +warpx.do_pml = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 +warpx.cfl = 0.9999 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# +particles.nspecies = 1 +particles.species_names = electrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = NUniformPerCell +electrons.num_particles_per_cell_each_dim = 1 1 +electrons.momentum_distribution_type = "gaussian" +electrons.xmin = -150.e-6 +electrons.xmax = 150.e-6 +electrons.ymin = -150.e-6 +electrons.ymax = 150.e-6 +electrons.zmin = 0.0 +electrons.zmax = 0.32 +electrons.profile = "predefined" +electrons.predefined_profile_name = "parabolic_channel" +# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 +electrons.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 + +################################# +########## DIAGNOSTIC ########### +################################# +diagnostics.diags_names = diag1 +diag1.diag_type = Full +diag1.fields_to_plot = rho +diag1.period = 1 diff --git a/LICENSE.txt b/LICENSE.txt index 6387ddab9b4..6919ef67a97 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -WarpX v20.06 Copyright (c) 2018, The Regents of the University of California, through Lawrence Berkeley National Laboratory, and Lawrence Livermore National Security, LLC, for the operation of Lawrence Livermore National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. +WarpX v20.07 Copyright (c) 2018, The Regents of the University of California, through Lawrence Berkeley National Laboratory, and Lawrence Livermore National Security, LLC, for the operation of Lawrence Livermore National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Python/pywarpx/PGroup.py b/Python/pywarpx/PGroup.py index 7383c88ef13..ad6b4b87823 100644 --- a/Python/pywarpx/PGroup.py +++ b/Python/pywarpx/PGroup.py @@ -125,27 +125,27 @@ def getgaminv(self): gaminv = property(getgaminv) def getex(self): - return _libwarpx.get_particle_Ex(self.ispecie, self.level)[self.igroup] + raise Exception('Particle E fields not supported') ex = property(getex) def getey(self): - return _libwarpx.get_particle_Ey(self.ispecie, self.level)[self.igroup] + raise Exception('Particle E fields not supported') ey = property(getey) def getez(self): - return _libwarpx.get_particle_Ez(self.ispecie, self.level)[self.igroup] + raise Exception('Particle E fields not supported') ez = property(getez) def getbx(self): - return _libwarpx.get_particle_Bx(self.ispecie, self.level)[self.igroup] + raise Exception('Particle B fields not supported') bx = property(getbx) def getby(self): - return _libwarpx.get_particle_By(self.ispecie, self.level)[self.igroup] + raise Exception('Particle B fields not supported') by = property(getby) def getbz(self): - return _libwarpx.get_particle_Bz(self.ispecie, self.level)[self.igroup] + raise Exception('Particle B fields not supported') bz = property(getbz) def gettheta(self): diff --git a/Python/pywarpx/_libwarpx.py b/Python/pywarpx/_libwarpx.py index 7fb85e261b8..953de06984d 100755 --- a/Python/pywarpx/_libwarpx.py +++ b/Python/pywarpx/_libwarpx.py @@ -248,6 +248,8 @@ def initialize(argv=None): argv = sys.argv amrex_init(argv) libwarpx.warpx_ConvertLabParamsToBoost() + if geometry_dim == 'rz': + libwarpx.warpx_CheckGriddingForRZSpectral() libwarpx.warpx_init() @@ -600,72 +602,6 @@ def get_particle_uz(species_number, level=0): return get_particle_arrays(species_number, 3, level) -def get_particle_Ex(species_number, level=0): - ''' - - Return a list of numpy arrays containing the particle - x electric field on each tile. - - ''' - - return get_particle_arrays(species_number, 4, level) - - -def get_particle_Ey(species_number, level=0): - ''' - - Return a list of numpy arrays containing the particle - y electric field on each tile. - - ''' - - return get_particle_arrays(species_number, 5, level) - - -def get_particle_Ez(species_number, level=0): - ''' - - Return a list of numpy arrays containing the particle - z electric field on each tile. - - ''' - - return get_particle_arrays(species_number, 6, level) - - -def get_particle_Bx(species_number, level=0): - ''' - - Return a list of numpy arrays containing the particle - x magnetic field on each tile. - - ''' - - return get_particle_arrays(species_number, 7, level) - - -def get_particle_By(species_number, level=0): - ''' - - Return a list of numpy arrays containing the particle - y magnetic field on each tile. - - ''' - - return get_particle_arrays(species_number, 8, level) - - -def get_particle_Bz(species_number, level=0): - ''' - - Return a list of numpy arrays containing the particle - z magnetic field on each tile. - - ''' - - return get_particle_arrays(species_number, 9, level) - - def get_particle_theta(species_number, level=0): ''' @@ -675,7 +611,7 @@ def get_particle_theta(species_number, level=0): ''' if geometry_dim == 'rz': - return get_particle_arrays(species_number, 10, level) + return get_particle_arrays(species_number, 4, level) elif geometry_dim == '3d': return [np.arctan2(struct['y'], struct['x']) for struct in structs] elif geometry_dim == '2d': diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 6f4d7a4bab9..df700a1248c 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -453,7 +453,7 @@ def init(self, kw): if self.method == 'PSATD': self.periodic_single_box_fft = kw.pop('warpx_periodic_single_box_fft', None) self.fftw_plan_measure = kw.pop('warpx_fftw_plan_measure', None) - self.do_current_correction = kw.pop('warpx_do_current_correction', None) + self.current_correction = kw.pop('warpx_current_correction', None) def initialize_inputs(self): @@ -466,7 +466,7 @@ def initialize_inputs(self): if self.method == 'PSATD': pywarpx.psatd.periodic_single_box_fft = self.periodic_single_box_fft pywarpx.psatd.fftw_plan_measure = self.fftw_plan_measure - pywarpx.psatd.do_current_correction = self.do_current_correction + pywarpx.psatd.current_correction = self.current_correction if self.stencil_order is not None: pywarpx.psatd.nox = self.stencil_order[0] diff --git a/Python/setup.py b/Python/setup.py index 5b98d039567..993ba70d5c1 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -28,7 +28,7 @@ package_data = {} setup (name = 'pywarpx', - version = '20.06', + version = '20.07', packages = ['pywarpx'], package_dir = {'pywarpx':'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/README.md b/README.md index 1135cc2df3d..d8205818d53 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # WarpX -[![Code Status master](https://img.shields.io/travis/ECP-WarpX/WarpX/master.svg?label=master)](https://travis-ci.com/ECP-WarpX/WarpX/branches) +[![Code Status development](https://img.shields.io/travis/ECP-WarpX/WarpX/development.svg?label=development)](https://travis-ci.com/ECP-WarpX/WarpX/branches) [![Documentation Status](https://readthedocs.org/projects/warpx/badge/?version=latest)](https://warpx.readthedocs.io/en/latest/?badge=latest) -[![GitHub commits since last release](https://img.shields.io/github/commits-since/ECP-WarpX/WarpX/latest/master.svg)](https://github.com/ECP-WarpX/WarpX/compare/master) +[![GitHub commits since last release](https://img.shields.io/github/commits-since/ECP-WarpX/WarpX/latest/development.svg)](https://github.com/ECP-WarpX/WarpX/compare/development) [![Language](https://img.shields.io/badge/language-C%2B%2B14-orange.svg)](https://isocpp.org/) [![Development Status](https://img.shields.io/badge/development%20status-alpha-orange.svg)]() [![Gitter](https://badges.gitter.im/ECP-WarpX/community.svg)](https://gitter.im/ECP-WarpX/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/Regression/Checksum/benchmarks_json/Langmuir_2d.json b/Regression/Checksum/benchmarks_json/Langmuir_2d.json index 85feaf80100..ae115ffbeb8 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_2d.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_2d.json @@ -1,6 +1,5 @@ { "electrons": { - "particle_Ex": 22673325741654.633, "particle_cpu": 4096.0, "particle_id": 17567744.0, "particle_momentum_x": 9.644382989451675e-21, diff --git a/Regression/Checksum/benchmarks_json/Langmuir_2d_nompi.json b/Regression/Checksum/benchmarks_json/Langmuir_2d_nompi.json index 921c28ccbf6..0ce183d3460 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_2d_nompi.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_2d_nompi.json @@ -1,8 +1,7 @@ { "electrons": { - "particle_Ex": 22673325741654.633, "particle_cpu": 0.0, - "particle_id": 52170752.0, + "particle_id": 35409920.0, "particle_momentum_x": 9.644382989451675e-21, "particle_position_x": 0.08204530071915744, "particle_position_y": 0.08192, diff --git a/Regression/Checksum/benchmarks_json/Langmuir_2d_single_precision.json b/Regression/Checksum/benchmarks_json/Langmuir_2d_single_precision.json index 55010669682..8e08ca0ed1a 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_2d_single_precision.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_2d_single_precision.json @@ -1,6 +1,5 @@ { "electrons": { - "particle_Ex": 22675062646016.0, "particle_cpu": 4096.0, "particle_id": 17305600.0, "particle_momentum_x": 9.646555269782835e-21, diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi.json b/Regression/Checksum/benchmarks_json/Langmuir_multi.json index 3a0ef145655..61ae10535c7 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi.json @@ -2,31 +2,31 @@ "electrons": { "particle_cpu": 131072.0, "particle_id": 18862440448.0, - "particle_momentum_x": 9.638052142962567e-20, + "particle_momentum_x": 9.638052142962566e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, "particle_position_z": 2.6214400000000007, "particle_weight": 128000000000.00002 }, "lev=0": { - "Bx": 12.117994152440259, - "By": 12.117994153623144, - "Bz": 12.117994153640254, - "Ex": 84779179148604.14, + "Bx": 12.117994152442217, + "By": 12.117994153638133, + "Bz": 12.117994153639632, + "Ex": 84779179148604.16, "Ey": 84779179148604.05, "Ez": 84779179148604.05, - "jx": 6.087467475688615e+16, - "jy": 6.0874674756883144e+16, - "jz": 6.087467475688317e+16, + "jx": 6.087467475688619e+16, + "jy": 6.087467475688316e+16, + "jz": 6.087467475688315e+16, "part_per_cell": 524288.0, - "rho": 771603348.7307231 + "rho": 702984843.3445112 }, "positrons": { - "particle_Ey": 84768126738155.34, "particle_cpu": 131072.0, "particle_id": 56518901760.0, + "particle_momentum_z": 9.638052142962866e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, "particle_position_z": 2.6214400000000007 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_nodal.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_nodal.json index 346f5ed33de..d3927b2abef 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_nodal.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_nodal.json @@ -1,8 +1,5 @@ { "electrons": { - "particle_Ex": 14986451875836.121, - "particle_Ey": 0.0, - "particle_Ez": 14986451875836.125, "particle_cpu": 32768.0, "particle_id": 1123057664.0, "particle_momentum_x": 5.668407425899705e-20, @@ -24,9 +21,6 @@ "jz": 1.0088447956385268e+16 }, "positrons": { - "particle_Ex": 14986451875836.117, - "particle_Ey": 0.0, - "particle_Ez": 14986451875836.125, "particle_cpu": 32768.0, "particle_id": 3371204608.0, "particle_momentum_x": 5.668407425899705e-20, diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json index c3570d38078..fbe79a0f122 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json @@ -1,36 +1,30 @@ { "electrons": { - "particle_Ex": 15085626458339.264, - "particle_Ey": 0.0, - "particle_Ez": 15085626458339.277, "particle_cpu": 32768.0, "particle_id": 1123057664.0, - "particle_momentum_x": 5.665639562608442e-20, + "particle_momentum_x": 5.663705661977914e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.665639562608442e-20, - "particle_position_x": 0.6553599999999999, - "particle_position_y": 0.6553600000000002, + "particle_momentum_z": 5.663705661977915e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553599999999999, "particle_weight": 3200000000000000.5 }, "lev=0": { - "Ex": 3771857291767.906, + "Ex": 3771082811318.298, "Ey": 0.0, - "Ez": 3771857291767.909, - "jx": 1.0097075028724462e+16, + "Ez": 3771082811318.301, + "jx": 1.009365330460902e+16, "jy": 0.0, - "jz": 1.0097075028724462e+16, + "jz": 1.009365330460902e+16, "part_per_cell": 131072.0 }, "positrons": { - "particle_Ex": 15085626458339.295, - "particle_Ey": 0.0, - "particle_Ez": 15085626458339.305, "particle_cpu": 32768.0, "particle_id": 3371204608.0, - "particle_momentum_x": 5.665639562608429e-20, + "particle_momentum_x": 5.663705661977916e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.665639562608428e-20, - "particle_position_x": 0.6553599999999999, + "particle_momentum_z": 5.663705661977916e-20, + "particle_position_x": 0.65536, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 } diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction.json index 52b54c8a301..ca054578e1e 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction.json @@ -1,39 +1,33 @@ { "electrons": { - "particle_Ex": 15186050664939.146, - "particle_Ey": 0.0, - "particle_Ez": 15186050664939.232, "particle_cpu": 0.0, "particle_id": 2229305344.0, - "particle_momentum_x": 5.658197625002215e-20, + "particle_momentum_x": 5.658193607299875e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.658197625002213e-20, + "particle_momentum_z": 5.658193607299919e-20, "particle_position_x": 0.65536, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 }, "lev=0": { - "Ex": 3796965799842.237, + "Ex": 3797003259305.1904, "Ey": 0.0, - "Ez": 3796965799842.259, - "divE": 2.38328254840169e+18, - "jx": 1.0086768992100014e+16, + "Ez": 3797003259305.2344, + "divE": 2.383282496736726e+18, + "jx": 1.0086760816184212e+16, "jy": 0.0, - "jz": 1.008676899210008e+16, + "jz": 1.0086760816184312e+16, "part_per_cell": 131072.0, - "rho": 21102031.294517133 + "rho": 21102030.83706584 }, "positrons": { - "particle_Ex": 15186050664939.146, - "particle_Ey": 0.0, - "particle_Ez": 15186050664939.232, "particle_cpu": 0.0, "particle_id": 6725599232.0, - "particle_momentum_x": 5.658197625002216e-20, + "particle_momentum_x": 5.658193607299875e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.658197625002213e-20, + "particle_momentum_z": 5.658193607299919e-20, "particle_position_x": 0.65536, - "particle_position_y": 0.6553600000000002, + "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction_nodal.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction_nodal.json new file mode 100644 index 00000000000..fd0f6a5df1e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_current_correction_nodal.json @@ -0,0 +1,33 @@ +{ + "electrons": { + "particle_cpu": 0.0, + "particle_id": 2229305344.0, + "particle_momentum_x": 5.666933339402305e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.66693333940231e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.65536, + "particle_weight": 3200000000000000.5 + }, + "lev=0": { + "Ex": 3754823063509.187, + "Ey": 0.0, + "Ez": 3754823063509.1787, + "divE": 2.3596320673507973e+18, + "jx": 1.0088801837356758e+16, + "jy": 0.0, + "jz": 1.008880183735675e+16, + "part_per_cell": 131072.0, + "rho": 20892625.49342951 + }, + "positrons": { + "particle_cpu": 0.0, + "particle_id": 6725599232.0, + "particle_momentum_x": 5.666933339402305e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.66693333940231e-20, + "particle_position_x": 0.6553599999999999, + "particle_position_y": 0.65536, + "particle_weight": 3200000000000000.5 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_momentum_conserving.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_momentum_conserving.json index 662bb2366e6..888d269d49a 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_momentum_conserving.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_momentum_conserving.json @@ -1,36 +1,30 @@ { "electrons": { - "particle_Ex": 14818473436129.969, - "particle_Ey": 0.0, - "particle_Ez": 14818473436129.988, "particle_cpu": 32768.0, "particle_id": 1123057664.0, - "particle_momentum_x": 5.67711146356253e-20, + "particle_momentum_x": 5.67689426271667e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.677111463562529e-20, - "particle_position_x": 0.65536, + "particle_momentum_z": 5.67689426271667e-20, + "particle_position_x": 0.6553599999999999, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 }, "lev=0": { - "Ex": 3705061266665.8633, + "Ex": 3704881857285.2974, "Ey": 0.0, - "Ez": 3705061266665.8696, - "jx": 1.011414995243282e+16, + "Ez": 3704881857285.295, + "jx": 1.0113762708404172e+16, "jy": 0.0, - "jz": 1.011414995243282e+16, + "jz": 1.0113762708404174e+16, "part_per_cell": 131072.0 }, "positrons": { - "particle_Ex": 14818473436129.98, - "particle_Ey": 0.0, - "particle_Ez": 14818473436130.002, "particle_cpu": 32768.0, "particle_id": 3371204608.0, - "particle_momentum_x": 5.677111463562513e-20, + "particle_momentum_x": 5.676894262716672e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.677111463562512e-20, - "particle_position_x": 0.6553599999999999, + "particle_momentum_z": 5.676894262716671e-20, + "particle_position_x": 0.65536, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 } diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_nodal.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_nodal.json index 4c39d0e2cee..47fb58664ff 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_nodal.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_nodal.json @@ -1,37 +1,31 @@ { "electrons": { - "particle_Ex": 14987240735672.16, - "particle_Ey": 0.0, - "particle_Ez": 14987238943607.992, "particle_cpu": 32768.0, "particle_id": 1123057664.0, - "particle_momentum_x": 5.668629836343162e-20, + "particle_momentum_x": 5.668411736485855e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.668629454460509e-20, - "particle_position_x": 0.6553600000000004, + "particle_momentum_z": 5.668411698135936e-20, + "particle_position_x": 0.6553600000000067, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 }, "lev=0": { - "Ex": 3747258146540.8506, + "Ex": 3747069178797.4033, "Ey": 0.0, - "Ez": 3747257700467.7314, - "jx": 1.008884394883478e+16, + "Ez": 3747069276792.466, + "jx": 1.0088455906415716e+16, "jy": 0.0, - "jz": 1.0088843331184268e+16, + "jz": 1.0088455813733344e+16, "part_per_cell": 131072.0 }, "positrons": { - "particle_Ex": 14987240722150.396, - "particle_Ey": 0.0, - "particle_Ez": 14987238943288.441, "particle_cpu": 32768.0, "particle_id": 3371204608.0, - "particle_momentum_x": 5.668629830474806e-20, + "particle_momentum_x": 5.668411738799645e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.668629460626188e-20, - "particle_position_x": 0.6553599999999996, - "particle_position_y": 0.6553600000000002, + "particle_momentum_z": 5.668411697191805e-20, + "particle_position_x": 0.6553599999999925, + "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_nodal.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_nodal.json index 042d0c9fb9b..5f86c2cb260 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_nodal.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_nodal.json @@ -9,9 +9,9 @@ "particle_weight": 128000000000.00002 }, "lev=0": { - "Bx": 17.67689485265776, - "By": 17.676894852668948, - "Bz": 17.676894852669562, + "Bx": 17.67689485265927, + "By": 17.676894852670383, + "Bz": 17.67689485267166, "Ex": 86079763548288.75, "Ey": 86079763548288.78, "Ez": 86079763548288.78, @@ -19,14 +19,14 @@ "jy": 5.803381905407015e+16, "jz": 5.803381905407016e+16, "part_per_cell": 524288.0, - "rho": 786759127.7574509 + "rho": 720713352.6087718 }, "positrons": { - "particle_Ey": 86072707299548.95, "particle_cpu": 131072.0, "particle_id": 56518901760.0, + "particle_momentum_z": 9.320505028112306e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, "particle_position_z": 2.6214399999999998 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd.json index e660cb6eea6..ef1a83eeebb 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd.json @@ -2,31 +2,31 @@ "electrons": { "particle_cpu": 131072.0, "particle_id": 18862440448.0, - "particle_momentum_x": 9.630000480732885e-20, - "particle_position_x": 2.621440000012802, - "particle_position_y": 2.621440000012803, - "particle_position_z": 2.6214399999999993, + "particle_momentum_x": 9.638052096688673e-20, + "particle_position_x": 2.621440000001177, + "particle_position_y": 2.6214400000011775, + "particle_position_z": 2.6214399999999998, "particle_weight": 128000000000.00002 }, "lev=0": { - "Bx": 11.916179266713513, - "By": 11.91617926663706, - "Bz": 11.917607032204629, - "Ex": 84733730662264.23, - "Ey": 84733730662262.27, - "Ez": 84733724096318.2, - "jx": 6.0822131615524e+16, - "jy": 6.0822131615524344e+16, - "jz": 6.082209355386128e+16, + "Bx": 11.927039871438884, + "By": 11.92703987044596, + "Bz": 11.929384183262627, + "Ex": 84779189387324.25, + "Ey": 84779189387324.56, + "Ez": 84779185961806.78, + "jx": 6.087467490676277e+16, + "jy": 6.08746749067624e+16, + "jz": 6.087467421885045e+16, "part_per_cell": 524288.0, - "rho": 702660848.6316689 + "rho": 702985675.5592988 }, "positrons": { - "particle_Ey": 84722686943095.3, "particle_cpu": 131072.0, "particle_id": 56518901760.0, - "particle_position_x": 2.621440000012802, - "particle_position_y": 2.6214400000128038, - "particle_position_z": 2.6214399999999993 + "particle_momentum_z": 9.638051962153948e-20, + "particle_position_x": 2.621440000001177, + "particle_position_y": 2.621440000001178, + "particle_position_z": 2.6214399999999998 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction.json index cda05fce112..96141ff129b 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction.json @@ -2,30 +2,30 @@ "electrons": { "particle_cpu": 0.0, "particle_id": 37409652736.0, - "particle_momentum_x": 9.585443568545556e-20, + "particle_momentum_x": 9.585443568545559e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, "particle_position_z": 2.6214400000000007, "particle_weight": 128000000000.00002 }, "lev=0": { - "Bx": 11.867039048390716, - "By": 11.867039053074166, - "Bz": 11.867039053083186, - "Ex": 85001549508327.45, - "Ey": 85001549508324.98, - "Ez": 85001549508324.94, + "Bx": 11.867039048376148, + "By": 11.86703905304863, + "Bz": 11.867039053065115, + "Ex": 85001549508327.53, + "Ey": 85001549508324.97, + "Ez": 85001549508324.9, "divE": 7.97321190060971e+19, - "jx": 6.039975332058757e+16, - "jy": 6.0399753320588824e+16, - "jz": 6.039975332058885e+16, + "jx": 6.039975332058763e+16, + "jy": 6.039975332058887e+16, + "jz": 6.039975332058881e+16, "part_per_cell": 524288.0, - "rho": 705963156.392504 + "rho": 705963156.3925041 }, "positrons": { - "particle_Ey": 84990471098283.88, "particle_cpu": 0.0, "particle_id": 112722575360.0, + "particle_momentum_z": 9.585443568545593e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, "particle_position_z": 2.6214400000000007 diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction_nodal.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction_nodal.json new file mode 100644 index 00000000000..19891e7074d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_current_correction_nodal.json @@ -0,0 +1,33 @@ +{ + "electrons": { + "particle_cpu": 0.0, + "particle_id": 37409652736.0, + "particle_momentum_x": 9.427560635869364e-20, + "particle_position_x": 2.6214400000000015, + "particle_position_y": 2.621440000000001, + "particle_position_z": 2.6214399999999998, + "particle_weight": 128000000000.00002 + }, + "lev=0": { + "Bx": 17.49996248979552, + "By": 17.499962489807935, + "Bz": 17.499962489796665, + "Ex": 85654801346583.62, + "Ey": 85654801346583.34, + "Ez": 85654801346583.31, + "divE": 8.073511509609898e+19, + "jx": 5.898369391324192e+16, + "jy": 5.898369391324414e+16, + "jz": 5.898369391324419e+16, + "part_per_cell": 524288.0, + "rho": 714843872.1488855 + }, + "positrons": { + "particle_cpu": 0.0, + "particle_id": 112722575360.0, + "particle_momentum_z": 9.427560635869377e-20, + "particle_position_x": 2.6214400000000015, + "particle_position_y": 2.621440000000001, + "particle_position_z": 2.6214399999999998 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_momentum_conserving.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_momentum_conserving.json index dd5243ea20f..a84855a06f6 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_momentum_conserving.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_momentum_conserving.json @@ -2,31 +2,31 @@ "electrons": { "particle_cpu": 131072.0, "particle_id": 18862440448.0, - "particle_momentum_x": 9.313026326765913e-20, - "particle_position_x": 2.6214400000125964, - "particle_position_y": 2.621440000012595, + "particle_momentum_x": 9.320946076473149e-20, + "particle_position_x": 2.621440000001092, + "particle_position_y": 2.621440000001092, "particle_position_z": 2.62144, "particle_weight": 128000000000.00002 }, "lev=0": { - "Bx": 15.076849705026993, - "By": 15.076849706218196, - "Bz": 15.077115911503535, - "Ex": 86034772984177.27, - "Ey": 86034772984177.08, - "Ez": 86034760818989.12, - "jx": 5.854806321272413e+16, - "jy": 5.85480632127243e+16, - "jz": 5.854804280396043e+16, + "Bx": 15.08683235643132, + "By": 15.086832354582613, + "Bz": 15.08865435829179, + "Ex": 86081132569285.61, + "Ey": 86081132569285.25, + "Ez": 86081129790492.31, + "jx": 5.859963709146216e+16, + "jy": 5.859963709146176e+16, + "jz": 5.859963721500062e+16, "part_per_cell": 524288.0, - "rho": 720374346.6048291 + "rho": 720713532.3806039 }, "positrons": { - "particle_Ey": 86027721802307.64, "particle_cpu": 131072.0, "particle_id": 56518901760.0, - "particle_position_x": 2.6214400000125964, - "particle_position_y": 2.6214400000125946, + "particle_momentum_z": 9.320946065812051e-20, + "particle_position_x": 2.621440000001092, + "particle_position_y": 2.621440000001092, "particle_position_z": 2.62144 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_nodal.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_nodal.json index e394c66860e..8ca75b994ee 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_nodal.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_nodal.json @@ -2,31 +2,31 @@ "electrons": { "particle_cpu": 131072.0, "particle_id": 18862440448.0, - "particle_momentum_x": 9.336540089113062e-20, + "particle_momentum_x": 9.320515831688772e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, - "particle_position_z": 2.621439999999999, + "particle_position_z": 2.62144, "particle_weight": 128000000000.00002 }, "lev=0": { - "Bx": 17.35580092401342, - "By": 17.355800924024127, - "Bz": 17.35580092400305, - "Ex": 86173786000018.14, - "Ey": 86173786000018.19, - "Ez": 86173786000018.17, - "jx": 5.813715216795644e+16, - "jy": 5.813715216795643e+16, - "jz": 5.8137152167956424e+16, + "Bx": 17.31887367426522, + "By": 17.318873674232954, + "Bz": 17.31887367425445, + "Ex": 86079988606567.34, + "Ey": 86079988606567.34, + "Ez": 86079988606567.33, + "jx": 5.803387776443088e+16, + "jy": 5.8033877764430856e+16, + "jz": 5.803387776443085e+16, "part_per_cell": 524288.0, - "rho": 721399745.4594865 + "rho": 720717033.3443888 }, "positrons": { - "particle_Ey": 86166716262545.0, "particle_cpu": 131072.0, "particle_id": 56518901760.0, + "particle_momentum_z": 9.32051583168877e-20, "particle_position_x": 2.6214400000000015, "particle_position_y": 2.621440000000001, - "particle_position_z": 2.621439999999999 + "particle_position_z": 2.62144 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_single_precision.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_single_precision.json index a0909be22f6..5c8da1dda21 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_single_precision.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_single_precision.json @@ -2,31 +2,31 @@ "electrons": { "particle_cpu": 131072.0, "particle_id": 18270519296.0, - "particle_momentum_x": 9.630175835453285e-20, - "particle_position_x": 2.6214399486056834, - "particle_position_y": 2.6214399661659513, - "particle_position_z": 2.621439966153872, + "particle_momentum_x": 9.638190038567394e-20, + "particle_position_x": 2.621439945236091, + "particle_position_y": 2.6214399563523614, + "particle_position_z": 2.621439956304812, "particle_weight": 127999991808.0 }, "lev=0": { - "Bx": 16.283409055462556, - "By": 27.847420256523446, - "Bz": 27.860402087754352, - "Ex": 84734483985576.88, - "Ey": 84733344104538.75, - "Ez": 84733290264251.75, - "jx": 6.08243835275767e+16, - "jy": 6.082200882039661e+16, - "jz": 6.082200213864928e+16, + "Bx": 14.48676500327366, + "By": 25.497343310142455, + "Bz": 25.49186985377491, + "Ex": 84779692998503.0, + "Ey": 84778783600995.75, + "Ez": 84778763578015.0, + "jx": 6.087711623376877e+16, + "jy": 6.087565247419194e+16, + "jz": 6.087569917678931e+16, "part_per_cell": 524288.0, - "rho": 702644169.0683594 + "rho": 702976066.7402344 }, "positrons": { - "particle_Ey": 84722307724362.25, "particle_cpu": 131072.0, "particle_id": 54811295744.0, - "particle_position_x": 2.6214399497206955, - "particle_position_y": 2.6214399663009544, - "particle_position_z": 2.6214399657276033 + "particle_momentum_z": 9.638099798267982e-20, + "particle_position_x": 2.6214399441530816, + "particle_position_y": 2.621439956467526, + "particle_position_z": 2.6214399562612414 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz.json index e5e311f830c..b6477c2a0ae 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz.json @@ -1,41 +1,31 @@ { "electrons": { - "particle_Bx": 58.8416263852574, - "particle_By": 59.16597702799425, - "particle_Ex": 6600700367966.209, - "particle_Ey": 6543561338956.482, - "particle_Ez": 14386276215464.854, "particle_cpu": 29440.0, - "particle_id": 1006524160.0, - "particle_momentum_x": 3.3268808500279122e-21, - "particle_momentum_y": 3.2997701493966797e-21, - "particle_momentum_z": 7.085667539965204e-21, - "particle_position_x": 0.529000844539088, - "particle_position_y": 0.5888000000000001, - "particle_theta": 92464.67381042684, + "particle_id": 913434880.0, + "particle_momentum_x": 3.3220167752726847e-21, + "particle_momentum_y": 3.3049663196342287e-21, + "particle_momentum_z": 7.085667539965317e-21, + "particle_position_x": 0.5290008445390879, + "particle_position_y": 0.5888, + "particle_theta": 92506.29869360026, "particle_weight": 81147583679.15044 }, "ions": { - "particle_Bx": 59.15077093467293, - "particle_By": 58.83567443208132, - "particle_Ex": 6556437015881.182, - "particle_Ey": 6595024813707.55, - "particle_Ez": 14386446938764.07, "particle_cpu": 29440.0, - "particle_id": 3026343680.0, - "particle_momentum_x": 1.6904985179025105e-20, - "particle_momentum_y": 1.7000305052184998e-20, - "particle_momentum_z": 3.6847066227487116e-20, - "particle_position_x": 0.5290000037284652, + "particle_id": 2743896320.0, + "particle_momentum_x": 1.6915725245148433e-20, + "particle_momentum_y": 1.6994902822356307e-20, + "particle_momentum_z": 3.6847066227487176e-20, + "particle_position_x": 0.5290000037284651, "particle_position_y": 0.5888, - "particle_theta": 92151.2820341227, + "particle_theta": 92089.14050622113, "particle_weight": 81147583679.15044 }, "lev=0": { - "By": 11.642563221842089, - "Ex": 1290320281471.9382, - "Ez": 1798383207735.961, - "jx": 252864052657298.47, - "jz": 347280901333530.0 + "By": 11.642563221840676, + "Ex": 1290320281471.9312, + "Ez": 1798383207735.9521, + "jx": 252864052657309.56, + "jz": 347280901333537.25 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_single_precision.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_single_precision.json index 1644dfadcc9..ec4f303d6d5 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_single_precision.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_single_precision.json @@ -2,31 +2,31 @@ "electrons": { "particle_cpu": 131072.0, "particle_id": 18270519296.0, - "particle_momentum_x": 9.638160416482527e-20, - "particle_position_x": 2.6214399456764, - "particle_position_y": 2.6214399582924273, - "particle_position_z": 2.6214399582613623, + "particle_momentum_x": 9.638160826190717e-20, + "particle_position_x": 2.6214399456919466, + "particle_position_y": 2.6214399582432577, + "particle_position_z": 2.6214399583116688, "particle_weight": 127999991808.0 }, "lev=0": { - "Bx": 13.749213500551377, - "By": 23.70175167118765, - "Bz": 23.697557306031285, - "Ex": 84779697208316.75, - "Ey": 84779379998572.12, - "Ez": 84779381031286.25, - "jx": 6.087465586195686e+16, - "jy": 6.087430695259379e+16, - "jz": 6.087430637996634e+16, + "Bx": 13.746930533642427, + "By": 23.700993117762437, + "Bz": 23.69751037016036, + "Ex": 84779699984212.88, + "Ey": 84779380102544.12, + "Ez": 84779381453679.5, + "jx": 6.08746566247641e+16, + "jy": 6.087430753764646e+16, + "jz": 6.087430519585869e+16, "part_per_cell": 524288.0, - "rho": 771605195.3974609 + "rho": 702985454.7099609 }, "positrons": { - "particle_Ey": 84768333819952.0, "particle_cpu": 131072.0, "particle_id": 54811295744.0, - "particle_position_x": 2.6214399459832123, - "particle_position_y": 2.621439958047347, - "particle_position_z": 2.6214399581733687 + "particle_momentum_z": 9.638010595476651e-20, + "particle_position_x": 2.6214399459793185, + "particle_position_y": 2.621439958043453, + "particle_position_z": 2.6214399581214423 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Larmor.json b/Regression/Checksum/benchmarks_json/Larmor.json index 0bed8ec6ede..8c19e7b43d3 100644 --- a/Regression/Checksum/benchmarks_json/Larmor.json +++ b/Regression/Checksum/benchmarks_json/Larmor.json @@ -1,11 +1,5 @@ { "electron": { - "particle_Bx": 0.0, - "particle_By": 0.0007789471980429488, - "particle_Bz": 0.0, - "particle_Ex": 19669.903641764264, - "particle_Ey": 0.0, - "particle_Ez": 537.4023205327452, "particle_cpu": 0.0, "particle_id": 1.0, "particle_momentum_x": 1.2084126370460285e-22, @@ -44,12 +38,6 @@ "part_per_cell": 0.0 }, "positron": { - "particle_Bx": 0.0, - "particle_By": 0.0007789471980429488, - "particle_Bz": 0.0, - "particle_Ex": 19669.90364176427, - "particle_Ey": 0.0, - "particle_Ez": 537.4023205327467, "particle_cpu": 0.0, "particle_id": 2.0, "particle_momentum_x": 1.2084126370460285e-22, diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration.json b/Regression/Checksum/benchmarks_json/LaserAcceleration.json index 27dac4b0272..e73cb6a56df 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration.json @@ -1,30 +1,24 @@ { "electrons": { - "particle_Bx": 1823612.763420117, - "particle_By": 1594.036464819075, - "particle_Bz": 47860.82957589165, - "particle_Ex": 3400199636487.5806, - "particle_Ey": 726835961770741.9, - "particle_Ez": 34114413425319.66, "particle_cpu": 69212.0, - "particle_id": 10198484142.0, - "particle_momentum_x": 1.7927320949196398e-20, - "particle_momentum_y": 7.226299296705388e-20, - "particle_momentum_z": 4.23296743571345e-20, - "particle_position_x": 0.7139122548608255, - "particle_position_y": 0.7150340442762904, - "particle_position_z": 1.3175770608646191, + "particle_id": 2655287162.0, + "particle_momentum_x": 1.7927320949194885e-20, + "particle_momentum_y": 7.226299296704682e-20, + "particle_momentum_z": 4.2329674357125244e-20, + "particle_position_x": 0.7139122548608258, + "particle_position_y": 0.7150340442762911, + "particle_position_z": 1.317577060864619, "particle_weight": 12926557617.187498 }, "lev=0": { - "Bx": 5863857.810242278, - "By": 2411.5707428261194, - "Bz": 116028.94525718055, - "Ex": 6268732518324.82, - "Ey": 1670757913492924.8, - "Ez": 104345842781113.34, - "jx": 555905302485612.94, - "jy": 1596092839924513.0, - "jz": 1045523884130469.9 + "Bx": 5863857.810242274, + "By": 2411.5707428261294, + "Bz": 116028.94525718097, + "Ex": 6268732518322.813, + "Ey": 1670757913492923.0, + "Ez": 104345842781112.31, + "jx": 555905302485608.94, + "jy": 1596092839924503.8, + "jz": 1045523884130288.4 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index c0adf0f38d7..4f8386a9b4f 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,13 +1,7 @@ { "beam": { - "particle_Bx": 2.0543216636907036, - "particle_By": 14.900417906604332, - "particle_Bz": 0.22233327031927602, - "particle_Ex": 4517976547.436662, - "particle_Ey": 633564460.6160239, - "particle_Ez": 15405468837.772211, "particle_cpu": 0.0, - "particle_id": 35060500.0, + "particle_id": 500500.0, "particle_momentum_x": 4.3244537833987973e-19, "particle_momentum_y": 4.51585729236226e-19, "particle_momentum_z": 2.666058436389631e-18, @@ -16,15 +10,9 @@ "particle_weight": 62415090744.60765 }, "electrons": { - "particle_Bx": 530.8105746418198, - "particle_By": 0.00010417687909189867, - "particle_Bz": 35.72584920547117, - "particle_Ex": 23373.585397386178, - "particle_Ey": 164837019826.8382, - "particle_Ez": 750684.6954455632, "particle_cpu": 1140.0, - "particle_id": 43482810.0, - "particle_momentum_x": 4.8513146378506466e-27, + "particle_id": 1254690.0, + "particle_momentum_x": 4.851314637850439e-27, "particle_momentum_y": 2.198172635551489e-22, "particle_momentum_z": 3.0976485630303994e-18, "particle_position_x": 0.06840000000361957, @@ -32,30 +20,24 @@ "particle_weight": 6.28658221073865e+16 }, "ions": { - "particle_Bx": 530.8105522074252, - "particle_By": 0.00010417687513907945, - "particle_Bz": 35.72584997453756, - "particle_Ex": 23373.584547084207, - "particle_Ey": 164837012558.85843, - "particle_Ez": 750684.6439845883, "particle_cpu": 1140.0, - "particle_id": 43628730.0, - "particle_momentum_x": 3.360051969962725e-30, + "particle_id": 1390110.0, + "particle_momentum_x": 3.360051969537704e-30, "particle_momentum_y": 2.1981727064697293e-22, - "particle_momentum_z": 5.687755728408194e-15, + "particle_momentum_z": 5.687755728408193e-15, "particle_position_x": 0.0684, "particle_position_y": 0.0393001381332615, "particle_weight": 6.28658221073865e+16 }, "lev=0": { "Bx": 7187.973258845824, - "By": 16.40364440449975, - "Bz": 426.103118917313, - "Ex": 4521887802.403418, + "By": 16.40364440449974, + "Bz": 426.10311891731305, + "Ex": 4521887802.403417, "Ey": 1984359719957.4739, - "Ez": 6310696230.623612, + "Ez": 6310696230.623689, "jx": 5122915921.273047, "jy": 551785946484956.7, - "jz": 448986139469.4084 + "jz": 448986139469.4807 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationMR.json b/Regression/Checksum/benchmarks_json/LaserAccelerationMR.json index a19f5592301..6ead57ab9fc 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationMR.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationMR.json @@ -1,11 +1,5 @@ { "beam": { - "particle_Bx": 66.36298890249402, - "particle_By": 6824.872059894314, - "particle_Bz": 6.1501945107515, - "particle_Ex": 2030566419114.0784, - "particle_Ey": 21378773475.605545, - "particle_Ez": 1446901337250.283, "particle_cpu": 0.0, "particle_id": 214748364400.0, "particle_momentum_x": 4.3785337042075496e-20, @@ -16,17 +10,11 @@ "particle_weight": 12483018148921.525 }, "electrons": { - "particle_Bx": 8419020.374016589, - "particle_By": 1492.5583084263199, - "particle_Bz": 263998.6119992592, - "particle_Ex": 536859063153.5126, - "particle_Ey": 2485904866731919.0, - "particle_Ez": 2341935724590.825, "particle_cpu": 4788.0, - "particle_id": 87752154.0, - "particle_momentum_x": 4.275117840757358e-20, - "particle_momentum_y": 1.9580305087551085e-19, - "particle_momentum_z": 8.913655221012208e-20, + "particle_id": 13694226.0, + "particle_momentum_x": 4.2751178407573573e-20, + "particle_momentum_y": 1.9580305087551053e-19, + "particle_momentum_z": 8.913655221012199e-20, "particle_position_x": 0.047419299682949334, "particle_position_y": 0.08483137297620588, "particle_weight": 119232421874999.98 @@ -38,18 +26,18 @@ "Ex": 23005837885800.03, "Ey": 6508545692004002.0, "Ez": 32121364643700.242, - "jx": 1131265582036581.0, + "jx": 1131265582036586.2, "jy": 2.325136123562568e+18, - "jz": 6769514298165735.0 + "jz": 6769514298165733.0 }, "lev=1": { - "Bx": 53303255.155727945, + "Bx": 53303255.15572796, "By": 214325.15712710656, "Bz": 1321596.3739210798, "Ex": 48108501141109.54, "Ey": 1.4551006821042612e+16, "Ez": 64901292650351.06, - "jx": 48050235573831.984, + "jx": 48050235573831.96, "jy": 9.274838877361125e+18, "jz": 1.92627519812193e+16 } diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json index 6ab7bd43868..3acf8e9c486 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json @@ -1,12 +1,7 @@ { "beam": { - "particle_Bx": 1068.7120965886738, - "particle_By": 902.9886284146921, - "particle_Ex": 142251245678.64517, - "particle_Ey": 172725505795.35986, - "particle_Ez": 855052975051.5054, "particle_cpu": 0.0, - "particle_id": 6865850.0, + "particle_id": 5050.0, "particle_momentum_x": 4.3910959229136746e-20, "particle_momentum_y": 4.5449319558312995e-20, "particle_momentum_z": 1.3496674867194975e-17, @@ -16,30 +11,26 @@ "particle_weight": 6241509.074460764 }, "electrons": { - "particle_Bx": 33032.2917421148, - "particle_By": 179.35906104395352, - "particle_Ex": 60899865362.08995, - "particle_Ey": 9897446784083.63, - "particle_Ez": 205437656917.69284, "particle_cpu": 4128.0, - "particle_id": 147732176.0, - "particle_momentum_x": 3.2048286176059313e-24, - "particle_momentum_y": 6.250807154005758e-22, - "particle_momentum_z": 1.366956054907569e-23, + "particle_id": 10445216.0, + "particle_momentum_x": 3.2048286176059666e-24, + "particle_momentum_y": 6.250807154005757e-22, + "particle_momentum_z": 1.3669560549075696e-23, "particle_position_x": 0.04160250000177469, - "particle_position_y": 0.047891250703679784, + "particle_position_y": 0.04789125070367979, "particle_theta": 6754.424205218055, - "particle_weight": 813672305.5321578 + "particle_weight": 813672305.5321579 }, "lev=0": { + "rho": 38984833.03605626, "Bx": 130251.30343202608, - "By": 1245.0385469006405, - "Bz": 5405.222819087962, - "Ex": 207583479874.01596, + "By": 1245.03854690064, + "Bz": 5405.2228190879605, + "Ex": 207583479874.0158, "Ey": 46261581432526.1, "Ez": 556826342131.5586, - "jx": 1194737793719.8267, - "jy": 2.0581597632100973e+17, + "jx": 1194737793717.6147, + "jy": 2.058159763210097e+17, "jz": 1796942413288391.8 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost2d.json b/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost2d.json index 8bbbf1631cc..04864b2d671 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost2d.json +++ b/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost2d.json @@ -1,13 +1,7 @@ { "beam": { - "particle_Bx": 1.2421837369762971, - "particle_By": 112.61474112317613, - "particle_Bz": 0.15372744675077343, - "particle_Ex": 31266390602.63783, - "particle_Ey": 425776260.82294077, - "particle_Ez": 341412999657.737, "particle_cpu": 0.0, - "particle_id": 18780500.0, + "particle_id": 1500500.0, "particle_momentum_x": 4.275414307401172e-19, "particle_momentum_y": 4.17353489404236e-19, "particle_momentum_z": 2.7466540338095267e-17, @@ -16,12 +10,6 @@ "particle_weight": 3744905444676.4585 }, "driver": { - "particle_Bx": 0.17162902577074082, - "particle_By": 2599.284470679273, - "particle_Bz": 0.057532769068605646, - "particle_Ex": 383225305850.66095, - "particle_Ey": 56852018.90139305, - "particle_Ez": 3125011859579.5933, "particle_cpu": 0.0, "particle_id": 500500.0, "particle_momentum_x": 4.3244537833987973e-19, @@ -40,34 +28,22 @@ "Ez": 3013327887369.4033, "jx": 5164332131.333552, "jy": 4628824062.592653, - "jz": 102420439518871.45 + "jz": 102420439518871.47 }, "plasma_e": { - "particle_Bx": 0.0005857990454885307, - "particle_By": 1.404488613959929, - "particle_Bz": 0.00034742934066197536, - "particle_Ex": 406103675.3903916, - "particle_Ey": 201138.6662916025, - "particle_Ez": 204395172.69021654, "particle_cpu": 504.0, - "particle_id": 8147196.0, - "particle_momentum_x": 3.372310398663099e-24, + "particle_id": 236124.0, + "particle_momentum_x": 3.3723103986630875e-24, "particle_momentum_y": 1.0980790746199993e-27, "particle_momentum_z": 1.3694876444851278e-18, "particle_position_x": 0.01771875610610223, - "particle_position_y": 0.02062005682722234, + "particle_position_y": 0.020620056827222344, "particle_weight": 677858073779801.2 }, "plasma_p": { - "particle_Bx": 0.0005858021375000984, - "particle_By": 1.4044879028870387, - "particle_Bz": 0.0003474290686070161, - "particle_Ex": 406103585.3352183, - "particle_Ey": 201139.68518534108, - "particle_Ez": 204396090.73847008, "particle_cpu": 504.0, - "particle_id": 8806140.0, - "particle_momentum_x": 3.3723091945226086e-24, + "particle_id": 272412.0, + "particle_momentum_x": 3.372309194522596e-24, "particle_momentum_y": 1.0980892491861113e-27, "particle_momentum_z": 2.514586742183712e-15, "particle_position_x": 0.0177187499966745, diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d.json b/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d.json index 00420b4f3ff..9dc47fee6c6 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d.json +++ b/Regression/Checksum/benchmarks_json/PlasmaAccelerationBoost3d.json @@ -1,13 +1,7 @@ { "beam": { - "particle_Bx": 3227.5589727359957, - "particle_By": 3088.1685105461247, - "particle_Bz": 4.57828126790789, - "particle_Ex": 309125717582.3938, - "particle_Ey": 328821757616.5381, - "particle_Ez": 6694077359865.185, "particle_cpu": 0.0, - "particle_id": 550364500.0, + "particle_id": 1500500.0, "particle_momentum_x": 4.275414307401172e-19, "particle_momentum_y": 4.17353489404236e-19, "particle_momentum_z": 2.7466540338095267e-17, @@ -17,12 +11,6 @@ "particle_weight": 3120754537.230381 }, "driver": { - "particle_Bx": 11659.346120355556, - "particle_By": 11215.848986117628, - "particle_Bz": 0.3070321484989495, - "particle_Ex": 530762394556.9341, - "particle_Ey": 552097091951.6976, - "particle_Ez": 5610847367213.652, "particle_cpu": 0.0, "particle_id": 500500.0, "particle_momentum_x": 4.747251762692475e+21, @@ -30,18 +18,12 @@ "particle_momentum_z": 2.9967719884827463e+25, "particle_position_x": 0.001529110176834359, "particle_position_y": 0.0016033921624255842, - "particle_position_z": 0.37579166250443896, + "particle_position_z": 0.3757916625044391, "particle_weight": 6241509074.460762 }, "driverback": { - "particle_Bx": 11688.980573229921, - "particle_By": 11159.188912235142, - "particle_Bz": 0.2544570753786498, - "particle_Ex": 523831138849.1566, - "particle_Ey": 551237711480.2166, - "particle_Ez": 5610277438559.053, "particle_cpu": 0.0, - "particle_id": 551364500.0, + "particle_id": 2500500.0, "particle_momentum_x": 4.6831534563358655e+21, "particle_momentum_y": 5.084648482174756e+21, "particle_momentum_z": 2.999729460061163e+25, @@ -53,46 +35,34 @@ "lev=0": { "Bx": 44941.49413640603, "By": 44979.43825578673, - "Bz": 13.934939072805973, + "Bz": 13.934939072805953, "Ex": 1826216104868.1426, - "Ey": 1823330229384.4048, - "Ez": 15903140534253.646, + "Ey": 1823330229384.404, + "Ez": 15903140534253.645, "jx": 478635383386.41437, "jy": 287752998229.3329, "jz": 1808527844770048.0 }, "plasma_e": { - "particle_Bx": 0.4955999326255563, - "particle_By": 0.5039645530718819, - "particle_Bz": 0.0001248192975851117, - "particle_Ex": 134686587.21869737, - "particle_Ey": 131645153.54607514, - "particle_Ez": 122964023.06369597, "particle_cpu": 3600.0, - "particle_id": 1461156360.0, - "particle_momentum_x": 2.0860691311063294e-24, - "particle_momentum_y": 2.0163771091033843e-24, + "particle_id": 9721800.0, + "particle_momentum_x": 2.0860691311063257e-24, + "particle_momentum_y": 2.016377109103382e-24, "particle_momentum_z": 9.78204978521714e-18, "particle_position_x": 0.12656250418442266, "particle_position_y": 0.12656250385223128, "particle_position_z": 0.06789177847668806, - "particle_weight": 3869103021.3870597 + "particle_weight": 3869103021.387059 }, "plasma_p": { - "particle_Bx": 0.49560143627080033, - "particle_By": 0.5039657942943742, - "particle_Bz": 0.00012481965039337357, - "particle_Ex": 134687136.7309273, - "particle_Ey": 131645780.12750848, - "particle_Ez": 122964266.16215158, "particle_cpu": 3600.0, - "particle_id": 1970570760.0, - "particle_momentum_x": 2.0860707893867245e-24, - "particle_momentum_y": 2.016379117137926e-24, + "particle_id": 16201800.0, + "particle_momentum_x": 2.086070789386721e-24, + "particle_momentum_y": 2.016379117137924e-24, "particle_momentum_z": 1.7961333877559267e-14, "particle_position_x": 0.1265624999977211, "particle_position_y": 0.12656249999790198, "particle_position_z": 0.0678917784374789, - "particle_weight": 3869103021.3870597 + "particle_weight": 3869103021.387059 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/PlasmaAccelerationMR.json b/Regression/Checksum/benchmarks_json/PlasmaAccelerationMR.json index 145053393d3..17942a1bf7c 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaAccelerationMR.json +++ b/Regression/Checksum/benchmarks_json/PlasmaAccelerationMR.json @@ -1,11 +1,5 @@ { "beam": { - "particle_Bx": 32.31407180126963, - "particle_By": 6411.5062725212565, - "particle_Bz": 0.5571695868179536, - "particle_Ex": 1636006861658.8643, - "particle_Ey": 10062002545.305954, - "particle_Ez": 1935862949965.4546, "particle_cpu": 0.0, "particle_id": 2147483644000.0, "particle_momentum_x": 4.322181912633372e-19, @@ -16,14 +10,8 @@ "particle_weight": 12483018148921.53 }, "driver": { - "particle_Bx": 32.609326565006015, - "particle_By": 20031.368639484455, - "particle_Bz": 1.5411503609728023, - "particle_Ex": 5626282517515.15, - "particle_Ey": 9877108921.802748, - "particle_Ez": 18226789365193.43, "particle_cpu": 0.0, - "particle_id": 10236500.0, + "particle_id": 1500500.0, "particle_momentum_x": 4.2756896805319457e-19, "particle_momentum_y": 4.1735339783006983e-19, "particle_momentum_z": 5.473681318004841e-16, @@ -37,8 +25,8 @@ "Bz": 4.774206155364002, "Ex": 36269846569014.32, "Ey": 3698302984.4105253, - "Ez": 41138364604231.73, - "jx": 2251964952778817.0, + "Ez": 41138364604231.72, + "jx": 2251964952778816.0, "jy": 304840114298.38477, "jz": 4252877895382316.0 }, @@ -47,25 +35,19 @@ "By": 142098.79841193178, "Bz": 4.658510073379045, "Ex": 49368488395341.13, - "Ey": 2329098185.3141475, - "Ez": 52339572250983.49, - "jx": 3016726519391277.5, - "jy": 162992754789.0484, + "Ey": 2329098185.314148, + "Ez": 52339572250983.5, + "jx": 3016726519391277.0, + "jy": 162992754789.04834, "jz": 4308416589615174.5 }, "plasma_e": { - "particle_Bx": 7.054183342141237, - "particle_By": 69097.79655524061, - "particle_Bz": 2.404677024126321, - "particle_Ex": 16213699604075.434, - "particle_Ey": 2276549918.021788, - "particle_Ez": 17965021901547.03, "particle_cpu": 3600.0, - "particle_id": 41596488.0, + "particle_id": 6546600.0, "particle_momentum_x": 1.6052131138342657e-19, "particle_momentum_y": 7.604979610181104e-24, "particle_momentum_z": 1.6653769795689454e-19, - "particle_position_x": 0.1341092742250025, + "particle_position_x": 0.13410927422500246, "particle_position_y": 0.10349153092104842, "particle_weight": 823974609374999.9 } diff --git a/Regression/Checksum/benchmarks_json/PlasmaMirror.json b/Regression/Checksum/benchmarks_json/PlasmaMirror.json index bc59de5d04e..23168961500 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaMirror.json +++ b/Regression/Checksum/benchmarks_json/PlasmaMirror.json @@ -1,13 +1,7 @@ { "electrons": { - "particle_Bx": 0.0, - "particle_By": 104961246.92730346, - "particle_Bz": 0.0, - "particle_Ex": 4.195946890380097e+16, - "particle_Ey": 0.0, - "particle_Ez": 5.398038858575869e+16, "particle_cpu": 3809.0, - "particle_id": 115986415.0, + "particle_id": 22462447.0, "particle_momentum_x": 2.0003633196988347e-17, "particle_momentum_y": 0.0, "particle_momentum_z": 2.446760440657976e-17, @@ -16,14 +10,8 @@ "particle_weight": 2.904173860599889e+18 }, "ions": { - "particle_Bx": 0.0, - "particle_By": 108988922.02755898, - "particle_Bz": 0.0, - "particle_Ex": 3.9621727637401224e+16, - "particle_Ey": 0.0, - "particle_Ez": 5.350781700255595e+16, "particle_cpu": 3834.0, - "particle_id": 631174650.0, + "particle_id": 61902330.0, "particle_momentum_x": 2.805687685002513e-17, "particle_momentum_y": 0.0, "particle_momentum_z": 9.605283978686312e-17, diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir.json b/Regression/Checksum/benchmarks_json/Python_Langmuir.json index 16fa4c9ab16..16e593cccca 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir.json @@ -1,6 +1,5 @@ { "electrons": { - "particle_Ex": 2.380866721344282e+16, "particle_cpu": 524288.0, "particle_id": 447927025664.0, "particle_momentum_x": 1.8915228266922305e-17, diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json index bcb47cc19fe..0ebb255d144 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json @@ -1,6 +1,5 @@ { "electrons": { - "particle_Ex": 219449752302051.72, "particle_cpu": 4096.0, "particle_id": 17567744.0, "particle_momentum_x": 1.0449418710231497e-19, diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_rz_multimode.json b/Regression/Checksum/benchmarks_json/Python_Langmuir_rz_multimode.json index 4ce2699ec0e..8739247ef35 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir_rz_multimode.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir_rz_multimode.json @@ -1,27 +1,21 @@ { "electrons": { - "particle_Bx": 47.223888719248514, - "particle_By": 81.29252077224889, - "particle_Bz": 44.825833191000626, - "particle_Ex": 32416882559812.16, - "particle_Ey": 36617948944833.2, - "particle_Ez": 47879188522350.67, "particle_cpu": 353280.0, - "particle_id": 158538771840.0, - "particle_momentum_x": 2.1038307987216882e-20, - "particle_momentum_y": 2.3745155589395634e-20, - "particle_momentum_z": 3.102444510250924e-20, - "particle_position_x": 6.61250119096399, - "particle_position_y": 14.719999999999999, - "particle_theta": 1156106.5707136206, - "particle_weight": 81147583679.15045 + "particle_id": 143802756480.0, + "particle_momentum_x": 2.10383079872169e-20, + "particle_momentum_y": 2.3745155589395647e-20, + "particle_momentum_z": 3.102444510250923e-20, + "particle_position_x": 6.612501190963988, + "particle_position_y": 14.720000000000002, + "particle_theta": 1156106.570713621, + "particle_weight": 81147583679.15047 }, "lev=0": { - "By": 2.4960978462233174, - "Ex": 879037550423.2676, - "Ez": 1707660097260.009, - "jx": 210208695620051.66, - "jz": 407196128263304.3, + "By": 2.4960978462281567, + "Ex": 879037550423.2649, + "Ez": 1707660097260.0088, + "jx": 210208695620033.72, + "jz": 407196128263304.25, "part_per_cell": 1472000.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationMR.json b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationMR.json index c8fea0d0b51..d12e9749597 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationMR.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationMR.json @@ -1,17 +1,17 @@ { "electrons": { "particle_cpu": 695224.0, - "particle_id": 1478232160988.0, - "particle_momentum_x": 2.3675577751195297e-19, - "particle_momentum_y": 1.929061463191686e-21, - "particle_momentum_z": 1.8938350321846786e-20, - "particle_position_x": 7.006554882095785, - "particle_position_y": 7.006554897199219, - "particle_position_z": 4.600065455619852, + "particle_id": 285949191772.0, + "particle_momentum_x": 2.367557775119529e-19, + "particle_momentum_y": 1.9290614631916863e-21, + "particle_momentum_z": 1.8938350321846783e-20, + "particle_position_x": 7.006554882095786, + "particle_position_y": 7.006554897199218, + "particle_position_z": 4.600065455619854, "particle_weight": 21640883789.062504 }, "lev=0": { - "Bx": 2.05709414269199, + "Bx": 2.0570941426919918, "By": 2449255.0148639255, "Bz": 71239.03584863919, "Ex": 752773924385165.6, @@ -20,6 +20,6 @@ "jx": 1.928774119411483e+18, "jy": 69804337605269.28, "jz": 742686862695719.5, - "rho": 27689135301.61659 + "rho": 27910721385.36522 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration.json b/Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration.json index 1165cc1d3e6..d4b11368679 100644 --- a/Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_PlasmaAcceleration.json @@ -1,14 +1,11 @@ { "beam": { - "particle_Ex": 5756179858433.416, - "particle_Ey": 5756179858433.41, - "particle_Ez": 16964931026248.906, "particle_cpu": 0.0, - "particle_id": 302285376.0, + "particle_id": 1290816.0, "particle_momentum_x": 5.708397145757236e-20, - "particle_momentum_y": 5.708397145757228e-20, + "particle_momentum_y": 5.708397145757229e-20, "particle_momentum_z": 7.574215683313995e-19, - "particle_position_x": 0.009793902444696391, + "particle_position_x": 0.009793902444696388, "particle_position_y": 0.00979390244469639, "particle_position_z": 0.1047802881481609, "particle_weight": 7031249999.999999 @@ -17,17 +14,14 @@ "Ex": 7074528555636.087, "Ey": 7074528078305.145, "Ez": 19091561484766.926, - "jx": 92775315721769.94, + "jx": 92775315721769.97, "jy": 92775315721769.94, "jz": 1267816446833707.5, "part_per_cell": 590976.0 }, "plasma": { - "particle_Ex": 0.0, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 589824.0, - "particle_id": 514063630336.0, + "particle_id": 185196642304.0, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, diff --git a/Regression/Checksum/benchmarks_json/Python_PlasmaAccelerationMR.json b/Regression/Checksum/benchmarks_json/Python_PlasmaAccelerationMR.json index 89e3cbdb138..2fbfc33a6c4 100644 --- a/Regression/Checksum/benchmarks_json/Python_PlasmaAccelerationMR.json +++ b/Regression/Checksum/benchmarks_json/Python_PlasmaAccelerationMR.json @@ -1,8 +1,5 @@ { "beam": { - "particle_Ex": 107128787405.89337, - "particle_Ey": 107128787405.88715, - "particle_Ez": 118963821845577.23, "particle_cpu": 0.0, "particle_id": 46454366187008.0, "particle_momentum_x": 3.8119259477932654e-21, @@ -32,11 +29,8 @@ "part_per_cell": 21632.0 }, "plasma": { - "particle_Ex": 0.0, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 8388608.0, - "particle_id": 1231405243432960.0, + "particle_id": 1161246952652800.0, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, diff --git a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json index fc66d50774b..6b868e8cea7 100644 --- a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json +++ b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 0.0002284627371099781, - "particle_By": 0.00021467707180826862, - "particle_Bz": 0.0002102644327743607, - "particle_Ex": 16981277.687556587, - "particle_Ey": 17009505.007470798, - "particle_Ez": 16939221.994517952, "particle_cpu": 0.0, "particle_id": 536887296.0, "particle_momentum_x": 6.797039213991982e-20, @@ -29,12 +23,6 @@ "part_per_cell": 65536.0 }, "protons": { - "particle_Bx": 0.00022692446773522987, - "particle_By": 0.00021360016225691481, - "particle_Bz": 0.00021031904641947448, - "particle_Ex": 16527170.45904002, - "particle_Ey": 16538759.196916314, - "particle_Ez": 16456286.211242234, "particle_cpu": 0.0, "particle_id": 1610629120.0, "particle_momentum_x": 3.199547530708718e-21, diff --git a/Regression/Checksum/benchmarks_json/RigidInjection_lab.json b/Regression/Checksum/benchmarks_json/RigidInjection_lab.json index 7b9fe6d7a94..ee32c737ed1 100644 --- a/Regression/Checksum/benchmarks_json/RigidInjection_lab.json +++ b/Regression/Checksum/benchmarks_json/RigidInjection_lab.json @@ -1,11 +1,5 @@ { "beam": { - "particle_Bx": 2.6566894511996866e-05, - "particle_By": 0.0008575397894793725, - "particle_Bz": 3.651332298602814e-06, - "particle_Ex": 257689.1138980855, - "particle_Ey": 8160.730985703307, - "particle_Ez": 51350.93704570909, "particle_cpu": 0.0, "particle_id": 2001000.0, "particle_momentum_x": 4.2999340454016905e-17, diff --git a/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json b/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json index f1bcdcc2c33..330cbc10066 100644 --- a/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json +++ b/Regression/Checksum/benchmarks_json/TwoParticle_electrostatic.json @@ -1,11 +1,5 @@ { "electron1": { - "particle_Bx": 2.5036214403545607e-33, - "particle_By": 3.6007341350514865e-33, - "particle_Bz": 1.0868573384860435e-33, - "particle_Ex": 3.4123501299430056e-08, - "particle_Ey": 3.412350127746802e-08, - "particle_Ez": 3.4123501245407426e-08, "particle_cpu": 0.0, "particle_id": 1.0, "particle_momentum_x": 3.346088201352191e-29, @@ -17,12 +11,6 @@ "particle_weight": 1.0 }, "electron2": { - "particle_Bx": 1.96192602420799e-33, - "particle_By": 1.6876606788059644e-33, - "particle_Bz": 3.644858398013685e-33, - "particle_Ex": 3.4123501222575865e-08, - "particle_Ey": 3.412350127595063e-08, - "particle_Ez": 3.412350126765692e-08, "particle_cpu": 0.0, "particle_id": 2.0, "particle_momentum_x": 3.346088199424244e-29, diff --git a/Regression/Checksum/benchmarks_json/Uniform_2d.json b/Regression/Checksum/benchmarks_json/Uniform_2d.json index 0099c107f2c..caa7b033e39 100644 --- a/Regression/Checksum/benchmarks_json/Uniform_2d.json +++ b/Regression/Checksum/benchmarks_json/Uniform_2d.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 25700.400626747654, - "particle_By": 38491.27943906533, - "particle_Bz": 25938.261862056585, - "particle_Ex": 44859167061465.125, - "particle_Ey": 6794395639950.69, - "particle_Ez": 44201725613366.95, "particle_cpu": 32768.0, "particle_id": 1123057664.0, "particle_momentum_x": 1.351015959339252e-19, diff --git a/Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd.json b/Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd.json new file mode 100644 index 00000000000..2b5b09496fa --- /dev/null +++ b/Regression/Checksum/benchmarks_json/averaged_galilean_2d_psatd.json @@ -0,0 +1,33 @@ +{ + "electrons": { + "particle_cpu": 0.0, + "particle_id": 2147516416.0, + "particle_momentum_x": 7.517481917937245e-21, + "particle_momentum_y": 3.896955095181759e-21, + "particle_momentum_z": 1.7807680056139682e-16, + "particle_position_x": 405588.5826369648, + "particle_position_y": 20127109.08237198, + "particle_weight": 6.917460794691972e+17 + }, + "ions": { + "particle_cpu": 0.0, + "particle_id": 6442483712.0, + "particle_momentum_x": 2.6093555711476015e-18, + "particle_momentum_y": 2.6179624992626143e-18, + "particle_momentum_z": 3.2697610954306833e-13, + "particle_position_x": 405588.43858521566, + "particle_position_y": 20127109.123472013, + "particle_weight": 6.917460794691972e+17 + }, + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } + } diff --git a/Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd.json b/Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd.json new file mode 100644 index 00000000000..b4e672f3739 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/averaged_galilean_3d_psatd.json @@ -0,0 +1,35 @@ +{ + "electrons": { + "particle_cpu": 0.0, + "particle_id": 536887296.0, + "particle_momentum_x": 5.299115645186443e-20, + "particle_momentum_y": 5.0895892013058854e-20, + "particle_momentum_z": 8.904212187695088e-17, + "particle_position_x": 158433.3678817281, + "particle_position_y": 158432.16553634303, + "particle_position_z": 15724303.92031937, + "particle_weight": 4.082754265421834e+18 + }, + "ions": { + "particle_cpu": 0.0, + "particle_id": 1610629120.0, + "particle_momentum_x": 1.31503018697119e-18, + "particle_momentum_y": 1.3124347226262318e-18, + "particle_momentum_z": 1.6348803463769262e-13, + "particle_position_x": 158433.36794743972, + "particle_position_y": 158432.13055468648, + "particle_position_z": 15724303.986768533, + "particle_weight": 4.082754265421834e+18 + }, + "lev=0": { + "Bx": 0.08500956549081025, + "By": 0.08720794527319246, + "Bz": 0.8241774802143333, + "Ex": 280427664.90059906, + "Ey": 268497758.3768232, + "Ez": 1054300.297795306, + "jx": 72051.25977792213, + "jy": 69843.47286978895, + "jz": 21538.610669061116 + } + } diff --git a/Regression/Checksum/benchmarks_json/bilinear_filter.json b/Regression/Checksum/benchmarks_json/bilinear_filter.json index 581fbd252e5..409e63bb2e8 100644 --- a/Regression/Checksum/benchmarks_json/bilinear_filter.json +++ b/Regression/Checksum/benchmarks_json/bilinear_filter.json @@ -1,11 +1,5 @@ { "electron": { - "particle_Bx": 0.0, - "particle_By": 1.9043507547185993e-19, - "particle_Bz": 0.0, - "particle_Ex": 1.235067153470628e-09, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 0.0, "particle_id": 1.0, "particle_momentum_x": 0.027309245307378237, diff --git a/Regression/Checksum/benchmarks_json/dive_cleaning_2d.json b/Regression/Checksum/benchmarks_json/dive_cleaning_2d.json new file mode 100644 index 00000000000..1a18ebe20ff --- /dev/null +++ b/Regression/Checksum/benchmarks_json/dive_cleaning_2d.json @@ -0,0 +1,29 @@ +{ + "beam": { + "particle_Bx": 0.0, + "particle_By": 1.4229748768905527e-19, + "particle_Bz": 0.0, + "particle_Ex": 210305.84591470752, + "particle_Ey": 0.0, + "particle_Ez": 210741.1714121227, + "particle_cpu": 0.0, + "particle_id": 200010000.0, + "particle_momentum_x": 1.111019933689776e-26, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.113270980745195e-26, + "particle_position_x": 0.03183627816789909, + "particle_position_y": 0.03171922054171794, + "particle_weight": 31207.545372303823 + }, + "lev=0": { + "Bx": 0.0, + "By": 1.5468538800972258e-20, + "Bz": 0.0, + "Ex": 8533.638650013556, + "Ey": 0.0, + "Ez": 8534.98921988922, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/dive_cleaning_3d.json b/Regression/Checksum/benchmarks_json/dive_cleaning_3d.json new file mode 100644 index 00000000000..a072f614b30 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/dive_cleaning_3d.json @@ -0,0 +1,30 @@ +{ + "beam": { + "particle_Bx": 1.6547943661629038e-20, + "particle_By": 1.7166945226626064e-20, + "particle_Bz": 1.851357799836734e-20, + "particle_Ex": 39363.91309372786, + "particle_Ey": 39331.17437154593, + "particle_Ez": 39396.18209787599, + "particle_cpu": 0.0, + "particle_id": 200010000.0, + "particle_momentum_x": 1.700384472207379e-27, + "particle_momentum_y": 1.69889110346099e-27, + "particle_momentum_z": 1.7017928140329036e-27, + "particle_position_x": 0.031880969779242374, + "particle_position_y": 0.03175704658379704, + "particle_position_z": 0.03183674192208247, + "particle_weight": 0.06241509074460764 + }, + "lev=0": { + "Bx": 2.2090009624207165e-20, + "By": 2.2307246822783936e-20, + "Bz": 2.1967888687392684e-20, + "Ex": 8888.956516621029, + "Ey": 8838.45337149105, + "Ez": 8837.421045658291, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd.json index e00b5a517c1..444a33d8761 100644 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd.json +++ b/Regression/Checksum/benchmarks_json/galilean_2d_psatd.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 0.0, - "particle_By": 0.009299012937832037, - "particle_Bz": 0.0, - "particle_Ex": 2816198.341358315, - "particle_Ey": 0.0, - "particle_Ez": 226824.31985528197, "particle_cpu": 0.0, "particle_id": 2160099328.0, "particle_momentum_x": 1.5318380864485464e-21, @@ -16,12 +10,6 @@ "particle_weight": 1.6888332018290936e+18 }, "ions": { - "particle_Bx": 0.0, - "particle_By": 0.00929911174906337, - "particle_Bz": 0.0, - "particle_Ex": 2816228.27643041, - "particle_Ey": 0.0, - "particle_Ez": 226827.1246175955, "particle_cpu": 0.0, "particle_id": 6522175488.0, "particle_momentum_x": 2.6093881778048826e-18, diff --git a/Regression/Checksum/benchmarks_json/galilean_3d_psatd.json b/Regression/Checksum/benchmarks_json/galilean_3d_psatd.json index d44296e2417..d422efd3070 100644 --- a/Regression/Checksum/benchmarks_json/galilean_3d_psatd.json +++ b/Regression/Checksum/benchmarks_json/galilean_3d_psatd.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 8.686865792841713e-05, - "particle_By": 0.0002939590693725135, - "particle_Bz": 3.452350267327678e-05, - "particle_Ex": 88208.54084575009, - "particle_Ey": 28448.58165631292, - "particle_Ez": 7238.699716136691, "particle_cpu": 0.0, "particle_id": 524800.0, "particle_momentum_x": 2.7514767750837127e-23, @@ -16,12 +10,6 @@ "particle_weight": 1.0555207511431835e+17 }, "ions": { - "particle_Bx": 8.68715702754361e-05, - "particle_By": 0.0002939977457230219, - "particle_Bz": 3.452290180102611e-05, - "particle_Ex": 88220.42522764044, - "particle_Ey": 28448.864182536563, - "particle_Ez": 7238.973235144771, "particle_cpu": 0.0, "particle_id": 1573376.0, "particle_momentum_x": 4.033320884535886e-20, diff --git a/Regression/Checksum/benchmarks_json/initial_distribution.json b/Regression/Checksum/benchmarks_json/initial_distribution.json index 246d3804197..96740820320 100644 --- a/Regression/Checksum/benchmarks_json/initial_distribution.json +++ b/Regression/Checksum/benchmarks_json/initial_distribution.json @@ -1,11 +1,5 @@ { "beam": { - "particle_Bx": 1.7649330608090273e-13, - "particle_By": 1.6201399701460411e-13, - "particle_Bz": 1.66810333771523e-13, - "particle_Ex": 788306.6243597023, - "particle_Ey": 807373.0427832225, - "particle_Ez": 849117.4848964723, "particle_cpu": 0.0, "particle_id": 869690834940.0, "particle_momentum_x": 1.0643551806277579e-16, @@ -17,12 +11,6 @@ "particle_weight": 0.059552651788017566 }, "gaussian": { - "particle_Bx": 1.7020260238260468e-13, - "particle_By": 1.7933856122499433e-13, - "particle_Bz": 1.709281128949862e-13, - "particle_Ex": 849360.3198012591, - "particle_Ey": 875082.5083100101, - "particle_Ez": 878926.7763904883, "particle_cpu": 0.0, "particle_id": 131072256000.0, "particle_momentum_x": 1.1148694854418848e-18, @@ -45,12 +33,6 @@ "jz": 135925010961.82391 }, "maxwell_boltzmann": { - "particle_Bx": 1.7042273054862176e-13, - "particle_By": 1.7958882714803126e-13, - "particle_Bz": 1.7106339244943236e-13, - "particle_Ex": 849630.7492019709, - "particle_Ey": 874259.2333479257, - "particle_Ez": 879811.2999356263, "particle_cpu": 0.0, "particle_id": 393216256000.0, "particle_momentum_x": 1.1174920939750226e-18, @@ -62,12 +44,6 @@ "particle_weight": 8e+21 }, "maxwell_juttner": { - "particle_Bx": 1.7022872090850356e-13, - "particle_By": 1.7950700813504624e-13, - "particle_Bz": 1.709136282800873e-13, - "particle_Ex": 849523.6100404036, - "particle_Ey": 875528.2217687981, - "particle_Ez": 879351.2082461334, "particle_cpu": 0.0, "particle_id": 655360256000.0, "particle_momentum_x": 2.2132580272612802e-16, diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index f5f59ae4663..869e901d475 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,35 +1,23 @@ { "electrons": { - "particle_Bx": 0.0, - "particle_By": 22263824.862206914, - "particle_Bz": 0.0, - "particle_Ex": 9291877034473320.0, - "particle_Ey": 0.0, - "particle_Ez": 5319.164340861062, - "particle_cpu": 20.0, - "particle_id": 4332886865.0, - "particle_momentum_x": 2.0857482219894723e-17, + "particle_cpu": 27.0, + "particle_id": 1309618160.0, + "particle_momentum_x": 2.1096313075249046e-17, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.762015026993497e-17, - "particle_position_x": 0.11036837993083386, - "particle_position_y": 1.7283871495895655, - "particle_weight": 3.071068957031251e-09 + "particle_momentum_z": 1.7264669463007245e-17, + "particle_position_x": 0.11048873689229222, + "particle_position_y": 1.7098255333343246, + "particle_weight": 3.0709523304687515e-09 }, "ions": { - "particle_Bx": 0.0, - "particle_By": 659186.9143798536, - "particle_Bz": 0.0, - "particle_Ex": 386286390605633.4, - "particle_Ey": 0.0, - "particle_Ez": 1130.542128053148, "particle_cpu": 0.0, - "particle_id": 587622528.0, - "particle_ionization_level": 52665.0, - "particle_momentum_x": 3.441120480256377e-18, + "particle_id": 89154368.0, + "particle_ionization_level": 52663.0, + "particle_momentum_x": 3.631023150973392e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.0432995298680571e-13, - "particle_position_x": 0.02143988984627425, - "particle_position_y": 0.4726766035525105, + "particle_momentum_z": 1.043299530138721e-13, + "particle_position_x": 0.02143992896238411, + "particle_position_y": 0.47267660340847956, "particle_weight": 5.000947000000002e-10 }, "lev=0": { @@ -38,9 +26,9 @@ "Bz": 0.0, "Ex": 5472946208912828.0, "Ey": 0.0, - "Ez": 829.4153550071132, + "Ez": 829.4153550071181, "jx": 12255073032842.39, "jy": 0.0, - "jz": 83416.00000110811 + "jz": 83416.00000112198 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 5f0a437fd05..768a3f5b079 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -1,35 +1,23 @@ { "electrons": { - "particle_Bx": 0.0, - "particle_By": 160958142.7106404, - "particle_Bz": 0.0, - "particle_Ex": 4.836891522729804e+16, - "particle_Ey": 0.0, - "particle_Ez": 6681.416407321183, - "particle_cpu": 17886.0, - "particle_id": 1440973250.0, - "particle_momentum_x": 4.345383730592745e-18, + "particle_cpu": 17826.0, + "particle_id": 835176910.0, + "particle_momentum_x": 4.423317346074595e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6255247164666637e-18, - "particle_position_x": 0.11067439437835992, - "particle_position_y": 0.6395301140823729, - "particle_weight": 3.45546875e-10 + "particle_momentum_z": 2.654287388285012e-18, + "particle_position_x": 0.11034345761508463, + "particle_position_y": 0.6415606590060071, + "particle_weight": 3.440703125e-10 }, "ions": { - "particle_Bx": 0.0, - "particle_By": 10930527.873315271, - "particle_Bz": 0.0, - "particle_Ex": 3270566810756972.0, - "particle_Ey": 0.0, - "particle_Ez": 1887.110708409788, "particle_cpu": 4864.0, - "particle_id": 193800448.0, - "particle_ionization_level": 72810.0, - "particle_momentum_x": 1.73229610377399e-18, + "particle_id": 45340928.0, + "particle_ionization_level": 72773.0, + "particle_momentum_x": 1.7335485753555193e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.509872456624198e-23, - "particle_position_x": 0.032000004532743265, - "particle_position_y": 0.12800000460884142, + "particle_momentum_z": 3.5110738038450676e-23, + "particle_position_x": 0.03199996960212278, + "particle_position_y": 0.12800000461301925, "particle_weight": 9.999999999999999e-11 }, "lev=0": { @@ -38,9 +26,9 @@ "Bz": 0.0, "Ex": 7878018647090106.0, "Ey": 0.0, - "Ez": 2740.9364504831315, + "Ez": 2740.936450483133, "jx": 1.2111228466502686e+16, "jy": 0.0, - "jz": 1.360852807841737e-07 + "jz": 1.362596673906476e-07 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/momentum-conserving-gather.json b/Regression/Checksum/benchmarks_json/momentum-conserving-gather.json index 3d8525b254a..04c09d6c123 100644 --- a/Regression/Checksum/benchmarks_json/momentum-conserving-gather.json +++ b/Regression/Checksum/benchmarks_json/momentum-conserving-gather.json @@ -1,11 +1,5 @@ { "beam": { - "particle_Bx": 30.446171922907517, - "particle_By": 3870.917493104042, - "particle_Bz": 0.34755106738400265, - "particle_Ex": 1014015529098.6418, - "particle_Ey": 19826252289.36772, - "particle_Ez": 1908690024876.2688, "particle_cpu": 0.0, "particle_id": 2147483644000.0, "particle_momentum_x": 4.323192428952614e-19, @@ -16,14 +10,8 @@ "particle_weight": 12483018148921.53 }, "driver": { - "particle_Bx": 31.940096497888742, - "particle_By": 15604.732356054985, - "particle_Bz": 1.2252572333665939, - "particle_Ex": 4364681742392.5117, - "particle_Ey": 9873744504.0512, - "particle_Ez": 18055631247141.04, "particle_cpu": 0.0, - "particle_id": 10236500.0, + "particle_id": 1500500.0, "particle_momentum_x": 4.273571972654775e-19, "particle_momentum_y": 4.1735282690939002e-19, "particle_momentum_z": 5.473699235390372e-16, @@ -32,36 +20,30 @@ "particle_weight": 93622636116911.45 }, "lev=0": { - "Bx": 10.733012173941601, - "By": 123705.21142513887, + "Bx": 10.733012173941603, + "By": 123705.21142513888, "Bz": 4.499473350064483, - "Ex": 32786118366421.47, + "Ex": 32786118366421.473, "Ey": 3709615080.588276, "Ez": 40656586163467.92, - "jx": 2141101370253514.8, + "jx": 2141101370253514.2, "jy": 306062373710.40106, - "jz": 4320228054997730.5 + "jz": 4320228054997731.0 }, "lev=1": { - "Bx": 5.2561348113069375, + "Bx": 5.256134811306936, "By": 137245.47590665444, - "Bz": 4.270745951650613, + "Bz": 4.270745951650612, "Ex": 44351722683610.33, "Ey": 3488288401.6961145, "Ez": 51479977260499.836, - "jx": 2867710882689230.0, - "jy": 165181878916.29016, - "jz": 4439666503460565.0 + "jx": 2867710882689230.5, + "jy": 165181878916.2902, + "jz": 4439666503460564.0 }, "plasma_e": { - "particle_Bx": 6.9776518155443075, - "particle_By": 65971.4138126626, - "particle_Bz": 2.2431702261360305, - "particle_Ex": 14972474416018.527, - "particle_Ey": 2292688361.624074, - "particle_Ez": 17838346022653.637, "particle_cpu": 3600.0, - "particle_id": 41596488.0, + "particle_id": 6546600.0, "particle_momentum_x": 1.5130313200302403e-19, "particle_momentum_y": 7.700274017007924e-24, "particle_momentum_z": 1.7302395904705489e-19, diff --git a/Regression/Checksum/benchmarks_json/parabolic_channel_initialization_2d_single_precision.json b/Regression/Checksum/benchmarks_json/parabolic_channel_initialization_2d_single_precision.json new file mode 100644 index 00000000000..b07bca40116 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/parabolic_channel_initialization_2d_single_precision.json @@ -0,0 +1,5 @@ +{ + "lev=0": { + "rho": 0.004023485417624635 + } +} diff --git a/Regression/Checksum/benchmarks_json/particle_pusher.json b/Regression/Checksum/benchmarks_json/particle_pusher.json index dd254882735..7c6c52032f8 100644 --- a/Regression/Checksum/benchmarks_json/particle_pusher.json +++ b/Regression/Checksum/benchmarks_json/particle_pusher.json @@ -11,12 +11,6 @@ "jz": 0.0 }, "positron": { - "particle_Bx": 0.0, - "particle_By": 0.0, - "particle_Bz": 1.0, - "particle_Ex": 299417482.9214179, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 0.0, "particle_id": 1.0, "particle_momentum_x": 6.55085314065218e-05, diff --git a/Regression/Checksum/benchmarks_json/qed_breit_wheeler_opt_depth_evolution.json b/Regression/Checksum/benchmarks_json/qed_breit_wheeler_opt_depth_evolution.json index 0e2c5e60f3e..2712d517fc1 100644 --- a/Regression/Checksum/benchmarks_json/qed_breit_wheeler_opt_depth_evolution.json +++ b/Regression/Checksum/benchmarks_json/qed_breit_wheeler_opt_depth_evolution.json @@ -1,15 +1,9 @@ { "ele_bw": { - "particle_Bx": 44488571428.57164, - "particle_By": 66732857142.85782, - "particle_Bz": 133465714285.71455, - "particle_Ex": 3.7889246226388705e+19, - "particle_Ey": 1.5155698490565198e+19, - "particle_Ez": 2.273354773582188e+19, "particle_cpu": 7868.0, - "particle_id": 909726920.0, + "particle_id": 646554824.0, "particle_momentum_x": 2.5371833905509472e-14, - "particle_momentum_y": 2.3900945928864857e-14, + "particle_momentum_y": 2.390094592886486e-14, "particle_momentum_z": 3.195830540752922e-14, "particle_position_x": 0.0038628680559308396, "particle_position_y": 0.0038691826866907985, @@ -64,14 +58,8 @@ "particle_position_z": 0.0016801197916666664 }, "pos_bw": { - "particle_Bx": 44488571428.57158, - "particle_By": 66732857142.85768, - "particle_Bz": 133465714285.71454, - "particle_Ex": 3.788924622638887e+19, - "particle_Ey": 1.5155698490565136e+19, - "particle_Ez": 2.2733547735821787e+19, "particle_cpu": 7868.0, - "particle_id": 911804493.0, + "particle_id": 648632397.0, "particle_momentum_x": 2.5161516094347012e-14, "particle_momentum_y": 2.4559377145598413e-14, "particle_momentum_z": 3.330888308559405e-14, diff --git a/Regression/Checksum/benchmarks_json/qed_breit_wheeler_tau_init.json b/Regression/Checksum/benchmarks_json/qed_breit_wheeler_tau_init.json index 553f043fba4..24663e19b2c 100644 --- a/Regression/Checksum/benchmarks_json/qed_breit_wheeler_tau_init.json +++ b/Regression/Checksum/benchmarks_json/qed_breit_wheeler_tau_init.json @@ -11,20 +11,14 @@ "jz": 0.0 }, "photons": { - "particle_Bx": 0.0, - "particle_By": 0.0, - "particle_Bz": 0.0, - "particle_Ex": 0.0, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 28800.0, - "particle_id": 993882240.0, + "particle_id": 897828480.0, "particle_momentum_x": 1.257725971841766e-15, "particle_momentum_y": 1.2533271477501288e-15, "particle_momentum_z": 1.2567654312332688e-15, - "particle_optical_depth_BW": 57886.05893096986, - "particle_position_x": 0.8640051523469108, - "particle_position_y": 0.8640331757950865, + "particle_optical_depth_BW": 57886.05893096988, + "particle_position_x": 0.8640051523469104, + "particle_position_y": 0.8640331757950863, "particle_weight": 36000000000.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/qed_quantum_sync_tau_init.json b/Regression/Checksum/benchmarks_json/qed_quantum_sync_tau_init.json index 26133ef698b..61c33ab3949 100644 --- a/Regression/Checksum/benchmarks_json/qed_quantum_sync_tau_init.json +++ b/Regression/Checksum/benchmarks_json/qed_quantum_sync_tau_init.json @@ -1,19 +1,13 @@ { "electrons": { - "particle_Bx": 0.23333485603512907, - "particle_By": 0.3306506464928457, - "particle_Bz": 0.23170804111682167, - "particle_Ex": 306898947.1037948, - "particle_Ey": 314678746.66859293, - "particle_Ez": 322524922.65796685, "particle_cpu": 28800.0, - "particle_id": 993882240.0, + "particle_id": 897828480.0, "particle_momentum_x": 1.2577259718364536e-15, - "particle_momentum_y": 1.253327147744764e-15, - "particle_momentum_z": 1.256765431227726e-15, - "particle_optical_depth_QSR": 57886.05893096986, - "particle_position_x": 0.8640051455090564, - "particle_position_y": 0.8640331765233291, + "particle_momentum_y": 1.2533271477447635e-15, + "particle_momentum_z": 1.2567654312277259e-15, + "particle_optical_depth_QSR": 57886.05893096988, + "particle_position_x": 0.8640051455090565, + "particle_position_y": 0.8640331765233288, "particle_weight": 36000000000.0 }, "lev=0": { @@ -28,20 +22,14 @@ "jz": 665944376473.1846 }, "positrons": { - "particle_Bx": 0.23364278767344948, - "particle_By": 0.3313944153647703, - "particle_Bz": 0.23186653729970977, - "particle_Ex": 307442525.02915114, - "particle_Ey": 313908687.7726069, - "particle_Ez": 322478933.98023397, "particle_cpu": 28800.0, - "particle_id": 2969792640.0, + "particle_id": 2682046080.0, "particle_momentum_x": 1.2564867705042074e-15, - "particle_momentum_y": 1.2522720470651855e-15, - "particle_momentum_z": 1.250680188440362e-15, + "particle_momentum_y": 1.2522720470651854e-15, + "particle_momentum_z": 1.2506801884403616e-15, "particle_optical_depth_QSR": 57542.97683982639, - "particle_position_x": 0.8641071175220608, - "particle_position_y": 0.8639817645288141, + "particle_position_x": 0.8641071175220609, + "particle_position_y": 0.863981764528814, "particle_weight": 36000000000.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger2.json b/Regression/Checksum/benchmarks_json/qed_schwinger2.json index cd3ecf6c9ac..e2a52492996 100644 --- a/Regression/Checksum/benchmarks_json/qed_schwinger2.json +++ b/Regression/Checksum/benchmarks_json/qed_schwinger2.json @@ -1,11 +1,5 @@ { "ele_schwinger": { - "particle_Bx": 3439183579241.8213, - "particle_By": 1076561948990.9731, - "particle_Bz": 3760851107750.2676, - "particle_Ex": 1.6138262991542085e+26, - "particle_Ey": 1.0214830999058134e+25, - "particle_Ez": 3.5684391886872975e+25, "particle_cpu": 1024.0, "particle_id": 1573888.0, "particle_momentum_x": 1.253856079274937e-09, @@ -28,12 +22,6 @@ "jz": 3.214948145600786e+30 }, "pos_schwinger": { - "particle_Bx": 3439183579241.8213, - "particle_By": 1076561948990.9731, - "particle_Bz": 3760851107750.2676, - "particle_Ex": 1.6138262991542085e+26, - "particle_Ey": 1.0214830999058134e+25, - "particle_Ez": 3.5684391886872975e+25, "particle_cpu": 1024.0, "particle_id": 2622464.0, "particle_momentum_x": 1.2705200005945566e-09, diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger3.json b/Regression/Checksum/benchmarks_json/qed_schwinger3.json index ea687e71191..d2de759a405 100644 --- a/Regression/Checksum/benchmarks_json/qed_schwinger3.json +++ b/Regression/Checksum/benchmarks_json/qed_schwinger3.json @@ -1,11 +1,5 @@ { "ele_schwinger": { - "particle_Bx": 1.348034558144351, - "particle_By": 0.0, - "particle_Bz": 0.33927654383456507, - "particle_Ex": 0.0, - "particle_Ey": 1.3909415199145668e+20, - "particle_Ez": 0.0, "particle_cpu": 631.0, "particle_id": 610043.0, "particle_momentum_x": 7.391180107793151e-28, @@ -28,12 +22,6 @@ "jz": 0.0 }, "pos_schwinger": { - "particle_Bx": 1.3398489034143053, - "particle_By": 0.0, - "particle_Bz": 0.3427233099177752, - "particle_Ex": 0.0, - "particle_Ey": 1.3909415199145551e+20, - "particle_Ez": 0.0, "particle_cpu": 631.0, "particle_id": 1017026.0, "particle_momentum_x": 7.723864257907636e-28, diff --git a/Regression/Checksum/benchmarks_json/qed_schwinger4.json b/Regression/Checksum/benchmarks_json/qed_schwinger4.json index db5ad389e13..b2276d5476e 100644 --- a/Regression/Checksum/benchmarks_json/qed_schwinger4.json +++ b/Regression/Checksum/benchmarks_json/qed_schwinger4.json @@ -1,11 +1,5 @@ { "ele_schwinger": { - "particle_Bx": 2.319332155934254, - "particle_By": 1707847966720000.0, - "particle_Bz": 0.6015864770059552, - "particle_Ex": 2139883430.888652, - "particle_Ey": 0.0, - "particle_Ez": 5.119999999998239e+23, "particle_cpu": 1024.0, "particle_id": 1573888.0, "particle_momentum_x": 6.569424596732217e-12, @@ -28,12 +22,6 @@ "jz": 1.5866782361655698e+16 }, "pos_schwinger": { - "particle_Bx": 2.3212042547187046, - "particle_By": 1707847966720000.0, - "particle_Bz": 0.6075072929142599, - "particle_Ex": 2146992296.4054935, - "particle_Ey": 0.0, - "particle_Ez": 5.119999999998239e+23, "particle_cpu": 1024.0, "particle_id": 2622464.0, "particle_momentum_x": 6.569424596732212e-12, diff --git a/Regression/Checksum/benchmarks_json/radiation_reaction.json b/Regression/Checksum/benchmarks_json/radiation_reaction.json index 9466924a28c..4bf97517407 100644 --- a/Regression/Checksum/benchmarks_json/radiation_reaction.json +++ b/Regression/Checksum/benchmarks_json/radiation_reaction.json @@ -1,11 +1,5 @@ { "ele_para0": { - "particle_Bx": 917978.2333474257, - "particle_By": 1376967.350021139, - "particle_Bz": 2753934.700042278, - "particle_Ex": 0.003842707741931274, - "particle_Ey": 0.015625799690926225, - "particle_Ez": 0.023446517674477488, "particle_cpu": 0.0, "particle_id": 1.0, "particle_momentum_x": 7.802641516393779e-20, @@ -17,12 +11,6 @@ "particle_weight": 1e-08 }, "ele_perp0": { - "particle_Bx": 917978.2333474257, - "particle_By": 1376967.350021139, - "particle_Bz": 2753934.700042278, - "particle_Ex": 0.1250313976349998, - "particle_Ey": 0.0776652150231756, - "particle_Ez": 0.06187021356968523, "particle_cpu": 0.0, "particle_id": 2.0, "particle_momentum_x": 8.11389431708072e-21, @@ -34,12 +22,6 @@ "particle_weight": 1e-08 }, "ele_perp1": { - "particle_Bx": 917978.2333474257, - "particle_By": 1376967.350021139, - "particle_Bz": 2753934.700042278, - "particle_Ex": 0.03710062023180817, - "particle_Ey": 0.002435032603780963, - "particle_Ez": 0.023704514044336038, "particle_cpu": 0.0, "particle_id": 3.0, "particle_momentum_x": 2.3334382603801444e-20, @@ -51,12 +33,6 @@ "particle_weight": 1e-08 }, "ele_perp2": { - "particle_Bx": 917978.2333474257, - "particle_By": 1376967.350021139, - "particle_Bz": 2753934.700042278, - "particle_Ex": 0.0002716012778477919, - "particle_Ey": 0.005385504253127432, - "particle_Ez": 0.016067456901663155, "particle_cpu": 0.0, "particle_id": 4.0, "particle_momentum_x": 1.81896887291916e-20, @@ -71,12 +47,6 @@ "rho": 0.00051269652288 }, "pos_perp2": { - "particle_Bx": 917978.2333474257, - "particle_By": 1376967.350021139, - "particle_Bz": 2753934.700042278, - "particle_Ex": 0.004491380475289906, - "particle_Ey": 0.012777393943318885, - "particle_Ez": 0.02007316097507273, "particle_cpu": 0.0, "particle_id": 5.0, "particle_momentum_x": 1.818968872919164e-20, diff --git a/Regression/Checksum/benchmarks_json/reduced_diags.json b/Regression/Checksum/benchmarks_json/reduced_diags.json index 023995db0ad..8a385327d7a 100644 --- a/Regression/Checksum/benchmarks_json/reduced_diags.json +++ b/Regression/Checksum/benchmarks_json/reduced_diags.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 0.08468252959119768, - "particle_By": 0.08433759166060013, - "particle_Bz": 0.0834983764785095, - "particle_Ex": 106389075.07051627, - "particle_Ey": 110696689.8493621, - "particle_Ez": 107474460.40560272, "particle_cpu": 16384.0, "particle_id": 268451840.0, "particle_momentum_x": 2.433513100910759e-19, @@ -28,12 +22,6 @@ "jz": 719566.2651192012 }, "photons": { - "particle_Bx": 0.08443156833972876, - "particle_By": 0.08498460548568723, - "particle_Bz": 0.08428210467483634, - "particle_Ex": 106054332.53718573, - "particle_Ey": 111005174.77433285, - "particle_Ez": 105476983.07484898, "particle_cpu": 16384.0, "particle_id": 1342193664.0, "particle_momentum_x": 1.4328434977988592e-18, @@ -45,12 +33,6 @@ "particle_weight": 800000000000000.0 }, "protons": { - "particle_Bx": 0.08405060899990621, - "particle_By": 0.08395398257418159, - "particle_Bz": 0.08318247160660851, - "particle_Ex": 102197583.14430666, - "particle_Ey": 106376868.79571314, - "particle_Ez": 102627734.35036321, "particle_cpu": 16384.0, "particle_id": 805322752.0, "particle_momentum_x": 1.4305311394194743e-19, diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_heuristic.json b/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_heuristic.json index 36eb4d4217e..65054341076 100644 --- a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_heuristic.json +++ b/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_heuristic.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 0.0, - "particle_By": 0.0, - "particle_Bz": 0.0, - "particle_Ex": 0.0, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 0.0, "particle_id": 8590000128.0, "particle_momentum_x": 0.0, diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers.json b/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers.json index 36eb4d4217e..65054341076 100644 --- a/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers.json +++ b/Regression/Checksum/benchmarks_json/reduced_diags_loadbalancecosts_timers.json @@ -1,11 +1,5 @@ { "electrons": { - "particle_Bx": 0.0, - "particle_By": 0.0, - "particle_Bz": 0.0, - "particle_Ex": 0.0, - "particle_Ey": 0.0, - "particle_Ez": 0.0, "particle_cpu": 0.0, "particle_id": 8590000128.0, "particle_momentum_x": 0.0, diff --git a/Regression/Checksum/benchmarks_json/subcyclingMR.json b/Regression/Checksum/benchmarks_json/subcyclingMR.json index ac83e2c4669..a2657cc966f 100644 --- a/Regression/Checksum/benchmarks_json/subcyclingMR.json +++ b/Regression/Checksum/benchmarks_json/subcyclingMR.json @@ -1,30 +1,18 @@ { "beam": { - "particle_Bx": 0.0, - "particle_By": 1295546.8975611941, - "particle_Bz": 0.0, - "particle_Ex": 344765556152058.25, - "particle_Ey": 0.0, - "particle_Ez": 1193101601971968.0, "particle_cpu": 0.0, "particle_id": 21474836440000.0, - "particle_momentum_x": 3.0267885065250823e-19, + "particle_momentum_x": 3.026788506525083e-19, "particle_momentum_y": 0.0, - "particle_momentum_z": 6.30193261465718e-17, - "particle_position_x": 0.00038109441622609754, + "particle_momentum_z": 6.301932614657182e-17, + "particle_position_x": 0.00038109441622609716, "particle_position_y": 0.27997390731251637, "particle_weight": 62415090744607.65 }, "driver": { - "particle_Bx": 0.0, - "particle_By": 7036528.904102431, - "particle_Bz": 0.0, - "particle_Ex": 2076986724754339.0, - "particle_Ey": 0.0, - "particle_Ez": 1520191782419694.0, "particle_cpu": 0.0, "particle_id": 50005000.0, - "particle_momentum_x": 7.231072222092701e-19, + "particle_momentum_x": 7.231072222092698e-19, "particle_momentum_y": 0.0, "particle_momentum_z": 2.730891621102868e-12, "particle_position_x": 0.0238454319116077, @@ -33,20 +21,20 @@ }, "lev=0": { "Bx": 0.0, - "By": 1042087.5215612946, + "By": 1042087.5215612948, "Bz": 0.0, - "Ex": 424746138822356.94, + "Ex": 424746138822357.06, "Ey": 0.0, "Ez": 469575222851079.0, "jx": 3.776918520059734e+17, "jy": 0.0, - "jz": 6.854243563864266e+17 + "jz": 6.854243563864268e+17 }, "lev=1": { "Bx": 0.0, - "By": 2277507.1426832867, + "By": 2277507.142683287, "Bz": 0.0, - "Ex": 883571288026587.6, + "Ex": 883571288026587.8, "Ey": 0.0, "Ez": 897344748434965.6, "jx": 1.2113167359612608e+17, @@ -54,33 +42,21 @@ "jz": 3.914141701645454e+17 }, "plasma_e": { - "particle_Bx": 0.0, - "particle_By": 3453725.2247103425, - "particle_Bz": 0.0, - "particle_Ex": 1427604131682412.5, - "particle_Ey": 0.0, - "particle_Ez": 1484821487586464.8, "particle_cpu": 0.0, - "particle_id": 22976894251680.0, + "particle_id": 22972039003792.0, "particle_momentum_x": 1.1529420490113797e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.5374313849366925e-18, - "particle_position_x": 0.242397647830779, - "particle_position_y": 0.8980075222163136, + "particle_momentum_z": 1.537431384936693e-18, + "particle_position_x": 0.24239764783077897, + "particle_position_y": 0.8980075222163137, "particle_weight": 5723510742187501.0 }, "plasma_p": { - "particle_Bx": 0.0, - "particle_By": 3766950.28595219, - "particle_Bz": 0.0, - "particle_Ex": 1592516445686014.5, - "particle_Ey": 0.0, - "particle_Ez": 1810017527838155.5, "particle_cpu": 0.0, - "particle_id": 23097613251264.0, - "particle_momentum_x": 1.5163562656560446e-18, + "particle_id": 23092453563072.0, + "particle_momentum_x": 1.5163562656560444e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6529164052371967e-18, + "particle_momentum_z": 2.652916405237197e-18, "particle_position_x": 0.2448077799219613, "particle_position_y": 0.944822140164403, "particle_weight": 5976562500000001.0 diff --git a/Regression/TestFillBoundary/compare_guard_cells.sh b/Regression/TestFillBoundary/compare_guard_cells.sh index 7387fa8652f..316704bf521 100755 --- a/Regression/TestFillBoundary/compare_guard_cells.sh +++ b/Regression/TestFillBoundary/compare_guard_cells.sh @@ -72,7 +72,7 @@ if [ $DO_GIT_CLONE == true ]; then # Clone all three repos rm -rf WarpX picsar amrex git clone https://github.com/ECP-WarpX/WarpX.git - git clone https://bitbucket.org/berkeleylab/picsar.git + git clone https://github.com/ECP-WarpX/picsar.git git clone --branch development https://github.com/AMReX-Codes/amrex.git fi diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 7fd4594f64a..c56b772f4a9 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -12,7 +12,6 @@ sourceTree = C_Src suiteName = WarpX-GPU COMP = g++ -FCOMP = gfortran add_to_c_make_command = TEST=TRUE USE_ASSERTION=TRUE WarpxBinDir= purge_output = 1 @@ -53,11 +52,11 @@ branch = development [source] dir = /home/regtester/git/WarpX -branch = master +branch = development [extra-PICSAR] dir = /home/regtester/git/picsar/ -branch = master +branch = development # individual problems follow @@ -237,7 +236,7 @@ compileTest = 0 doVis = 0 compareParticles = 0 particleTypes = electrons -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir2d.py analysisOutputImage = langmuir2d_analysis.png tolerance = 1e-12 @@ -245,7 +244,7 @@ tolerance = 1e-12 [Langmuir_2d_single_precision] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_3d_rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux dim = 2 addToCompileString = USE_GPU=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE restartTest = 0 @@ -275,7 +274,7 @@ compileTest = 0 doVis = 0 compareParticles = 0 particleTypes = electrons -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir2d.py analysisOutputImage = langmuir2d_analysis.png tolerance = 1e-12 @@ -294,7 +293,7 @@ compileTest = 0 doVis = 0 compareParticles = 0 particleTypes = electrons -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex jx diag1.electrons.variables=w ux analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir.py analysisOutputImage = langmuir_x_analysis.png tolerance = 5e-11 @@ -313,7 +312,7 @@ compileTest = 0 doVis = 0 compareParticles = 0 particleTypes = electrons -runtime_params = electrons.uy=0.01 electrons.ymax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ey jy diag1.electrons.variables=w uy Ey +runtime_params = electrons.uy=0.01 electrons.ymax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ey jy diag1.electrons.variables=w uy analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir.py analysisOutputImage = langmuir_y_analysis.png tolerance = 5e-11 @@ -332,7 +331,7 @@ compileTest = 0 doVis = 0 compareParticles = 0 particleTypes = electrons -runtime_params = electrons.uz=0.01 electrons.zmax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ez jz diag1.electrons.variables=w uz Ez +runtime_params = electrons.uz=0.01 electrons.zmax=0.e-6 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ez jz diag1.electrons.variables=w uz analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir.py analysisOutputImage = langmuir_z_analysis.png tolerance = 5e-11 @@ -426,7 +425,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez +runtime_params = warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz particleTypes = electrons positrons analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -445,7 +444,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = psatd.fftw_plan_measure=0 diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell +runtime_params = psatd.fftw_plan_measure=0 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell particleTypes = electrons positrons analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -464,7 +463,7 @@ tolerance = 5e-11 # compileTest = 0 # doVis = 0 # compareParticles = 0 -# runtime_params = psatd.fftw_plan_measure=0 warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell +# runtime_params = psatd.fftw_plan_measure=0 warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell # particleTypes = electrons positrons # analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py # analysisOutputImage = langmuir_multi_2d_analysis.png @@ -482,7 +481,7 @@ tolerance = 5e-11 # numthreads = 1 # compileTest = 0 # doVis = 0 -# runtime_params = diag1.electrons.variables=w ux uy uz Ex Ey Ez Bx By diag1.ions.variables=w ux uy uz Ex Ey Ez Bx By +# runtime_params = diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz # compareParticles = 0 # particleTypes = electrons ions # analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir_multi_rz.py @@ -624,7 +623,7 @@ compileTest = 0 doVis = 0 compareParticles = 0 particleTypes = electrons -outputFile = diags/plotfiles/plt00040 +outputFile = diags/diag200040 tolerance = 5e-11 [uniform_plasma_restart] diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 4a1c8c6a885..77a049f3a0a 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -12,7 +12,6 @@ sourceTree = C_Src suiteName = WarpX COMP = g++ -FCOMP = gfortran add_to_c_make_command = TEST=TRUE USE_ASSERTION=TRUE WarpxBinDir= purge_output = 1 @@ -44,7 +43,7 @@ check_file_name = none # email sendEmailWhenFail = 1 -emailTo = weiqunzhang@lbl.gov, jlvay@lbl.gov, rlehe@lbl.gov, atmyers@lbl.gov, mthevenet@lbl.gov, oshapoval@lbl.gov, henri.vincenti@cea.fr, ldianaamorim@lbl.gov, rjambunathan@lbl.gov, axelhuebl@lbl.gov, ezoni@lbl.gov, yinjianzhao@lbl.gov +emailTo = weiqunzhang@lbl.gov, jlvay@lbl.gov, rlehe@lbl.gov, atmyers@lbl.gov, oshapoval@lbl.gov, henri.vincenti@cea.fr, rjambunathan@lbl.gov, yinjianzhao@lbl.gov emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more details. [AMReX] @@ -53,7 +52,7 @@ branch = development [source] dir = /home/regtester/AMReX_RegTesting/warpx -branch = master +branch = development [extra-PICSAR] dir = /home/regtester/AMReX_RegTesting/picsar/ @@ -96,7 +95,7 @@ tolerance = 1.e-14 [pml_x_psatd] buildDir = . inputFile = Examples/Tests/PML/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE +runtime_params = psatd.update_with_rho=1 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -249,7 +248,7 @@ tolerance = 1.e-14 [Langmuir_2d] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_3d_rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux dim = 2 addToCompileString = restartTest = 0 @@ -268,7 +267,7 @@ tolerance = 1.e-14 [Langmuir_2d_single_precision] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_3d_rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux dim = 2 addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE restartTest = 0 @@ -287,7 +286,7 @@ tolerance = 1.0e-4 [Langmuir_2d_nompi] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_3d_rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux Ex +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 diag1.fields_to_plot=Ex jx diag1.electrons.variables=w ux dim = 2 addToCompileString = restartTest = 0 @@ -382,7 +381,26 @@ tolerance = 5.e-11 [Langmuir_multi_psatd_current_correction] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_3d_multi_rt -runtime_params = psatd.fftw_plan_measure=0 psatd.periodic_single_box_fft=1 psatd.do_current_correction=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE +runtime_params = algo.current_deposition=esirkepov psatd.fftw_plan_measure=0 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE +dim = 3 +addToCompileString = USE_PSATD=TRUE +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +tolerance = 5.e-11 +particleTypes = electrons positrons +analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir_multi.py +analysisOutputImage = langmuir_multi_analysis.png + +[Langmuir_multi_psatd_current_correction_nodal] +buildDir = . +inputFile = Examples/Tests/Langmuir/inputs_3d_multi_rt +runtime_params = algo.current_deposition=direct psatd.fftw_plan_measure=0 psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.do_nodal=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE dim = 3 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -458,7 +476,7 @@ tolerance = 5.e-7 [Langmuir_multi_2d_nodal] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rt -runtime_params = warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez +runtime_params = warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz dim = 2 addToCompileString = restartTest = 0 @@ -477,7 +495,7 @@ tolerance = 1.e-14 [Langmuir_multi_2d_psatd] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rt -runtime_params = psatd.fftw_plan_measure=0 diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell +runtime_params = psatd.fftw_plan_measure=0 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -496,7 +514,7 @@ tolerance = 1.e-14 [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rt -runtime_params = algo.field_gathering=momentum-conserving psatd.fftw_plan_measure=0 diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell +runtime_params = algo.field_gathering=momentum-conserving psatd.fftw_plan_measure=0 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -515,7 +533,25 @@ tolerance = 1.e-14 [Langmuir_multi_2d_psatd_current_correction] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rt -runtime_params = amr.max_grid_size=128 psatd.fftw_plan_measure=0 psatd.periodic_single_box_fft=1 psatd.do_current_correction=1 diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE +runtime_params = amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.fftw_plan_measure=0 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE +dim = 2 +addToCompileString = USE_PSATD=TRUE +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons positrons +analysisRoutine = Examples/Tests/Langmuir/analysis_langmuir_multi_2d.py +analysisOutputImage = langmuir_multi_2d_analysis.png + +[Langmuir_multi_2d_psatd_current_correction_nodal] +buildDir = . +inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rt +runtime_params = amr.max_grid_size=128 algo.current_deposition=direct psatd.fftw_plan_measure=0 psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.do_nodal=1 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -533,7 +569,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_nodal] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rt -runtime_params = psatd.fftw_plan_measure=0 warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz Ex Ey Ez diag1.positrons.variables=w ux uy uz Ex Ey Ez diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell +runtime_params = psatd.fftw_plan_measure=0 warpx.do_nodal=1 algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -552,7 +588,7 @@ tolerance = 1.e-14 [Langmuir_multi_rz] buildDir = . inputFile = Examples/Tests/Langmuir/inputs_2d_multi_rz_rt -runtime_params = diag1.electrons.variables=w ux uy uz Ex Ey Ez Bx By diag1.ions.variables=w ux uy uz Ex Ey Ez Bx By diag1.dump_rz_modes=0 +runtime_params = diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 dim = 2 addToCompileString = USE_RZ=TRUE restartTest = 0 @@ -786,6 +822,59 @@ analysisRoutine = Examples/Modules/relativistic_space_charge_initialization/anal analysisOutputImage = Comparison.png tolerance = 1.e-14 +[parabolic_channel_initialization_2d_single_precision] +buildDir = . +inputFile = Examples/Tests/initial_plasma_profile/inputs +dim = 2 +addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +runtime_params = warpx.do_dynamic_scheduling=0 +analysisRoutine = Examples/Tests/initial_plasma_profile/analysis.py +tolerance = 1.e-14 + +[dive_cleaning_2d] +buildDir = . +inputFile = Examples/Modules/dive_cleaning/inputs_3d +dim = 2 +addToCompileString = +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +runtime_params = warpx.do_dynamic_scheduling=0 diag1.file_prefix=dive_cleaning_2d_plt +analysisRoutine = Examples/Modules/dive_cleaning/analysis.py +analysisOutputImage = Comparison.png +tolerance = 1.e-14 + +[dive_cleaning_3d] +buildDir = . +inputFile = Examples/Modules/dive_cleaning/inputs_3d +dim = 3 +addToCompileString = +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +runtime_params = warpx.do_dynamic_scheduling=0 diag1.file_prefix=dive_cleaning_3d_plt +analysisRoutine = Examples/Modules/dive_cleaning/analysis.py +analysisOutputImage = Comparison.png +tolerance = 1.e-14 + [particles_in_pml_2d] buildDir = . inputFile = Examples/Tests/particles_in_PML/inputs_2d @@ -1106,7 +1195,7 @@ tolerance = 1.e-14 [LaserAccelerationRZ] buildDir = . inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d_rz -runtime_params = diag1.electrons.variables=w ux uy uz Ex Ey Ez Bx By diag1.beam.variables=w ux uy uz Ex Ey Ez Bx By max_step=10 +runtime_params = diag1.electrons.variables=w ux uy uz diag1.beam.variables=w ux uy uz max_step=10 dim = 2 addToCompileString = USE_RZ=TRUE restartTest = 0 @@ -1371,6 +1460,40 @@ particleTypes = electrons ions analysisRoutine = Examples/Tests/galilean/analysis_3d.py tolerance = 1.e-14 +[averaged_galilean_2d_psatd] +buildDir = . +inputFile = Examples/Tests/averaged_galilean/inputs_avg_2d +dim = 2 +addToCompileString = USE_PSATD=TRUE +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons ions +analysisRoutine = Examples/Tests/averaged_galilean/analysis_avg_2d.py +tolerance = 1e-6 + +[averaged_galilean_3d_psatd] +buildDir = . +inputFile = Examples/Tests/averaged_galilean/inputs_avg_3d +dim = 3 +addToCompileString = USE_PSATD=TRUE +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons ions +analysisRoutine = Examples/Tests/averaged_galilean/analysis_avg_3d.py +tolerance = 1e-4 + [TwoParticle_electrostatic] buildDir = . inputFile = Examples/Tests/TwoParticle_electrostatic/inputs_3d diff --git a/Source/BoundaryConditions/CMakeLists.txt b/Source/BoundaryConditions/CMakeLists.txt new file mode 100644 index 00000000000..265f5c210d7 --- /dev/null +++ b/Source/BoundaryConditions/CMakeLists.txt @@ -0,0 +1,5 @@ +target_sources(WarpX + PRIVATE + PML.cpp + WarpXEvolvePML.cpp +) diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H new file mode 100644 index 00000000000..3033c86f006 --- /dev/null +++ b/Source/Diagnostics/BTDiagnostics.H @@ -0,0 +1,140 @@ +#ifndef WARPX_BTDIAGNOSTICS_H_ +#define WARPX_BTDIAGNOSTICS_H_ + +#include "Diagnostics.H" + +class +BTDiagnostics final : public Diagnostics +{ +public: + + BTDiagnostics (int i, std::string name); + +private: + /** Read relevant parameters for BTD */ + void ReadParameters (); + /** Flush m_mf_output and particles to file. */ + // Currently this is an empty function. When OpenPMD/Plotfile is supported + // this function can be moved to the base class Diagnostics. + void Flush (int i_buffer) override {} ; + /** whether to write output files at this time step + * The data is flushed when the buffer is full and/or + * when the simulation ends or when forced. + * \param[in] step current time step + * \param[in] force_flush if true, return true for any step + * The return bool is used to determine when to write buffer data out + */ + bool DoDump (int step, int i_buffer, bool force_flush=false) override; + bool DoComputeAndPack (int step, bool force_flush=false) override; + /** Initialize Data required to compute back-transformed diagnostics */ + void DerivedInitData (); + /** Initialize functors that store pointers to the fields requested by the user. + * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. + */ + void InitializeFieldFunctors (int lev); + /** This function initialized and allocates particle buffers for all the snapshots. + * The particle containers required for this must be added to populate this function. + */ + void InitializeParticleBuffer () override {}; + /** Compute the cell-centered data for all user-requested fields and store in + * the multi-level cell-centered multifab, m_mf_cc. This MultiFab extends + * over the entire domain and is coarsened using the user-defined crse_ratio + */ + void PrepareFieldDataForOutput (); + /** Initialize buffer domain, buffer box and lab-frame parameters such as + * m_t_lab, and z-positions for the i^th snapshot, i_buffer, and level, lev. + * \param[in] i_buffer i^th snapshot or buffer + * \param[in] lev level + */ + void InitializeFieldBufferData ( int i_buffer , int lev) override; + /** Whether to compute back-tranformed values for field-data */ + bool m_do_back_transformed_fields = true; + /** Whether to compute back-tranformed values for particle-data */ + bool m_do_back_transformed_particles = true; + + // boost parameters + amrex::Real m_gamma_boost; + amrex::Real m_beta_boost; + int m_moving_window_dir; + + /** Number of back-transformed snapshots in the lab-frame requested by the user */ + int m_num_snapshots_lab = std::numeric_limits::lowest(); + /** Time interval in lab-frame between the back-transformed snapshots */ + amrex::Real m_dt_snapshots_lab = std::numeric_limits::lowest(); + /** Distance between the back-transformed snapshots in lab-frame + * m_dz_snapshots_lab = m_dt_snapshots_lab/c + */ + amrex::Real m_dz_snapshots_lab; + + /** Number of z-slices in each buffer of the snapshot */ + int m_buffer_size = 256; + /** max grid size used to generate BoxArray to define output MultiFabs */ + int m_max_box_size = 256; + + /** Time in lab-frame of the back-transformed snapshot */ + amrex::Vector m_t_lab; + /** Physical domain with lab-frame co-ordinates of the back-transformed snapshot */ + amrex::Vector m_buffer_domain_lab; + /** Number of cells in the lab-frame for all back-transformed snapshots */ + amrex::Vector m_buffer_ncells_lab; + /** Box-dimension in boosted-frame index space for each back-transformed snapshot */ + amrex::Vector m_buffer_box; + /** Lab-frame z co-ordinate of the back-transformed snapshot at the current timestep */ + amrex::Vector m_current_z_lab; + /** Boosted-frame z co-ordinate corresponding to the back-transformed + snapshot at the current timestep */ + amrex::Vector m_current_z_boost; + /** Number of back-transformed z-slices in the output buffer multifab */ + amrex::Vector m_buffer_counter; + /** Multi-level cell-centered multifab with all field-data components + which will be sliced and transformed. The same multifab will be used + for all snapshots. + */ + amrex::Vector > m_cell_centered_data; + std::vector m_file_name; + /** Vector of pointers to compute cell-centered data, per level, per component + * using the coarsening-ratio provided by the user. + */ + amrex::Vector< amrex::Vector > > m_cell_center_functors; + + /** Customized function to write back-transformed data */ + void TMP_writeMetaData (); + /** Customized functions to write back-transformed data */ + void TMP_createLabFrameDirectories (int i_buffer, int lev); + /** Define the cell-centered multi-component MultiFab at level, lev. + * \param[in] lev level at which the cell-centered MultiFab is defined. + */ + void DefineCellCenteredMultiFab(int lev); + /** Returns z-position in the boosted-frame + * \param[in] lab-frame time of the snapshot + * \param[in] boosted-frame time at level, lev + */ + amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) + { + amrex::Real current_z_boost = (t_lab / m_gamma_boost - t_boost) * PhysConst::c / m_beta_boost; + return current_z_boost; + } + /** Returns z-position in the lab-frame + * \param[in] lab-frame time of the snapshot + * \param[in] boosted-frame time at level, lev + */ + amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) + { + amrex::Real current_z_lab = (t_lab - t_boost / m_gamma_boost ) * PhysConst::c / m_beta_boost; + return current_z_lab; + } + /** Returns cell-size of the grid in z-direction in the lab-frame at level, lev + * \param[in] dt in boosted-frame at level, lev + * \param[in] ref_ratio, refinement ratio at level, lev + */ + amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio); + /** whether field buffer is full + * \param[in] i_buffer buffer id for which the buffer size is checked. + * returns bool = true is buffer is full + */ + bool buffer_full (int i_buffer) { + return ( m_buffer_counter[i_buffer] == m_buffer_size ); + } +}; + +#endif // WARPX_BTDIAGNOSTICS_H_ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp new file mode 100644 index 00000000000..5782cf58e97 --- /dev/null +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -0,0 +1,388 @@ +#include "BTDiagnostics.H" +#include "WarpX.H" +#include "Utils/WarpXConst.H" +#include "ComputeDiagFunctors/ComputeDiagFunctor.H" +#include "ComputeDiagFunctors/CellCenterFunctor.H" +#include "ComputeDiagFunctors/BackTransformFunctor.H" +#include "Utils/CoarsenIO.H" +#include +#include +#include +using namespace amrex::literals; + +BTDiagnostics::BTDiagnostics (int i, std::string name) + : Diagnostics(i, name) +{ + ReadParameters(); +} + + +void BTDiagnostics::DerivedInitData () +{ + // storing BTD related member variables + auto & warpx = WarpX::GetInstance(); + m_gamma_boost = WarpX::gamma_boost; + m_beta_boost = std::sqrt( 1._rt - 1._rt/( m_gamma_boost * m_gamma_boost) ); + m_moving_window_dir = warpx.moving_window_dir; + // Currently, for BTD, all the data is averaged+coarsened to coarsest level + // and then sliced+back-transformed+filled_to_buffer. + // The number of levels to be output is nlev_output. + nlev_output = 1; + // temporary function related to customized output from previous BTD to verify accuracy + TMP_writeMetaData(); + + // allocate vector of m_t_lab with m_num_buffers; + m_t_lab.resize(m_num_buffers); + // allocate vector of RealBox of the diag domain + m_buffer_domain_lab.resize(m_num_buffers); + // define box correctly (one for all snapshots) + m_buffer_box.resize(m_num_buffers); + // allocate vector of m_current_z_lab + m_current_z_lab.resize(m_num_buffers); + // allocate vector of m_num_buffers + m_current_z_boost.resize(m_num_buffers); + // allocate vector of m_buff_counter + m_buffer_counter.resize(m_num_buffers); + // allocate vector of num_Cells in the lab-frame + m_buffer_ncells_lab.resize(m_num_buffers); + // allocate vector of file names for each buffer + m_file_name.resize(m_num_buffers); + // allocate vector of cell centered multifabs for nlevels + m_cell_centered_data.resize(nmax_lev); + // allocate vector of cell-center functors for nlevels + m_cell_center_functors.resize(nmax_lev); + + for (int i = 0; i < m_num_buffers; ++i) { + // TMP variable name for customized BTD output to verify accuracy + m_file_name[i] = amrex::Concatenate(m_file_prefix +"/snapshots/snapshot",i,5); + } + for (int lev = 0; lev < nmax_lev; ++lev) { + // Define cell-centered multifab over the whole domain with + // user-defined crse_ratio for nlevels + DefineCellCenteredMultiFab(lev); + } + +} + +void +BTDiagnostics::ReadParameters () +{ + BaseReadParameters(); + auto & warpx = WarpX::GetInstance(); + // Read list of back-transform diag parameters requested by the user // + amrex::Print() << " in read parameters for BTD \n"; + + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ( warpx.do_back_transformed_diagnostics==true), + "the do_back_transformed_diagnostics flag must be set to true for BTDiagnostics"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( warpx.gamma_boost > 1.0_rt, + "gamma_boost must be > 1 to use the back-transformed diagnostics"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( warpx.boost_direction[2] == 1, + "The back transformed diagnostics currently only works if the boost is in the z-direction"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( warpx.do_moving_window, + "The moving window should be on if using the boosted frame diagnostic."); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( warpx.moving_window_dir == 2, + "The boosted frame diagnostic currently only works if the moving window is in the z direction."); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + m_format == "plotfile" || m_format == "openpmd", + ".format must be plotfile or openpmd for back transformed diagnostics"); + amrex::ParmParse pp(m_diag_name); + + m_file_prefix = "diags/lab_frame_data/" + m_diag_name; + pp.query("file_prefix", m_file_prefix); + pp.query("do_back_transformed_fields", m_do_back_transformed_fields); + pp.query("do_back_transformed_particles", m_do_back_transformed_particles); + AMREX_ALWAYS_ASSERT(m_do_back_transformed_fields or m_do_back_transformed_particles); + + pp.get("num_snapshots_lab", m_num_snapshots_lab); + m_num_buffers = m_num_snapshots_lab; + + // Read either dz_snapshots_lab or dt_snapshots_lab + bool snapshot_interval_is_specified = false; + amrex::Real m_dz_snapshots_lab = 0.0_rt; + snapshot_interval_is_specified = pp.query("dt_snapshots_lab", m_dt_snapshots_lab); + if ( pp.query("dz_snapshots_lab", m_dz_snapshots_lab) ) { + m_dt_snapshots_lab = m_dz_snapshots_lab/PhysConst::c; + snapshot_interval_is_specified = true; + } + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(snapshot_interval_is_specified, + "For back-transformed diagnostics, user should specify either dz_snapshots_lab or dt_snapshots_lab"); + +} + +void +BTDiagnostics::TMP_writeMetaData () +{ + // This function will have the same functionality as writeMetaData in + // previously used BackTransformedDiagnostics class to write + // back-transformed data in a customized format + + if (amrex::ParallelDescriptor::IOProcessor()) { + const std::string fullpath = m_file_prefix + "/snapshots"; + if (!amrex::UtilCreateDirectory(fullpath, 0755)) amrex::CreateDirectoryFailed(fullpath); + + amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::IO_Buffer_Size); + std::ofstream HeaderFile; + HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + std::string HeaderFileName(m_file_prefix + "/snapshots/Header"); + HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | + std::ofstream::trunc | + std::ofstream::binary); + if (!HeaderFile.good()) amrex::FileOpenFailed( HeaderFileName ); + + HeaderFile.precision(17); + HeaderFile << m_num_snapshots_lab << "\n"; + HeaderFile << m_dt_snapshots_lab << "\n"; + HeaderFile << m_gamma_boost << "\n"; + HeaderFile << m_beta_boost << "\n"; + + } + +} + +bool +BTDiagnostics::DoDump (int step, int i_buffer, bool force_flush) +{ + // check if buffer is full (m_buffer_counter[i_buffer] == m_) or if force_flush == true + if ( buffer_full(i_buffer) || force_flush) { + return true; + } + return false; +} + + +bool +BTDiagnostics::DoComputeAndPack (int step, bool force_flush) +{ + // always set to true for BTDiagnostics since back-transform buffers are potentially + // computed and packed every timstep. + return true; +} + +void +BTDiagnostics::InitializeFieldBufferData ( int i_buffer , int lev) +{ + auto & warpx = WarpX::GetInstance(); + // 1. Lab-frame time for the i^th snapshot + m_t_lab.at(i_buffer) = i_buffer * m_dt_snapshots_lab; + // 2. Define domain in boosted frame at level, lev + amrex::RealBox diag_dom; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) { + // Setting lo-coordinate for the diag domain by taking the max of user-defined + // lo-cordinate and lo-coordinat of the simulation domain at level, lev + diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); + // Setting hi-coordinate for the diag domain by taking the max of user-defined + // hi-cordinate and hi-coordinate of the simulation domain at level, lev + diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); + } + // 3. Initializing the m_buffer_box for the i^th snapshot. + // At initialization, the Box has the same index space as the boosted-frame + // As time-progresses, the z-dimension indices will be modified based on + // current_z_lab + amrex::IntVect lo(0); + amrex::IntVect hi(-1); + for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { + // lo index with same cell-size as simulation at level, lev. + const int lo_index = static_cast( floor( + ( diag_dom.lo(idim) - warpx.Geom(lev).ProbLo(idim) ) / + warpx.Geom(lev).CellSize(idim) ) ); + // Taking max of (0,lo_index) because lo_index must always be >=0 + lo[idim] = std::max( 0, lo_index ); + // hi index with same cell-size as simulation at level, lev. + const int hi_index = static_cast( ceil( + ( diag_dom.hi(idim) - warpx.Geom(lev).ProbLo(idim) ) / + warpx.Geom(lev).CellSize(idim) ) ); + // Taking max of (0,hi_index) because hi_index must always be >=0 + // Subtracting by 1 because lo,hi indices are set to cell-centered staggering. + hi[idim] = std::max( 0, hi_index) - 1; + // if hi<=lo, then hi = lo + 1, to ensure one cell in that dimension + if ( hi[idim] <= lo[idim] ) { + hi[idim] = lo[idim] + 1; + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + m_crse_ratio[idim]==1, "coarsening ratio in reduced dimension must be 1." + ); + } + } + amrex::Box diag_box( lo, hi ); + // The box is not coarsened yet. Should this be coarsened here or when + // the buffer multifab is initialized? + m_buffer_box[i_buffer] = diag_box; + + // 4. Define buffer_domain in lab-frame for the i^th snapshot. + // Replace z-dimension with lab-frame co-ordinates. + amrex::Real zmin_lab = diag_dom.lo(m_moving_window_dir) + / ( (1.0_rt + m_beta_boost) * m_gamma_boost); + amrex::Real zmax_lab = diag_dom.hi(m_moving_window_dir) + / ( (1.0_rt + m_beta_boost) * m_gamma_boost); + + + m_buffer_domain_lab[i_buffer] = warpx.Geom(lev).ProbDomain(); + m_buffer_domain_lab[i_buffer].setLo(m_moving_window_dir, zmin_lab + warpx.moving_window_v * m_t_lab[i_buffer]); + m_buffer_domain_lab[i_buffer].setHi(m_moving_window_dir, zmin_lab + warpx.moving_window_v * m_t_lab[i_buffer]); + + // 5. Initialize buffer counter and z-positions of the i^th snapshot in + // boosted-frame and lab-frame + m_buffer_counter[i_buffer] = 0; + m_current_z_lab[i_buffer] = 0.0_rt; + m_current_z_boost[i_buffer] = 0.0_rt; + // Now Update Current Z Positions + m_current_z_boost[i_buffer] = UpdateCurrentZBoostCoordinate(m_t_lab[i_buffer], + warpx.gett_new(lev) ); + m_current_z_lab[i_buffer] = UpdateCurrentZLabCoordinate(m_t_lab[i_buffer], + warpx.gett_new(lev) ); + + + // 6. Compute ncells_lab required for writing Header file and potentially to generate + // Back-Transform geometry to ensure compatibility with plotfiles // + // For the z-dimension, number of cells in the lab-frame is + // computed using the coarsened cell-size in the lab-frame obtained using + // the ref_ratio at level, lev. + amrex::IntVect ref_ratio = WarpX::RefRatio(lev); + // Number of lab-frame cells in z-direction at level, lev + const int num_zcells_lab = static_cast( floor ( + ( zmax_lab - zmin_lab) + / dz_lab(warpx.getdt(lev), ref_ratio[AMREX_SPACEDIM-1] ) ) ); + // Take the max of 0 and num_zcells_lab + int Nz_lab = std::max( 0, num_zcells_lab ); + // Number of lab-frame cells in x-direction at level, lev + const int num_xcells_lab = static_cast( floor ( + ( diag_dom.hi(0) - diag_dom.lo(0) ) + / warpx.Geom(lev).CellSize(0) + ) ); + // Take the max of 0 and num_ycells_lab + int Nx_lab = std::max( 0, num_xcells_lab); +#if (AMREX_SPACEDIM == 3) + // Number of lab-frame cells in the y-direction at level, lev + const int num_ycells_lab = static_cast( floor ( + ( diag_dom.hi(1) - diag_dom.lo(1) ) + / warpx.Geom(lev).CellSize(1) + ) ); + // Take the max of 0 and num_xcells_lab + int Ny_lab = std::max( 0, num_ycells_lab ); + m_buffer_ncells_lab[i_buffer] = {Nx_lab, Ny_lab, Nz_lab}; +#else + int Ny_lab = 0; + m_buffer_ncells_lab[i_buffer] = {Nx_lab, Nz_lab}; +#endif + + // 7. Call funtion to create directories for customized output format + TMP_createLabFrameDirectories(i_buffer, lev); +} + +void +BTDiagnostics::DefineCellCenteredMultiFab(int lev) +{ + // Creating MultiFab to store cell-centered data in boosted-frame for the entire-domain + // This MultiFab will store all the user-requested fields in the boosted-frame + auto & warpx = WarpX::GetInstance(); + // The BoxArray is coarsened based on the user-defined coarsening ratio + amrex::BoxArray ba = warpx.boxArray(lev); + ba.coarsen(m_crse_ratio); + amrex::DistributionMapping dmap = warpx.DistributionMap(lev); + int ngrow = 1; + m_cell_centered_data[lev].reset( new amrex::MultiFab(ba, dmap, m_varnames.size(), ngrow) ); + +} + +void +BTDiagnostics::InitializeFieldFunctors (int lev) +{ + auto & warpx = WarpX::GetInstance(); + // Clear any pre-existing vector to release stored data + // This ensures that when domain is load-balanced, the functors point + // to the correct field-data pointers + m_all_field_functors[lev].clear(); + // For back-transformed data, all the components are cell-centered and stored in a single multifab. Therefore, size of functors at all levels is 1. + int num_BT_functors = 1; + m_all_field_functors[lev].resize(num_BT_functors); + m_cell_center_functors[lev].clear(); + m_cell_center_functors[lev].resize( m_varnames.size() ); + // 1. create an object of class BackTransformFunctor + for (int i = 0; i < num_BT_functors; ++i) + { + // coarsening ratio is not provided since the source MultiFab, m_cell_centered_data + // is coarsened based on the user-defined m_crse_ratio + m_all_field_functors[lev][i] = std::make_unique( + m_cell_centered_data[lev].get(), lev, m_varnames.size() ); + } + + // 2. Define all cell-centered functors required to compute cell-centere data + // Fill vector of cell-center functors for all components + // Only E,B, j, and rho are included in the cell-center functors for BackTransform Diags + for (int comp=0, n=m_cell_center_functors[lev].size(); comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Ey" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Ez" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Bx" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "By" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Bz" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jx" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jy" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jz" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "rho" ){ + // rho_new is stored in component 1 of rho_fp when using PSATD +#ifdef WARPX_USE_PSATD + amrex::MultiFab* rho_new = new amrex::MultiFab(*warpx.get_pointer_rho_fp(lev), amrex::make_alias, 1, 1); + m_cell_center_functors[lev][comp] = std::make_unique(rho_new, lev, m_crse_ratio); +#else + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_rho_fp(lev), lev, m_crse_ratio); +#endif + } + } + +} + +// Temporary function only to debug the current implementation. +// Will be replaced with plotfile/OpenPMD functionality +void +BTDiagnostics::TMP_createLabFrameDirectories(int i_buffer, int lev) +{ + // This function will include relevant code from class BackTransformedDiagnostics + // to create lab-frame directories. Note that here we will also add level, lev + // if required. + +} + +void +BTDiagnostics::PrepareFieldDataForOutput () +{ + auto & warpx = WarpX::GetInstance(); + // In this function, we will get cell-centered data for every level, lev, + // using the cell-center functors and their respective opeators() + // Call m_cell_center_functors->operator + for (int lev = 0; lev < nmax_lev; ++lev) { + int icomp_dst = 0; + for (int icomp = 0, n=m_cell_center_functors[0].size(); icompoperator()(*m_cell_centered_data[lev], icomp_dst); + icomp_dst += m_cell_center_functors[lev][icomp]->nComp(); + } + // Check that the proper number of user-requested components are cell-centered + AMREX_ALWAYS_ASSERT( icomp_dst == m_varnames.size() ); + // fill boundary call is required to average_down (flatten) data to + // the coarsest level. + m_cell_centered_data[lev]->FillBoundary(warpx.Geom(lev).periodicity() ); + } + // Flattening out MF over levels + + for (int lev = warpx.finestLevel(); lev > 0; --lev) { + CoarsenIO::Coarsen( *m_cell_centered_data[lev-1], *m_cell_centered_data[lev], 0, 0, m_varnames.size(), 0, WarpX::RefRatio(lev-1) ); + } +} + + +amrex::Real +BTDiagnostics::dz_lab (amrex::Real dt, amrex::Real ref_ratio) +{ + return PhysConst::c * dt * 1._rt/m_beta_boost * 1._rt/m_gamma_boost * 1._rt/ref_ratio; +} diff --git a/Source/Diagnostics/BackTransformedDiagnostic.cpp b/Source/Diagnostics/BackTransformedDiagnostic.cpp index d4db011b6f6..8318f455ef3 100644 --- a/Source/Diagnostics/BackTransformedDiagnostic.cpp +++ b/Source/Diagnostics/BackTransformedDiagnostic.cpp @@ -115,7 +115,7 @@ namespace /* Creates a dataset with the given cell dimensions, at the path "/native_fields/(field_name)". - Should be run only by the master rank. + Should be run only by the root rank. */ void output_create_field(const std::string& file_path, const std::string& field_path, const unsigned nx, const unsigned ny, const unsigned nz) @@ -175,7 +175,7 @@ namespace /* Resize an extendible dataset, suitable for storing particle data. - Should be run only by the master rank. + Should be run only by the root rank. */ long output_resize_particle_field(const std::string& file_path, const std::string& field_path, const long num_to_add) diff --git a/Source/Diagnostics/CMakeLists.txt b/Source/Diagnostics/CMakeLists.txt new file mode 100644 index 00000000000..223027478d8 --- /dev/null +++ b/Source/Diagnostics/CMakeLists.txt @@ -0,0 +1,18 @@ +target_sources(WarpX + PRIVATE + BackTransformedDiagnostic.cpp + Diagnostics.cpp + FieldIO.cpp + FullDiagnostics.cpp + MultiDiagnostics.cpp + ParticleIO.cpp + SliceDiagnostic.cpp + WarpXIO.cpp + WarpXOpenPMD.cpp + BTDiagnostics.cpp +) + +add_subdirectory(ComputeDiagFunctors) +add_subdirectory(FlushFormats) +add_subdirectory(ParticleDiag) +add_subdirectory(ReducedDiags) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H new file mode 100644 index 00000000000..b8702e715b2 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -0,0 +1,46 @@ +#ifndef WARPX_BACKTRANSFORMFUNCTOR_H_ +#define WARPX_BACKTRANSFORMFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +/** + * \brief Functor to back-transform cell-centered data and store result in mf_out + */ + +class +BackTransformFunctor final : public ComputeDiagFunctor +{ +public: + // This class will handle the slicing of the cell-centered field-data (stored as mf_src) + // It will Lorentz-Transform the sliced data from boosted-frame at z=zboost + // to lab-frame at z=zlab and add the lab-frame slice to the destination + // multifab that will be provided in the operator () function, for all ncomponents + // starting from icomp_dst. + // The operator() function will also need a few boosted-frame parameters, such as, + // current_z_boost of the corresponding snapshot, to generate appropriate slices + // in boosted-frame and lab-frame.i + // Unlike cell-center functor, coarsening ratio is not provided to this functor, + // since the data is coarsened when preparing fields in m_mf_cc (mf_src). + + /** Constructor description + * \param[in] mf_src cell-centered multifab containing all user-requested + fields in boosted-frame + * \param[in] lev level of multifab. + * \param[in] ncom number of components of mf_src to Lorentz-Transform + and store in destination multifab. + */ + + // initializing crse_ratio with 1, to accurately initialize ComputeDiagFunctor + + BackTransformFunctor( const amrex::MultiFab * const mf_src, const int lev, + const int ncomp, const amrex::IntVect crse_ratio= amrex::IntVect(1)); + + void operator()(amrex::MultiFab& mf_dst, int dcomp) const override {}; +private: + /** pointer to source multifab (cell-centered multi-component multifab) */ + amrex::MultiFab const * const m_mf_src = nullptr; + /** level at which m_mf_src is defined */ + int const m_lev; +}; + +#endif diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp new file mode 100644 index 00000000000..2d8651efe4b --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -0,0 +1,8 @@ +#include "BackTransformFunctor.H" + +BackTransformFunctor::BackTransformFunctor(amrex::MultiFab const * mf_src, int lev, + const int ncomp, const amrex::IntVect crse_ratio) + : ComputeDiagFunctor(ncomp, crse_ratio), m_mf_src(mf_src), m_lev(lev) +{ + // Get the right slice of each field in the CC MultiFab, transform it and store it in the output. +} diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt new file mode 100644 index 00000000000..0439c9d6798 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -0,0 +1,10 @@ +target_sources(WarpX + PRIVATE + CellCenterFunctor.cpp + DivBFunctor.cpp + DivEFunctor.cpp + RhoFunctor.cpp + PartPerCellFunctor.cpp + PartPerGridFunctor.cpp + BackTransformFunctor.cpp +) diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp index 3ee86dd7d58..a821861bb00 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp @@ -1,8 +1,6 @@ #include "CellCenterFunctor.H" #include "Utils/CoarsenIO.H" -using namespace amrex; - CellCenterFunctor::CellCenterFunctor(amrex::MultiFab const * mf_src, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) @@ -21,12 +19,12 @@ CellCenterFunctor::operator()(amrex::MultiFab& mf_dst, int dcomp) const nComp()==1, "The RZ averaging over modes must write into 1 single component"); auto& warpx = WarpX::GetInstance(); - MultiFab mf_dst_stag(m_mf_src->boxArray(), warpx.DistributionMap(m_lev), 1, m_mf_src->nGrowVect()); + amrex::MultiFab mf_dst_stag(m_mf_src->boxArray(), warpx.DistributionMap(m_lev), 1, m_mf_src->nGrowVect()); // Mode 0 - MultiFab::Copy(mf_dst_stag, *m_mf_src, 0, 0, 1, m_mf_src->nGrowVect()); + amrex::MultiFab::Copy(mf_dst_stag, *m_mf_src, 0, 0, 1, m_mf_src->nGrowVect()); for (int ic=1 ; ic < m_mf_src->nComp() ; ic += 2) { // All modes > 0 - MultiFab::Add(mf_dst_stag, *m_mf_src, ic, 0, 1, m_mf_src->nGrowVect()); + amrex::MultiFab::Add(mf_dst_stag, *m_mf_src, ic, 0, 1, m_mf_src->nGrowVect()); } CoarsenIO::Coarsen( mf_dst, mf_dst_stag, dcomp, 0, nComp(), 0, m_crse_ratio); } else { diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp index b0e4c48e8f9..3b2889b83d1 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.cpp @@ -1,8 +1,6 @@ #include "DivBFunctor.H" #include "Utils/CoarsenIO.H" -using namespace amrex; - DivBFunctor::DivBFunctor(const std::array arr_mf_src, const int lev, const amrex::IntVect crse_ratio, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_arr_mf_src(arr_mf_src), m_lev(lev) {} @@ -17,7 +15,7 @@ DivBFunctor::operator()(amrex::MultiFab& mf_dst, int dcomp) const constexpr int ng = 1; // A cell-centered divB multifab spanning the entire domain is generated // and divB is computed on the cell-center, with ng=1. - MultiFab divB( warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng ); + amrex::MultiFab divB( warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng ); warpx.ComputeDivB(divB, 0, m_arr_mf_src, WarpX::CellSize(m_lev) ); // Coarsen and Interpolate from divB to coarsened/reduced_domain mf_dst CoarsenIO::Coarsen( mf_dst, divB, dcomp, 0, nComp(), 0, m_crse_ratio); diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp index 418a1fa868b..adf78b3291a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.cpp @@ -1,8 +1,6 @@ #include "DivEFunctor.H" #include "Utils/CoarsenIO.H" -using namespace amrex; - DivEFunctor::DivEFunctor(const std::array arr_mf_src, const int lev, const amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, const int ncomp) @@ -20,8 +18,9 @@ DivEFunctor::operator()(amrex::MultiFab& mf_dst, const int dcomp) const constexpr int ng = 1; // For staggered and nodal calculations, divE is computed on the nodes. // The temporary divE MultiFab is generated to comply with the location of divE. - const BoxArray& ba = amrex::convert(warpx.boxArray(m_lev),IntVect::TheUnitVector()); - MultiFab divE(ba, warpx.DistributionMap(m_lev), 2*warpx.n_rz_azimuthal_modes-1, ng ); + const amrex::BoxArray& ba = amrex::convert(warpx.boxArray(m_lev), + amrex::IntVect::TheUnitVector()); + amrex::MultiFab divE(ba, warpx.DistributionMap(m_lev), 2*warpx.n_rz_azimuthal_modes-1, ng); warpx.ComputeDivE(divE, m_lev); #ifdef WARPX_DIM_RZ @@ -31,12 +30,12 @@ DivEFunctor::operator()(amrex::MultiFab& mf_dst, const int dcomp) const AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nComp()==1, "The RZ averaging over modes must write into 1 single component"); - MultiFab mf_dst_stag(divE.boxArray(), warpx.DistributionMap(m_lev), 1, divE.nGrowVect()); + amrex::MultiFab mf_dst_stag(divE.boxArray(), warpx.DistributionMap(m_lev), 1, divE.nGrowVect()); // Mode 0 - MultiFab::Copy(mf_dst_stag, divE, 0, 0, 1, divE.nGrowVect()); + amrex::MultiFab::Copy(mf_dst_stag, divE, 0, 0, 1, divE.nGrowVect()); for (int ic=1 ; ic < divE.nComp() ; ic += 2) { // Real part of all modes > 0 - MultiFab::Add(mf_dst_stag, divE, ic, 0, 1, divE.nGrowVect()); + amrex::MultiFab::Add(mf_dst_stag, divE, ic, 0, 1, divE.nGrowVect()); } CoarsenIO::Coarsen( mf_dst, mf_dst_stag, dcomp, 0, nComp(), 0, m_crse_ratio); } else { diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index 7006fb87c8a..297eee57686 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -3,5 +3,7 @@ CEXE_sources += PartPerCellFunctor.cpp CEXE_sources += PartPerGridFunctor.cpp CEXE_sources += DivBFunctor.cpp CEXE_sources += DivEFunctor.cpp +CEXE_sources += RhoFunctor.cpp +CEXE_sources += BackTransformFunctor.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics/ComputeDiagFunctors diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.cpp index ce41cc64ead..232a2348f2c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.cpp @@ -2,7 +2,7 @@ #include "WarpX.H" #include "Utils/CoarsenIO.H" -using namespace amrex; +using namespace amrex::literals; PartPerCellFunctor::PartPerCellFunctor(const amrex::MultiFab* mf_src, const int lev, amrex::IntVect crse_ratio, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev) @@ -22,7 +22,7 @@ PartPerCellFunctor::operator()(amrex::MultiFab& mf_dst, const int dcomp) const // the operations performend in the CoarsenAndInterpolate function. constexpr int ng = 1; // Temporary cell-centered, single-component MultiFab for storing particles per cell. - MultiFab ppc_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); + amrex::MultiFab ppc_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); // Set value to 0, and increment the value in each cell with ppc. ppc_mf.setVal(0._rt); // Compute ppc which includes a summation over all species. diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.cpp index 0dc3003d119..b61748a18e6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.cpp @@ -1,8 +1,6 @@ #include "PartPerGridFunctor.H" #include "Utils/CoarsenIO.H" -using namespace amrex; - PartPerGridFunctor::PartPerGridFunctor(const amrex::MultiFab * const mf_src, const int lev, const amrex::IntVect crse_ratio, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev) { @@ -16,19 +14,19 @@ void PartPerGridFunctor::operator()(amrex::MultiFab& mf_dst, const int dcomp) const { auto& warpx = WarpX::GetInstance(); - const Vector& npart_in_grid = warpx.GetPartContainer().NumberOfParticlesInGrid(m_lev); + const amrex::Vector& npart_in_grid = warpx.GetPartContainer().NumberOfParticlesInGrid(m_lev); // Guard cell is set to 1 for generality. However, for a cell-centered // output Multifab, mf_dst, the guard-cell data is not needed especially considering // the operations performend in the CoarsenAndInterpolate function. constexpr int ng = 1; // Temporary MultiFab containing number of particles per grid. // (stored as constant for all cells in each grid) - MultiFab ppg_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); + amrex::MultiFab ppg_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); #ifdef _OPENMP #pragma omp parallel #endif - for (MFIter mfi(ppg_mf); mfi.isValid(); ++mfi) { - ppg_mf[mfi].setVal(static_cast(npart_in_grid[mfi.index()])); + for (amrex::MFIter mfi(ppg_mf); mfi.isValid(); ++mfi) { + ppg_mf[mfi].setVal(static_cast(npart_in_grid[mfi.index()])); } // Coarsen and interpolate from ppg_mf to the output diagnostic MultiFab, mf_dst. diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H new file mode 100644 index 00000000000..ed31b845b6f --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H @@ -0,0 +1,45 @@ +#ifndef WARPX_RHOFUNCTOR_H_ +#define WARPX_RHOFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +/** + * \brief Functor to compute charge density rho into mf_out + */ +class +RhoFunctor final : public ComputeDiagFunctor +{ + +public: + + /** + * \brief Constructor + * + * \param[in] lev level of MultiFab + * \param[in] crse_ratio coarsening ratio for interpolation of field values + * from simulation MultiFabs to the output MultiFab mf_dst + * \param[in] convertRZmodes2cartesian if true, all RZ modes are averaged into one component + * \param[in] ncomp optional number of component of source MultiFab mf_src + * to be cell-centered in output MultiFab mf_dst + */ + RhoFunctor ( const int lev, const amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, const int ncomp=1 ); + + /** + * \brief Compute rho directly into mf_dst + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered data are stored + */ + virtual void operator() ( amrex::MultiFab& mf_dst, const int dcomp ) const override; + +private: + + // Level on which source MultiFab mf_src is defined in RZ geometry + int const m_lev; + + // Whether to average all modes into one component in RZ geometry + bool m_convertRZmodes2cartesian; +}; + +#endif // WARPX_RHOFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp new file mode 100644 index 00000000000..c5739ef21a1 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -0,0 +1,43 @@ +#include "WarpX.H" +#include "RhoFunctor.H" +#include "Utils/CoarsenIO.H" + +RhoFunctor::RhoFunctor ( const int lev, const amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian, const int ncomp ) + : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), + m_convertRZmodes2cartesian(convertRZmodes2cartesian) +{} + +void +RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp ) const +{ + auto& warpx = WarpX::GetInstance(); + auto& mypc = warpx.GetPartContainer(); + + // Deposit charge density + std::unique_ptr rho = mypc.GetChargeDensity( m_lev ); + +#ifdef WARPX_DIM_RZ + if (m_convertRZmodes2cartesian) { + // In cylindrical geometry, sum real part of all modes of rho in + // temporary MultiFab mf_dst_stag, and cell-center it to mf_dst + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + nComp()==1, + "The RZ averaging over modes must write into one single component"); + amrex::MultiFab mf_dst_stag( rho->boxArray(), warpx.DistributionMap(m_lev), 1, rho->nGrowVect() ); + // Mode 0 + amrex::MultiFab::Copy( mf_dst_stag, *rho, 0, 0, 1, rho->nGrowVect() ); + for (int ic=1 ; ic < rho->nComp() ; ic += 2) { + // Real part of all modes > 0 + amrex::MultiFab::Add( mf_dst_stag, *rho, ic, 0, 1, rho->nGrowVect() ); + } + CoarsenIO::Coarsen( mf_dst, mf_dst_stag, dcomp, 0, nComp(), 0, m_crse_ratio ); + } else { + CoarsenIO::Coarsen( mf_dst, *rho, dcomp, 0, nComp(), 0, m_crse_ratio ); + } +#else + // In Cartesian geometry, coarsen and interpolate from temporary MultiFab rho + // to output diagnostic MultiFab mf_dst + CoarsenIO::Coarsen( mf_dst, *rho, dcomp, 0, nComp(), 0, m_crse_ratio ); +#endif +} diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 3eb5b66a37b..3997c3b4b2f 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -18,82 +18,119 @@ class Diagnostics { public: Diagnostics (int i, std::string name); - ~Diagnostics (); + /** Virtual Destructor to handle clean destruction of derived classes */ + virtual ~Diagnostics () ; /** Pack (stack) all fields in the cell-centered output MultiFab m_mf_output. - * Fields are computed (e.g., cell-centered) on-the-fly using a functor.*/ + * + * Fields are computed (e.g., cell-centered or back-transformed) + on-the-fly using a functor. */ void ComputeAndPack (); - /** Flush m_mf_output and particles to file. */ - void Flush (); - /** Flush raw data */ - void FlushRaw (); + /** \brief Flush particle and field buffers to file using the FlushFormat member variable. + * + * This function should belong to class Diagnostics and not be virtual, as it flushes + * the particle buffers (name not found yet) and the field buffers (m_mf_output), both + * of which are members of Diagnostics. Yet, the implementation is left to derived classes + * for now because: + * - The functions underlying FlushFormat::WriteToFile expect a geometry object, which is + * WarpX::geom for FullDiagnostics but should be constructed for BTDiagnostics; + * - The functions underlying FlushFormat::WriteToFile do not support writing a buffer to file + * multiple times yet. + * When these are fixed, the implementation of Flush should be in Diagnostics.cpp + * \param[in] i_buffer index of the buffer data to be flushed. + */ + virtual void Flush (int i_buffer) = 0; /** Initialize pointers to main fields and allocate output multifab m_mf_output. */ void InitData (); + /** Initialize functors that store pointers to the fields requested by the user. + * + * Derived classes MUST implement this function, and it must allocate m_all_field_functors + * and fill it with ComputeDiagFunctor objects. + * The functions is called at intiailization and when the domain is decomposed + * during the simulation to load-balance. + * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. + */ + virtual void InitializeFieldFunctors (int lev) = 0; + /** whether to compute and pack data in output buffers at this time step + * \param[in] step current time step + * \param[in] force_flush if true, return true for any step + * \return bool, whether to compute and pack data + */ + virtual bool DoComputeAndPack(int step, bool force_flush=false) = 0; /** whether to flush at this time step * \param[in] step current time step * \param[in] force_flush if true, return true for any step * \return bool, whether to flush */ - bool DoDump (int step, bool force_flush=false); - /** Initialize functors that store pointers to the fields requested by the user. - * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. - */ - void InitializeFieldFunctors (int lev); + virtual bool DoDump (int step, int i_buffer, bool force_flush=false) = 0; /** Start a new iteration, i.e., dump has not been done yet. */ - void NewIteration () {m_already_done = false;}; + void NewIteration () {m_already_done = false;} + /** Perform necessary operations with user-defined diagnostic parameters + * to filter (coarsen, slice), compute (cell-center, back-transform), + * and flush the output data stored in buffers, m_mf_output. + * \param[in] step current timestep + * \param[in] force_flush used to force-fully write data stored in buffers. + */ + void FilterComputePackFlush (int step, bool force_flush=false); -private: - void ReadParameters (); - /** Append varnames with names for all modes of a field - * \param[in] field field name (includes component, e.g., Er) - * \param[in] ncomp number of components (modes, real and imag) +protected: + /** Read Parameters of the base Diagnostics class */ + bool BaseReadParameters (); + /** Initialize member variables of the base Diagnostics class. */ + void InitBaseData (); + /** Initialize m_mf_output vectors and data required to construct the buffers + * \param[in] i_buffer index of buffer for which the output MultiFab is defined. + * \param[in] lev level on which the output MultiFab is defined + */ + virtual void InitializeFieldBufferData (int i_buffer, int lev) = 0; + /** Initialize member variables and arrays specific to the diagnostics in the + * derived classes.(FullDiagnostics, BTDiagnostics) */ - void AddRZModesToOutputNames (const std::string& field, int ncomp); - /** Append m_all_field_functors[lev] with all modes of all components of E B and j - * and add name to varnames. - * \param[in] lev level on which source multifabs are defined. + virtual void DerivedInitData () {}; + /** This function initialized particle buffers (not implemented in diagnostics, yet) */ + virtual void InitializeParticleBuffer () = 0; + /** Prepare data (either fill-boundary or cell-centered data for + back-transform diagnostics) to be processed for diagnostics. */ - void AddRZModesToDiags (int lev); + virtual void PrepareFieldDataForOutput () {}; /** Name of diagnostics: runtime parameter given in the input file. */ std::string m_diag_name; /** Prefix for output directories */ std::string m_file_prefix; /** Index of diagnostics in MultiDiagnostics::alldiags */ int m_diag_index; - /** Determines timesteps at which diagnostics are written to file */ - IntervalsParser m_intervals; /** Names of each component requested by the user. * in cylindrical geometry, this list is appended with * automatically-constructed names for all modes of all fields */ amrex::Vector< std::string > m_varnames; - /** Vector of (pointers to) functors to compute output fields, per level, - * per component. This allows for simple operations (averaging to - * cell-center for standard EB fields) as well as more involved operations - * (back-transformed diagnostics, filtering, reconstructing cartesian - * fields in cylindrical). */ - amrex::Vector< amrex::Vector > > m_all_field_functors; - /** output multifab, where all fields are cell-centered and stacked */ - amrex::Vector< amrex::MultiFab > m_mf_output; - int nlev; /**< number of levels to output */ - int nmax_lev; /**< max_level to allocate output multifab and vector of field functors. */ - /** This class is responsible for flushing the data to file */ - FlushFormat* m_flush_format; - /** Whether to plot raw (i.e., NOT cell-centered) fields */ - bool m_plot_raw_fields = false; - /** Whether to plot guard cells of raw fields */ - bool m_plot_raw_fields_guards = false; - /** Whether to plot charge density rho in raw fields */ - bool m_plot_raw_rho = false; - /** Whether to plot F (charge conservation error) in raw fields */ - bool m_plot_raw_F = false; - /** Whether to dump the RZ modes */ - bool m_dump_rz_modes = false; + /** format for output files, "plotfile" or "openpmd" or "sensei" or "ascent" + * The checkpoint format is applicable for FullDiagnostics only. + */ + std::string m_format = "plotfile"; /** Whether this iteration has already been dumped, to avoid writing data twice */ int m_already_done = false; + /** This class is responsible for flushing the data to file */ + FlushFormat* m_flush_format; + /** output multifab, where all fields are computed (cell-centered or back-transformed) + * and stacked. + * The first vector is for total number of snapshots. (=1 for FullDiagnostics) + * The second vector is loops over the total number of levels. + */ + amrex::Vector< amrex::Vector< amrex::MultiFab > > m_mf_output; + // a particle buffer here? + int nlev; /**< number of levels to output */ + int nmax_lev; /**< max_level to allocate output multifab and vector of field functors. */ + /** Number of levels to be output*/ + int nlev_output; /** Name of species to write to file */ std::vector< std::string > m_species_names; /** Each element of this vector handles output for 1 species */ amrex::Vector< ParticleDiag > m_all_species; - + /** Vector of (pointers to) functors to compute output fields, per level, + * per component. This allows for simple operations (averaging to + * cell-center for standard EB fields) as well as more involved operations + * (back-transformed diagnostics, filtering, reconstructing cartesian + * fields in cylindrical). */ + amrex::Vector< amrex::Vector > > m_all_field_functors; /** Coarsening ratio such that, fields are averaged to the coarsened grid. * The ratio should render the grid to be coarsenable (as defined by AMReX). */ amrex::IntVect m_crse_ratio = amrex::IntVect(1); @@ -101,16 +138,8 @@ private: amrex::Vector< amrex::Real> m_lo; /** Higher corner of the diagnostics output, in physical coordinates */ amrex::Vector< amrex::Real> m_hi; - /** Define the cell-centered multifab m_mf_output depending on user-defined - * lo and hi and coarsening ratio. This MultiFab may have a different BoxArray and - * DistributionMap than field MultiFabs in the simulation. - * - * \param[in] lev level on which source multifabs are defined - */ - void DefineDiagMultiFab ( int lev ); - - /** format for output files, "plotfile" or "openpmd" */ - std::string m_format = "plotfile"; + /** Number of output buffers. The value is set to 1 for all FullDiagnostics */ + int m_num_buffers; }; #endif // WARPX_DIAGNOSTICS_H_ diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 5a0f29113c4..484ea88c935 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -14,12 +14,10 @@ #include "WarpX.H" #include "Utils/WarpXUtil.H" -using namespace amrex; Diagnostics::Diagnostics (int i, std::string name) : m_diag_name(name), m_diag_index(i) { - ReadParameters(); } Diagnostics::~Diagnostics () @@ -27,32 +25,24 @@ Diagnostics::~Diagnostics () delete m_flush_format; } -void -Diagnostics::ReadParameters () +bool +Diagnostics::BaseReadParameters () { auto & warpx = WarpX::GetInstance(); // Read list of fields requested by the user. - ParmParse pp(m_diag_name); + amrex::ParmParse pp(m_diag_name); m_file_prefix = "diags/" + m_diag_name; pp.query("file_prefix", m_file_prefix); - std::string period_string = "0"; - pp.query("period", period_string); - m_intervals = IntervalsParser(period_string); pp.query("format", m_format); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - m_format == "plotfile" || m_format == "openpmd" || - m_format == "checkpoint" || m_format == "ascent" || - m_format == "sensei", - ".format must be plotfile or openpmd or checkpoint or ascent or sensei"); - bool raw_specified = pp.query("plot_raw_fields", m_plot_raw_fields); - raw_specified += pp.query("plot_raw_fields_guards", m_plot_raw_fields_guards); bool varnames_specified = pp.queryarr("fields_to_plot", m_varnames); if (!varnames_specified){ m_varnames = {"Ex", "Ey", "Ez", "Bx", "By", "Bz", "jx", "jy", "jz"}; } - // set plot_rho to true of the users requests it, so that - // rho is computed at each iteration. - if (WarpXUtilStr::is_in(m_varnames, "rho")) warpx.setplot_rho(true); + // If user requests rho with back-transformed diagnostics, we set plot_rho=true + // and compute rho at each iteration + if (WarpXUtilStr::is_in(m_varnames, "rho") && WarpX::do_back_transformed_diagnostics) { + warpx.setplot_rho(true); + } // Sanity check if user requests to plot F if (WarpXUtilStr::is_in(m_varnames, "F")){ AMREX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -61,14 +51,11 @@ Diagnostics::ReadParameters () } // If user requests to plot proc_number for a serial run, // delete proc_number from fields_to_plot - if (ParallelDescriptor::NProcs() == 1){ + if (amrex::ParallelDescriptor::NProcs() == 1){ m_varnames.erase( std::remove(m_varnames.begin(), m_varnames.end(), "proc_number"), m_varnames.end()); } -#ifdef WARPX_DIM_RZ - pp.query("dump_rz_modes", m_dump_rz_modes); -#endif // Read user-defined physical extents for the output and store in m_lo and m_hi. m_lo.resize(AMREX_SPACEDIM); @@ -89,7 +76,7 @@ Diagnostics::ReadParameters () } // Initialize cr_ratio with default value of 1 for each dimension. - Vector cr_ratio(AMREX_SPACEDIM, 1); + amrex::Vector cr_ratio(AMREX_SPACEDIM, 1); // Read user-defined coarsening ratio for the output MultiFab. bool cr_specified = pp.queryarr("coarsening_ratio", cr_ratio); if (cr_specified) { @@ -100,54 +87,60 @@ Diagnostics::ReadParameters () bool species_specified = pp.queryarr("species", m_species_names); + bool checkpoint_compatibility = false; if (m_format == "checkpoint"){ - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - raw_specified == false && - varnames_specified == false && + if ( varnames_specified == false && lo_specified == false && hi_specified == false && cr_specified == false && - species_specified == false, - "For a checkpoint output, cannot specify these parameters as all data must be dumped " - "to file for a restart"); + species_specified == false ) checkpoint_compatibility = true; } + return checkpoint_compatibility; } + void Diagnostics::InitData () { - Print()<<"Diagnostics::InitData\n"; + // initialize member variables and arrays in base class::Diagnostics + InitBaseData(); + // initialize member variables and arrays specific to each derived class + // (FullDiagnostics, BTDiagnostics, etc.) + DerivedInitData(); + // loop over all buffers + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + // loop over all levels + for (int lev = 0; lev < nmax_lev; ++lev) { + // allocate and initialize m_all_field_functors depending on diag type + InitializeFieldFunctors(lev); + // Initialize field buffer data, m_mf_output + InitializeFieldBufferData(i_buffer, lev); + } + } + // When particle buffers, m_particle_buffers are included, they will be initialized here + InitializeParticleBuffer(); + +} + + +void +Diagnostics::InitBaseData () +{ auto & warpx = WarpX::GetInstance(); - // Number of levels + // Number of levels in the simulation at the current timestep nlev = warpx.finestLevel() + 1; + // default number of levels to be output = nlev + nlev_output = nlev; // Maximum number of levels that will be allocated in the simulation nmax_lev = warpx.maxLevel() + 1; - m_mf_output.resize( nmax_lev ); m_all_field_functors.resize( nmax_lev ); - for ( int lev=0; lev= m_all_field_functors[0].size() - - // Initialize member variable m_mf_output depending on m_crse_ratio, m_lo and m_hi - DefineDiagMultiFab( lev ); - } - - const MultiParticleContainer& mpc = warpx.GetPartContainer(); - // If not specified, dump all species - if (m_species_names.size() == 0) m_species_names = mpc.GetSpeciesNames(); - // Initialize one ParticleDiag per species requested - for (int i=0; ioperator()(m_mf_output[lev], icomp_dst); - // update the index of the next component to fill - icomp_dst += m_all_field_functors[lev][icomp]->nComp(); - } - // Check that the proper number of components of mf_avg were updated. - AMREX_ALWAYS_ASSERT( icomp_dst == m_varnames.size() ); - } -} - -void -Diagnostics::Flush () -{ - auto & warpx = WarpX::GetInstance(); - m_flush_format->WriteToFile( - m_varnames, m_mf_output, warpx.Geom(), warpx.getistep(), - warpx.gett_new(0), m_all_species, nlev, m_file_prefix, - m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F); -} - -void -Diagnostics::FlushRaw () {} - -bool -Diagnostics::DoDump (int step, bool force_flush) -{ - if (m_already_done) return false; - if ( force_flush || (m_intervals.contains(step+1)) ){ - m_already_done = true; - return true; + // allocate vector of buffers then allocate vector of levels for each buffer + m_mf_output.resize( m_num_buffers ); + for (int i = 0; i < m_num_buffers; ++i) { + m_mf_output[i].resize( nmax_lev ); } - return false; } void -Diagnostics::AddRZModesToDiags (int lev) +Diagnostics::ComputeAndPack () { -#ifdef WARPX_DIM_RZ - - if (!m_dump_rz_modes) return; - - auto & warpx = WarpX::GetInstance(); - int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); - // Make sure all multifabs have the same number of components - for (int dim=0; dim<3; dim++){ - AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); - AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); - AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); - } - - // Check if divE is requested - // If so, all components will be written out - bool divE_requested = false; - for (int comp = 0; comp < m_varnames.size(); comp++) { - if ( m_varnames[comp] == "divE" ) { - divE_requested = true; - } - } - - // First index of m_all_field_functors[lev] where RZ modes are stored - int icomp = m_all_field_functors[0].size(); - const std::array coord {"r", "theta", "z"}; - - // Er, Etheta, Ez, Br, Btheta, Bz, jr, jtheta, jz - // Each of them being a multi-component multifab - int n_new_fields = 9; - if (divE_requested) { - n_new_fields += 1; - } - m_all_field_functors[lev].resize( m_all_field_functors[0].size() + n_new_fields ); - // E - for (int dim=0; dim<3; dim++){ - // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Efield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); - AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.get_pointer_Efield_aux(0, 0)->nComp()); - icomp += 1; - } - // B - for (int dim=0; dim<3; dim++){ - // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Bfield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); - AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.get_pointer_Bfield_aux(0, 0)->nComp()); - icomp += 1; - } - // j - for (int dim=0; dim<3; dim++){ - // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_current_fp(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); - icomp += 1; - AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.get_pointer_current_fp(0, 0)->nComp()); - } - // divE - if (divE_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, - m_crse_ratio, false, ncomp_multimodefab); - icomp += 1; - AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); - } - // Sum the number of components in input vector m_all_field_functors - // and check that it corresponds to the number of components in m_varnames - // and m_mf_output - int ncomp_from_src = 0; - for (int i=0; inComp(); - } - AMREX_ALWAYS_ASSERT( ncomp_from_src == m_varnames.size() ); -#endif -} - -void -Diagnostics::AddRZModesToOutputNames (const std::string& field, int ncomp){ -#ifdef WARPX_DIM_RZ - // In cylindrical geometry, real and imag part of each mode are also - // dumped to file separately, so they need to be added to m_varnames - m_varnames.push_back( field + "_0_real" ); - for (int ic=1 ; ic < (ncomp+1)/2 ; ic += 1) { - m_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); - m_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); - } -#endif -} - -void -Diagnostics::DefineDiagMultiFab ( int lev ) { - auto & warpx = WarpX::GetInstance(); - amrex::RealBox diag_dom; - bool use_warpxba = true; - const IntVect blockingFactor = warpx.blockingFactor( lev ); - - // Default BoxArray and DistributionMap for initializing the output MultiFab, m_mf_output. - BoxArray ba = warpx.boxArray(lev); - DistributionMapping dmap = warpx.DistributionMap(lev); - - // Check if warpx BoxArray is coarsenable. - AMREX_ALWAYS_ASSERT_WITH_MESSAGE ( - ba.coarsenable(m_crse_ratio), - "Invalid coarsening ratio for warpx boxArray. Must be an integer divisor of the blocking factor." - ); - - // Find if user-defined physical dimensions are different from the simulation domain. - for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - // To ensure that the diagnostic lo and hi are within the domain defined at level, lev. - diag_dom.setLo(idim, max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); - diag_dom.setHi(idim, min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); - if ( fabs(warpx.Geom(lev).ProbLo(idim) - diag_dom.lo(idim)) - > warpx.Geom(lev).CellSize(idim) ) - use_warpxba = false; - if ( fabs(warpx.Geom(lev).ProbHi(idim) - diag_dom.hi(idim)) - > warpx.Geom(lev).CellSize(idim) ) - use_warpxba = false; - - // User-defined value for coarsening should be an integer divisor of - // blocking factor at level, lev. - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( blockingFactor[idim] % m_crse_ratio[idim]==0, - " coarsening ratio must be integer divisor of blocking factor"); - } - - - if (use_warpxba == false) { - // Following are the steps to compute the lo and hi index corresponding to user-defined - // m_lo and m_hi using the same resolution as the simulation at level, lev. - IntVect lo(0); - IntVect hi(1); - for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - // lo index with same cell-size as simulation at level, lev. - lo[idim] = max( static_cast( floor ( - ( diag_dom.lo(idim) - warpx.Geom(lev).ProbLo(idim)) / - warpx.Geom(lev).CellSize(idim)) ), 0 ); - // hi index with same cell-size as simulation at level, lev. - hi[idim] = max( static_cast ( ceil ( - ( diag_dom.hi(idim) - warpx.Geom(lev).ProbLo(idim)) / - warpx.Geom(lev).CellSize(idim) ) ), 0) - 1 ; - // if hi<=lo, then hi = lo + 1, to ensure one cell in that dimension - if ( hi[idim] <= lo[idim] ) { - hi[idim] = lo[idim] + 1; - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - m_crse_ratio[idim]==1, "coarsening ratio in reduced dimension must be 1." - ); + // prepare the field-data necessary to compute output data + PrepareFieldDataForOutput(); + // compute the necessary fields and stiore result in m_mf_output. + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + for(int lev=0; levoperator()(m_mf_output[i_buffer][lev], icomp_dst); + // update the index of the next component to fill + icomp_dst += m_all_field_functors[lev][icomp]->nComp(); } - } - - // Box for the output MultiFab corresponding to the user-defined physical co-ordinates at lev. - Box diag_box( lo, hi ); - // Define box array - BoxArray diag_ba; - diag_ba.define(diag_box); - ba = diag_ba.maxSize( warpx.maxGridSize( lev ) ); - // At this point in the code, the BoxArray, ba, is defined with the same index space and - // resolution as the simulation, at level, lev. - // Coarsen and refine so that the new BoxArray is coarsenable. - ba.coarsen(m_crse_ratio).refine(m_crse_ratio); - - // Update the physical co-ordinates m_lo and m_hi using the final index values - // from the coarsenable, cell-centered BoxArray, ba. - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - m_lo[idim] = warpx.Geom(lev).ProbLo(idim) + warpx.Geom(lev).CellSize(idim)/2.0_rt + - ba.getCellCenteredBox(0).smallEnd(idim) * warpx.Geom(lev).CellSize(idim); - m_hi[idim] = warpx.Geom(lev).ProbLo(idim) + warpx.Geom(lev).CellSize(idim)/2.0_rt + - ba.getCellCenteredBox( ba.size()-1 ).bigEnd(idim) * warpx.Geom(lev).CellSize(idim); + // Check that the proper number of components of mf_avg were updated. + AMREX_ALWAYS_ASSERT( icomp_dst == m_varnames.size() ); } } - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - m_crse_ratio.min() > 0, "Coarsening ratio must be non-zero."); - // The BoxArray is coarsened based on the user-defined coarsening ratio. - ba.coarsen(m_crse_ratio); - // Generate a new distribution map if the physical m_lo and m_hi for the output - // is different from the lo and hi physical co-ordinates of the simulation domain. - if (use_warpxba == false) dmap = DistributionMapping{ba}; - // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers. - int ngrow = (m_format == "sensei") ? 1 : 0; - m_mf_output[lev] = MultiFab(ba, dmap, m_varnames.size(), ngrow); } void -Diagnostics::InitializeFieldFunctors (int lev) +Diagnostics::FilterComputePackFlush (int step, bool force_flush) { - auto & warpx = WarpX::GetInstance(); - // Clear any pre-existing vector to release stored data. - m_all_field_functors[lev].clear(); + if ( DoComputeAndPack (step, force_flush) ) { + ComputeAndPack(); - m_all_field_functors[lev].resize( m_varnames.size() ); - // Fill vector of functors for all components except individual cylindrical modes. - for (int comp=0, n=m_all_field_functors[lev].size(); comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jy" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "rho" ){ - // rho_new is stored in component 1 of rho_fp when using PSATD -#ifdef WARPX_USE_PSATD - MultiFab* rho_new = new MultiFab(*warpx.get_pointer_rho_fp(lev), amrex::make_alias, 1, 1); - m_all_field_functors[lev][comp] = std::make_unique(rho_new, lev, m_crse_ratio); -#else - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_rho_fp(lev), lev, m_crse_ratio); -#endif - } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "part_per_cell" ){ - m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "part_per_grid" ){ - m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio); + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + if ( !DoDump (step, i_buffer, force_flush) ) continue; + Flush(i_buffer); } + } - AddRZModesToDiags( lev ); + } diff --git a/Source/Diagnostics/FieldIO.H b/Source/Diagnostics/FieldIO.H index 7533c847b37..cfe58d9272c 100644 --- a/Source/Diagnostics/FieldIO.H +++ b/Source/Diagnostics/FieldIO.H @@ -25,63 +25,6 @@ AverageAndPackScalarField( amrex::MultiFab& mf_avg, const amrex::DistributionMapping& dm, const int dcomp, const int ngrow ); -void -WriteRawField( const amrex::MultiFab& F, - const amrex::DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const std::string& field_name, - const int lev, const bool plot_guards ); - -void -WriteZeroRawField( const amrex::MultiFab& F, - const amrex::DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const std::string& field_name, - const int lev, const int ng ); - -void -WriteCoarseScalar( const std::string field_name, - const std::unique_ptr& F_cp, - const std::unique_ptr& F_fp, - const amrex::DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const int lev, const bool plot_guards, - const int icomp=0 ); - -void -WriteCoarseVector( const std::string field_name, - const std::unique_ptr& Fx_cp, - const std::unique_ptr& Fy_cp, - const std::unique_ptr& Fz_cp, - const std::unique_ptr& Fx_fp, - const std::unique_ptr& Fy_fp, - const std::unique_ptr& Fz_fp, - const amrex::DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const int lev, const bool plot_guards ); - -std::unique_ptr -getInterpolatedScalar( - const amrex::MultiFab& F_cp, const amrex::MultiFab& F_fp, - const amrex::DistributionMapping& dm, const int r_ratio, - const amrex::Real* dx, const int ngrow ); - -std::array, 3> -getInterpolatedVector( - const std::unique_ptr& Fx_cp, - const std::unique_ptr& Fy_cp, - const std::unique_ptr& Fz_cp, - const std::unique_ptr& Fx_fp, - const std::unique_ptr& Fy_fp, - const std::unique_ptr& Fz_fp, - const amrex::DistributionMapping& dm, - const int r_ratio, const amrex::Real* dx, - const int ngrow ); - #ifdef WARPX_USE_OPENPMD void setOpenPMDUnit( openPMD::Mesh mesh, const std::string field_name ); diff --git a/Source/Diagnostics/FieldIO.cpp b/Source/Diagnostics/FieldIO.cpp index beff4828ed0..3ee88866502 100644 --- a/Source/Diagnostics/FieldIO.cpp +++ b/Source/Diagnostics/FieldIO.cpp @@ -14,9 +14,6 @@ # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif -#include -#include - #ifdef WARPX_USE_OPENPMD # include #endif @@ -215,271 +212,3 @@ AverageAndPackScalarField (MultiFab& mf_avg, amrex::Abort("Unknown staggering."); } } - -/** \brief Write the data from MultiFab `F` into the file `filename` - * as a raw field (i.e. no interpolation to cell centers). - * Write guard cells if `plot_guards` is True. - */ -void -WriteRawField( const MultiFab& F, const DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const std::string& field_name, - const int lev, const bool plot_guards ) -{ - std::string prefix = amrex::MultiFabFileFullPrefix(lev, - filename, level_prefix, field_name); - - if (plot_guards) { - // Dump original MultiFab F - VisMF::Write(F, prefix); - } else { - // Copy original MultiFab into one that does not have guard cells - MultiFab tmpF( F.boxArray(), dm, F.nComp(), 0); - MultiFab::Copy(tmpF, F, 0, 0, F.nComp(), 0); - VisMF::Write(tmpF, prefix); - } - -} - -/** \brief Write a multifab of the same shape as `F` but filled with 0. - * (The shape includes guard cells if `plot_guards` is True.) - * This is mainly needed because the yt reader requires all levels of the - * coarse/fine patch to be written, but WarpX does not have data for - * the coarse patch of level 0 (meaningless). - */ -void -WriteZeroRawField( const MultiFab& F, const DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const std::string& field_name, - const int lev, const int ng ) -{ - std::string prefix = amrex::MultiFabFileFullPrefix(lev, - filename, level_prefix, field_name); - - MultiFab tmpF(F.boxArray(), dm, F.nComp(), ng); - tmpF.setVal(0.); - VisMF::Write(tmpF, prefix); -} - -/** \brief Write the coarse scalar multifab `F_cp` to the file `filename` - * *after* sampling/interpolating its value on the fine grid corresponding - * to `F_fp`. This is mainly needed because the yt reader requires the - * coarse and fine patch to have the same shape. - */ -void -WriteCoarseScalar( const std::string field_name, - const std::unique_ptr& F_cp, - const std::unique_ptr& F_fp, - const DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const int lev, const bool plot_guards, - const int icomp ) -{ - int ng = 0; - if (plot_guards) ng = F_fp->nGrow(); - - if (lev == 0) { - // No coarse field for level 0: instead write a MultiFab - // filled with 0, with the same number of cells as the _fp field - WriteZeroRawField( *F_fp, dm, filename, level_prefix, field_name+"_cp", lev, ng ); - } else { - // Create an alias to the component `icomp` of F_cp - MultiFab F_comp(*F_cp, amrex::make_alias, icomp, 1); - // Interpolate coarse data onto fine grid - const int r_ratio = WarpX::GetInstance().refRatio(lev-1)[0]; - const Real* dx = WarpX::GetInstance().Geom(lev-1).CellSize(); - auto F = getInterpolatedScalar( F_comp, *F_fp, dm, r_ratio, dx, ng ); - // Write interpolated raw data - WriteRawField( *F, dm, filename, level_prefix, field_name+"_cp", lev, plot_guards ); - } -} - -/** \brief Write the coarse vector multifab `F*_cp` to the file `filename` - * *after* sampling/interpolating its value on the fine grid corresponding - * to `F*_fp`. This is mainly needed because the yt reader requires the - * coarse and fine patch to have the same shape. - */ -void -WriteCoarseVector( const std::string field_name, - const std::unique_ptr& Fx_cp, - const std::unique_ptr& Fy_cp, - const std::unique_ptr& Fz_cp, - const std::unique_ptr& Fx_fp, - const std::unique_ptr& Fy_fp, - const std::unique_ptr& Fz_fp, - const DistributionMapping& dm, - const std::string& filename, - const std::string& level_prefix, - const int lev, const bool plot_guards ) -{ - int ng = 0; - if (plot_guards) ng = Fx_fp->nGrow(); - - if (lev == 0) { - // No coarse field for level 0: instead write a MultiFab - // filled with 0, with the same number of cells as the _fp field - WriteZeroRawField( *Fx_fp, dm, filename, level_prefix, field_name+"x_cp", lev, ng ); - WriteZeroRawField( *Fy_fp, dm, filename, level_prefix, field_name+"y_cp", lev, ng ); - WriteZeroRawField( *Fz_fp, dm, filename, level_prefix, field_name+"z_cp", lev, ng ); - } else { - // Interpolate coarse data onto fine grid - const int r_ratio = WarpX::GetInstance().refRatio(lev-1)[0]; - const Real* dx = WarpX::GetInstance().Geom(lev-1).CellSize(); - auto F = getInterpolatedVector( Fx_cp, Fy_cp, Fz_cp, Fx_fp, Fy_fp, Fz_fp, - dm, r_ratio, dx, ng ); - // Write interpolated raw data - WriteRawField( *F[0], dm, filename, level_prefix, field_name+"x_cp", lev, plot_guards ); - WriteRawField( *F[1], dm, filename, level_prefix, field_name+"y_cp", lev, plot_guards ); - WriteRawField( *F[2], dm, filename, level_prefix, field_name+"z_cp", lev, plot_guards ); - } -} - -/** \brief Samples/Interpolates the coarse scalar multifab `F_cp` on the - * fine grid associated with the fine multifab `F_fp`. - */ -std::unique_ptr -getInterpolatedScalar( - const MultiFab& F_cp, const MultiFab& F_fp, - const DistributionMapping& dm, const int r_ratio, - const Real* /*dx*/, const int ngrow ) -{ - // Prepare the structure that will contain the returned fields - std::unique_ptr interpolated_F; - interpolated_F.reset( new MultiFab(F_fp.boxArray(), dm, 1, ngrow) ); - interpolated_F->setVal(0.); - - // Loop through the boxes and interpolate the values from the _cp data -#ifdef _OPENMP -#pragma omp parallel -#endif - { - FArrayBox ffab; // Temporary array ; contains interpolated fields - for (MFIter mfi(*interpolated_F); mfi.isValid(); ++mfi) - { - Box finebx = mfi.fabbox(); - finebx.coarsen(r_ratio).refine(r_ratio); // so that finebx is coarsenable - - const FArrayBox& cfab = (F_cp)[mfi]; - ffab.resize(finebx); - - // - Fully nodal - if ( F_fp.is_nodal() ){ - IntVect refinement_vector{AMREX_D_DECL(r_ratio, r_ratio, r_ratio)}; - node_bilinear_interp.interp(cfab, 0, ffab, 0, 1, - finebx, refinement_vector, {}, {}, {}, 0, 0, RunOn::Cpu); - } else { - amrex::Abort("Unknown field staggering."); - } - - // Add temporary array to the returned structure - const Box& bx = (*interpolated_F)[mfi].box(); - (*interpolated_F)[mfi].plus(ffab, bx, bx, 0, 0, 1); - } - } - return interpolated_F; -} - -/** \brief Samples/Interpolates the coarse vector multifab `F*_cp` on the - * fine grid associated with the fine multifab `F*_fp`. - */ -std::array, 3> -getInterpolatedVector( - const std::unique_ptr& Fx_cp, - const std::unique_ptr& Fy_cp, - const std::unique_ptr& Fz_cp, - const std::unique_ptr& Fx_fp, - const std::unique_ptr& Fy_fp, - const std::unique_ptr& Fz_fp, - const DistributionMapping& dm, const int r_ratio, - const Real* dx, const int ngrow ) -{ - - // Prepare the structure that will contain the returned fields - std::array, 3> interpolated_F; - interpolated_F[0].reset( new MultiFab(Fx_fp->boxArray(), dm, 1, ngrow) ); - interpolated_F[1].reset( new MultiFab(Fy_fp->boxArray(), dm, 1, ngrow) ); - interpolated_F[2].reset( new MultiFab(Fz_fp->boxArray(), dm, 1, ngrow) ); - for (int i=0; i<3; i++) interpolated_F[i]->setVal(0.); - - // Loop through the boxes and interpolate the values from the _cp data - const int use_limiter = 0; -#ifdef _OPENMP -#pragma omp parallel -#endif - { - std::array ffab; // Temporary array ; contains interpolated fields - for (MFIter mfi(*interpolated_F[0]); mfi.isValid(); ++mfi) - { - Box ccbx = mfi.fabbox(); - ccbx.enclosedCells(); - ccbx.coarsen(r_ratio).refine(r_ratio); // so that ccbx is coarsenable - - const FArrayBox& cxfab = (*Fx_cp)[mfi]; - const FArrayBox& cyfab = (*Fy_cp)[mfi]; - const FArrayBox& czfab = (*Fz_cp)[mfi]; - ffab[0].resize(amrex::convert(ccbx,(*Fx_fp)[mfi].box().type())); - ffab[1].resize(amrex::convert(ccbx,(*Fy_fp)[mfi].box().type())); - ffab[2].resize(amrex::convert(ccbx,(*Fz_fp)[mfi].box().type())); - - // - Face centered, in the same way as B on a Yee grid - if ( (*Fx_fp)[mfi].box().type() == IntVect{AMREX_D_DECL(1,0,0)} ){ -#if (AMREX_SPACEDIM == 3) - amrex_interp_div_free_bfield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(cyfab), - BL_TO_FORTRAN_ANYD(czfab), - dx, &r_ratio, &use_limiter); -#else - amrex_interp_div_free_bfield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(czfab), - dx, &r_ratio, &use_limiter); - amrex_interp_cc_bfield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(cyfab), - &r_ratio, &use_limiter); -#endif - // - Edge centered, in the same way as E on a Yee grid - } else if ( (*Fx_fp)[mfi].box().type() == IntVect{AMREX_D_DECL(0,1,1)} ){ -#if (AMREX_SPACEDIM == 3) - amrex_interp_efield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(cyfab), - BL_TO_FORTRAN_ANYD(czfab), - &r_ratio, &use_limiter); -#else - amrex_interp_efield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(czfab), - &r_ratio,&use_limiter); - amrex_interp_nd_efield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(cyfab), - &r_ratio); -#endif - } else { - amrex::Abort("Unknown field staggering."); - } - - // Add temporary array to the returned structure - for (int i = 0; i < 3; ++i) { - const Box& bx = (*interpolated_F[i])[mfi].box(); - (*interpolated_F[i])[mfi].plus(ffab[i], bx, bx, 0, 0, 1); - } - } - } - return interpolated_F; -} diff --git a/Source/Diagnostics/FlushFormats/CMakeLists.txt b/Source/Diagnostics/FlushFormats/CMakeLists.txt new file mode 100644 index 00000000000..478db3750d4 --- /dev/null +++ b/Source/Diagnostics/FlushFormats/CMakeLists.txt @@ -0,0 +1,13 @@ +target_sources(WarpX + PRIVATE + FlushFormatAscent.cpp + FlushFormatCheckpoint.cpp + FlushFormatPlotfile.cpp +) + +if(WarpX_HAVE_OPENPMD) + target_sources(WarpX + PRIVATE + FlushFormatOpenPMD.cpp + ) +endif() diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index fc685dc056d..de548a84847 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -275,14 +275,6 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); - real_names.push_back("Ex"); - real_names.push_back("Ey"); - real_names.push_back("Ez"); - - real_names.push_back("Bx"); - real_names.push_back("By"); - real_names.push_back("Bz"); - #ifdef WARPX_DIM_RZ real_names.push_back("theta"); #endif diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 0979c02164c..4fdabf9e4d1 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -3,10 +3,68 @@ #include "Diagnostics.H" -class FullDiagnostics : public Diagnostics +class +FullDiagnostics final : public Diagnostics { public: FullDiagnostics (int i, std::string name); +private: + /** Read user-requested parameters for full diagnostics */ + void ReadParameters (); + /** Determines timesteps at which full diagnostics are written to file */ + IntervalsParser m_intervals; + /** Whether to plot raw (i.e., NOT cell-centered) fields */ + bool m_plot_raw_fields = false; + /** Whether to plot guard cells of raw fields */ + bool m_plot_raw_fields_guards = false; + /** Whether to plot charge density rho in raw fields */ + bool m_plot_raw_rho = false; + /** Whether to plot F (charge conservation error) in raw fields */ + bool m_plot_raw_F = false; + /** Flush m_mf_output and particles to file for the i^th buffer */ + void Flush (int i_buffer) override; + /** Flush raw data */ + void FlushRaw (); + /** whether to compute and pack cell-centered data in m_mf_output + * \param[in] step current time step + * \param[in] force_flush if true, return true for any step since output must be + computed in this case + * \return bool, whether to flush + */ + bool DoComputeAndPack (int step, bool force_flush=false) override; + /** whether to flush at this time step + * \param[in] step current time step + * \param[in] force_flush if true, return true for any step + * \return bool, whether to flush + */ + bool DoDump (int step, int i_buffer, bool force_flush=false) override; + /** Append varnames with names for all modes of a field + * \param[in] field field name (includes component, e.g., Er) + * \param[in] ncomp number of components (modes, real and imag) + */ + void AddRZModesToOutputNames (const std::string& field, int ncomp); + /** Append m_all_field_functors[lev] with all modes of all components of E B and j + * and add name to varnames. + * \param[in] lev level on which source multifabs are defined. + */ + void AddRZModesToDiags (int lev); + /** Whether to dump the RZ modes */ + bool m_dump_rz_modes = false; + /** Define the cell-centered multifab m_mf_output depending on user-defined + * lo and hi and coarsening ratio. This MultiFab may have a different BoxArray and + * DistributionMap than field MultiFabs in the simulation. + * + * \param[in] lev level on which source multifabs are defined + */ + void InitializeFieldBufferData ( int i_buffer, int lev ) override; + /** Initialize functors that store pointers to the fields requested by the user. + * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. + */ + void InitializeFieldFunctors (int lev) override; + void InitializeParticleBuffer () override; + /** Prepare field data to be used for diagnostics */ + void PrepareFieldDataForOutput () override; + }; #endif // WARPX_FULLDIAGNOSTICS_H_ diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index c115a88bcc8..889ec6d3c5d 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -1,7 +1,397 @@ #include "FullDiagnostics.H" +#include "WarpX.H" +#include "ComputeDiagFunctors/ComputeDiagFunctor.H" +#include "ComputeDiagFunctors/CellCenterFunctor.H" +#include "ComputeDiagFunctors/PartPerCellFunctor.H" +#include "ComputeDiagFunctors/PartPerGridFunctor.H" +#include "ComputeDiagFunctors/DivBFunctor.H" +#include "ComputeDiagFunctors/DivEFunctor.H" +#include "ComputeDiagFunctors/RhoFunctor.H" +#include "FlushFormats/FlushFormat.H" +#include "FlushFormats/FlushFormatPlotfile.H" +#include "FlushFormats/FlushFormatCheckpoint.H" +#include "FlushFormats/FlushFormatAscent.H" +#ifdef WARPX_USE_OPENPMD +# include "FlushFormats/FlushFormatOpenPMD.H" +#endif +#include +#include +using namespace amrex::literals; -using namespace amrex; FullDiagnostics::FullDiagnostics (int i, std::string name) : Diagnostics(i, name) -{} +{ + ReadParameters(); +} + +void +FullDiagnostics::InitializeParticleBuffer () +{ + // When particle buffers are included, the vector of particle containers + // must be allocated in this function. + // Initialize data in the base class Diagnostics + auto & warpx = WarpX::GetInstance(); + + const MultiParticleContainer& mpc = warpx.GetPartContainer(); + // If not specified, dump all species + if (m_species_names.size() == 0) m_species_names = mpc.GetSpeciesNames(); + // Initialize one ParticleDiag per species requested + for (int i=0; i.format must be plotfile or openpmd or checkpoint or ascent or sensei"); + std::string period_string = "0"; + pp.query("period", period_string); + m_intervals = IntervalsParser(period_string); + bool raw_specified = pp.query("plot_raw_fields", m_plot_raw_fields); + raw_specified += pp.query("plot_raw_fields_guards", m_plot_raw_fields_guards); + +#ifdef WARPX_DIM_RZ + pp.query("dump_rz_modes", m_dump_rz_modes); +#endif + + if (m_format == "checkpoint"){ + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + raw_specified == false && + checkpoint_compatibility == true, + "For a checkpoint output, cannot specify these parameters as all data must be dumped " + "to file for a restart"); + } + // Number of buffers = 1 for FullDiagnostics. + // It is used to allocate the number of output multu-level MultiFab, m_mf_output + m_num_buffers = 1; +} + +void +FullDiagnostics::Flush ( int i_buffer ) +{ + // This function should be moved to Diagnostics when plotfiles/openpmd format + // is supported for BackTransformed Diagnostics, in BTDiagnostics class. + auto & warpx = WarpX::GetInstance(); + m_flush_format->WriteToFile( + m_varnames, m_mf_output[i_buffer], warpx.Geom(), warpx.getistep(), + warpx.gett_new(0), m_all_species, nlev_output, m_file_prefix, + m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F); + + FlushRaw(); +} + +void +FullDiagnostics::FlushRaw () {} + + +bool +FullDiagnostics::DoDump (int step, int i_buffer, bool force_flush) +{ + if (m_already_done) return false; + if ( force_flush || (m_intervals.contains(step+1)) ){ + m_already_done = true; + return true; + } + return false; +} + +bool +FullDiagnostics::DoComputeAndPack (int step, bool force_flush) +{ + // Data must be computed and packed for full diagnostics + // whenever the data needs to be flushed. + if (force_flush || m_intervals.contains(step+1) ){ + return true; + } + return false; +} + + +void +FullDiagnostics::AddRZModesToDiags (int lev) +{ +#ifdef WARPX_DIM_RZ + + if (!m_dump_rz_modes) return; + + auto & warpx = WarpX::GetInstance(); + int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + // Make sure all multifabs have the same number of components + for (int dim=0; dim<3; dim++){ + AMREX_ALWAYS_ASSERT( + warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + AMREX_ALWAYS_ASSERT( + warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + AMREX_ALWAYS_ASSERT( + warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + } + + // Check if divE is requested + // If so, all components will be written out + bool divE_requested = false; + for (int comp = 0; comp < m_varnames.size(); comp++) { + if ( m_varnames[comp] == "divE" ) { + divE_requested = true; + } + } + + // If rho is requested, all components will be written out + bool rho_requested = WarpXUtilStr::is_in( m_varnames, "rho" ); + + // First index of m_all_field_functors[lev] where RZ modes are stored + int icomp = m_all_field_functors[0].size(); + const std::array coord {"r", "theta", "z"}; + + // Er, Etheta, Ez, Br, Btheta, Bz, jr, jtheta, jz + // Each of them being a multi-component multifab + int n_new_fields = 9; + if (divE_requested) { + n_new_fields += 1; + } + if (rho_requested) { + n_new_fields += 1; + } + m_all_field_functors[lev].resize( m_all_field_functors[0].size() + n_new_fields ); + // E + for (int dim=0; dim<3; dim++){ + // 3 components, r theta z + m_all_field_functors[lev][icomp] = + std::make_unique(warpx.get_pointer_Efield_aux(lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab); + AddRZModesToOutputNames(std::string("E") + coord[dim], + warpx.get_pointer_Efield_aux(0, 0)->nComp()); + icomp += 1; + } + // B + for (int dim=0; dim<3; dim++){ + // 3 components, r theta z + m_all_field_functors[lev][icomp] = + std::make_unique(warpx.get_pointer_Bfield_aux(lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab); + AddRZModesToOutputNames(std::string("B") + coord[dim], + warpx.get_pointer_Bfield_aux(0, 0)->nComp()); + icomp += 1; + } + // j + for (int dim=0; dim<3; dim++){ + // 3 components, r theta z + m_all_field_functors[lev][icomp] = + std::make_unique(warpx.get_pointer_current_fp(lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab); + icomp += 1; + AddRZModesToOutputNames(std::string("J") + coord[dim], + warpx.get_pointer_current_fp(0, 0)->nComp()); + } + // divE + if (divE_requested) { + m_all_field_functors[lev][icomp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, + m_crse_ratio, false, ncomp_multimodefab); + icomp += 1; + AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); + } + // rho + if (rho_requested) { + m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, false, ncomp_multimodefab); + icomp += 1; + AddRZModesToOutputNames(std::string("rho"), ncomp_multimodefab); + } + // Sum the number of components in input vector m_all_field_functors + // and check that it corresponds to the number of components in m_varnames + // and m_mf_output + int ncomp_from_src = 0; + for (int i=0; inComp(); + } + AMREX_ALWAYS_ASSERT( ncomp_from_src == m_varnames.size() ); +#endif +} + +void +FullDiagnostics::AddRZModesToOutputNames (const std::string& field, int ncomp){ +#ifdef WARPX_DIM_RZ + // In cylindrical geometry, real and imag part of each mode are also + // dumped to file separately, so they need to be added to m_varnames + m_varnames.push_back( field + "_0_real" ); + for (int ic=1 ; ic < (ncomp+1)/2 ; ic += 1) { + m_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); + m_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); + } +#endif +} + + +void +FullDiagnostics::InitializeFieldBufferData (int i_buffer, int lev ) { + auto & warpx = WarpX::GetInstance(); + amrex::RealBox diag_dom; + bool use_warpxba = true; + const amrex::IntVect blockingFactor = warpx.blockingFactor( lev ); + + // Default BoxArray and DistributionMap for initializing the output MultiFab, m_mf_output. + amrex::BoxArray ba = warpx.boxArray(lev); + amrex::DistributionMapping dmap = warpx.DistributionMap(lev); + + // Check if warpx BoxArray is coarsenable. + AMREX_ALWAYS_ASSERT_WITH_MESSAGE ( + ba.coarsenable(m_crse_ratio), + "Invalid coarsening ratio for warpx boxArray. Must be an integer divisor of the blocking factor." + ); + + // Find if user-defined physical dimensions are different from the simulation domain. + for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { + // To ensure that the diagnostic lo and hi are within the domain defined at level, lev. + diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); + diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); + if ( fabs(warpx.Geom(lev).ProbLo(idim) - diag_dom.lo(idim)) + > warpx.Geom(lev).CellSize(idim) ) + use_warpxba = false; + if ( fabs(warpx.Geom(lev).ProbHi(idim) - diag_dom.hi(idim)) + > warpx.Geom(lev).CellSize(idim) ) + use_warpxba = false; + + // User-defined value for coarsening should be an integer divisor of + // blocking factor at level, lev. + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( blockingFactor[idim] % m_crse_ratio[idim]==0, + " coarsening ratio must be integer divisor of blocking factor"); + } + + if (use_warpxba == false) { + // Following are the steps to compute the lo and hi index corresponding to user-defined + // m_lo and m_hi using the same resolution as the simulation at level, lev. + amrex::IntVect lo(0); + amrex::IntVect hi(1); + for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { + // lo index with same cell-size as simulation at level, lev. + lo[idim] = std::max( static_cast( floor ( + ( diag_dom.lo(idim) - warpx.Geom(lev).ProbLo(idim)) / + warpx.Geom(lev).CellSize(idim)) ), 0 ); + // hi index with same cell-size as simulation at level, lev. + hi[idim] = std::max( static_cast ( ceil ( + ( diag_dom.hi(idim) - warpx.Geom(lev).ProbLo(idim)) / + warpx.Geom(lev).CellSize(idim) ) ), 0) - 1 ; + // if hi<=lo, then hi = lo + 1, to ensure one cell in that dimension + if ( hi[idim] <= lo[idim] ) { + hi[idim] = lo[idim] + 1; + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + m_crse_ratio[idim]==1, "coarsening ratio in reduced dimension must be 1." + ); + } + } + + // Box for the output MultiFab corresponding to the user-defined physical co-ordinates at lev. + amrex::Box diag_box( lo, hi ); + // Define box array + amrex::BoxArray diag_ba; + diag_ba.define(diag_box); + ba = diag_ba.maxSize( warpx.maxGridSize( lev ) ); + // At this point in the code, the BoxArray, ba, is defined with the same index space and + // resolution as the simulation, at level, lev. + // Coarsen and refine so that the new BoxArray is coarsenable. + ba.coarsen(m_crse_ratio).refine(m_crse_ratio); + + // Update the physical co-ordinates m_lo and m_hi using the final index values + // from the coarsenable, cell-centered BoxArray, ba. + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + m_lo[idim] = warpx.Geom(lev).ProbLo(idim) + warpx.Geom(lev).CellSize(idim)/2.0_rt + + ba.getCellCenteredBox(0).smallEnd(idim) * warpx.Geom(lev).CellSize(idim); + m_hi[idim] = warpx.Geom(lev).ProbLo(idim) + warpx.Geom(lev).CellSize(idim)/2.0_rt + + ba.getCellCenteredBox( ba.size()-1 ).bigEnd(idim) * warpx.Geom(lev).CellSize(idim); + } + } + + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + m_crse_ratio.min() > 0, "Coarsening ratio must be non-zero."); + // The BoxArray is coarsened based on the user-defined coarsening ratio. + ba.coarsen(m_crse_ratio); + // Generate a new distribution map if the physical m_lo and m_hi for the output + // is different from the lo and hi physical co-ordinates of the simulation domain. + if (use_warpxba == false) dmap = amrex::DistributionMapping{ba}; + // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers. + int ngrow = (m_format == "sensei") ? 1 : 0; + // The zero is hard-coded since the number of output buffers = 1 for FullDiagnostics + m_mf_output[i_buffer][lev] = amrex::MultiFab(ba, dmap, m_varnames.size(), ngrow); + +} + + +void +FullDiagnostics::InitializeFieldFunctors (int lev) +{ + auto & warpx = WarpX::GetInstance(); + // Clear any pre-existing vector to release stored data. + m_all_field_functors[lev].clear(); + + m_all_field_functors[lev].resize( m_varnames.size() ); + // Fill vector of functors for all components except individual cylindrical modes. + for (int comp=0, n=m_all_field_functors[lev].size(); comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Ey" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Ez" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Bx" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "By" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Bz" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jx" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jy" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jz" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "rho" ){ + if ( WarpX::do_back_transformed_diagnostics ) { +#ifdef WARPX_USE_PSATD + // rho_new is stored in component 1 of rho_fp when using PSATD + amrex::MultiFab* rho_new = new amrex::MultiFab(*warpx.get_pointer_rho_fp(lev), amrex::make_alias, 1, 1); + m_all_field_functors[lev][comp] = std::make_unique(rho_new, lev, m_crse_ratio); +#else + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_rho_fp(lev), lev, m_crse_ratio); +#endif + } + else { + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); + } + } else if ( m_varnames[comp] == "F" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "part_per_cell" ){ + m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); + } else if ( m_varnames[comp] == "part_per_grid" ){ + m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); + } else if ( m_varnames[comp] == "divB" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "divE" ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio); + } + } + AddRZModesToDiags( lev ); +} + + +void +FullDiagnostics::PrepareFieldDataForOutput () +{ + // First, make sure all guard cells are properly filled + // Probably overkill/unnecessary, but safe and shouldn't happen often !! + auto & warpx = WarpX::GetInstance(); + warpx.FillBoundaryE(warpx.getngE(), warpx.getngExtra()); + warpx.FillBoundaryB(warpx.getngE(), warpx.getngExtra()); +#ifndef WARPX_USE_PSATD + warpx.FillBoundaryAux(warpx.getngUpdateAux()); +#endif + warpx.UpdateAuxilaryData(); +} diff --git a/Source/Diagnostics/Make.package b/Source/Diagnostics/Make.package index af328109947..a4f03bbb911 100644 --- a/Source/Diagnostics/Make.package +++ b/Source/Diagnostics/Make.package @@ -6,6 +6,7 @@ CEXE_sources += BackTransformedDiagnostic.cpp CEXE_sources += ParticleIO.cpp CEXE_sources += FieldIO.cpp CEXE_sources += SliceDiagnostic.cpp +CEXE_sources += BTDiagnostics.cpp ifeq ($(USE_OPENPMD), TRUE) CEXE_sources += WarpXOpenPMD.cpp diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index fc917a1aa38..d0c64e623ce 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -2,9 +2,10 @@ #define WARPX_MULTIDIAGNOSTICS_H_ #include "FullDiagnostics.H" +#include "BTDiagnostics.H" /** All types of diagnostics. */ -enum struct DiagTypes {Full}; +enum struct DiagTypes {Full, BackTransformed}; /** * \brief This class contains a vector of all diagnostics in the simulation. diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index 7f5ef4c5263..e67586d17d2 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -13,6 +13,8 @@ MultiDiagnostics::MultiDiagnostics () for (int i=0; iDoDump( step, force_flush ) ) continue; - diag->ComputeAndPack(); - diag->Flush(); - diag->FlushRaw(); + diag->FilterComputePackFlush (step, force_flush); } } diff --git a/Source/Diagnostics/ParticleDiag/CMakeLists.txt b/Source/Diagnostics/ParticleDiag/CMakeLists.txt new file mode 100644 index 00000000000..e682608afa2 --- /dev/null +++ b/Source/Diagnostics/ParticleDiag/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + ParticleDiag.cpp +) diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 15bd01e86e4..b06817f91f4 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -10,7 +10,7 @@ ParticleDiag::ParticleDiag(std::string diag_name, std::string name, WarpXParticl { ParmParse pp(diag_name + "." + name); if (!pp.queryarr("variables", variables)){ - variables = {"Ex", "Ey", "Ez", "Bx", "By", "Bz", "ux", "uy", "uz", "w"}; + variables = {"ux", "uy", "uz", "w"}; } //variable to set plot_flags size diff --git a/Source/Diagnostics/ReducedDiags/CMakeLists.txt b/Source/Diagnostics/ReducedDiags/CMakeLists.txt new file mode 100644 index 00000000000..559cac5fb8b --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/CMakeLists.txt @@ -0,0 +1,10 @@ +target_sources(WarpX + PRIVATE + BeamRelevant.cpp + FieldEnergy.cpp + LoadBalanceCosts.cpp + MultiReducedDiags.cpp + ParticleEnergy.cpp + ParticleHistogram.cpp + ReducedDiags.cpp +) diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 8076fb2fa53..20dc3d69973 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -8,7 +8,6 @@ #include "SliceDiagnostic.H" #include #include -#include #include diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 402fd03bac0..c9d8edc1ed8 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #ifdef BL_USE_SENSEI_INSITU diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index a9b0951b064..78a5bddea12 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -53,8 +53,8 @@ public: WarpXParticleCounter(WarpXParticleContainer* pc); unsigned long GetTotalNumParticles() {return m_Total;} - std::vector m_ParticleOffsetAtRank;; - std::vector m_ParticleSizeAtRank;; + std::vector m_ParticleOffsetAtRank; + std::vector m_ParticleSizeAtRank; private: /** get the offset in the overall particle id collection * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 0412eada498..7340d568e3b 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -174,12 +174,12 @@ WarpXOpenPMDPlot::~WarpXOpenPMDPlot() // void WarpXOpenPMDPlot::GetFileName(std::string& filename) { - filename.append(m_OpenPMDFileType).append("/simData"); + filename.append("/openpmd"); // // OpenPMD supports timestepped names // if (m_OneFilePerTS) - filename = filename.append("_%07T"); + filename = filename.append("_%06T"); filename.append(".").append(m_OpenPMDFileType); } diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt new file mode 100644 index 00000000000..fea38b47d8e --- /dev/null +++ b/Source/Evolve/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + WarpXEvolve.cpp +) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 87d39ffc5f1..c210c6f0a57 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -113,12 +113,16 @@ WarpX::Evolve (int numsteps) FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ng_Extra); // E and B: enough guard cells to update Aux or call Field Gather in fp and cp // Need to update Aux on lower levels, to interpolate to higher levels. + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather, guard_cells.ng_Extra); + FillBoundaryB_avg(guard_cells.ng_FieldGather, guard_cells.ng_Extra); + } #ifndef WARPX_USE_PSATD FillBoundaryAux(guard_cells.ng_UpdateAux); #endif UpdateAuxilaryData(); } - if (do_subcycling == 0 || finest_level == 0) { OneStep_nosub(cur_time); // E : guard cells are up-to-date @@ -162,6 +166,8 @@ WarpX::Evolve (int numsteps) cur_time += dt[0]; + ShiftGalileanBoundary(); + if (do_back_transformed_diagnostics) { std::unique_ptr cell_centered_data = nullptr; if (WarpX::do_back_transformed_fields) { @@ -174,8 +180,6 @@ WarpX::Evolve (int numsteps) // If is_synchronized we need to shift j too so that next step we can evolve E by dt/2. // We might need to move j because we are going to make a plotfile. - ShiftGalileanBoundary(); - int num_moved = MoveWindow(move_j); // Electrostatic solver: particles can move by an arbitrary number of cells @@ -267,7 +271,8 @@ WarpX::OneStep_nosub (Real cur_time) // Loop over species. For each ionizable species, create particles in // product species. - mypc->doFieldIonization(); + doFieldIonization(); + mypc->doCoulombCollisions(); #ifdef WARPX_QED mypc->doQEDSchwinger(); @@ -287,33 +292,27 @@ WarpX::OneStep_nosub (Real cur_time) if (warpx_py_afterdeposition) warpx_py_afterdeposition(); #endif +// TODO +// Apply current correction in Fourier space: for domain decomposition with local +// FFTs over guard cells, apply this before calling SyncCurrent #ifdef WARPX_USE_PSATD - // Apply current correction in Fourier space - // (equation (19) of https://doi.org/10.1016/j.jcp.2013.03.010) - if ( fft_periodic_single_box == false ) { - // For domain decomposition with local FFT over guard cells, - // apply this before `SyncCurrent`, i.e. before exchanging guard cells for J - if ( do_current_correction ) CurrentCorrection(); - } + if ( !fft_periodic_single_box && current_correction ) + amrex::Abort("\nCurrent correction does not guarantee charge conservation with local FFTs over guard cells:\n" + "set psatd.periodic_single_box_fft=1 too, in order to guarantee charge conservation"); #endif #ifdef WARPX_QED - //Do QED processes - mypc->doQedEvents(); + doQEDEvents(); #endif // Synchronize J and rho SyncCurrent(); SyncRho(); +// Apply current correction in Fourier space: for periodic single-box global FFTs +// without guard cells, apply this after calling SyncCurrent #ifdef WARPX_USE_PSATD - // Apply current correction in Fourier space - // (equation (19) of https://doi.org/10.1016/j.jcp.2013.03.010) - if ( fft_periodic_single_box == true ) { - // For periodic, single-box FFT (FFT without guard cells) - // apply this after `SyncCurrent`, i.e. after exchanging guard cells for J - if ( do_current_correction ) CurrentCorrection(); - } + if ( fft_periodic_single_box && current_correction ) CurrentCorrection(); #endif @@ -408,11 +407,10 @@ WarpX::OneStep_sub1 (Real curtime) // TODO: we could save some charge depositions // Loop over species. For each ionizable species, create particles in // product species. - mypc->doFieldIonization(); + doFieldIonization(); #ifdef WARPX_QED - //Do QED processes - mypc->doQedEvents(); + doQEDEvents(); #endif AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 1, "Must have exactly two levels"); @@ -549,6 +547,40 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver); } +void +WarpX::doFieldIonization () +{ + for (int lev = 0; lev <= finest_level; ++lev) { + doFieldIonization(lev); + } +} + +void +WarpX::doFieldIonization (int lev) +{ + mypc->doFieldIonization(lev, + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); +} + +#ifdef WARPX_QED +void +WarpX::doQEDEvents () +{ + for (int lev = 0; lev <= finest_level; ++lev) { + doQEDEvents(lev); + } +} + +void +WarpX::doQEDEvents (int lev) +{ + mypc->doQedEvents(lev, + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); +} +#endif + void WarpX::PushParticlesandDepose (amrex::Real cur_time) { @@ -565,6 +597,8 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type) mypc->Evolve(lev, *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], + *Efield_avg_aux[lev][0],*Efield_avg_aux[lev][1],*Efield_avg_aux[lev][2], + *Bfield_avg_aux[lev][0],*Bfield_avg_aux[lev][1],*Bfield_avg_aux[lev][2], *current_fp[lev][0],*current_fp[lev][1],*current_fp[lev][2], current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), rho_fp[lev].get(), charge_buf[lev].get(), diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt new file mode 100644 index 00000000000..210bfa1b250 --- /dev/null +++ b/Source/FieldSolver/CMakeLists.txt @@ -0,0 +1,11 @@ +target_sources(WarpX + PRIVATE + ElectrostaticSolver.cpp + WarpXPushFieldsEM.cpp + WarpX_QED_Field_Pushers.cpp +) + +add_subdirectory(FiniteDifferenceSolver) +if(WarpX_PSATD) + add_subdirectory(SpectralSolver) +endif() diff --git a/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt b/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt new file mode 100644 index 00000000000..93dbd793892 --- /dev/null +++ b/Source/FieldSolver/FiniteDifferenceSolver/CMakeLists.txt @@ -0,0 +1,14 @@ +target_sources(WarpX + PRIVATE + ComputeDivE.cpp + EvolveB.cpp + EvolveBPML.cpp + EvolveE.cpp + EvolveEPML.cpp + EvolveF.cpp + EvolveFPML.cpp + FiniteDifferenceSolver.cpp + MacroscopicEvolveE.cpp +) + +add_subdirectory(MacroscopicProperties) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/CMakeLists.txt b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/CMakeLists.txt new file mode 100644 index 00000000000..3d3d5613665 --- /dev/null +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + MacroscopicProperties.cpp +) diff --git a/Source/FieldSolver/SpectralSolver/CMakeLists.txt b/Source/FieldSolver/SpectralSolver/CMakeLists.txt new file mode 100644 index 00000000000..34012aabd3e --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/CMakeLists.txt @@ -0,0 +1,30 @@ +target_sources(WarpX + PRIVATE + SpectralFieldData.cpp + SpectralKSpace.cpp + SpectralSolver.cpp +) + +if(ENABLE_CUDA) + target_sources(WarpX + PRIVATE + WrapCuFFT.cpp + ) +else() + target_sources(WarpX + PRIVATE + WrapFFTW.cpp + ) +endif() + +if(WarpX_DIMS STREQUAL RZ) + target_sources(WarpX + PRIVATE + SpectralSolverRZ.cpp + SpectralFieldDataRZ.cpp + SpectralKSpaceRZ.cpp + ) + add_subdirectory(SpectralHankelTransform) +endif() + +add_subdirectory(SpectralAlgorithms) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.H new file mode 100644 index 00000000000..f2ae61f33ea --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.H @@ -0,0 +1,35 @@ +#ifndef WARPX_AVG_GALILEAN_ALGORITHM_H_ +#define WARPX_AVG_GALILEAN_ALGORITHM_H_ + +#include "FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H" + +/* \brief Class that updates the field in spectral space + * and stores the coefficients of the corresponding update equation. + */ +class AvgGalileanAlgorithm : public SpectralBaseAlgorithm +{ + public: + AvgGalileanAlgorithm (const SpectralKSpace& spectral_kspace, + const amrex::DistributionMapping& dm, + const int norder_x, const int norder_y, + const int norder_z, const bool nodal, + const amrex::Array& v_galilean, + const amrex::Real dt); + // Redefine update equation from base class + virtual void pushSpectralFields (SpectralFieldData& f) const override final; + virtual int getRequiredNumberOfFields () const override final { + return SpectralAvgFieldIndex::n_fields; + }; + void InitializeSpectralCoefficients( + const SpectralKSpace& spectral_kspace, + const amrex::DistributionMapping& dm, + const amrex::Array& v_galilean, + const amrex::Real dt); + + private: + SpectralRealCoefficients C_coef, S_ck_coef, C1_coef, C3_coef, S1_coef,S3_coef; + SpectralComplexCoefficients Theta2_coef, X1_coef, X2_coef, X3_coef, X4_coef, Psi1_coef, Psi2_coef, Psi3_coef, Psi4_coef, A1_coef, A2_coef, Rhoold_coef, Rhonew_coef, Jcoef_coef; + +}; + +#endif // WARPX_GALILEAN_ALGORITHM_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.cpp new file mode 100644 index 00000000000..1ddcd675883 --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.cpp @@ -0,0 +1,382 @@ +#include "FieldSolver/SpectralSolver/SpectralAlgorithms/AvgGalileanAlgorithm.H" +#include "Utils/WarpXConst.H" +#include + +using namespace amrex; + +/* \brief Initialize coefficients for the update equation */ +AvgGalileanAlgorithm::AvgGalileanAlgorithm(const SpectralKSpace& spectral_kspace, + const DistributionMapping& dm, + const int norder_x, const int norder_y, + const int norder_z, const bool nodal, + const amrex::Array& v_galilean, + const Real dt) + // Initialize members of base class + : SpectralBaseAlgorithm( spectral_kspace, dm, + norder_x, norder_y, norder_z, nodal ) +{ + const BoxArray& ba = spectral_kspace.spectralspace_ba; + + // Allocate the arrays of coefficients + C_coef = SpectralRealCoefficients(ba, dm, 1, 0); + S_ck_coef = SpectralRealCoefficients(ba, dm, 1, 0); + + C1_coef = SpectralRealCoefficients(ba, dm, 1, 0); + S1_coef = SpectralRealCoefficients(ba, dm, 1, 0); + C3_coef = SpectralRealCoefficients(ba, dm, 1, 0); + S3_coef = SpectralRealCoefficients(ba, dm, 1, 0); + + Psi1_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + Psi2_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + Psi3_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + + + X1_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + X2_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + X3_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + X4_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + Theta2_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + + A1_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + A2_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + + Rhoold_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + Rhonew_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + Jcoef_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + + InitializeSpectralCoefficients(spectral_kspace, dm, v_galilean, dt); + +} + +void AvgGalileanAlgorithm::InitializeSpectralCoefficients( + const SpectralKSpace& spectral_kspace, + const amrex::DistributionMapping& dm, + const Array& v_galilean, + const amrex::Real dt) +{ + const BoxArray& ba = spectral_kspace.spectralspace_ba; + // Fill them with the right values: + // Loop over boxes and allocate the corresponding coefficients + // for each box owned by the local MPI proc + for (MFIter mfi(ba, dm); mfi.isValid(); ++mfi){ + + const Box& bx = ba[mfi]; + + // Extract pointers for the k vectors + const Real* modified_kx = modified_kx_vec[mfi].dataPtr(); +#if (AMREX_SPACEDIM==3) + const Real* modified_ky = modified_ky_vec[mfi].dataPtr(); +#endif + const Real* modified_kz = modified_kz_vec[mfi].dataPtr(); + // Extract arrays for the coefficients + Array4 C = C_coef[mfi].array(); + Array4 S_ck = S_ck_coef[mfi].array(); + Array4 C1 = C1_coef[mfi].array(); + Array4 S1 = S1_coef[mfi].array(); + Array4 C3 = C3_coef[mfi].array(); + Array4 S3 = S3_coef[mfi].array(); + + Array4 Psi1 = Psi1_coef[mfi].array(); + Array4 Psi2 = Psi2_coef[mfi].array(); + Array4 Psi3 = Psi3_coef[mfi].array(); + Array4 X1 = X1_coef[mfi].array(); + Array4 X2 = X2_coef[mfi].array(); + Array4 X3 = X3_coef[mfi].array(); + Array4 X4 = X4_coef[mfi].array(); + Array4 Theta2 = Theta2_coef[mfi].array(); + Array4 A1 = A1_coef[mfi].array(); + Array4 A2 = A2_coef[mfi].array(); + + Array4 CRhoold = Rhoold_coef[mfi].array(); + Array4 CRhonew = Rhonew_coef[mfi].array(); + Array4 Jcoef = Jcoef_coef[mfi].array(); + // Extract reals (for portability on GPU) + + Real vx = v_galilean[0]; + Real vy = v_galilean[1]; + Real vz = v_galilean[2]; + + // Loop over indices within one box + ParallelFor(bx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + // Calculate norm of vector + const Real k_norm = std::sqrt( + std::pow(modified_kx[i], 2) + +#if (AMREX_SPACEDIM==3) + std::pow(modified_ky[j], 2) + + std::pow(modified_kz[k], 2)); +#else + std::pow(modified_kz[j], 2)); +#endif + + // Calculate coefficients + constexpr Real c = PhysConst::c; + constexpr Real c2 = PhysConst::c*PhysConst::c; + constexpr Real ep0 = PhysConst::ep0; + const Complex I{0.,1.}; + if (k_norm != 0){ + + C(i,j,k) = std::cos(c*k_norm*dt); + S_ck(i,j,k) = std::sin(c*k_norm*dt)/(c*k_norm); + + C1(i,j,k) = std::cos(0.5_rt*c*k_norm*dt); + S1(i,j,k) = std::sin(0.5_rt*c*k_norm*dt); + C3(i,j,k) = std::cos(1.5_rt*c*k_norm*dt); + S3(i,j,k) = std::sin(1.5_rt*c*k_norm*dt); + + // Calculate dot product with galilean velocity + const Real kv = modified_kx[i]*vx + +#if (AMREX_SPACEDIM==3) + modified_ky[j]*vy + + modified_kz[k]*vz; +#else + modified_kz[j]*vz; +#endif + + const Real nu = kv/(k_norm*c); + const Complex theta = amrex::exp( 0.5_rt*I*kv*dt ); + const Complex theta_star = amrex::exp( -0.5_rt*I*kv*dt ); + const Complex e_theta = amrex::exp( I*c*k_norm*dt ); + + Theta2(i,j,k) = theta*theta; + + if ( (nu != 1.) && (nu != 0) ) { + + // Note: the coefficients X1, X2, X3 do not correspond + // exactly to the original Galilean paper, but the + // update equation have been modified accordingly so that + // the expressions/ below (with the update equations) + // are mathematically equivalent to those of the paper. + Complex x1 = 1._rt/(1._rt-nu*nu) * + (theta_star - C(i,j,k)*theta + I*kv*S_ck(i,j,k)*theta); + + Complex C_rho = I* c2 /( (1._rt-theta*theta) * ep0); + + Psi1(i,j,k) = theta * ((S1(i,j,k) + I*nu*C1(i,j,k)) + - Theta2(i,j,k) * (S3(i,j,k) + I*nu*C3(i,j,k))) /(c*k_norm*dt * (nu*nu - 1._rt)); + Psi2(i,j,k) = theta * ((C1(i,j,k) - I*nu*S1(i,j,k)) + - Theta2(i,j,k) * (C3(i,j,k) - I*nu*S3(i,j,k))) /(c2*k_norm*k_norm*dt * (nu*nu - 1._rt)); + Psi3(i,j,k) = I * theta * (1._rt - theta*theta) /(c*k_norm*dt*nu); + + A1(i,j,k) = (Psi1(i,j,k) - 1._rt + I * kv*Psi2(i,j,k) )/ (c2* k_norm*k_norm * (nu*nu - 1._rt)); + A2(i,j,k) = (Psi3(i,j,k) - Psi1(i,j,k)) / (c2*k_norm*k_norm); + + CRhoold(i,j,k) = C_rho * (theta*theta * A1(i,j,k) - A2(i,j,k)); + CRhonew(i,j,k) = C_rho * (A2(i,j,k) - A1(i,j,k)); + Jcoef(i,j,k) = (I*kv*A1(i,j,k) + Psi2(i,j,k))/ep0; + // x1, above, is identical to the original paper + X1(i,j,k) = theta*x1/(ep0*c*c*k_norm*k_norm); + // The difference betwen X2 and X3 below, and those + // from the original paper is the factor ep0*k_norm*k_norm + X2(i,j,k) = (x1 - theta*(1._rt - C(i,j,k))) + /(theta_star-theta)/(ep0*k_norm*k_norm); + X3(i,j,k) = (x1 - theta_star*(1._rt - C(i,j,k))) + /(theta_star-theta)/(ep0*k_norm*k_norm); + X4(i,j,k) = I*kv*X1(i,j,k) - theta*theta*S_ck(i,j,k)/ep0; + } + if ( nu == 0) { + X1(i,j,k) = (1._rt - C(i,j,k)) / (ep0*c*c*k_norm*k_norm); + X2(i,j,k) = (1._rt - S_ck(i,j,k)/dt) / (ep0*k_norm*k_norm); + X3(i,j,k) = (C(i,j,k) - S_ck(i,j,k)/dt) / (ep0*k_norm*k_norm); + X4(i,j,k) = -S_ck(i,j,k)/ep0; + + Psi1(i,j,k) = (-S1(i,j,k) + S3(i,j,k)) / (c*k_norm*dt); + Psi2(i,j,k) = (-C1(i,j,k) + C3(i,j,k)) / (c2*k_norm*k_norm*dt); + Psi3(i,j,k) = 1._rt; + A1(i,j,k) = (c*k_norm*dt + S1(i,j,k) - S3(i,j,k)) / (c*c2 * k_norm*k_norm*k_norm * dt); + A2(i,j,k) = (c*k_norm*dt + S1(i,j,k) - S3(i,j,k)) / (c*c2 * k_norm*k_norm*k_norm * dt); + CRhoold(i,j,k) = 2._rt * I * S1(i,j,k) * ( dt*C(i,j,k) - S_ck(i,j,k)) + / (c*k_norm*k_norm*k_norm*dt*dt*ep0); + CRhonew(i,j,k) = - I * (c2* k_norm*k_norm * dt*dt - C1(i,j,k) + C3(i,j,k)) + / (c2 * k_norm*k_norm*k_norm*k_norm * ep0 * dt*dt); + Jcoef(i,j,k) = (-C1(i,j,k) + C3(i,j,k)) / (c2*ep0*k_norm*k_norm*dt); + } + if ( nu == 1.) { + X1(i,j,k) = (1._rt - e_theta*e_theta + 2._rt*I*c*k_norm*dt) / (4._rt*c*c*ep0*k_norm*k_norm); + X2(i,j,k) = (3._rt - 4._rt*e_theta + e_theta*e_theta + 2._rt*I*c*k_norm*dt) / (4._rt*ep0*k_norm*k_norm*(1._rt- e_theta)); + X3(i,j,k) = (3._rt - 2._rt/e_theta - 2._rt*e_theta + e_theta*e_theta - 2._rt*I*c*k_norm*dt) / (4._rt*ep0*(e_theta - 1._rt)*k_norm*k_norm); + X4(i,j,k) = I*(-1._rt + e_theta*e_theta + 2._rt*I*c*k_norm*dt) / (4._rt*ep0*c*k_norm); + } + + } else { // Handle k_norm = 0, by using the analytical limit + C(i,j,k) = 1._rt; + S_ck(i,j,k) = dt; + C1(i,j,k) = 1._rt; + S1(i,j,k) = 0._rt; + C3(i,j,k) = 1._rt; + S3(i,j,k) = 0._rt; + + X1(i,j,k) = dt*dt/(2._rt * ep0); + X2(i,j,k) = c2*dt*dt/(6._rt * ep0); + X3(i,j,k) = - c2*dt*dt/(3._rt * ep0); + X4(i,j,k) = -dt/ep0; + Theta2(i,j,k) = 1._rt; + + Psi1(i,j,k) = 1._rt; + Psi2(i,j,k) = -dt; + Psi3(i,j,k) = 1._rt; + A1(i,j,k) = 13._rt * dt*dt /24._rt; + A2(i,j,k) = 13._rt * dt*dt /24._rt; + CRhoold(i,j,k) = -I*c2 * dt*dt / (3._rt * ep0); + CRhonew(i,j,k) = -5._rt*I*c2 * dt*dt / (24._rt * ep0); + Jcoef(i,j,k) = -dt/ep0; + } + + }); + } +}; + +/* Advance the E and B field in spectral space (stored in `f`) + * over one time step */ +void +AvgGalileanAlgorithm::pushSpectralFields(SpectralFieldData& f) const{ + + // Loop over boxes + for (MFIter mfi(f.fields); mfi.isValid(); ++mfi){ + + const Box& bx = f.fields[mfi].box(); + + // Extract arrays for the fields to be updated + Array4 fields = f.fields[mfi].array(); + // Extract arrays for the coefficients + Array4 C_arr = C_coef[mfi].array(); + Array4 S_ck_arr = S_ck_coef[mfi].array(); + Array4 X1_arr = X1_coef[mfi].array(); + Array4 X2_arr = X2_coef[mfi].array(); + Array4 X3_arr = X3_coef[mfi].array(); + Array4 X4_arr = X4_coef[mfi].array(); + Array4 Theta2_arr = Theta2_coef[mfi].array(); + Array4 Psi1_arr = Psi1_coef[mfi].array(); + Array4 Psi2_arr = Psi2_coef[mfi].array(); + Array4 Psi3_arr = Psi3_coef[mfi].array(); + Array4 C1_arr = C1_coef[mfi].array(); + Array4 S1_arr = S1_coef[mfi].array(); + Array4 C3_arr = C3_coef[mfi].array(); + Array4 S3_arr = S3_coef[mfi].array(); + + + Array4 A1_arr = A1_coef[mfi].array(); + Array4 A2_arr = A2_coef[mfi].array(); + Array4 Rhonew_arr = Rhonew_coef[mfi].array(); + Array4 Rhoold_arr = Rhoold_coef[mfi].array(); + Array4 Jcoef_arr =Jcoef_coef[mfi].array(); + // Extract pointers for the k vectors + const Real* modified_kx_arr = modified_kx_vec[mfi].dataPtr(); +#if (AMREX_SPACEDIM==3) + const Real* modified_ky_arr = modified_ky_vec[mfi].dataPtr(); +#endif + const Real* modified_kz_arr = modified_kz_vec[mfi].dataPtr(); + + // Loop over indices within one box + ParallelFor(bx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + // Record old values of the fields to be updated + using Idx = SpectralAvgFieldIndex; + + const Complex Ex_old = fields(i,j,k,Idx::Ex); + const Complex Ey_old = fields(i,j,k,Idx::Ey); + const Complex Ez_old = fields(i,j,k,Idx::Ez); + const Complex Bx_old = fields(i,j,k,Idx::Bx); + const Complex By_old = fields(i,j,k,Idx::By); + const Complex Bz_old = fields(i,j,k,Idx::Bz); + + // Shortcut for the values of J and rho + const Complex Jx = fields(i,j,k,Idx::Jx); + const Complex Jy = fields(i,j,k,Idx::Jy); + const Complex Jz = fields(i,j,k,Idx::Jz); + const Complex rho_old = fields(i,j,k,Idx::rho_old); + const Complex rho_new = fields(i,j,k,Idx::rho_new); + + const Complex Ex_avg = fields(i,j,k,Idx::Ex_avg); + const Complex Ey_avg= fields(i,j,k,Idx::Ey_avg); + const Complex Ez_avg = fields(i,j,k,Idx::Ez_avg); + const Complex Bx_avg = fields(i,j,k,Idx::Bx_avg); + const Complex By_avg = fields(i,j,k,Idx::By_avg); + const Complex Bz_avg = fields(i,j,k,Idx::Bz_avg); + // k vector values, and coefficients + const Real kx = modified_kx_arr[i]; + +#if (AMREX_SPACEDIM==3) + const Real ky = modified_ky_arr[j]; + const Real kz = modified_kz_arr[k]; +#else + constexpr Real ky = 0; + const Real kz = modified_kz_arr[j]; +#endif + constexpr Real c = PhysConst::c; + constexpr Real c2 = PhysConst::c*PhysConst::c; + constexpr Real inv_ep0 = 1._rt/PhysConst::ep0; + constexpr Complex I = Complex{0,1}; + + const Real C = C_arr(i,j,k); + const Real S_ck = S_ck_arr(i,j,k); + + const Real C1 = C1_arr(i,j,k); + const Real C3 = C3_arr(i,j,k); + const Real S1 = S1_arr(i,j,k); + const Real S3 = S3_arr(i,j,k); + + const Complex X1 = X1_arr(i,j,k); + const Complex X2 = X2_arr(i,j,k); + const Complex X3 = X3_arr(i,j,k); + const Complex X4 = X4_arr(i,j,k); + const Complex T2 = Theta2_arr(i,j,k); + + const Complex Psi1 = Psi1_arr(i,j,k); + const Complex Psi2 = Psi2_arr(i,j,k); + const Complex Psi3 = Psi3_arr(i,j,k); + const Complex A1 = A1_arr(i,j,k); + const Complex A2 = A2_arr(i,j,k); + const Complex CRhoold= Rhoold_arr(i,j,k); + const Complex CRhonew= Rhonew_arr(i,j,k); + const Complex Jcoef = Jcoef_arr(i,j,k); + + + //Update E (see the original Galilean article) + fields(i,j,k,Idx::Ex) = T2*C*Ex_old + + T2*S_ck*c2*I*(ky*Bz_old - kz*By_old) + + X4*Jx - I*(X2*rho_new - T2*X3*rho_old)*kx; + fields(i,j,k,Idx::Ey) = T2*C*Ey_old + + T2*S_ck*c2*I*(kz*Bx_old - kx*Bz_old) + + X4*Jy - I*(X2*rho_new - T2*X3*rho_old)*ky; + fields(i,j,k,Idx::Ez) = T2*C*Ez_old + + T2*S_ck*c2*I*(kx*By_old - ky*Bx_old) + + X4*Jz - I*(X2*rho_new - T2*X3*rho_old)*kz; + // Update B (see the original Galilean article) + // Note: here X1 is T2*x1/(ep0*c*c*k_norm*k_norm), where + // x1 has the same definition as in the original paper + fields(i,j,k,Idx::Bx) = T2*C*Bx_old + - T2*S_ck*I*(ky*Ez_old - kz*Ey_old) + + X1*I*(ky*Jz - kz*Jy); + fields(i,j,k,Idx::By) = T2*C*By_old + - T2*S_ck*I*(kz*Ex_old - kx*Ez_old) + + X1*I*(kz*Jx - kx*Jz); + fields(i,j,k,Idx::Bz) = T2*C*Bz_old + - T2*S_ck*I*(kx*Ey_old - ky*Ex_old) + + X1*I*(kx*Jy - ky*Jx); + +//Update the averaged E,B fields in time on the interval [(n-1/2)dx, (n+1/2)dx] + fields(i,j,k,Idx::Ex_avg) = Psi1*Ex_old + - Psi2*c2*I*(ky*Bz_old - kz*By_old) + + Jcoef*Jx + ( CRhonew * rho_new + CRhoold*rho_old )*kx; + fields(i,j,k,Idx::Ey_avg) = Psi1*Ey_old + - Psi2*c2*I*(kz*Bx_old - kx*Bz_old) + + Jcoef*Jy +( CRhonew * rho_new + CRhoold*rho_old )*ky; + fields(i,j,k,Idx::Ez_avg) = Psi1*Ez_old + - Psi2*c2*I*(kx*By_old - ky*Bx_old) + + Jcoef*Jz + ( CRhonew * rho_new + CRhoold*rho_old )*kz; + + fields(i,j,k,Idx::Bx_avg) = Psi1*Bx_old + + I*Psi2*(ky*Ez_old - kz*Ey_old) + + A1*I*(ky*Jz - kz*Jy)*inv_ep0; + fields(i,j,k,Idx::By_avg) = Psi1*By_old + + I*Psi2*(kz*Ex_old - kx*Ez_old) + + A1*I*(kz*Jx - kx*Jz)*inv_ep0; + fields(i,j,k,Idx::Bz_avg) = Psi1*Bz_old + + I*Psi2*(kx*Ey_old - ky*Ex_old) + + A1*I*(kx*Jy - ky*Jx)*inv_ep0; + }); + } +}; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/CMakeLists.txt b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/CMakeLists.txt new file mode 100644 index 00000000000..c5ebf1c3e80 --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/CMakeLists.txt @@ -0,0 +1,16 @@ +target_sources(WarpX + PRIVATE + GalileanAlgorithm.cpp + PMLPsatdAlgorithm.cpp + PsatdAlgorithm.cpp + SpectralBaseAlgorithm.cpp + AvgGalileanAlgorithm.cpp +) + +if(WarpX_DIMS STREQUAL RZ) + target_sources(WarpX + PRIVATE + SpectralBaseAlgorithmRZ.cpp + PsatdAlgorithmRZ.cpp + ) +endif() diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/Make.package b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/Make.package index 436d99adf3f..f316abefadd 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/Make.package +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/Make.package @@ -1,6 +1,7 @@ CEXE_sources += SpectralBaseAlgorithm.cpp CEXE_sources += PsatdAlgorithm.cpp CEXE_sources += GalileanAlgorithm.cpp +CEXE_sources += AvgGalileanAlgorithm.cpp CEXE_sources += PMLPsatdAlgorithm.cpp ifeq ($(USE_RZ),TRUE) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H index eb440d11896..b492d9a18b2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H @@ -25,7 +25,8 @@ class PsatdAlgorithm : public SpectralBaseAlgorithm const amrex::DistributionMapping& dm, const int norder_x, const int norder_y, const int norder_z, const bool nodal, - const amrex::Real dt); + const amrex::Real dt, + const bool update_with_rho); // Redefine functions from base class virtual void pushSpectralFields(SpectralFieldData& f) const override final; virtual int getRequiredNumberOfFields() const override final { @@ -55,6 +56,7 @@ class PsatdAlgorithm : public SpectralBaseAlgorithm private: SpectralRealCoefficients C_coef, S_ck_coef, X1_coef, X2_coef, X3_coef; amrex::Real m_dt; + bool m_update_with_rho; }; #endif // WARPX_USE_PSATD diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp index 9684fde27de..b8b64b80c15 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp @@ -21,10 +21,12 @@ using namespace amrex; PsatdAlgorithm::PsatdAlgorithm(const SpectralKSpace& spectral_kspace, const DistributionMapping& dm, const int norder_x, const int norder_y, - const int norder_z, const bool nodal, const Real dt) - // Initialize members of base class - : SpectralBaseAlgorithm( spectral_kspace, dm, - norder_x, norder_y, norder_z, nodal ) + const int norder_z, const bool nodal, const Real dt, + const bool update_with_rho) + // Initialize members of base class + : m_dt( dt ), + m_update_with_rho( update_with_rho ), + SpectralBaseAlgorithm( spectral_kspace, dm, norder_x, norder_y, norder_z, nodal ) { const BoxArray& ba = spectral_kspace.spectralspace_ba; @@ -37,8 +39,6 @@ PsatdAlgorithm::PsatdAlgorithm(const SpectralKSpace& spectral_kspace, // Initialize coefficients for update equations InitializeSpectralCoefficients(spectral_kspace, dm, dt); - - m_dt = dt; } /** @@ -47,6 +47,8 @@ PsatdAlgorithm::PsatdAlgorithm(const SpectralKSpace& spectral_kspace, void PsatdAlgorithm::pushSpectralFields(SpectralFieldData& f) const{ + const bool update_with_rho = m_update_with_rho; + // Loop over boxes for (MFIter mfi(f.fields); mfi.isValid(); ++mfi){ @@ -68,23 +70,24 @@ PsatdAlgorithm::pushSpectralFields(SpectralFieldData& f) const{ const Real* modified_kz_arr = modified_kz_vec[mfi].dataPtr(); // Loop over indices within one box - ParallelFor(bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // Record old values of the fields to be updated using Idx = SpectralFieldIndex; + const Complex Ex_old = fields(i,j,k,Idx::Ex); const Complex Ey_old = fields(i,j,k,Idx::Ey); const Complex Ez_old = fields(i,j,k,Idx::Ez); const Complex Bx_old = fields(i,j,k,Idx::Bx); const Complex By_old = fields(i,j,k,Idx::By); const Complex Bz_old = fields(i,j,k,Idx::Bz); + // Shortcut for the values of J and rho const Complex Jx = fields(i,j,k,Idx::Jx); const Complex Jy = fields(i,j,k,Idx::Jy); const Complex Jz = fields(i,j,k,Idx::Jz); const Complex rho_old = fields(i,j,k,Idx::rho_old); const Complex rho_new = fields(i,j,k,Idx::rho_new); + // k vector values, and coefficients const Real kx = modified_kx_arr[i]; #if (AMREX_SPACEDIM==3) @@ -95,36 +98,51 @@ PsatdAlgorithm::pushSpectralFields(SpectralFieldData& f) const{ const Real kz = modified_kz_arr[j]; #endif constexpr Real c2 = PhysConst::c*PhysConst::c; - constexpr Real inv_ep0 = 1./PhysConst::ep0; + constexpr Real inv_eps0 = 1.0_rt/PhysConst::ep0; + const Complex I = Complex{0,1}; + const Real C = C_arr(i,j,k); const Real S_ck = S_ck_arr(i,j,k); const Real X1 = X1_arr(i,j,k); const Real X2 = X2_arr(i,j,k); const Real X3 = X3_arr(i,j,k); - // Update E (see WarpX online documentation: theory section) - fields(i,j,k,Idx::Ex) = C*Ex_old - + S_ck*(c2*I*(ky*Bz_old - kz*By_old) - inv_ep0*Jx) - - I*(X2*rho_new - X3*rho_old)*kx; - fields(i,j,k,Idx::Ey) = C*Ey_old - + S_ck*(c2*I*(kz*Bx_old - kx*Bz_old) - inv_ep0*Jy) - - I*(X2*rho_new - X3*rho_old)*ky; - fields(i,j,k,Idx::Ez) = C*Ez_old - + S_ck*(c2*I*(kx*By_old - ky*Bx_old) - inv_ep0*Jz) - - I*(X2*rho_new - X3*rho_old)*kz; + + if (update_with_rho) { + + fields(i,j,k,Idx::Ex) = C*Ex_old + S_ck*(c2*I*(ky*Bz_old-kz*By_old)-inv_eps0*Jx) + - I*(X2*rho_new-X3*rho_old)*kx; + + fields(i,j,k,Idx::Ey) = C*Ey_old + S_ck*(c2*I*(kz*Bx_old-kx*Bz_old)-inv_eps0*Jy) + - I*(X2*rho_new-X3*rho_old)*ky; + + fields(i,j,k,Idx::Ez) = C*Ez_old + S_ck*(c2*I*(kx*By_old-ky*Bx_old)-inv_eps0*Jz) + - I*(X2*rho_new-X3*rho_old)*kz; + } else { + + Complex k_dot_J = kx*Jx + ky*Jy + kz*Jz; + Complex k_dot_E = kx*Ex_old + ky*Ey_old + kz*Ez_old; + + fields(i,j,k,Idx::Ex) = C*Ex_old + S_ck*(c2*I*(ky*Bz_old-kz*By_old)-inv_eps0*Jx) + + X2*k_dot_E*kx + X3*inv_eps0*k_dot_J*kx; + + fields(i,j,k,Idx::Ey) = C*Ey_old + S_ck*(c2*I*(kz*Bx_old-kx*Bz_old)-inv_eps0*Jy) + + X2*k_dot_E*ky + X3*inv_eps0*k_dot_J*ky; + + fields(i,j,k,Idx::Ez) = C*Ez_old + S_ck*(c2*I*(kx*By_old-ky*Bx_old)-inv_eps0*Jz) + + X2*k_dot_E*kz + X3*inv_eps0*k_dot_J*kz; + } + // Update B (see WarpX online documentation: theory section) - fields(i,j,k,Idx::Bx) = C*Bx_old - - S_ck*I*(ky*Ez_old - kz*Ey_old) - + X1*I*(ky*Jz - kz*Jy); - fields(i,j,k,Idx::By) = C*By_old - - S_ck*I*(kz*Ex_old - kx*Ez_old) - + X1*I*(kz*Jx - kx*Jz); - fields(i,j,k,Idx::Bz) = C*Bz_old - - S_ck*I*(kx*Ey_old - ky*Ex_old) - + X1*I*(kx*Jy - ky*Jx); - }); + + fields(i,j,k,Idx::Bx) = C*Bx_old - S_ck*I*(ky*Ez_old-kz*Ey_old) + X1*I*(ky*Jz-kz*Jy); + + fields(i,j,k,Idx::By) = C*By_old - S_ck*I*(kz*Ex_old-kx*Ez_old) + X1*I*(kz*Jx-kx*Jz); + + fields(i,j,k,Idx::Bz) = C*Bz_old - S_ck*I*(kx*Ey_old-ky*Ex_old) + X1*I*(kx*Jy-ky*Jx); + } ); } }; @@ -135,8 +153,10 @@ void PsatdAlgorithm::InitializeSpectralCoefficients(const SpectralKSpace& spectr const amrex::DistributionMapping& dm, const amrex::Real dt) { + const bool update_with_rho = m_update_with_rho; + const BoxArray& ba = spectral_kspace.spectralspace_ba; - // Fill them with the right values: + // Loop over boxes and allocate the corresponding coefficients // for each box owned by the local MPI proc for (MFIter mfi(ba, dm); mfi.isValid(); ++mfi){ @@ -149,6 +169,7 @@ void PsatdAlgorithm::InitializeSpectralCoefficients(const SpectralKSpace& spectr const Real* modified_ky = modified_ky_vec[mfi].dataPtr(); #endif const Real* modified_kz = modified_kz_vec[mfi].dataPtr(); + // Extract arrays for the coefficients Array4 C = C_coef[mfi].array(); Array4 S_ck = S_ck_coef[mfi].array(); @@ -157,37 +178,44 @@ void PsatdAlgorithm::InitializeSpectralCoefficients(const SpectralKSpace& spectr Array4 X3 = X3_coef[mfi].array(); // Loop over indices within one box - ParallelFor(bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Calculate norm of vector const Real k_norm = std::sqrt( - std::pow(modified_kx[i], 2) + + std::pow(modified_kx[i],2) + #if (AMREX_SPACEDIM==3) - std::pow(modified_ky[j], 2) + - std::pow(modified_kz[k], 2)); + std::pow(modified_ky[j],2) + std::pow(modified_kz[k],2)); #else - std::pow(modified_kz[j], 2)); + std::pow(modified_kz[j],2)); #endif - - // Calculate coefficients constexpr Real c = PhysConst::c; - constexpr Real ep0 = PhysConst::ep0; - if (k_norm != 0){ - C(i,j,k) = std::cos(c*k_norm*dt); + constexpr Real eps0 = PhysConst::ep0; + + if (k_norm != 0) { + C(i,j,k) = std::cos(c*k_norm*dt); S_ck(i,j,k) = std::sin(c*k_norm*dt)/(c*k_norm); - X1(i,j,k) = (1. - C(i,j,k))/(ep0 * c*c * k_norm*k_norm); - X2(i,j,k) = (1. - S_ck(i,j,k)/dt)/(ep0 * k_norm*k_norm); - X3(i,j,k) = (C(i,j,k) - S_ck(i,j,k)/dt)/(ep0 * k_norm*k_norm); - } else { // Handle k_norm = 0, by using the analytical limit - C(i,j,k) = 1.; + X1(i,j,k) = (1.0_rt-C(i,j,k))/(eps0*c*c*k_norm*k_norm); + if (update_with_rho) { + X2(i,j,k) = (1.0_rt-S_ck(i,j,k)/dt)/(eps0*k_norm*k_norm); + X3(i,j,k) = (C(i,j,k)-S_ck(i,j,k)/dt)/(eps0*k_norm*k_norm); + } else { + X2(i,j,k) = (1.0_rt-C(i,j,k))/(k_norm*k_norm); + X3(i,j,k) = (S_ck(i,j,k)-dt)/(k_norm*k_norm); + } + } else { // Handle k_norm = 0 with analytical limit + C(i,j,k) = 1.0_rt; S_ck(i,j,k) = dt; - X1(i,j,k) = 0.5 * dt*dt / ep0; - X2(i,j,k) = c*c * dt*dt / (6.*ep0); - X3(i,j,k) = - c*c * dt*dt / (3.*ep0); + X1(i,j,k) = 0.5_rt*dt*dt/eps0; + if (update_with_rho) { + X2(i,j,k) = c*c*dt*dt/(6.0_rt*eps0); + X3(i,j,k) = -c*c*dt*dt/(3.0_rt*eps0); + } else { + X2(i,j,k) = 0.5_rt*dt*dt*c*c; + X3(i,j,k) = -c*c*dt*dt*dt/6.0_rt; + } } - }); + } ); } } @@ -214,6 +242,7 @@ PsatdAlgorithm::CurrentCorrection( SpectralFieldData& field_data, // Extract arrays for the fields to be updated Array4 fields = field_data.fields[mfi].array(); + // Extract pointers for the k vectors const Real* const modified_kx_arr = modified_kx_vec[mfi].dataPtr(); #if (AMREX_SPACEDIM==3) @@ -225,17 +254,18 @@ PsatdAlgorithm::CurrentCorrection( SpectralFieldData& field_data, const Real dt = m_dt; // Loop over indices within one box - ParallelFor(bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { // Record old values of the fields to be updated using Idx = SpectralFieldIndex; + // Shortcuts for the values of J and rho const Complex Jx = fields(i,j,k,Idx::Jx); const Complex Jy = fields(i,j,k,Idx::Jy); const Complex Jz = fields(i,j,k,Idx::Jz); const Complex rho_old = fields(i,j,k,Idx::rho_old); const Complex rho_new = fields(i,j,k,Idx::rho_new); + // k vector values, and coefficients const Real kx = modified_kx_arr[i]; #if (AMREX_SPACEDIM==3) @@ -245,10 +275,10 @@ PsatdAlgorithm::CurrentCorrection( SpectralFieldData& field_data, constexpr Real ky = 0; const Real kz = modified_kz_arr[j]; #endif - constexpr Complex I = Complex{0,1}; - const Real k_norm = std::sqrt( kx*kx + ky*ky + kz*kz ); + constexpr Complex I = Complex{0,1}; + // div(J) in Fourier space const Complex k_dot_J = kx*Jx + ky*Jy + kz*Jz; @@ -259,7 +289,7 @@ PsatdAlgorithm::CurrentCorrection( SpectralFieldData& field_data, fields(i,j,k,Idx::Jy) = Jy - (k_dot_J-I*(rho_new-rho_old)/dt)*ky/(k_norm*k_norm); fields(i,j,k,Idx::Jz) = Jz - (k_dot_J-I*(rho_new-rho_old)/dt)*kz/(k_norm*k_norm); } - }); + } ); } // Backward Fourier transform of J diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H index f618fda3570..b445054cc9d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H @@ -27,7 +27,13 @@ struct SpectralFieldIndex { enum { Ex=0, Ey, Ez, Bx, By, Bz, Jx, Jy, Jz, rho_old, rho_new, n_fields, divE=3 }; }; -/** Index for the PML fields, when stored in spectral space */ +/* Index for the regular fields + averaged fields, when stored in spectral space */ +struct SpectralAvgFieldIndex { + enum { Ex=0, Ey, Ez, Bx, By, Bz, Jx, Jy, Jz, rho_old, rho_new, Ex_avg, Ey_avg, Ez_avg, Bx_avg, By_avg, Bz_avg,n_fields }; + // n_fields is automatically the total number of fields +}; + +/* Index for the PML fields, when stored in spectral space */ struct SpectralPMLIndex { enum { Exy=0, Exz, Eyx, Eyz, Ezx, Ezy, Bxy, Bxz, Byx, Byz, Bzx, Bzy, n_fields }; diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt new file mode 100644 index 00000000000..858ffb137ec --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt @@ -0,0 +1,5 @@ +target_sources(WarpX + PRIVATE + SpectralHankelTransformer.cpp + HankelTransform.cpp +) diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package index a3c22d64a12..8bb1d7ef7b4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package @@ -1,6 +1,4 @@ -ifeq ($(USE_RZ),TRUE) - CEXE_sources += SpectralHankelTransformer.cpp - CEXE_sources += HankelTransform.cpp +CEXE_sources += SpectralHankelTransformer.cpp +CEXE_sources += HankelTransform.cpp - VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/SpectralHankelTransform -endif +VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/SpectralHankelTransform diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 89771a3f730..8de6de185a7 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -10,7 +10,6 @@ #include - using namespace amrex; using namespace Gpu; @@ -217,11 +216,11 @@ SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, // contains only the positive k, and the Nyquist frequency is // the last element of the array. modified_k[k.size()-1] = 0.0_rt; - } else { + } else { // The other axes contains both positive and negative k ; // the Nyquist frequency is in the middle of the array. modified_k[k.size()/2] = 0.0_rt; - } + } } } return modified_k_comp; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 6a6410d49fc..bdec4499d5b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -35,7 +35,8 @@ class SpectralSolver const amrex::Array& v_galilean, const amrex::RealVect dx, const amrex::Real dt, const bool pml=false, - const bool periodic_single_box=false ); + const bool periodic_single_box=false, + const bool update_with_rho=false ); /** * \brief Transform the component `i_comp` of MultiFab `mf` @@ -82,7 +83,10 @@ class SpectralSolver algorithm->CurrentCorrection( field_data, current, rho ); }; + bool fft_do_time_averaging = false; + private: + void ReadParameters (); // Store field in spectral space and perform the Fourier transforms SpectralFieldData field_data; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 7b6ab83e816..16894da78ff 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -8,10 +8,11 @@ #include "SpectralSolver.H" #include "SpectralAlgorithms/PsatdAlgorithm.H" #include "SpectralAlgorithms/GalileanAlgorithm.H" +#include "SpectralAlgorithms/AvgGalileanAlgorithm.H" #include "SpectralAlgorithms/PMLPsatdAlgorithm.H" #include "WarpX.H" #include "Utils/WarpXProfilerWrapper.H" - +#include "Utils/WarpXUtil.H" #if WARPX_USE_PSATD @@ -37,7 +38,8 @@ SpectralSolver::SpectralSolver( const int norder_z, const bool nodal, const amrex::Array& v_galilean, const amrex::RealVect dx, const amrex::Real dt, - const bool pml, const bool periodic_single_box ) { + const bool pml, const bool periodic_single_box, + const bool update_with_rho ) { // Initialize all structures using the same distribution mapping dm @@ -49,19 +51,30 @@ SpectralSolver::SpectralSolver( // - Select the algorithm depending on the input parameters // Initialize the corresponding coefficients over k space + amrex::ParmParse pp("psatd"); + pp.query("do_time_averaging", fft_do_time_averaging); + if (pml) { algorithm = std::unique_ptr( new PMLPsatdAlgorithm( k_space, dm, norder_x, norder_y, norder_z, nodal, dt ) ); - } else if ((v_galilean[0]==0) && (v_galilean[1]==0) && (v_galilean[2]==0)){ - // v_galilean is 0: use standard PSATD algorithm - algorithm = std::unique_ptr( new PsatdAlgorithm( - k_space, dm, norder_x, norder_y, norder_z, nodal, dt ) ); - } else { - // Otherwise: use the Galilean algorithm - algorithm = std::unique_ptr( new GalileanAlgorithm( - k_space, dm, norder_x, norder_y, norder_z, nodal, v_galilean, dt )); - } - + } + else { + if (fft_do_time_averaging){ + algorithm = std::unique_ptr( new AvgGalileanAlgorithm( + k_space, dm, norder_x, norder_y, norder_z, nodal, v_galilean, dt ) ); + } + else { + if ((v_galilean[0]==0) && (v_galilean[1]==0) && (v_galilean[2]==0)){ + // v_galilean is 0: use standard PSATD algorithm + algorithm = std::unique_ptr( new PsatdAlgorithm( + k_space, dm, norder_x, norder_y, norder_z, nodal, dt, update_with_rho ) ); + } + else { + algorithm = std::unique_ptr( new GalileanAlgorithm( + k_space, dm, norder_x, norder_y, norder_z, nodal, v_galilean, dt ) ); + } + } + } // - Initialize arrays for fields in spectral space + FFT plans field_data = SpectralFieldData( realspace_ba, k_space, dm, @@ -95,4 +108,5 @@ SpectralSolver::pushSpectralFields(){ // initialized in the constructor of `SpectralSolver` algorithm->pushSpectralFields( field_data ); } + #endif // WARPX_USE_PSATD diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 1925bd8830b..a6487cf1309 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -36,10 +36,12 @@ namespace { #endif std::array,3>& Efield, std::array,3>& Bfield, + std::array,3>& Efield_avg, + std::array,3>& Bfield_avg, std::array,3>& current, std::unique_ptr& rho ) { - using Idx = SpectralFieldIndex; + using Idx = SpectralAvgFieldIndex; // Perform forward Fourier transform #ifdef WARPX_DIM_RZ @@ -87,6 +89,18 @@ namespace { solver.BackwardTransform(*Bfield[1], Idx::By); #endif solver.BackwardTransform(*Bfield[2], Idx::Bz); + +#ifndef WARPX_DIM_RZ + if (solver.fft_do_time_averaging){ + solver.BackwardTransform(*Efield_avg[0], Idx::Ex_avg); + solver.BackwardTransform(*Efield_avg[1], Idx::Ey_avg); + solver.BackwardTransform(*Efield_avg[2], Idx::Ez_avg); + + solver.BackwardTransform(*Bfield_avg[0], Idx::Bx_avg); + solver.BackwardTransform(*Bfield_avg[1], Idx::By_avg); + solver.BackwardTransform(*Bfield_avg[2], Idx::Bz_avg); + } +#endif } } @@ -109,10 +123,10 @@ WarpX::PushPSATD (int lev, amrex::Real /* dt */) { // Update the fields on the fine and coarse patch PushPSATDSinglePatch( *spectral_solver_fp[lev], - Efield_fp[lev], Bfield_fp[lev], current_fp[lev], rho_fp[lev] ); + Efield_fp[lev], Bfield_fp[lev], Efield_avg_fp[lev], Bfield_avg_fp[lev], current_fp[lev], rho_fp[lev] ); if (spectral_solver_cp[lev]) { PushPSATDSinglePatch( *spectral_solver_cp[lev], - Efield_cp[lev], Bfield_cp[lev], current_cp[lev], rho_cp[lev] ); + Efield_cp[lev], Bfield_cp[lev], Efield_avg_cp[lev], Bfield_avg_cp[lev], current_cp[lev], rho_cp[lev] ); } } #endif diff --git a/Source/Filter/CMakeLists.txt b/Source/Filter/CMakeLists.txt new file mode 100644 index 00000000000..801e721dd0c --- /dev/null +++ b/Source/Filter/CMakeLists.txt @@ -0,0 +1,6 @@ +target_sources(WarpX + PRIVATE + BilinearFilter.cpp + Filter.cpp + NCIGodfreyFilter.cpp +) diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt new file mode 100644 index 00000000000..b6cab076ca7 --- /dev/null +++ b/Source/Initialization/CMakeLists.txt @@ -0,0 +1,7 @@ +target_sources(WarpX + PRIVATE + InjectorDensity.cpp + InjectorMomentum.cpp + PlasmaInjector.cpp + WarpXInitData.cpp +) diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H index 38f49270a2e..b60f1195960 100644 --- a/Source/Initialization/InjectorDensity.H +++ b/Source/Initialization/InjectorDensity.H @@ -1,4 +1,5 @@ /* Copyright 2019 Axel Huebl, Maxence Thevenet, Weiqun Zhang + * Michael Rowan * * * This file is part of WarpX. @@ -65,15 +66,21 @@ struct InjectorDensityPredefined { case Profile::parabolic_channel: { - amrex::Real z_start = p[0]; - amrex::Real ramp_up = p[1]; - amrex::Real plateau = p[2]; - amrex::Real ramp_down = p[3]; - amrex::Real rc = p[4]; - amrex::Real n0 = p[5]; - amrex::Real n; - amrex::Real kp = PhysConst::q_e/PhysConst::c - *std::sqrt( n0/(PhysConst::m_e*PhysConst::ep0) ); + // These are cast as double to ensure sufficient precision in the + // initialized profile density profile; without these, single and + // double precision versions of the executable can show disagreement + // From testing, it seems that (for at least some setups), it is only + // necessary to case n0 as double to get good agreement between single + // and double precision, but just in case, all are cast as double. + double z_start = p[0]; + double ramp_up = p[1]; + double plateau = p[2]; + double ramp_down = p[3]; + double rc = p[4]; + double n0 = p[5]; + double n; + double kp = PhysConst::q_e/PhysConst::c + *amrex::Math::sqrt( n0/(PhysConst::m_e*PhysConst::ep0) ); // Longitudinal profile, normalized to 1 if ((z-z_start)>=0 and diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 0ef6c0390c9..e8331cb6002 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -134,6 +134,16 @@ struct InjectorPosition z < zmax and z >= zmin); } + // bool: whether the region defined by lo and hi overaps with the plasma region + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + bool + overlapsWith (const amrex::XDim3& lo, const amrex::XDim3& hi) const noexcept + { + return ! ( (xmin > hi.x) || (xmax < lo.x) + || (ymin > hi.y) || (ymax < lo.y) + || (zmin > hi.z) || (zmax < lo.z) ); + } + private: enum struct Type { random, regular }; Type type; diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 4d0011aa038..9e2df164f20 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -41,8 +41,12 @@ public: PlasmaInjector (int ispecies, const std::string& name); + // bool: whether the point (x, y, z) is inside the plasma region bool insideBounds (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; + // bool: whether the region defined by lo and hi overlaps with the plasma region + bool overlapsWith (const amrex::XDim3& lo, const amrex::XDim3& hi) const noexcept; + int num_particles_per_cell; amrex::Vector num_particles_per_cell_each_dim; @@ -76,6 +80,7 @@ public: int do_symmetrize = 0; bool external_file = false; //! initialize from an openPMD file + amrex::Real z_shift = 0.0; //! additional z offset for particle positions #ifdef WARPX_USE_OPENPMD //! openPMD::Series to load from in external_file injection std::unique_ptr m_openpmd_input_series; diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 90324789c59..ca3cc340cf2 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -79,7 +79,7 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) pp.query("radially_weighted", radially_weighted); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(radially_weighted, "ERROR: Only radially_weighted=true is supported"); - // parse plasma boundaries + // Unlimited boundaries xmin = std::numeric_limits::lowest(); ymin = std::numeric_limits::lowest(); zmin = std::numeric_limits::lowest(); @@ -88,6 +88,30 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) ymax = std::numeric_limits::max(); zmax = std::numeric_limits::max(); + // NOTE: When periodic boundaries are used, default injection range is set to mother grid dimensions. + const Geometry& geom = WarpX::GetInstance().Geom(0); + if( geom.isPeriodic(0) ) { + xmin = geom.ProbLo(0); + xmax = geom.ProbHi(0); + } + + if( geom.isPeriodic(1) ) { +# ifndef WARPX_DIM_3D + zmin = geom.ProbLo(1); + zmax = geom.ProbHi(1); +# else + ymin = geom.ProbLo(1); + ymax = geom.ProbHi(1); +# endif + } + +# ifdef WARPX_DIM_3D + if( geom.isPeriodic(2) ) { + zmin = geom.ProbLo(2); + zmax = geom.ProbHi(2); + } +# endif + pp.query("xmin", xmin); pp.query("ymin", ymin); pp.query("zmin", zmin); @@ -230,11 +254,6 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) parseDensity(pp); parseMomentum(pp); } else if (part_pos_s == "external_file") { -#ifdef WARPX_DIM_RZ - amrex::Abort("The option of reading particle data from an external " - "file has not been implemented nor tested in RZ geometry"); -#endif - pp.query("q_tot", q_tot); // optional #ifndef WARPX_USE_OPENPMD amrex::Abort("WarpX has to be compiled with USE_OPENPMD=TRUE to be able" " to read the external openPMD file with species data"); @@ -242,6 +261,9 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) external_file = true; std::string str_injection_file; pp.get("injection_file", str_injection_file); + // optional parameters + pp.query("q_tot", q_tot); + pp.query("z_shift",z_shift); #ifdef WARPX_USE_OPENPMD if (ParallelDescriptor::IOProcessor()) { @@ -499,6 +521,14 @@ bool PlasmaInjector::insideBounds (Real x, Real y, Real z) const noexcept z < zmax and z >= zmin); } +bool PlasmaInjector::overlapsWith (const amrex::XDim3& lo, + const amrex::XDim3& hi) const noexcept +{ + return ! ( (xmin > hi.x) || (xmax < lo.x) + || (ymin > hi.y) || (ymax < lo.y) + || (zmin > hi.z) || (zmax < lo.z) ); +} + InjectorPosition* PlasmaInjector::getInjectorPosition () { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 7ee7e1902dd..60121b3f4e6 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -65,7 +65,7 @@ WarpX::InitData () if (restart_chkfile.empty()) { - multi_diags->FilterComputePackFlush( 0, true ); + multi_diags->FilterComputePackFlush( -1, true ); // Write reduced diagnostics before the first iteration. if (reduced_diags->m_plot_rd != 0) @@ -247,6 +247,11 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (E_ext_grid_s == "constant") pp.getarr("E_external_grid", E_external_grid); + // initialize the averaged fields only if the averaged algorithm + // is activated ('psatd.do_time_averaging=1') + ParmParse ppsatd("psatd"); + ppsatd.query("do_time_averaging", fft_do_time_averaging ); + for (int i = 0; i < 3; ++i) { current_fp[lev][i]->setVal(0.0); if (lev > 0) @@ -254,16 +259,33 @@ WarpX::InitLevelData (int lev, Real /*time*/) if (B_ext_grid_s == "constant" || B_ext_grid_s == "default") { Bfield_fp[lev][i]->setVal(B_external_grid[i]); + if (fft_do_time_averaging) { + Bfield_avg_fp[lev][i]->setVal(B_external_grid[i]); + } + if (lev > 0) { Bfield_aux[lev][i]->setVal(B_external_grid[i]); Bfield_cp[lev][i]->setVal(B_external_grid[i]); + if (fft_do_time_averaging) { + Bfield_avg_aux[lev][i]->setVal(B_external_grid[i]); + Bfield_avg_cp[lev][i]->setVal(B_external_grid[i]); + } } } if (E_ext_grid_s == "constant" || E_ext_grid_s == "default") { Efield_fp[lev][i]->setVal(E_external_grid[i]); + if (fft_do_time_averaging) { + Efield_avg_fp[lev][i]->setVal(E_external_grid[i]); + } + if (lev > 0) { Efield_aux[lev][i]->setVal(E_external_grid[i]); Efield_cp[lev][i]->setVal(E_external_grid[i]); + if (fft_do_time_averaging) { + Efield_avg_aux[lev][i]->setVal(E_external_grid[i]); + Efield_avg_cp[lev][i]->setVal(E_external_grid[i]); + } + } } } @@ -403,9 +425,9 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& tbx = mfi.growntilebox(x_nodal_flag); - const Box& tby = mfi.growntilebox(y_nodal_flag); - const Box& tbz = mfi.growntilebox(z_nodal_flag); + const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); auto const& mfxfab = mfx->array(mfi); auto const& mfyfab = mfy->array(mfi); diff --git a/Source/Laser/CMakeLists.txt b/Source/Laser/CMakeLists.txt new file mode 100644 index 00000000000..859b5b05a6f --- /dev/null +++ b/Source/Laser/CMakeLists.txt @@ -0,0 +1,6 @@ +target_sources(WarpX + PRIVATE + LaserParticleContainer.cpp +) + +add_subdirectory(LaserProfilesImpl) diff --git a/Source/Laser/LaserParticleContainer.H b/Source/Laser/LaserParticleContainer.H index 0ac494cc6da..95dc050941b 100644 --- a/Source/Laser/LaserParticleContainer.H +++ b/Source/Laser/LaserParticleContainer.H @@ -39,6 +39,8 @@ public: virtual void InitData () final; virtual void Evolve (int lev, + const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, + const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 6100a4408d2..c80f64d2ebf 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -379,6 +379,8 @@ LaserParticleContainer::InitData (int lev) void LaserParticleContainer::Evolve (int lev, + const MultiFab&, const MultiFab&, const MultiFab&, + const MultiFab&, const MultiFab&, const MultiFab&, const MultiFab&, const MultiFab&, const MultiFab&, const MultiFab&, const MultiFab&, const MultiFab&, MultiFab& jx, MultiFab& jy, MultiFab& jz, diff --git a/Source/Laser/LaserProfilesImpl/CMakeLists.txt b/Source/Laser/LaserProfilesImpl/CMakeLists.txt new file mode 100644 index 00000000000..f5ce9f5fb3b --- /dev/null +++ b/Source/Laser/LaserProfilesImpl/CMakeLists.txt @@ -0,0 +1,7 @@ +target_sources(WarpX + PRIVATE + LaserProfileFieldFunction.cpp + LaserProfileFromTXYEFile.cpp + LaserProfileGaussian.cpp + LaserProfileHarris.cpp +) diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 298f008efd3..6be61e9d078 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -6,6 +6,7 @@ CXXSTD = c++14 USE_MPI = TRUE USE_PARTICLES = TRUE USE_RPATH = TRUE +BL_NO_FORT = TRUE ifeq ($(USE_GPU),TRUE) USE_OMP = FALSE @@ -21,9 +22,9 @@ ifeq ($(USE_RZ),TRUE) endif ifeq ($(USE_ASCENT_INSITU),TRUE) - ASCENT_HOME ?= NOT_SET - ifneq ($(ASCENT_HOME),NOT_SET) - include $(ASCENT_HOME)/share/ascent/ascent_config.mk + ASCENT_DIR ?= NOT_SET + ifneq ($(ASCENT_DIR),NOT_SET) + include $(ASCENT_DIR)/share/ascent/ascent_config.mk endif USE_CONDUIT = TRUE USE_ASCENT = TRUE @@ -185,14 +186,7 @@ ifeq ($(USE_RZ),TRUE) endif ifeq ($(USE_HDF5),TRUE) - HDF5_HOME ?= NOT_SET - ifneq ($(HDF5_HOME),NOT_SET) - VPATH_LOCATIONS += $(HDF5_HOME)/include - INCLUDE_LOCATIONS += $(HDF5_HOME)/include - LIBRARY_LOCATIONS += $(HDF5_HOME)/lib - endif DEFINES += -DWARPX_USE_HDF5 - libraries += -lhdf5 -lz endif # job_info support diff --git a/Source/Parallelization/CMakeLists.txt b/Source/Parallelization/CMakeLists.txt new file mode 100644 index 00000000000..3588f6f31e9 --- /dev/null +++ b/Source/Parallelization/CMakeLists.txt @@ -0,0 +1,6 @@ +target_sources(WarpX + PRIVATE + GuardCellManager.cpp + WarpXComm.cpp + WarpXRegrid.cpp +) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 95ed848f14f..79bcce2f462 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -352,6 +352,25 @@ WarpX::FillBoundaryF (IntVect ng) } } +void +WarpX::FillBoundaryB_avg (IntVect ng, IntVect ng_extra_fine) +{ + for (int lev = 0; lev <= finest_level; ++lev) + { + FillBoundaryB_avg(lev, ng, ng_extra_fine); + } +} + +void +WarpX::FillBoundaryE_avg (IntVect ng, IntVect ng_extra_fine) +{ + for (int lev = 0; lev <= finest_level; ++lev) + { + FillBoundaryE_avg(lev, ng, ng_extra_fine); + } +} + + void WarpX::FillBoundaryE(int lev, IntVect ng, IntVect ng_extra_fine) { @@ -474,6 +493,112 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) } } +void +WarpX::FillBoundaryE_avg(int lev, IntVect ng, IntVect ng_extra_fine) +{ + FillBoundaryE_avg(lev, PatchType::fine, ng+ng_extra_fine); + if (lev > 0) FillBoundaryE_avg(lev, PatchType::coarse, ng); +} + +void +WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) +{ + if (patch_type == PatchType::fine) + { + if (do_pml && pml[lev]->ok()) + { + amrex::Abort("Averaged Galilean PSATD with PML is not yet implemented"); + } + + const auto& period = Geom(lev).periodicity(); + if ( safe_guard_cells ){ + Vector mf{Efield_avg_fp[lev][0].get(),Efield_avg_fp[lev][1].get(),Efield_avg_fp[lev][2].get()}; + amrex::FillBoundary(mf, period); + } else { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Efield_avg_fp[lev][0]->nGrowVect(), + "Error: in FillBoundaryE_avg, requested more guard cells than allocated"); + Efield_avg_fp[lev][0]->FillBoundary(ng, period); + Efield_avg_fp[lev][1]->FillBoundary(ng, period); + Efield_avg_fp[lev][2]->FillBoundary(ng, period); + } + } + else if (patch_type == PatchType::coarse) + { + if (do_pml && pml[lev]->ok()) + { + amrex::Abort("Averaged Galilean PSATD with PML is not yet implemented"); + } + + const auto& cperiod = Geom(lev-1).periodicity(); + if ( safe_guard_cells ) { + Vector mf{Efield_avg_cp[lev][0].get(),Efield_avg_cp[lev][1].get(),Efield_avg_cp[lev][2].get()}; + amrex::FillBoundary(mf, cperiod); + + } else { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Efield_avg_cp[lev][0]->nGrowVect(), + "Error: in FillBoundaryE, requested more guard cells than allocated"); + Efield_avg_cp[lev][0]->FillBoundary(ng, cperiod); + Efield_avg_cp[lev][1]->FillBoundary(ng, cperiod); + Efield_avg_cp[lev][2]->FillBoundary(ng, cperiod); + } + } +} + + +void +WarpX::FillBoundaryB_avg (int lev, IntVect ng, IntVect ng_extra_fine) +{ + FillBoundaryB_avg(lev, PatchType::fine, ng + ng_extra_fine); + if (lev > 0) FillBoundaryB_avg(lev, PatchType::coarse, ng); +} + +void +WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) +{ + if (patch_type == PatchType::fine) + { + if (do_pml && pml[lev]->ok()) + { + amrex::Abort("Averaged Galilean PSATD with PML is not yet implemented"); + } + const auto& period = Geom(lev).periodicity(); + if ( safe_guard_cells ) { + Vector mf{Bfield_avg_fp[lev][0].get(),Bfield_avg_fp[lev][1].get(),Bfield_avg_fp[lev][2].get()}; + amrex::FillBoundary(mf, period); + } else { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Bfield_fp[lev][0]->nGrowVect(), + "Error: in FillBoundaryB, requested more guard cells than allocated"); + Bfield_avg_fp[lev][0]->FillBoundary(ng, period); + Bfield_avg_fp[lev][1]->FillBoundary(ng, period); + Bfield_avg_fp[lev][2]->FillBoundary(ng, period); + } + } + else if (patch_type == PatchType::coarse) + { + if (do_pml && pml[lev]->ok()) + { + amrex::Abort("Averaged Galilean PSATD with PML is not yet implemented"); + } + + const auto& cperiod = Geom(lev-1).periodicity(); + if ( safe_guard_cells ){ + Vector mf{Bfield_avg_cp[lev][0].get(),Bfield_avg_cp[lev][1].get(),Bfield_avg_cp[lev][2].get()}; + amrex::FillBoundary(mf, cperiod); + } else { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Bfield_avg_cp[lev][0]->nGrowVect(), + "Error: in FillBoundaryB_avg, requested more guard cells than allocated"); + Bfield_avg_cp[lev][0]->FillBoundary(ng, cperiod); + Bfield_avg_cp[lev][1]->FillBoundary(ng, cperiod); + Bfield_avg_cp[lev][2]->FillBoundary(ng, cperiod); + } + } +} + + void WarpX::FillBoundaryF (int lev, IntVect ng) { diff --git a/Source/Parser/CMakeLists.txt b/Source/Parser/CMakeLists.txt new file mode 100644 index 00000000000..2c9a93942d3 --- /dev/null +++ b/Source/Parser/CMakeLists.txt @@ -0,0 +1,8 @@ +target_sources(WarpX + PRIVATE + WarpXParser.cpp + wp_parser_c.cpp + wp_parser.lex.cpp + wp_parser.tab.cpp + wp_parser_y.cpp +) diff --git a/Source/Particles/CMakeLists.txt b/Source/Particles/CMakeLists.txt new file mode 100644 index 00000000000..0a8abe4eb52 --- /dev/null +++ b/Source/Particles/CMakeLists.txt @@ -0,0 +1,16 @@ +target_sources(WarpX + PRIVATE + MultiParticleContainer.cpp + PhotonParticleContainer.cpp + PhysicalParticleContainer.cpp + RigidInjectedParticleContainer.cpp + WarpXParticleContainer.cpp +) + +add_subdirectory(Collision) +#add_subdirectory(Deposition) +add_subdirectory(ElementaryProcess) +add_subdirectory(Gather) +add_subdirectory(ParticleCreation) +#add_subdirectory(Pusher) +add_subdirectory(Sorting) diff --git a/Source/Particles/Collision/CMakeLists.txt b/Source/Particles/Collision/CMakeLists.txt new file mode 100644 index 00000000000..f77a13b2c6d --- /dev/null +++ b/Source/Particles/Collision/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + CollisionType.cpp +) diff --git a/Source/Particles/ElementaryProcess/CMakeLists.txt b/Source/Particles/ElementaryProcess/CMakeLists.txt new file mode 100644 index 00000000000..bc3bff03117 --- /dev/null +++ b/Source/Particles/ElementaryProcess/CMakeLists.txt @@ -0,0 +1,13 @@ +target_sources(WarpX + PRIVATE + Ionization.cpp +) + +if(WarpX_HAVE_QED) + target_sources(WarpX + PRIVATE + QEDPairGeneration.cpp + QEDPhotonEmission.cpp + ) + add_subdirectory(QEDInternals) +endif() diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 6cf30bd4dd9..6275c7af477 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -10,38 +10,96 @@ #include "Utils/WarpXConst.H" #include "Particles/WarpXParticleContainer.H" +#include "Particles/Gather/GetExternalFields.H" +#include "Particles/Gather/FieldGather.H" +#include "Particles/Pusher/GetAndSetPosition.H" struct IonizationFilterFunc { - const amrex::Real* const AMREX_RESTRICT m_ionization_energies; - const amrex::Real* const AMREX_RESTRICT m_adk_prefactor; - const amrex::Real* const AMREX_RESTRICT m_adk_exp_prefactor; - const amrex::Real* const AMREX_RESTRICT m_adk_power; + const amrex::Real* AMREX_RESTRICT m_ionization_energies; + const amrex::Real* AMREX_RESTRICT m_adk_prefactor; + const amrex::Real* AMREX_RESTRICT m_adk_exp_prefactor; + const amrex::Real* AMREX_RESTRICT m_adk_power; int comp; int m_atomic_number; + GetParticlePosition m_get_position; + GetExternalEField m_get_externalE; + GetExternalBField m_get_externalB; + + amrex::Array4 m_ex_arr; + amrex::Array4 m_ey_arr; + amrex::Array4 m_ez_arr; + amrex::Array4 m_bx_arr; + amrex::Array4 m_by_arr; + amrex::Array4 m_bz_arr; + + amrex::IndexType m_ex_type; + amrex::IndexType m_ey_type; + amrex::IndexType m_ez_type; + amrex::IndexType m_bx_type; + amrex::IndexType m_by_type; + amrex::IndexType m_bz_type; + + amrex::GpuArray m_dx_arr; + amrex::GpuArray m_xyzmin_arr; + + int m_l_lower_order_in_v; + int m_nox; + int m_n_rz_azimuthal_modes; + + amrex::Dim3 m_lo; + + IonizationFilterFunc (const WarpXParIter& a_pti, int lev, int ngE, + amrex::FArrayBox const& exfab, + amrex::FArrayBox const& eyfab, + amrex::FArrayBox const& ezfab, + amrex::FArrayBox const& bxfab, + amrex::FArrayBox const& byfab, + amrex::FArrayBox const& bzfab, + amrex::Array v_galilean, + const amrex::Real* const AMREX_RESTRICT a_ionization_energies, + const amrex::Real* const AMREX_RESTRICT a_adk_prefactor, + const amrex::Real* const AMREX_RESTRICT a_adk_exp_prefactor, + const amrex::Real* const AMREX_RESTRICT a_adk_power, + int a_comp, + int a_atomic_number, + int a_offset = 0) noexcept; + template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator() (const PData& ptd, int i) const noexcept { + using namespace amrex::literals; + const int ion_lev = ptd.m_runtime_idata[comp][i]; if (ion_lev < m_atomic_number) { constexpr amrex::Real c = PhysConst::c; constexpr amrex::Real c2_inv = 1./c/c; + // gather E and B + amrex::ParticleReal xp, yp, zp; + m_get_position(i, xp, yp, zp); + + amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; + m_get_externalE(i, ex, ey, ez); + + amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + m_get_externalB(i, bx, by, bz); + + doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, + m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, + m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, + m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_nox, m_l_lower_order_in_v); + // Compute electric field amplitude in the particle's frame of // reference (particularly important when in boosted frame). amrex::ParticleReal ux = ptd.m_rdata[PIdx::ux][i]; amrex::ParticleReal uy = ptd.m_rdata[PIdx::uy][i]; amrex::ParticleReal uz = ptd.m_rdata[PIdx::uz][i]; - amrex::ParticleReal ex = ptd.m_rdata[PIdx::Ex][i]; - amrex::ParticleReal ey = ptd.m_rdata[PIdx::Ey][i]; - amrex::ParticleReal ez = ptd.m_rdata[PIdx::Ez][i]; - amrex::ParticleReal bx = ptd.m_rdata[PIdx::Bx][i]; - amrex::ParticleReal by = ptd.m_rdata[PIdx::By][i]; - amrex::ParticleReal bz = ptd.m_rdata[PIdx::Bz][i]; amrex::Real ga = std::sqrt(1. + (ux*ux + uy*uy + uz*uz) * c2_inv); amrex::Real E = std::sqrt( diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp new file mode 100644 index 00000000000..4c30987f417 --- /dev/null +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -0,0 +1,72 @@ +/* Copyright 2019-2020 Andrew Myers, Axel Huebl, + * Maxence Thevenet + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpX.H" +#include "Particles/ElementaryProcess/Ionization.H" + +IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, int ngE, + amrex::FArrayBox const& exfab, + amrex::FArrayBox const& eyfab, + amrex::FArrayBox const& ezfab, + amrex::FArrayBox const& bxfab, + amrex::FArrayBox const& byfab, + amrex::FArrayBox const& bzfab, + amrex::Array v_galilean, + const amrex::Real* const AMREX_RESTRICT a_ionization_energies, + const amrex::Real* const AMREX_RESTRICT a_adk_prefactor, + const amrex::Real* const AMREX_RESTRICT a_adk_exp_prefactor, + const amrex::Real* const AMREX_RESTRICT a_adk_power, + int a_comp, + int a_atomic_number, + int a_offset) noexcept +{ + m_ionization_energies = a_ionization_energies; + m_adk_prefactor = a_adk_prefactor; + m_adk_exp_prefactor = a_adk_exp_prefactor; + m_adk_power = a_adk_power; + comp = a_comp; + m_atomic_number = a_atomic_number; + + m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_externalE = GetExternalEField (a_pti, a_offset); + m_get_externalB = GetExternalBField (a_pti, a_offset); + + m_ex_arr = exfab.array(); + m_ey_arr = eyfab.array(); + m_ez_arr = ezfab.array(); + m_bx_arr = bxfab.array(); + m_by_arr = byfab.array(); + m_bz_arr = bzfab.array(); + + m_ex_type = exfab.box().ixType(); + m_ey_type = eyfab.box().ixType(); + m_ez_type = ezfab.box().ixType(); + m_bx_type = bxfab.box().ixType(); + m_by_type = byfab.box().ixType(); + m_bz_type = bzfab.box().ixType(); + + amrex::Box box = a_pti.tilebox(); + box.grow(ngE); + + const std::array& dx = WarpX::CellSize(std::max(lev, 0)); + m_dx_arr = {dx[0], dx[1], dx[2]}; + + // Lower corner of tile box physical domain (take into account Galilean shift) + amrex::Real cur_time = WarpX::GetInstance().gett_new(lev); + const auto& time_of_last_gal_shift = WarpX::GetInstance().time_of_last_gal_shift; + amrex::Real time_shift = (cur_time - time_of_last_gal_shift); + amrex::Array galilean_shift = { v_galilean[0]*time_shift, v_galilean[1]*time_shift, v_galilean[2]*time_shift }; + const std::array& xyzmin = WarpX::LowerCorner(box, galilean_shift, lev); + m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + m_l_lower_order_in_v = WarpX::l_lower_order_in_v; + m_nox = WarpX::nox; + m_n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + m_lo = amrex::lbound(box); +} diff --git a/Source/Particles/ElementaryProcess/Make.package b/Source/Particles/ElementaryProcess/Make.package index 588d017e65a..cc0610731b5 100644 --- a/Source/Particles/ElementaryProcess/Make.package +++ b/Source/Particles/ElementaryProcess/Make.package @@ -1,4 +1,8 @@ +CEXE_sources += Ionization.cpp + ifeq ($(QED),TRUE) + CEXE_sources += QEDPairGeneration.cpp + CEXE_sources += QEDPhotonEmission.cpp include $(WARPX_HOME)/Source/Particles/ElementaryProcess/QEDInternals/Make.package VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/ElementaryProcess/QEDInternals/ endif diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H index bed6337c33b..7056ac884d9 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H @@ -84,6 +84,9 @@ public: class BreitWheelerEvolveOpticalDepth { public: + + BreitWheelerEvolveOpticalDepth () = default; + /** * Constructor acquires a reference to control parameters and * lookup tables data. @@ -138,10 +141,10 @@ public: } private: - //laser wavelenght is not used with SI units - const amrex::Real m_dummy_lambda{1.0}; + //laser wavelength is not used with SI units + amrex::Real m_dummy_lambda{1.0}; - const PicsarBreitWheelerCtrl m_ctrl; + PicsarBreitWheelerCtrl m_ctrl; //lookup table data size_t m_TTfunc_size; diff --git a/Source/Particles/ElementaryProcess/QEDInternals/CMakeLists.txt b/Source/Particles/ElementaryProcess/QEDInternals/CMakeLists.txt new file mode 100644 index 00000000000..1bd803cdea7 --- /dev/null +++ b/Source/Particles/ElementaryProcess/QEDInternals/CMakeLists.txt @@ -0,0 +1,12 @@ +target_sources(WarpX + PRIVATE + BreitWheelerEngineWrapper.cpp + QuantumSyncEngineWrapper.cpp +) + +#if(WARPX_QED_TABLE_GEN) +# target_sources(WarpX +# PRIVATE +# BreitWheelerEngineTableBuilder.cpp +# QuantumSyncEngineTableBuilder.cpp +#endif() diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H index 6d46c901971..fe7d560cd02 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H @@ -84,6 +84,9 @@ public: class QuantumSynchrotronEvolveOpticalDepth { public: + + QuantumSynchrotronEvolveOpticalDepth () = default; + /** * Constructor acquires pointers to control parameters and * lookup tables data. @@ -139,10 +142,10 @@ public: } private: - //laser wavelenght is not used with SI units - const amrex::Real m_dummy_lambda{1.0}; + //laser wavelength is not used with SI units + amrex::Real m_dummy_lambda{1.0}; - const PicsarQuantumSynchrotronCtrl m_ctrl; + PicsarQuantumSynchrotronCtrl m_ctrl; //lookup table data size_t m_KKfunc_size; diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index 22f37a3515e..838afcaaccd 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -10,7 +10,9 @@ #include "Utils/WarpXConst.H" #include "Particles/WarpXParticleContainer.H" - +#include "Particles/Gather/GetExternalFields.H" +#include "Particles/Gather/FieldGather.H" +#include "Particles/Pusher/GetAndSetPosition.H" #include "QEDInternals/BreitWheelerEngineWrapper.H" /** @file @@ -32,9 +34,9 @@ public: * * @param[in] opt_depth_runtime_comp index of the optical depth component */ - PairGenerationFilterFunc(int const opt_depth_runtime_comp): - m_opt_depth_runtime_comp{opt_depth_runtime_comp} - {} + PairGenerationFilterFunc(int const opt_depth_runtime_comp) + : m_opt_depth_runtime_comp(opt_depth_runtime_comp) + {} /** * \brief Functor call. This method determines if a given (photon) particle @@ -75,9 +77,16 @@ public: * * @param[in] generate_functor functor to be called to determine the properties of the generated pairs */ - PairGenerationTransformFunc(BreitWheelerGeneratePairs const generate_functor): - m_generate_functor{generate_functor} - {} + PairGenerationTransformFunc(BreitWheelerGeneratePairs const generate_functor, + const WarpXParIter& a_pti, int lev, int ngE, + amrex::FArrayBox const& exfab, + amrex::FArrayBox const& eyfab, + amrex::FArrayBox const& ezfab, + amrex::FArrayBox const& bxfab, + amrex::FArrayBox const& byfab, + amrex::FArrayBox const& bzfab, + amrex::Array v_galilean, + int a_offset = 0); /** * \brief Functor call. It determines the properties of the generated pair @@ -104,12 +113,22 @@ public: const ParticleReal ux = src.m_rdata[PIdx::ux][i_src]; const ParticleReal uy = src.m_rdata[PIdx::uy][i_src]; const ParticleReal uz = src.m_rdata[PIdx::uz][i_src]; - const ParticleReal ex = src.m_rdata[PIdx::Ex][i_src]; - const ParticleReal ey = src.m_rdata[PIdx::Ey][i_src]; - const ParticleReal ez = src.m_rdata[PIdx::Ez][i_src]; - const ParticleReal bx = src.m_rdata[PIdx::Bx][i_src]; - const ParticleReal by = src.m_rdata[PIdx::By][i_src]; - const ParticleReal bz = src.m_rdata[PIdx::Bz][i_src]; + + // gather E and B + amrex::ParticleReal xp, yp, zp; + m_get_position(i_src, xp, yp, zp); + + amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; + m_get_externalE(i_src, ex, ey, ez); + + amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + m_get_externalB(i_src, bx, by, bz); + + doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, + m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, + m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, + m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_nox, m_l_lower_order_in_v); const auto px = ux*me; const auto py = uy*me; @@ -153,7 +172,34 @@ public: private: const BreitWheelerGeneratePairs - m_generate_functor; /*!< A copy of the functor to generate pairs. It contains only pointers to the lookup tables.*/ + m_generate_functor; /*!< A copy of the functor to generate pairs. It contains only pointers to the lookup tables.*/ + + GetParticlePosition m_get_position; + GetExternalEField m_get_externalE; + GetExternalBField m_get_externalB; + + amrex::Array4 m_ex_arr; + amrex::Array4 m_ey_arr; + amrex::Array4 m_ez_arr; + amrex::Array4 m_bx_arr; + amrex::Array4 m_by_arr; + amrex::Array4 m_bz_arr; + + amrex::IndexType m_ex_type; + amrex::IndexType m_ey_type; + amrex::IndexType m_ez_type; + amrex::IndexType m_bx_type; + amrex::IndexType m_by_type; + amrex::IndexType m_bz_type; + + amrex::GpuArray m_dx_arr; + amrex::GpuArray m_xyzmin_arr; + + int m_l_lower_order_in_v; + int m_nox; + int m_n_rz_azimuthal_modes; + + amrex::Dim3 m_lo; }; #endif //QED_PAIR_GENERATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp new file mode 100644 index 00000000000..cdb88d281ef --- /dev/null +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp @@ -0,0 +1,62 @@ +/* Copyright 2019-2020 Andrew Myers, Axel Huebl, + * Maxence Thevenet + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpX.H" +#include "Particles/ElementaryProcess/QEDPairGeneration.H" + +PairGenerationTransformFunc:: +PairGenerationTransformFunc (BreitWheelerGeneratePairs const generate_functor, + const WarpXParIter& a_pti, int lev, int ngE, + amrex::FArrayBox const& exfab, + amrex::FArrayBox const& eyfab, + amrex::FArrayBox const& ezfab, + amrex::FArrayBox const& bxfab, + amrex::FArrayBox const& byfab, + amrex::FArrayBox const& bzfab, + amrex::Array v_galilean, + int a_offset) +: m_generate_functor(generate_functor) +{ + m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_externalE = GetExternalEField (a_pti, a_offset); + m_get_externalB = GetExternalBField (a_pti, a_offset); + + m_ex_arr = exfab.array(); + m_ey_arr = eyfab.array(); + m_ez_arr = ezfab.array(); + m_bx_arr = bxfab.array(); + m_by_arr = byfab.array(); + m_bz_arr = bzfab.array(); + + m_ex_type = exfab.box().ixType(); + m_ey_type = eyfab.box().ixType(); + m_ez_type = ezfab.box().ixType(); + m_bx_type = bxfab.box().ixType(); + m_by_type = byfab.box().ixType(); + m_bz_type = bzfab.box().ixType(); + + amrex::Box box = a_pti.tilebox(); + box.grow(ngE); + + const std::array& dx = WarpX::CellSize(std::max(lev, 0)); + m_dx_arr = {dx[0], dx[1], dx[2]}; + + // Lower corner of tile box physical domain (take into account Galilean shift) + amrex::Real cur_time = WarpX::GetInstance().gett_new(lev); + const auto& time_of_last_gal_shift = WarpX::GetInstance().time_of_last_gal_shift; + amrex::Real time_shift = (cur_time - time_of_last_gal_shift); + amrex::Array galilean_shift = { v_galilean[0]*time_shift, v_galilean[1]*time_shift, v_galilean[2]*time_shift }; + const std::array& xyzmin = WarpX::LowerCorner(box, galilean_shift, lev); + m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + m_l_lower_order_in_v = WarpX::l_lower_order_in_v; + m_nox = WarpX::nox; + m_n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + m_lo = amrex::lbound(box); +} diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index c7c801eebfe..4dc573f60be 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -10,7 +10,9 @@ #include "Utils/WarpXConst.H" #include "Particles/WarpXParticleContainer.H" - +#include "Particles/Gather/GetExternalFields.H" +#include "Particles/Gather/FieldGather.H" +#include "Particles/Pusher/GetAndSetPosition.H" #include "QEDInternals/QuantumSyncEngineWrapper.H" /** @file @@ -32,9 +34,9 @@ public: * * @param[in] opt_depth_runtime_comp Index of the optical depth component */ - PhotonEmissionFilterFunc(int const opt_depth_runtime_comp): - m_opt_depth_runtime_comp{opt_depth_runtime_comp} - {} + PhotonEmissionFilterFunc(int const opt_depth_runtime_comp) + : m_opt_depth_runtime_comp(opt_depth_runtime_comp) + {} /** * \brief Functor call. This method determines if a given (electron or positron) @@ -56,7 +58,7 @@ public: } private: - int m_opt_depth_runtime_comp /*!< Index of the optical depth component of the source species*/; + int m_opt_depth_runtime_comp; /*!< Index of the optical depth component of the source species*/ }; /** @@ -81,15 +83,19 @@ public: * @param[in] opt_depth_runtime_comp index of the optical depth component of the source species * @param[in] emission_functor functor to generate photons and update momentum of the source particles */ - PhotonEmissionTransformFunc( + PhotonEmissionTransformFunc ( QuantumSynchrotronGetOpticalDepth opt_depth_functor, int const opt_depth_runtime_comp, - QuantumSynchrotronGeneratePhotonAndUpdateMomentum const emission_functor - ): - m_opt_depth_functor{opt_depth_functor}, - m_opt_depth_runtime_comp{opt_depth_runtime_comp}, - m_emission_functor{emission_functor} - {} + QuantumSynchrotronGeneratePhotonAndUpdateMomentum const emission_functor, + const WarpXParIter& a_pti, int lev, int ngE, + amrex::FArrayBox const& exfab, + amrex::FArrayBox const& eyfab, + amrex::FArrayBox const& ezfab, + amrex::FArrayBox const& bxfab, + amrex::FArrayBox const& byfab, + amrex::FArrayBox const& bzfab, + amrex::Array v_galilean, + int a_offset = 0); /** * \brief Functor call. It determines the properties of the generated photon @@ -113,12 +119,22 @@ public: const ParticleReal ux = src.m_rdata[PIdx::ux][i_src]; const ParticleReal uy = src.m_rdata[PIdx::uy][i_src]; const ParticleReal uz = src.m_rdata[PIdx::uz][i_src]; - const ParticleReal ex = src.m_rdata[PIdx::Ex][i_src]; - const ParticleReal ey = src.m_rdata[PIdx::Ey][i_src]; - const ParticleReal ez = src.m_rdata[PIdx::Ez][i_src]; - const ParticleReal bx = src.m_rdata[PIdx::Bx][i_src]; - const ParticleReal by = src.m_rdata[PIdx::By][i_src]; - const ParticleReal bz = src.m_rdata[PIdx::Bz][i_src]; + + // gather E and B + amrex::ParticleReal xp, yp, zp; + m_get_position(i_src, xp, yp, zp); + + amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; + m_get_externalE(i_src, ex, ey, ez); + + amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + m_get_externalB(i_src, bx, by, bz); + + doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, + m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, + m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, + m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_nox, m_l_lower_order_in_v); // Particle momentum is stored as gamma * velocity. // Convert to m * gamma * velocity before applying the emission functor. @@ -160,6 +176,33 @@ private: const QuantumSynchrotronGeneratePhotonAndUpdateMomentum m_emission_functor; /*!< A copy of the functor to generate photons. It contains only pointers to the lookup tables.*/ const int m_opt_depth_runtime_comp = 0; /*!< Index of the optical depth component of source species*/ + + GetParticlePosition m_get_position; + GetExternalEField m_get_externalE; + GetExternalBField m_get_externalB; + + amrex::Array4 m_ex_arr; + amrex::Array4 m_ey_arr; + amrex::Array4 m_ez_arr; + amrex::Array4 m_bx_arr; + amrex::Array4 m_by_arr; + amrex::Array4 m_bz_arr; + + amrex::IndexType m_ex_type; + amrex::IndexType m_ey_type; + amrex::IndexType m_ez_type; + amrex::IndexType m_bx_type; + amrex::IndexType m_by_type; + amrex::IndexType m_bz_type; + + amrex::GpuArray m_dx_arr; + amrex::GpuArray m_xyzmin_arr; + + int m_l_lower_order_in_v; + int m_nox; + int m_n_rz_azimuthal_modes; + + amrex::Dim3 m_lo; }; diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp new file mode 100644 index 00000000000..6134ff54267 --- /dev/null +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp @@ -0,0 +1,66 @@ +/* Copyright 2019-2020 Andrew Myers, Axel Huebl, + * Maxence Thevenet + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpX.H" +#include "Particles/ElementaryProcess/QEDPhotonEmission.H" + +PhotonEmissionTransformFunc:: +PhotonEmissionTransformFunc (QuantumSynchrotronGetOpticalDepth opt_depth_functor, + int const opt_depth_runtime_comp, + QuantumSynchrotronGeneratePhotonAndUpdateMomentum const emission_functor, + const WarpXParIter& a_pti, int lev, int ngE, + amrex::FArrayBox const& exfab, + amrex::FArrayBox const& eyfab, + amrex::FArrayBox const& ezfab, + amrex::FArrayBox const& bxfab, + amrex::FArrayBox const& byfab, + amrex::FArrayBox const& bzfab, + amrex::Array v_galilean, + int a_offset) +:m_opt_depth_functor{opt_depth_functor}, + m_opt_depth_runtime_comp{opt_depth_runtime_comp}, + m_emission_functor{emission_functor} +{ + m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_externalE = GetExternalEField (a_pti, a_offset); + m_get_externalB = GetExternalBField (a_pti, a_offset); + + m_ex_arr = exfab.array(); + m_ey_arr = eyfab.array(); + m_ez_arr = ezfab.array(); + m_bx_arr = bxfab.array(); + m_by_arr = byfab.array(); + m_bz_arr = bzfab.array(); + + m_ex_type = exfab.box().ixType(); + m_ey_type = eyfab.box().ixType(); + m_ez_type = ezfab.box().ixType(); + m_bx_type = bxfab.box().ixType(); + m_by_type = byfab.box().ixType(); + m_bz_type = bzfab.box().ixType(); + + amrex::Box box = a_pti.tilebox(); + box.grow(ngE); + + const std::array& dx = WarpX::CellSize(std::max(lev, 0)); + m_dx_arr = {dx[0], dx[1], dx[2]}; + + // Lower corner of tile box physical domain (take into account Galilean shift) + amrex::Real cur_time = WarpX::GetInstance().gett_new(lev); + const auto& time_of_last_gal_shift = WarpX::GetInstance().time_of_last_gal_shift; + amrex::Real time_shift = (cur_time - time_of_last_gal_shift); + amrex::Array galilean_shift = { v_galilean[0]*time_shift, v_galilean[1]*time_shift, v_galilean[2]*time_shift }; + const std::array& xyzmin = WarpX::LowerCorner(box, galilean_shift, lev); + m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + m_l_lower_order_in_v = WarpX::l_lower_order_in_v; + m_nox = WarpX::nox; + m_n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + m_lo = amrex::lbound(box); +} diff --git a/Source/Particles/Gather/CMakeLists.txt b/Source/Particles/Gather/CMakeLists.txt new file mode 100644 index 00000000000..8ede378f2e8 --- /dev/null +++ b/Source/Particles/Gather/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + GetExternalFields.cpp +) diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index fa92a7ae9d3..551c96ac179 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -16,7 +16,7 @@ /** * \brief Field gather for a single particle * - * /param xp, yp, zp : Particle position coordinates + * \param xp, yp, zp : Particle position coordinates * \param Exp, Eyp, Ezp : Electric field on particles. * \param Bxp, Byp, Bzp : Magnetic field on particles. * \param ex_arr ey_arr ez_arr : Array4 of the electric field, either full array or tile. @@ -430,4 +430,87 @@ void doGatherShapeN(const GetParticlePosition& getPosition, ); } +/** + * \brief Field gather for a single particle + * + * /param xp, yp, zp : Particle position coordinates + * \param Exp, Eyp, Ezp : Electric field on particles. + * \param Bxp, Byp, Bzp : Magnetic field on particles. + * \param ex_arr ey_arr ez_arr : Array4 of the electric field, either full array or tile. + * \param bx_arr by_arr bz_arr : Array4 of the magnetic field, either full array or tile. + * \param ex_type, ey_type, ez_type : IndexType of the electric field + * \param bx_type, by_type, bz_type : IndexType of the magnetic field + * \param dx : 3D cell spacing + * \param xyzmin : Physical lower bounds of domain in x, y, z. + * \param lo : Index lower bounds of domain. + * \param n_rz_azimuthal_modes : Number of azimuthal modes when using RZ geometry + * \param nox : order of the particle shape function + * \param l_lower_order_in_v : whether to use lower order in v + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeN (const amrex::ParticleReal xp, + const amrex::ParticleReal yp, + const amrex::ParticleReal zp, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& ex_arr, + amrex::Array4 const& ey_arr, + amrex::Array4 const& ez_arr, + amrex::Array4 const& bx_arr, + amrex::Array4 const& by_arr, + amrex::Array4 const& bz_arr, + const amrex::IndexType ex_type, + const amrex::IndexType ey_type, + const amrex::IndexType ez_type, + const amrex::IndexType bx_type, + const amrex::IndexType by_type, + const amrex::IndexType bz_type, + const amrex::GpuArray& dx_arr, + const amrex::GpuArray& xyzmin_arr, + const amrex::Dim3& lo, + const long n_rz_azimuthal_modes, + const int nox, + const int l_lower_order_in_v) +{ + if (l_lower_order_in_v) { + if (nox == 1) { + doGatherShapeN<1,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeN<2,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeN<3,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } else { + if (nox == 1) { + doGatherShapeN<1,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeN<2,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeN<3,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } +} + #endif // FIELDGATHER_H_ diff --git a/Source/Particles/Gather/GetExternalFields.H b/Source/Particles/Gather/GetExternalFields.H index 8e08fc10104..f762febd5bc 100644 --- a/Source/Particles/Gather/GetExternalFields.H +++ b/Source/Particles/Gather/GetExternalFields.H @@ -2,6 +2,7 @@ #define WARPX_PARTICLES_GATHER_GETEXTERNALFIELDS_H_ #include "Particles/WarpXParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" #include @@ -60,27 +61,9 @@ struct GetExternalField */ struct GetExternalEField : GetExternalField { - GetExternalEField (const WarpXParIter& a_pti, int a_offset = 0) noexcept - { - auto& warpx = WarpX::GetInstance(); - auto& mypc = warpx.GetPartContainer(); - if (mypc.m_E_ext_particle_s=="constant" || mypc.m_E_ext_particle_s=="default") - { - m_type = Constant; - m_field_value[0] = mypc.m_E_external_particle[0]; - m_field_value[1] = mypc.m_E_external_particle[1]; - m_field_value[2] = mypc.m_E_external_particle[2]; - } - else if (mypc.m_E_ext_particle_s=="parse_e_ext_particle_function") - { - m_type = Parser; - m_time = warpx.gett_new(a_pti.GetLevel()); - m_get_position = GetParticlePosition(a_pti, a_offset); - m_xfield_partparser = mypc.m_Ex_particle_parser.get(); - m_yfield_partparser = mypc.m_Ey_particle_parser.get(); - m_zfield_partparser = mypc.m_Ez_particle_parser.get(); - } - } + GetExternalEField () = default; + + GetExternalEField (const WarpXParIter& a_pti, int a_offset = 0) noexcept; }; /** \brief Functor that can be used to assign the external @@ -88,27 +71,9 @@ struct GetExternalEField : GetExternalField */ struct GetExternalBField : GetExternalField { - GetExternalBField (const WarpXParIter& a_pti, int a_offset = 0) noexcept - { - auto& warpx = WarpX::GetInstance(); - auto& mypc = warpx.GetPartContainer(); - if (mypc.m_B_ext_particle_s=="constant" || mypc.m_B_ext_particle_s=="default") - { - m_type = Constant; - m_field_value[0] = mypc.m_B_external_particle[0]; - m_field_value[1] = mypc.m_B_external_particle[1]; - m_field_value[2] = mypc.m_B_external_particle[2]; - } - else if (mypc.m_B_ext_particle_s=="parse_e_ext_particle_function") - { - m_type = Parser; - m_time = warpx.gett_new(a_pti.GetLevel()); - m_get_position = GetParticlePosition(a_pti, a_offset); - m_xfield_partparser = mypc.m_Bx_particle_parser.get(); - m_yfield_partparser = mypc.m_By_particle_parser.get(); - m_zfield_partparser = mypc.m_Bz_particle_parser.get(); - } - } + GetExternalBField () = default; + + GetExternalBField (const WarpXParIter& a_pti, int a_offset = 0) noexcept; }; #endif diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp new file mode 100644 index 00000000000..a5ba6698f5d --- /dev/null +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -0,0 +1,46 @@ +#include "WarpX.H" +#include "Particles/Gather/GetExternalFields.H" + +GetExternalEField::GetExternalEField (const WarpXParIter& a_pti, int a_offset) noexcept +{ + auto& warpx = WarpX::GetInstance(); + auto& mypc = warpx.GetPartContainer(); + if (mypc.m_E_ext_particle_s=="constant" || mypc.m_E_ext_particle_s=="default") + { + m_type = Constant; + m_field_value[0] = mypc.m_E_external_particle[0]; + m_field_value[1] = mypc.m_E_external_particle[1]; + m_field_value[2] = mypc.m_E_external_particle[2]; + } + else if (mypc.m_E_ext_particle_s=="parse_e_ext_particle_function") + { + m_type = Parser; + m_time = warpx.gett_new(a_pti.GetLevel()); + m_get_position = GetParticlePosition(a_pti, a_offset); + m_xfield_partparser = mypc.m_Ex_particle_parser.get(); + m_yfield_partparser = mypc.m_Ey_particle_parser.get(); + m_zfield_partparser = mypc.m_Ez_particle_parser.get(); + } +} + +GetExternalBField::GetExternalBField (const WarpXParIter& a_pti, int a_offset) noexcept +{ + auto& warpx = WarpX::GetInstance(); + auto& mypc = warpx.GetPartContainer(); + if (mypc.m_B_ext_particle_s=="constant" || mypc.m_B_ext_particle_s=="default") + { + m_type = Constant; + m_field_value[0] = mypc.m_B_external_particle[0]; + m_field_value[1] = mypc.m_B_external_particle[1]; + m_field_value[2] = mypc.m_B_external_particle[2]; + } + else if (mypc.m_B_ext_particle_s=="parse_b_ext_particle_function") + { + m_type = Parser; + m_time = warpx.gett_new(a_pti.GetLevel()); + m_get_position = GetParticlePosition(a_pti, a_offset); + m_xfield_partparser = mypc.m_Bx_particle_parser.get(); + m_yfield_partparser = mypc.m_By_particle_parser.get(); + m_zfield_partparser = mypc.m_Bz_particle_parser.get(); + } +} diff --git a/Source/Particles/Gather/Make.package b/Source/Particles/Gather/Make.package index 19eb98e1435..c9a87821718 100644 --- a/Source/Particles/Gather/Make.package +++ b/Source/Particles/Gather/Make.package @@ -1 +1,3 @@ +CEXE_sources += GetExternalFields.cpp + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Gather diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H new file mode 100644 index 00000000000..cc94fac2934 --- /dev/null +++ b/Source/Particles/Gather/ScaleFields.H @@ -0,0 +1,63 @@ +#ifndef WARPX_PARTICLES_GATHER_SCALEFIELDS_H_ +#define WARPX_PARTICLES_GATHER_SCALEFIELDS_H_ + +#include "Particles/WarpXParticleContainer.H" + +#include + +#include + +/** \brief Functor that scales E and B by a factor before pushing the particles. + * This is used for rigid injection. + */ +struct ScaleFields +{ + amrex::Real m_do_scale; + amrex::Real m_dt; + amrex::Real m_z_plane_previous; + amrex::Real m_vz_ave_boosted; + amrex::Real m_v_boost; + + ScaleFields(bool do_scale) noexcept + : m_do_scale(do_scale) + {} + + ScaleFields (bool do_scale, amrex::Real dt, amrex::Real z_plane_previous, + amrex::Real vz_ave_boosted, amrex::Real v_boost) noexcept + : m_do_scale(do_scale), m_dt(dt), m_z_plane_previous(z_plane_previous), + m_vz_ave_boosted(vz_ave_boosted), m_v_boost(v_boost) + {} + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator () (amrex::ParticleReal /*xp*/, + amrex::ParticleReal /*yp*/, + amrex::ParticleReal zp, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp) const noexcept + { + using namespace amrex::literals; + + if (not m_do_scale) return; + + // Scale the fields of particles about to cross the injection plane. + // This only approximates what should be happening. The particles + // should by advanced a fraction of a time step instead. + // Scaling the fields is much easier and may be good enough. + const amrex::Real dtscale = m_dt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost); + if (0._rt < dtscale && dtscale < m_dt) + { + Exp *= dtscale; + Eyp *= dtscale; + Ezp *= dtscale; + Bxp *= dtscale; + Byp *= dtscale; + Bzp *= dtscale; + } + } +}; + +#endif diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 194c28e9e9a..20af8126d42 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -78,15 +78,6 @@ public: void InitData (); - /// - /// Performs the field gather operation using the input fields E and B, for all the species - /// in the MultiParticleContainer. This is the electromagnetic version of the field gather. - /// - void FieldGather (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, - const amrex::MultiFab& By, const amrex::MultiFab& Bz); - /// /// This evolves all the particles by one PIC time step, including current deposition, the /// field solve, and pushing the particles, for all the species in the MultiParticleContainer. @@ -95,6 +86,8 @@ public: void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + const amrex::MultiFab& Ex_avg, const amrex::MultiFab& Ey_avg, const amrex::MultiFab& Ez_avg, + const amrex::MultiFab& Bx_avg, const amrex::MultiFab& By_avg, const amrex::MultiFab& Bz_avg, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, amrex::MultiFab* rho, amrex::MultiFab* crho, @@ -125,7 +118,9 @@ public: /// std::unique_ptr GetChargeDensity(int lev, bool local = false); - void doFieldIonization (); + void doFieldIonization (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); void doCoulombCollisions (); @@ -219,7 +214,13 @@ public: /** * \brief Performs QED events (Breit-Wheeler process and photon emission) */ - void doQedEvents(); + void doQedEvents (int lev, + const amrex::MultiFab& Ex, + const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, + const amrex::MultiFab& By, + const amrex::MultiFab& Bz); #endif int getSpeciesID (std::string product_str) const; @@ -230,12 +231,24 @@ protected: /** * \brief Performs Breit-Wheeler process for the species for which it is enabled */ - void doQedBreitWheeler(); + void doQedBreitWheeler (int lev, + const amrex::MultiFab& Ex, + const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, + const amrex::MultiFab& By, + const amrex::MultiFab& Bz); /** * \brief Performs QED photon emission for the species for which it is enabled */ - void doQedQuantumSync(); + void doQedQuantumSync (int lev, + const amrex::MultiFab& Ex, + const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, + const amrex::MultiFab& By, + const amrex::MultiFab& Bz); #endif // Particle container types diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index b22a6f79d6b..9f015124bb0 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -312,21 +312,12 @@ MultiParticleContainer::InitData () } -void -MultiParticleContainer::FieldGather (int lev, - const MultiFab& Ex, const MultiFab& Ey, - const MultiFab& Ez, const MultiFab& Bx, - const MultiFab& By, const MultiFab& Bz) -{ - for (auto& pc : allcontainers) { - pc->FieldGather(lev, Ex, Ey, Ez, Bx, By, Bz); - } -} - void MultiParticleContainer::Evolve (int lev, const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + const MultiFab& Ex_avg, const MultiFab& Ey_avg, const MultiFab& Ez_avg, + const MultiFab& Bx_avg, const MultiFab& By_avg, const MultiFab& Bz_avg, MultiFab& jx, MultiFab& jy, MultiFab& jz, MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, MultiFab* rho, MultiFab* crho, @@ -343,7 +334,7 @@ MultiParticleContainer::Evolve (int lev, if (rho) rho->setVal(0.0); if (crho) crho->setVal(0.0); for (auto& pc : allcontainers) { - pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, + pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, Ex_avg, Ey_avg, Ez_avg, Bx_avg, By_avg, Bz_avg, jx, jy, jz, cjx, cjy, cjz, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type); } } @@ -634,7 +625,13 @@ MultiParticleContainer::getSpeciesID (std::string product_str) const } void -MultiParticleContainer::doFieldIonization () +MultiParticleContainer::doFieldIonization (int lev, + const MultiFab& Ex, + const MultiFab& Ey, + const MultiFab& Ez, + const MultiFab& Bx, + const MultiFab& By, + const MultiFab& Bz) { WARPX_PROFILE("MPC::doFieldIonization"); @@ -649,31 +646,31 @@ MultiParticleContainer::doFieldIonization () SmartCopyFactory copy_factory(*pc_source, *pc_product); auto phys_pc_ptr = static_cast(pc_source.get()); - auto Filter = phys_pc_ptr->getIonizationFunc(); auto Copy = copy_factory.getSmartCopy(); auto Transform = IonizationTransformFunc(); pc_source ->defineAllParticleTiles(); pc_product->defineAllParticleTiles(); - for (int lev = 0; lev <= pc_source->finestLevel(); ++lev) - { - const auto info = getMFItInfo(*pc_source, *pc_product); + auto info = getMFItInfo(*pc_source, *pc_product); #ifdef _OPENMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi = pc_source->MakeMFIter(lev, info); mfi.isValid(); ++mfi) - { - auto& src_tile = pc_source ->ParticlesAt(lev, mfi); - auto& dst_tile = pc_product->ParticlesAt(lev, mfi); + for (WarpXParIter pti(*pc_source, lev, info); pti.isValid(); ++pti) + { + auto& src_tile = pc_source ->ParticlesAt(lev, pti); + auto& dst_tile = pc_product->ParticlesAt(lev, pti); - const auto np_dst = dst_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, - Filter, Copy, Transform); + auto Filter = phys_pc_ptr->getIonizationFunc(pti, lev, Ex.nGrow(), + Ex[pti], Ey[pti], Ez[pti], + Bx[pti], By[pti], Bz[pti]); - setNewParticleIDs(dst_tile, np_dst, num_added); - } + const auto np_dst = dst_tile.numParticles(); + const auto num_added = filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + Filter, Copy, Transform); + + setNewParticleIDs(dst_tile, np_dst, num_added); } } } @@ -1126,15 +1123,27 @@ MultiParticleContainer::doQEDSchwinger () } } -void MultiParticleContainer::doQedEvents() +void MultiParticleContainer::doQedEvents (int lev, + const MultiFab& Ex, + const MultiFab& Ey, + const MultiFab& Ez, + const MultiFab& Bx, + const MultiFab& By, + const MultiFab& Bz) { WARPX_PROFILE("MPC::doQedEvents"); - doQedBreitWheeler(); - doQedQuantumSync(); + doQedBreitWheeler(lev, Ex, Ey, Ez, Bx, By, Bz); + doQedQuantumSync(lev, Ex, Ey, Ez, Bx, By, Bz); } -void MultiParticleContainer::doQedBreitWheeler() +void MultiParticleContainer::doQedBreitWheeler (int lev, + const MultiFab& Ex, + const MultiFab& Ey, + const MultiFab& Ez, + const MultiFab& Bx, + const MultiFab& By, + const MultiFab& Bz) { WARPX_PROFILE("MPC::doQedBreitWheeler"); @@ -1160,40 +1169,48 @@ void MultiParticleContainer::doQedBreitWheeler() const auto CopyPos = copy_factory_pos.getSmartCopy(); const auto pair_gen_functor = m_shr_p_bw_engine->build_pair_functor(); - auto Transform = PairGenerationTransformFunc(pair_gen_functor); pc_source ->defineAllParticleTiles(); pc_product_pos->defineAllParticleTiles(); pc_product_ele->defineAllParticleTiles(); - for (int lev = 0; lev <= pc_source->finestLevel(); ++lev) - { - const auto info = getMFItInfo(*pc_source, *pc_product_ele, *pc_product_pos); + auto info = getMFItInfo(*pc_source, *pc_product_ele, *pc_product_pos); #ifdef _OPENMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi = pc_source->MakeMFIter(lev, info); mfi.isValid(); ++mfi) - { - auto& src_tile = pc_source->ParticlesAt(lev, mfi); - auto& dst_ele_tile = pc_product_ele->ParticlesAt(lev, mfi); - auto& dst_pos_tile = pc_product_pos->ParticlesAt(lev, mfi); - - const auto np_dst_ele = dst_ele_tile.numParticles(); - const auto np_dst_pos = dst_pos_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>( - dst_ele_tile, dst_pos_tile, - src_tile, np_dst_ele, np_dst_pos, - Filter, CopyEle, CopyPos, Transform); - - setNewParticleIDs(dst_ele_tile, np_dst_ele, num_added); - setNewParticleIDs(dst_pos_tile, np_dst_pos, num_added); - } + for (WarpXParIter pti(*pc_source, lev, info); pti.isValid(); ++pti) + { + auto Transform = PairGenerationTransformFunc(pair_gen_functor, + pti, lev, Ex.nGrow(), + Ex[pti], Ey[pti], Ez[pti], + Bx[pti], By[pti], Bz[pti], + pc_source->get_v_galilean()); + + auto& src_tile = pc_source->ParticlesAt(lev, pti); + auto& dst_ele_tile = pc_product_ele->ParticlesAt(lev, pti); + auto& dst_pos_tile = pc_product_pos->ParticlesAt(lev, pti); + + const auto np_dst_ele = dst_ele_tile.numParticles(); + const auto np_dst_pos = dst_pos_tile.numParticles(); + const auto num_added = filterCopyTransformParticles<1>( + dst_ele_tile, dst_pos_tile, + src_tile, np_dst_ele, np_dst_pos, + Filter, CopyEle, CopyPos, Transform); + + setNewParticleIDs(dst_ele_tile, np_dst_ele, num_added); + setNewParticleIDs(dst_pos_tile, np_dst_pos, num_added); } - } + } } -void MultiParticleContainer::doQedQuantumSync() +void MultiParticleContainer::doQedQuantumSync (int lev, + const MultiFab& Ex, + const MultiFab& Ey, + const MultiFab& Ez, + const MultiFab& Bx, + const MultiFab& By, + const MultiFab& Bz) { WARPX_PROFILE("MPC::doQedEvents::doQedQuantumSync"); @@ -1204,7 +1221,7 @@ void MultiParticleContainer::doQedQuantumSync() for (auto& pc_source : allcontainers){ if(!pc_source->has_quantum_sync()){ continue; } - // Get product species + // Get product species auto& pc_product_phot = allcontainers[pc_source->m_qed_quantum_sync_phot_product]; @@ -1215,42 +1232,41 @@ void MultiParticleContainer::doQedQuantumSync() const auto Filter = phys_pc_ptr->getPhotonEmissionFilterFunc(); const auto CopyPhot = copy_factory_phot.getSmartCopy(); - auto Transform = PhotonEmissionTransformFunc( - m_shr_p_qs_engine->build_optical_depth_functor(), - pc_source->particle_runtime_comps["optical_depth_QSR"], - m_shr_p_qs_engine->build_phot_em_functor()); - pc_source ->defineAllParticleTiles(); pc_product_phot->defineAllParticleTiles(); - for (int lev = 0; lev <= pc_source->finestLevel(); ++lev) - { - const auto info = getMFItInfo(*pc_source, *pc_product_phot); + auto info = getMFItInfo(*pc_source, *pc_product_phot); #ifdef _OPENMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi = pc_source->MakeMFIter(lev, info); mfi.isValid(); ++mfi) - { - auto& src_tile = pc_source->ParticlesAt(lev, mfi); - auto& dst_tile = pc_product_phot->ParticlesAt(lev, mfi); + for (WarpXParIter pti(*pc_source, lev, info); pti.isValid(); ++pti) + { + auto Transform = PhotonEmissionTransformFunc( + m_shr_p_qs_engine->build_optical_depth_functor(), + pc_source->particle_runtime_comps["optical_depth_QSR"], + m_shr_p_qs_engine->build_phot_em_functor(), + pti, lev, Ex.nGrow(), + Ex[pti], Ey[pti], Ez[pti], + Bx[pti], By[pti], Bz[pti], + pc_source->get_v_galilean()); - const auto np_dst = dst_tile.numParticles(); + auto& src_tile = pc_source->ParticlesAt(lev, pti); + auto& dst_tile = pc_product_phot->ParticlesAt(lev, pti); - const auto num_added = - filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, - Filter, CopyPhot, Transform); + const auto np_dst = dst_tile.numParticles(); - setNewParticleIDs(dst_tile, np_dst, num_added); + const auto num_added = + filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + Filter, CopyPhot, Transform); - cleanLowEnergyPhotons( - dst_tile, np_dst, num_added, - m_quantum_sync_photon_creation_energy_threshold); + setNewParticleIDs(dst_tile, np_dst, num_added); - } + cleanLowEnergyPhotons( + dst_tile, np_dst, num_added, + m_quantum_sync_photon_creation_energy_threshold); } } - } void MultiParticleContainer::CheckQEDProductSpecies() diff --git a/Source/Particles/ParticleCreation/CMakeLists.txt b/Source/Particles/ParticleCreation/CMakeLists.txt new file mode 100644 index 00000000000..41669173336 --- /dev/null +++ b/Source/Particles/ParticleCreation/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + SmartUtils.cpp +) diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 9a8d9f7ee0c..0278c711820 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -41,12 +41,6 @@ static std::map initialization_policies = { {"ux", InitializationPolicy::Zero }, {"uy", InitializationPolicy::Zero }, {"uz", InitializationPolicy::Zero }, - {"Ex", InitializationPolicy::Zero }, - {"Ey", InitializationPolicy::Zero }, - {"Ez", InitializationPolicy::Zero }, - {"Bx", InitializationPolicy::Zero }, - {"By", InitializationPolicy::Zero }, - {"Bz", InitializationPolicy::Zero }, #ifdef WARPX_DIM_RZ {"theta", InitializationPolicy::Zero}, #endif diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index ee4ea2dd883..0cc8a7fe5ae 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -82,7 +82,7 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind copy(dst_data, src_data, i, N*p_offsets[i] + dst_index + j); transform(dst_data, src_data, i, N*p_offsets[i] + dst_index); } - }); + }) Gpu::streamSynchronize(); return num_added; @@ -143,7 +143,7 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, AMREX_HOST_DEVICE_FOR_1D(np, i, { p_mask[i] = filter(src_data, i); - }); + }) return filterCopyTransformParticles(dst, src, mask.dataPtr(), dst_index, std::forward(copy), @@ -235,7 +235,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, N*p_offsets[i] + dst1_index, N*p_offsets[i] + dst2_index); } - }); + }) Gpu::streamSynchronize(); return num_added; @@ -303,7 +303,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, AMREX_HOST_DEVICE_FOR_1D(np, i, { p_mask[i] = filter(src_data, i); - }); + }) return filterCopyTransformParticles(dst1, dst2, src, mask.dataPtr(), dst1_index, dst2_index, diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 4cdffd3853a..0bfe38f42ae 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -36,6 +36,12 @@ public: const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + const amrex::MultiFab& Ex_avg, + const amrex::MultiFab& Ey_avg, + const amrex::MultiFab& Ez_avg, + const amrex::MultiFab& Bx_avg, + const amrex::MultiFab& By_avg, + const amrex::MultiFab& Bz_avg, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, @@ -55,7 +61,18 @@ public: DtType a_dt_type=DtType::Full) override; virtual void PushPX(WarpXParIter& pti, - amrex::Real dt, DtType a_dt_type=DtType::Full) override; + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int /*e_is_nodal*/, + const long offset, + const long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type) override; // Do nothing virtual void PushP (int lev, @@ -84,19 +101,6 @@ public: int lev, int depos_lev, amrex::Real dt) override {}; - -#ifdef WARPX_QED - /** - * This function evolves the optical depth of the photons if QED effects - * are enabled. - * @param[in,out] pti particle iterator (optical depth will be modified) - * @param[in] dt temporal step - */ - virtual void EvolveOpticalDepth(WarpXParIter& pti, - amrex::Real dt) override; - -#endif - }; #endif // #ifndef WARPX_PhotonParticleContainer_H_ diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index bd80b8e438f..ff1107eeaf2 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -13,9 +13,11 @@ #include "Particles/Pusher/UpdatePositionPhoton.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/CopyParticleAttribs.H" +#include "Particles/Gather/FieldGather.H" +#include "Particles/Gather/GetExternalFields.H" #ifdef _OPENMP -# include +#include #endif #include @@ -63,10 +65,35 @@ void PhotonParticleContainer::InitData() } void -PhotonParticleContainer::PushPX(WarpXParIter& pti, Real dt, DtType a_dt_type) +PhotonParticleContainer::PushPX (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int /*e_is_nodal*/, + const long offset, + const long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields /*scaleFields*/, DtType a_dt_type) { + // Get cell size on gather_lev + const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + + // Get box from which field is gathered. + // If not gathering from the finest level, the box is coarsened. + amrex::Box box; + if (lev == gather_lev) { + box = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); + box = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + // Add guard cells to the box. + box.grow(ngE); - // This wraps the momentum and position advance so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); // Extract pointers to the different particle quantities @@ -74,19 +101,91 @@ PhotonParticleContainer::PushPX(WarpXParIter& pti, Real dt, DtType a_dt_type) ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); +#ifdef WARPX_QED + AMREX_ASSERT(has_breit_wheeler()); + + BreitWheelerEvolveOpticalDepth evolve_opt; + amrex::Real* AMREX_RESTRICT p_optical_depth_BW = nullptr; + const bool local_has_breit_wheeler = has_breit_wheeler(); + if (local_has_breit_wheeler) { + evolve_opt = m_shr_p_bw_engine->build_evolve_functor(); + p_optical_depth_BW = pti.GetAttribs(particle_comps["optical_depth_BW"]).dataPtr(); + } + + const auto me = PhysConst::m_e; +#endif + auto copyAttribs = CopyParticleAttribs(pti, tmp_particle_data); int do_copy = (WarpX::do_back_transformed_diagnostics && do_back_transformed_diagnostics && a_dt_type!=DtType::SecondHalf); - const auto GetPosition = GetParticlePosition(pti); - auto SetPosition = SetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti, offset); + auto SetPosition = SetParticlePosition(pti, offset); + + const auto getExternalE = GetExternalEField(pti, offset); + const auto getExternalB = GetExternalBField(pti, offset); + + // Lower corner of tile box physical domain (take into account Galilean shift) + amrex::Real cur_time = WarpX::GetInstance().gett_new(lev); + const auto& time_of_last_gal_shift = WarpX::GetInstance().time_of_last_gal_shift; + amrex::Real time_shift = (cur_time - time_of_last_gal_shift); + amrex::Array galilean_shift = { v_galilean[0]*time_shift, v_galilean[1]*time_shift, v_galilean[2]*time_shift }; + const std::array& xyzmin = WarpX::LowerCorner(box, galilean_shift, gather_lev); + + const Dim3 lo = lbound(box); + + int l_lower_order_in_v = WarpX::l_lower_order_in_v; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab->array(); + amrex::Array4 const& ey_arr = eyfab->array(); + amrex::Array4 const& ez_arr = ezfab->array(); + amrex::Array4 const& bx_arr = bxfab->array(); + amrex::Array4 const& by_arr = byfab->array(); + amrex::Array4 const& bz_arr = bzfab->array(); + + amrex::IndexType const ex_type = exfab->box().ixType(); + amrex::IndexType const ey_type = eyfab->box().ixType(); + amrex::IndexType const ez_type = ezfab->box().ixType(); + amrex::IndexType const bx_type = bxfab->box().ixType(); + amrex::IndexType const by_type = byfab->box().ixType(); + amrex::IndexType const bz_type = bzfab->box().ixType(); amrex::ParallelFor( - pti.numParticles(), + np_to_push, [=] AMREX_GPU_DEVICE (long i) { if (do_copy) copyAttribs(i); ParticleReal x, y, z; GetPosition(i, x, y, z); + + amrex::ParticleReal Exp, Eyp, Ezp; + getExternalE(i, Exp, Eyp, Ezp); + + amrex::ParticleReal Bxp, Byp, Bzp; + getExternalB(i, Bxp, Byp, Bzp); + + // first gather E and B to the particle positions + doGatherShapeN(x, y, z, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, l_lower_order_in_v); + +#ifdef WARPX_QED + if (local_has_breit_wheeler) { + const ParticleReal px = me * ux[i]; + const ParticleReal py = me * uy[i]; + const ParticleReal pz = me * uz[i]; + + bool has_event_happened = evolve_opt(px, py, pz, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + dt, p_optical_depth_BW[i]); + } +#endif + UpdatePositionPhoton( x, y, z, ux[i], uy[i], uz[i], dt ); SetPosition(i, x, y, z); } @@ -97,6 +196,8 @@ void PhotonParticleContainer::Evolve (int lev, const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + const MultiFab& Ex_avg, const MultiFab& Ey_avg, const MultiFab& Ez_avg, + const MultiFab& Bx_avg, const MultiFab& By_avg, const MultiFab& Bz_avg, MultiFab& jx, MultiFab& jy, MultiFab& jz, MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, MultiFab* rho, MultiFab* crho, @@ -110,6 +211,8 @@ PhotonParticleContainer::Evolve (int lev, PhysicalParticleContainer::Evolve (lev, Ex, Ey, Ez, Bx, By, Bz, + Ex_avg, Ey_avg, Ez_avg, + Bx_avg, By_avg, Bz_avg, jx, jy, jz, cjx, cjy, cjz, rho, crho, @@ -118,49 +221,3 @@ PhotonParticleContainer::Evolve (int lev, t, dt); } - -#ifdef WARPX_QED - -void -PhotonParticleContainer::EvolveOpticalDepth( - WarpXParIter& pti,amrex::Real dt) -{ - if(!has_breit_wheeler()) - return; - - auto& attribs = pti.GetAttribs(); - ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); - const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); - - BreitWheelerEvolveOpticalDepth evolve_opt = - m_shr_p_bw_engine->build_evolve_functor(); - - amrex::Real* AMREX_RESTRICT p_optical_depth_BW = - pti.GetAttribs(particle_comps["optical_depth_BW"]).dataPtr(); - - const auto me = PhysConst::m_e; - - amrex::ParallelFor( - pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - const ParticleReal px = me * ux[i]; - const ParticleReal py = me * uy[i]; - const ParticleReal pz = me * uz[i]; - - bool has_event_happened = evolve_opt( - px, py, pz, - Ex[i], Ey[i], Ez[i], - Bx[i], By[i], Bz[i], - dt, p_optical_depth_BW[i]); - } - ); -} - -#endif diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 5e076a2b5e0..c259e1e12e5 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -14,6 +14,7 @@ #include "WarpXParticleContainer.H" #include "Filter/NCIGodfreyFilter.H" #include "Particles/ElementaryProcess/Ionization.H" +#include "Particles/Gather/ScaleFields.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H" @@ -63,33 +64,6 @@ public: void InitIonizationModule (); - virtual void FieldGather (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz) final; - - void FieldGather (WarpXParIter& pti, - RealVector& Exp, - RealVector& Eyp, - RealVector& Ezp, - RealVector& Bxp, - RealVector& Byp, - RealVector& Bzp, - amrex::FArrayBox const * exfab, - amrex::FArrayBox const * eyfab, - amrex::FArrayBox const * ezfab, - amrex::FArrayBox const * bxfab, - amrex::FArrayBox const * byfab, - amrex::FArrayBox const * bzfab, - const int ngE, const int e_is_nodal, - const long offset, - const long np_to_gather, - int lev, - int depos_lev); - /** * \brief Evolve is the central function PhysicalParticleContainer that * advances plasma particles for a time dt (typically one timestep). @@ -130,6 +104,12 @@ public: const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + const amrex::MultiFab& Ex_avg, + const amrex::MultiFab& Ey_avg, + const amrex::MultiFab& Ez_avg, + const amrex::MultiFab& Bx_avg, + const amrex::MultiFab& By_avg, + const amrex::MultiFab& Bz_avg, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, @@ -148,7 +128,19 @@ public: amrex::Real dt, DtType a_dt_type=DtType::Full) override; - virtual void PushPX (WarpXParIter& pti, amrex::Real dt, DtType a_dt_type=DtType::Full); + virtual void PushPX (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int /*e_is_nodal*/, + const long offset, + const long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type=DtType::Full); virtual void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, @@ -175,11 +167,29 @@ public: void SplitParticles (int lev); - IonizationFilterFunc getIonizationFunc (); + IonizationFilterFunc getIonizationFunc (const WarpXParIter& pti, + int lev, + int ngE, + const amrex::FArrayBox& Ex, + const amrex::FArrayBox& Ey, + const amrex::FArrayBox& Ez, + const amrex::FArrayBox& Bx, + const amrex::FArrayBox& By, + const amrex::FArrayBox& Bz); // Inject particles in Box 'part_box' virtual void AddParticles (int lev); + /** + * Create new macroparticles for this species, with a fixed + * number of particles per cell (in the cells of `part_realbox`). + * The new particles are only created inside the intersection of `part_realbox` + * with the local grid for the current proc. + * @param[in] lev the index of the refinement level + * @param[in] part_realbox the box in which new particles should be created + * (this box should correspond to an integer number of cells in each direction, + * but its boundaries need not be aligned with the actual cells of the simulation) + */ void AddPlasma (int lev, amrex::RealBox part_realbox = amrex::RealBox()); void MapParticletoBoostedFrame (amrex::Real& x, amrex::Real& y, amrex::Real& z, std::array& u); @@ -191,10 +201,11 @@ public: const amrex::Real q_tot, long npart, const int do_symmetrize); /** Load a particle beam from an external file - * * @param[in] q_tot total charge of the particle species to be initialized + * @param[in] z_shift optional shift to the z position of particles (useful for boosted frame runs) */ - void AddPlasmaFromFile (amrex::ParticleReal q_tot); + void AddPlasmaFromFile (amrex::ParticleReal q_tot, + amrex::ParticleReal z_shift); void CheckAndAddParticle ( amrex::Real x, amrex::Real y, amrex::Real z, @@ -303,15 +314,6 @@ public: (std::shared_ptr ptr) override; //__________ - /** - * This function evolves the optical depth of the particles if QED effects - * are enabled. - * @param[in,out] pti particle iterator (optical depth will be modified) - * @param[in] dt temporal step - */ - virtual void EvolveOpticalDepth (WarpXParIter& pti, - amrex::Real dt); - PhotonEmissionFilterFunc getPhotonEmissionFilterFunc (); PairGenerationFilterFunc getPairGenerationFilterFunc (); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 094a8e556ee..4d8dc67cf8c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -34,9 +34,60 @@ #include #include - using namespace amrex; +namespace +{ + // Since the user provides the density distribution + // at t_lab=0 and in the lab-frame coordinates, + // we need to find the lab-frame position of this + // particle at t_lab=0, from its boosted-frame coordinates + // Assuming ballistic motion, this is given by: + // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) + // where betaz_lab is the speed of the particle in the lab frame + // + // In order for this equation to be solvable, betaz_lab + // is explicitly assumed to have no dependency on z0_lab + // + // Note that we use the bulk momentum to perform the ballistic correction + // Assume no z0_lab dependency + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + Real applyBallisticCorrection(const XDim3& pos, const InjectorMomentum* inj_mom, + Real gamma_boost, Real beta_boost, Real t) noexcept + { + const XDim3 u_bulk = inj_mom->getBulkMomentum(pos.x, pos.y, pos.z); + const Real gamma_bulk = std::sqrt(1._rt + + (u_bulk.x*u_bulk.x+u_bulk.y*u_bulk.y+u_bulk.z*u_bulk.z)); + const Real betaz_bulk = u_bulk.z/gamma_bulk; + const Real z0 = gamma_boost * ( pos.z*(1.0_rt-beta_boost*betaz_bulk) + - PhysConst::c*t*(betaz_bulk-beta_boost) ); + return z0; + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + XDim3 getCellCoords (const GpuArray& lo_corner, + const GpuArray& dx, + const XDim3& r, const IntVect& iv) noexcept + { + XDim3 pos; +#if (AMREX_SPACEDIM == 3) + pos.x = lo_corner[0] + (iv[0]+r.x)*dx[0]; + pos.y = lo_corner[1] + (iv[1]+r.y)*dx[1]; + pos.z = lo_corner[2] + (iv[2]+r.z)*dx[2]; +#else + pos.x = lo_corner[0] + (iv[0]+r.x)*dx[0]; + pos.y = 0.0_rt; +#if defined WARPX_DIM_XZ + pos.z = lo_corner[1] + (iv[1]+r.y)*dx[1]; +#elif defined WARPX_DIM_RZ + // Note that for RZ, r.y will be theta + pos.z = lo_corner[1] + (iv[1]+r.z)*dx[1]; +#endif +#endif + return pos; + } +} + PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int ispecies, const std::string& name) : WarpXParticleContainer(amr_core, ispecies), @@ -185,11 +236,11 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame ( Real zpr = WarpX::gamma_boost*z - uz_boost*t_lab; // transform u and gamma to the boosted frame - Real gamma_lab = std::sqrt(1. + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(PhysConst::c*PhysConst::c)); + Real gamma_lab = std::sqrt(1._rt + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(PhysConst::c*PhysConst::c)); // u[0] = u[0]; // u[1] = u[1]; u[2] = WarpX::gamma_boost*u[2] - uz_boost*gamma_lab; - Real gammapr = std::sqrt(1. + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(PhysConst::c*PhysConst::c)); + Real gammapr = std::sqrt(1._rt + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(PhysConst::c*PhysConst::c)); Real vxpr = u[0]/gammapr; Real vypr = u[1]/gammapr; @@ -239,7 +290,7 @@ PhysicalParticleContainer::AddGaussianBeam ( npart /= 4; } for (long i = 0; i < npart; ++i) { -#if (defined WARPX_DIM_3D) || (WARPX_DIM_RZ) +#if (defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ) const Real weight = q_tot/(npart*charge); const Real x = distx(mt); const Real y = disty(mt); @@ -247,7 +298,7 @@ PhysicalParticleContainer::AddGaussianBeam ( #elif (defined WARPX_DIM_XZ) const Real weight = q_tot/(npart*charge*y_rms); const Real x = distx(mt); - constexpr Real y = 0.; + constexpr Real y = 0._prt; const Real z = distz(mt); #endif if (plasma_injector->insideBounds(x, y, z) && @@ -294,7 +345,8 @@ PhysicalParticleContainer::AddGaussianBeam ( } void -PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot) +PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, + ParticleReal z_shift) { // Declare temporary vectors on the CPU Gpu::HostVector particle_x; @@ -327,7 +379,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot) double const momentum_unit_x = ps["momentum"]["x"].unitSI(); std::shared_ptr ptr_uz = ps["momentum"]["z"].loadChunk(); double const momentum_unit_z = ps["momentum"]["z"].unitSI(); -# ifdef WARPX_DIM_3D +# ifndef WARPX_DIM_XZ std::shared_ptr ptr_y = ps["position"]["y"].loadChunk(); double const position_unit_y = ps["position"]["y"].unitSI(); # endif @@ -359,11 +411,11 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot) for (auto i = decltype(npart){0}; iinsideBounds(x, y, z)) { ParticleReal const ux = ptr_ux.get()[i]*momentum_unit_x/PhysConst::m_e; @@ -373,9 +425,9 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot) uy = ptr_uy.get()[i]*momentum_unit_y/PhysConst::m_e; } CheckAndAddParticle(x, y, z, { ux, uy, uz}, weight, - particle_x, particle_y, particle_z, - particle_ux, particle_uy, particle_uz, - particle_w); + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); } } auto const np = particle_z.size(); @@ -453,7 +505,8 @@ PhysicalParticleContainer::AddParticles (int lev) } if (plasma_injector->external_file) { - AddPlasmaFromFile(plasma_injector->q_tot); + AddPlasmaFromFile(plasma_injector->q_tot, + plasma_injector->z_shift); return; } @@ -462,16 +515,6 @@ PhysicalParticleContainer::AddParticles (int lev) } } -/** - * Create new macroparticles for this species, with a fixed - * number of particles per cell (in the cells of `part_realbox`). - * The new particles are only created inside the intersection of `part_realbox` - * with the local grid for the current proc. - * @param lev the index of the refinement level - * @param part_realbox the box in which new particles should be created - * (this box should correspond to an integer number of cells in each direction, - * but its boundaries need not be aligned with the actual cells of the simulation) - */ void PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) { @@ -586,35 +629,52 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); - // Max number of new particles, if particles are created in the whole - // overlap_box. All of them are created, and invalid ones are then - // discaded - int max_new_particles = overlap_box.numPts() * num_ppc; + const GpuArray overlap_corner + {AMREX_D_DECL(overlap_realbox.lo(0), + overlap_realbox.lo(1), + overlap_realbox.lo(2))}; - // If refine injection, build pointer dp_cellid that holds pointer to - // array of refined cell IDs. - Vector cellid_v; - if (refine_injection and lev == 0) + // count the number of particles that each cell in overlap_box could add + Gpu::DeviceVector counts(overlap_box.numPts()+1, 0); + Gpu::DeviceVector offset(overlap_box.numPts()+1, 0); + auto pcounts = counts.data(); + int lrrfac = rrfac; + int lrefine_injection = refine_injection; + Box lfine_box = fine_injection_box; + amrex::ParallelFor(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - // then how many new particles will be injected is not that simple - // We have to shift fine_injection_box because overlap_box has been shifted. - Box fine_overlap_box = overlap_box & amrex::shift(fine_injection_box,shifted); - if (fine_overlap_box.ok()) { - max_new_particles += fine_overlap_box.numPts() * num_ppc - * (AMREX_D_TERM(rrfac,*rrfac,*rrfac)-1); - for (int icell = 0, ncells = overlap_box.numPts(); icell < ncells; ++icell) { - IntVect iv = overlap_box.atOffset(icell); - int r = (fine_overlap_box.contains(iv)) ? AMREX_D_TERM(rrfac,*rrfac,*rrfac) : 1; - for (int ipart = 0; ipart < r; ++ipart) { - cellid_v.push_back(icell); - cellid_v.push_back(ipart); + IntVect iv(AMREX_D_DECL(i, j, k)); + auto lo = getCellCoords(overlap_corner, dx, {0._rt, 0._rt, 0._rt}, iv); + auto hi = getCellCoords(overlap_corner, dx, {1._rt, 1._rt, 1._rt}, iv); + + lo.z = applyBallisticCorrection(lo, inj_mom, gamma_boost, beta_boost, t); + hi.z = applyBallisticCorrection(hi, inj_mom, gamma_boost, beta_boost, t); + + if (inj_pos->overlapsWith(lo, hi)) + { + auto index = overlap_box.index(iv); + if (lrefine_injection) { + Box fine_overlap_box = overlap_box & amrex::shift(lfine_box, shifted); + if (fine_overlap_box.ok()) { + int r = (fine_overlap_box.contains(iv)) ? + AMREX_D_TERM(lrrfac,*lrrfac,*lrrfac) : 1; + pcounts[index] = num_ppc*r; } + } else { + pcounts[index] = num_ppc; } } - } - int const* hp_cellid = (cellid_v.empty()) ? nullptr : cellid_v.data(); - amrex::AsyncArray cellid_aa(hp_cellid, cellid_v.size()); - int const* dp_cellid = cellid_aa.data(); + }); + Gpu::exclusive_scan(counts.begin(), counts.end(), offset.begin()); + + // Max number of new particles. All of them are created, + // and invalid ones are then discarded + int max_new_particles; +#ifdef AMREX_USE_GPU + Gpu::dtoh_memcpy(&max_new_particles, offset.dataPtr()+overlap_box.numPts(), sizeof(int)); +#else + std::memcpy(&max_new_particles, offset.dataPtr()+overlap_box.numPts(), sizeof(int)); +#endif // Update NextID to include particles created in this function int pid; @@ -625,6 +685,9 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) pid = ParticleType::NextID(); ParticleType::NextID(pid+max_new_particles); } + WarpXUtilMsg::AlwaysAssert(static_cast(pid + max_new_particles) > 0, + "ERROR: overflow on particle id numbers"); + const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; @@ -678,13 +741,6 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } #endif - const GpuArray overlap_corner - {AMREX_D_DECL(overlap_realbox.lo(0), - overlap_realbox.lo(1), - overlap_realbox.lo(2))}; - - int lrrfac = rrfac; - bool loc_do_field_ionization = do_field_ionization; int loc_ionization_initial_level = ionization_initial_level; @@ -692,197 +748,159 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - amrex::For(max_new_particles, [=] AMREX_GPU_DEVICE (int ip) noexcept + const auto poffset = offset.data(); + amrex::For(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; - - int cellid, i_part; - Real fac; - if (dp_cellid == nullptr) { - cellid = ip/num_ppc; - i_part = ip - cellid*num_ppc; - fac = 1.0; - } else { - cellid = dp_cellid[2*ip]; - i_part = dp_cellid[2*ip+1]; - fac = lrrfac; - } + IntVect iv = IntVect(AMREX_D_DECL(i, j, k)); + const auto index = overlap_box.index(iv); + for (int i_part = 0; i_part < pcounts[index]; ++i_part) + { + long ip = poffset[index] + i_part; + ParticleType& p = pp[ip]; + p.id() = pid+ip; + p.cpu() = cpuid; - IntVect iv = overlap_box.atOffset(cellid); + const XDim3 r = + inj_pos->getPositionUnitBox(i_part, lrrfac); + auto pos = getCellCoords(overlap_corner, dx, r, iv); - const XDim3 r = - inj_pos->getPositionUnitBox(i_part, static_cast(fac)); #if (AMREX_SPACEDIM == 3) - Real x = overlap_corner[0] + (iv[0]+r.x)*dx[0]; - Real y = overlap_corner[1] + (iv[1]+r.y)*dx[1]; - Real z = overlap_corner[2] + (iv[2]+r.z)*dx[2]; -#else - Real x = overlap_corner[0] + (iv[0]+r.x)*dx[0]; - Real y = 0.0; -#if defined WARPX_DIM_XZ - Real z = overlap_corner[1] + (iv[1]+r.y)*dx[1]; -#elif defined WARPX_DIM_RZ - // Note that for RZ, r.y will be theta - Real z = overlap_corner[1] + (iv[1]+r.z)*dx[1]; -#endif -#endif - -#if (AMREX_SPACEDIM == 3) - if (!tile_realbox.contains(XDim3{x,y,z})) { - p.id() = -1; - return; - } + if (!tile_realbox.contains(XDim3{pos.x,pos.y,pos.z})) { + p.id() = -1; + continue; + } #else - if (!tile_realbox.contains(XDim3{x,z,0.0})) { - p.id() = -1; - return; - } + if (!tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt})) { + p.id() = -1; + continue; + } #endif - // Save the x and y values to use in the insideBounds checks. - // This is needed with WARPX_DIM_RZ since x and y are modified. - Real xb = x; - Real yb = y; + // Save the x and y values to use in the insideBounds checks. + // This is needed with WARPX_DIM_RZ since x and y are modified. + Real xb = pos.x; + Real yb = pos.y; #ifdef WARPX_DIM_RZ - // Replace the x and y, setting an angle theta. - // These x and y are used to get the momentum and density - Real theta; - if (nmodes == 1) { - // With only 1 mode, the angle doesn't matter so - // choose it randomly. - theta = 2.*MathConst::pi*amrex::Random(); - } else { - theta = 2.*MathConst::pi*r.y; - } - x = xb*std::cos(theta); - y = xb*std::sin(theta); + // Replace the x and y, setting an angle theta. + // These x and y are used to get the momentum and density + Real theta; + if (nmodes == 1) { + // With only 1 mode, the angle doesn't matter so + // choose it randomly. + theta = 2._rt*MathConst::pi*amrex::Random(); + } else { + theta = 2._rt*MathConst::pi*r.y; + } + pos.x = xb*std::cos(theta); + pos.y = xb*std::sin(theta); #endif - Real dens; - XDim3 u; - if (gamma_boost == 1.) { - // Lab-frame simulation - // If the particle is not within the species's - // xmin, xmax, ymin, ymax, zmin, zmax, go to - // the next generated particle. - - // include ballistic correction for plasma species with bulk motion - const XDim3 u_bulk = inj_mom->getBulkMomentum(x, y, z); - const Real gamma_bulk = std::sqrt(1.+(u_bulk.x*u_bulk.x+u_bulk.y*u_bulk.y+u_bulk.z*u_bulk.z)); - const Real betaz_bulk = u_bulk.z/gamma_bulk; - const Real z0 = z - PhysConst::c*t*betaz_bulk; - - if (!inj_pos->insideBounds(xb, yb, z0)) { - p.id() = -1; - return; - } + Real dens; + XDim3 u; + if (gamma_boost == 1._rt) { + // Lab-frame simulation + // If the particle is not within the species's + // xmin, xmax, ymin, ymax, zmin, zmax, go to + // the next generated particle. + + // include ballistic correction for plasma species with bulk motion + const Real z0 = applyBallisticCorrection(pos, inj_mom, gamma_boost, + beta_boost, t); + if (!inj_pos->insideBounds(xb, yb, z0)) { + p.id() = -1; + continue; + } - u = inj_mom->getMomentum(x, y, z0); - dens = inj_rho->getDensity(x, y, z0); - // Remove particle if density below threshold - if ( dens < density_min ){ - p.id() = -1; - return; - } - // Cut density if above threshold - dens = amrex::min(dens, density_max); - } else { - // Boosted-frame simulation - // Since the user provides the density distribution - // at t_lab=0 and in the lab-frame coordinates, - // we need to find the lab-frame position of this - // particle at t_lab=0, from its boosted-frame coordinates - // Assuming ballistic motion, this is given by: - // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) - // where betaz_lab is the speed of the particle in the lab frame - // - // In order for this equation to be solvable, betaz_lab - // is explicitly assumed to have no dependency on z0_lab - // - // Note that we use the bulk momentum to perform the ballastic correction - const XDim3 u_bulk = inj_mom->getBulkMomentum(x, y, 0.); // No z0_lab dependency - // At this point u is the lab-frame momentum - // => Apply the above formula for z0_lab - const Real gamma_lab_bulk = std::sqrt(1.+(u_bulk.x*u_bulk.x+u_bulk.y*u_bulk.y+u_bulk.z*u_bulk.z)); - const Real betaz_lab_bulk = u_bulk.z/(gamma_lab_bulk); - const Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab_bulk) - - PhysConst::c*t*(betaz_lab_bulk-beta_boost) ); - // If the particle is not within the lab-frame zmin, zmax, etc. - // go to the next generated particle. - if (!inj_pos->insideBounds(xb, yb, z0_lab)) { - p.id() = -1; - return; - } - // call `getDensity` with lab-frame parameters - dens = inj_rho->getDensity(x, y, z0_lab); - // Remove particle if density below threshold - if ( dens < density_min ){ - p.id() = -1; - return; + u = inj_mom->getMomentum(pos.x, pos.y, z0); + dens = inj_rho->getDensity(pos.x, pos.y, z0); + + // Remove particle if density below threshold + if ( dens < density_min ){ + p.id() = -1; + continue; + } + // Cut density if above threshold + dens = amrex::min(dens, density_max); + } else { + // Boosted-frame simulation + const Real z0_lab = applyBallisticCorrection(pos, inj_mom, gamma_boost, + beta_boost, t); + + // If the particle is not within the lab-frame zmin, zmax, etc. + // go to the next generated particle. + if (!inj_pos->insideBounds(xb, yb, z0_lab)) { + p.id() = -1; + continue; + } + // call `getDensity` with lab-frame parameters + dens = inj_rho->getDensity(pos.x, pos.y, z0_lab); + // Remove particle if density below threshold + if ( dens < density_min ){ + p.id() = -1; + continue; + } + // Cut density if above threshold + dens = amrex::min(dens, density_max); + + // get the full momentum, including thermal motion + u = inj_mom->getMomentum(pos.x, pos.y, 0._rt); + const Real gamma_lab = std::sqrt( 1._rt+(u.x*u.x+u.y*u.y+u.z*u.z) ); + const Real betaz_lab = u.z/(gamma_lab); + + // At this point u and dens are the lab-frame quantities + // => Perform Lorentz transform + dens = gamma_boost * dens * ( 1.0_rt - beta_boost*betaz_lab ); + u.z = gamma_boost * ( u.z -beta_boost*gamma_lab ); } - // Cut density if above threshold - dens = amrex::min(dens, density_max); - - // get the full momentum, including thermal motion - u = inj_mom->getMomentum(x, y, 0.); - const Real gamma_lab = std::sqrt( 1.+(u.x*u.x+u.y*u.y+u.z*u.z) ); - const Real betaz_lab = u.z/(gamma_lab); - - // At this point u and dens are the lab-frame quantities - // => Perform Lorentz transform - dens = gamma_boost * dens * ( 1.0 - beta_boost*betaz_lab ); - u.z = gamma_boost * ( u.z -beta_boost*gamma_lab ); - } - if (loc_do_field_ionization) { - pi[ip] = loc_ionization_initial_level; - } + if (loc_do_field_ionization) { + pi[ip] = loc_ionization_initial_level; + } #ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth_QSR[ip] = quantum_sync_get_opt(); - } + if(loc_has_quantum_sync){ + p_optical_depth_QSR[ip] = quantum_sync_get_opt(); + } - if(loc_has_breit_wheeler){ - p_optical_depth_BW[ip] = breit_wheeler_get_opt(); - } + if(loc_has_breit_wheeler){ + p_optical_depth_BW[ip] = breit_wheeler_get_opt(); + } #endif - u.x *= PhysConst::c; - u.y *= PhysConst::c; - u.z *= PhysConst::c; + u.x *= PhysConst::c; + u.y *= PhysConst::c; + u.z *= PhysConst::c; - // Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); - Real weight = dens * scale_fac; + // Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); + Real weight = dens * scale_fac; #ifdef WARPX_DIM_RZ - if (radially_weighted) { - weight *= 2.*MathConst::pi*xb; - } else { - // This is not correct since it might shift the particle - // out of the local grid - x = std::sqrt(xb*rmax); - weight *= dx[0]; - } + if (radially_weighted) { + weight *= 2._rt*MathConst::pi*xb; + } else { + // This is not correct since it might shift the particle + // out of the local grid + pos.x = std::sqrt(xb*rmax); + weight *= dx[0]; + } #endif - pa[PIdx::w ][ip] = weight; - pa[PIdx::ux][ip] = u.x; - pa[PIdx::uy][ip] = u.y; - pa[PIdx::uz][ip] = u.z; + pa[PIdx::w ][ip] = weight; + pa[PIdx::ux][ip] = u.x; + pa[PIdx::uy][ip] = u.y; + pa[PIdx::uz][ip] = u.z; #if (AMREX_SPACEDIM == 3) - p.pos(0) = x; - p.pos(1) = y; - p.pos(2) = z; + p.pos(0) = pos.x; + p.pos(1) = pos.y; + p.pos(2) = pos.z; #elif (AMREX_SPACEDIM == 2) #ifdef WARPX_DIM_RZ - pa[PIdx::theta][ip] = theta; + pa[PIdx::theta][ip] = theta; #endif - p.pos(0) = xb; - p.pos(1) = z; + p.pos(0) = xb; + p.pos(1) = pos.z; #endif + } }); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) @@ -891,78 +909,18 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) wt = amrex::second() - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } + amrex::Gpu::synchronize(); } // The function that calls this is responsible for redistributing particles. } -void -PhysicalParticleContainer::FieldGather (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz) -{ - BL_ASSERT(OnSameGrids(lev,Ex)); - - amrex::LayoutData* cost = WarpX::getCosts(lev); - -#ifdef _OPENMP -#pragma omp parallel -#endif - { - for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) - { - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - } - Real wt = amrex::second(); - - auto& attribs = pti.GetAttribs(); - - auto& Exp = attribs[PIdx::Ex]; - auto& Eyp = attribs[PIdx::Ey]; - auto& Ezp = attribs[PIdx::Ez]; - auto& Bxp = attribs[PIdx::Bx]; - auto& Byp = attribs[PIdx::By]; - auto& Bzp = attribs[PIdx::Bz]; - - const long np = pti.numParticles(); - - // Data on the grid - const FArrayBox& exfab = Ex[pti]; - const FArrayBox& eyfab = Ey[pti]; - const FArrayBox& ezfab = Ez[pti]; - const FArrayBox& bxfab = Bx[pti]; - const FArrayBox& byfab = By[pti]; - const FArrayBox& bzfab = Bz[pti]; - - // - // Field Gather - // - int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, - 0, np, lev, lev); - - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - wt = amrex::second() - wt; - amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); - } - } - } -} - void PhysicalParticleContainer::Evolve (int lev, const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + const MultiFab& Ex_avg, const MultiFab& Ey_avg, const MultiFab& Ez_avg, + const MultiFab& Bx_avg, const MultiFab& By_avg, const MultiFab& Bz_avg, MultiFab& jx, MultiFab& jy, MultiFab& jz, MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, MultiFab* rho, MultiFab* crho, @@ -970,11 +928,13 @@ PhysicalParticleContainer::Evolve (int lev, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, Real /*t*/, Real dt, DtType a_dt_type) { + + bool fft_do_time_averaging = false; + ParmParse pp("psatd"); + pp.query("do_time_averaging", fft_do_time_averaging); + WARPX_PROFILE("PPC::Evolve()"); - WARPX_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); - WARPX_PROFILE_VAR_NS("PPC::FieldGather", blp_fg); - WARPX_PROFILE_VAR_NS("PPC::EvolveOpticalDepth", blp_ppc_qed_ev); - WARPX_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); + WARPX_PROFILE_VAR_NS("PPC::GatherAndPush", blp_fg); BL_ASSERT(OnSameGrids(lev,jx)); @@ -1027,22 +987,16 @@ PhysicalParticleContainer::Evolve (int lev, auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; auto& uzp = attribs[PIdx::uz]; - auto& Exp = attribs[PIdx::Ex]; - auto& Eyp = attribs[PIdx::Ey]; - auto& Ezp = attribs[PIdx::Ez]; - auto& Bxp = attribs[PIdx::Bx]; - auto& Byp = attribs[PIdx::By]; - auto& Bzp = attribs[PIdx::Bz]; const long np = pti.numParticles(); // Data on the grid - FArrayBox const* exfab = &(Ex[pti]); - FArrayBox const* eyfab = &(Ey[pti]); - FArrayBox const* ezfab = &(Ez[pti]); - FArrayBox const* bxfab = &(Bx[pti]); - FArrayBox const* byfab = &(By[pti]); - FArrayBox const* bzfab = &(Bz[pti]); + FArrayBox const* exfab = fft_do_time_averaging ? &(Ex_avg[pti]) : &(Ex[pti]); + FArrayBox const* eyfab = fft_do_time_averaging ? &(Ey_avg[pti]) : &(Ey[pti]); + FArrayBox const* ezfab = fft_do_time_averaging ? &(Ez_avg[pti]) : &(Ez[pti]); + FArrayBox const* bxfab = fft_do_time_averaging ? &(Bx_avg[pti]) : &(Bx[pti]); + FArrayBox const* byfab = fft_do_time_averaging ? &(By_avg[pti]) : &(By[pti]); + FArrayBox const* bzfab = fft_do_time_averaging ? &(Bz_avg[pti]) : &(Bz[pti]); Elixir exeli, eyeli, ezeli, bxeli, byeli, bzeli; @@ -1101,13 +1055,13 @@ PhysicalParticleContainer::Evolve (int lev, int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); // - // Field Gather of Aux Data (i.e., the full solution) + // Gather and push for particles not in the buffer // WARPX_PROFILE_VAR_START(blp_fg); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - Ex.nGrow(), e_is_nodal, - 0, np_gather, lev, lev); + PushPX(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrow(), e_is_nodal, + 0, np_gather, lev, lev, dt, ScaleFields(false), a_dt_type); if (np_gather < np) { @@ -1136,34 +1090,17 @@ PhysicalParticleContainer::Evolve (int lev, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab); } - // Field gather for particles in gather buffers + // Field gather and push for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - cexfab, ceyfab, cezfab, - cbxfab, cbyfab, cbzfab, - cEx->nGrow(), e_is_nodal, - nfine_gather, np-nfine_gather, - lev, lev-1); + PushPX(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrow(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); } WARPX_PROFILE_VAR_STOP(blp_fg); -#ifdef WARPX_QED - // - //Evolve Optical Depth - // - WARPX_PROFILE_VAR_START(blp_ppc_qed_ev); - EvolveOpticalDepth(pti, dt); - WARPX_PROFILE_VAR_STOP(blp_ppc_qed_ev); -#endif - - // - // Particle Push - // - WARPX_PROFILE_VAR_START(blp_ppc_pp); - PushPX(pti, dt, a_dt_type); - WARPX_PROFILE_VAR_STOP(blp_ppc_pp); - // // Current Deposition (only needed for electromagnetic solver) // @@ -1464,131 +1401,24 @@ PhysicalParticleContainer::SplitParticles (int lev) } void -PhysicalParticleContainer::PushPX (WarpXParIter& pti, Real dt, DtType a_dt_type) -{ - - // This wraps the momentum and position advance so that inheritors can modify the call. - auto& attribs = pti.GetAttribs(); - // Extract pointers to the different particle quantities - - const auto GetPosition = GetParticlePosition(pti); - auto SetPosition = SetParticlePosition(pti); - - ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); - const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); - - auto copyAttribs = CopyParticleAttribs(pti, tmp_particle_data); - int do_copy = (WarpX::do_back_transformed_diagnostics && - do_back_transformed_diagnostics && - (a_dt_type!=DtType::SecondHalf)); - - int* AMREX_RESTRICT ion_lev = nullptr; - if (do_field_ionization){ - ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); - } - - // Loop over the particles and update their momentum - const Real q = this->charge; - const Real m = this-> mass; - - const auto pusher_algo = WarpX::particle_pusher_algo; - const auto do_crr = do_classical_radiation_reaction; -#ifdef WARPX_QED - const auto do_sync = m_do_qed_quantum_sync; - amrex::Real t_chi_max = 0.0; - if (do_sync) t_chi_max = m_shr_p_qs_engine->get_ref_ctrl().chi_part_min; -#endif - - amrex::ParallelFor(pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - doParticlePush(GetPosition, SetPosition, copyAttribs, i, - ux[i], uy[i], uz[i], - Ex[i], Ey[i], Ez[i], - Bx[i], By[i], Bz[i], - ion_lev ? ion_lev[i] : 0, - m, q, pusher_algo, do_crr, do_copy, -#ifdef WARPX_QED - do_sync, - t_chi_max, -#endif - dt); - }); -} - -#ifdef WARPX_QED -void PhysicalParticleContainer::EvolveOpticalDepth( - WarpXParIter& pti, amrex::Real dt) -{ - if(!has_quantum_sync()) - return; - - QuantumSynchrotronEvolveOpticalDepth evolve_opt = - m_shr_p_qs_engine->build_evolve_functor(); - - auto& attribs = pti.GetAttribs(); - const ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - const ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - const ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); - const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); - - ParticleReal* const AMREX_RESTRICT p_optical_depth_QSR = - pti.GetAttribs(particle_comps["optical_depth_QSR"]).dataPtr(); - - const ParticleReal m = this->mass; - - amrex::ParallelFor(pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - const ParticleReal px = m * ux[i]; - const ParticleReal py = m * uy[i]; - const ParticleReal pz = m * uz[i]; - - bool has_event_happened = evolve_opt( - px, py, pz, - Ex[i], Ey[i], Ez[i], - Bx[i], By[i], Bz[i], - dt, p_optical_depth_QSR[i]); - } - ); - -} -#endif - -void -PhysicalParticleContainer::PushP ( - int lev, Real dt, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz) +PhysicalParticleContainer::PushP (int lev, Real dt, + const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, + const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz) { WARPX_PROFILE("PhysicalParticleContainer::PushP"); if (do_not_push) return; + const std::array& dx = WarpX::CellSize(std::max(lev,0)); + #ifdef _OPENMP #pragma omp parallel #endif { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - auto& attribs = pti.GetAttribs(); - - auto& Exp = attribs[PIdx::Ex]; - auto& Eyp = attribs[PIdx::Ey]; - auto& Ezp = attribs[PIdx::Ez]; - auto& Bxp = attribs[PIdx::Bx]; - auto& Byp = attribs[PIdx::By]; - auto& Bzp = attribs[PIdx::Bz]; + amrex::Box box = pti.tilebox(); + box.grow(Ex.nGrow()); const long np = pti.numParticles(); @@ -1600,83 +1430,100 @@ PhysicalParticleContainer::PushP ( const FArrayBox& byfab = By[pti]; const FArrayBox& bzfab = Bz[pti]; - int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, - 0, np, lev, lev); + const auto getPosition = GetParticlePosition(pti); + auto setPosition = SetParticlePosition(pti); + + const auto getExternalE = GetExternalEField(pti); + const auto getExternalB = GetExternalBField(pti); + + const auto& xyzmin = WarpX::GetInstance().LowerCornerWithGalilean(box,v_galilean,lev); + + const Dim3 lo = lbound(box); + + int l_lower_order_in_v = WarpX::l_lower_order_in_v; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab.array(); + amrex::Array4 const& ey_arr = eyfab.array(); + amrex::Array4 const& ez_arr = ezfab.array(); + amrex::Array4 const& bx_arr = bxfab.array(); + amrex::Array4 const& by_arr = byfab.array(); + amrex::Array4 const& bz_arr = bzfab.array(); - // This wraps the momentum advance so that inheritors can modify the call. - // Extract pointers to the different particle quantities + amrex::IndexType const ex_type = exfab.box().ixType(); + amrex::IndexType const ey_type = eyfab.box().ixType(); + amrex::IndexType const ez_type = ezfab.box().ixType(); + amrex::IndexType const bx_type = bxfab.box().ixType(); + amrex::IndexType const by_type = byfab.box().ixType(); + amrex::IndexType const bz_type = bzfab.box().ixType(); + + auto& attribs = pti.GetAttribs(); ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - const ParticleReal* const AMREX_RESTRICT Expp = Exp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Eypp = Eyp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bypp = Byp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bzpp = Bzp.dataPtr(); - - // Loop over the particles and update their momentum - const Real q = this->charge; - const Real m = this-> mass; int* AMREX_RESTRICT ion_lev = nullptr; - if (do_field_ionization){ + if (do_field_ionization) { ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } - //Assumes that all consistency checks have been done at initialization - if(do_classical_radiation_reaction){ - amrex::ParallelFor(pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i){ - Real qp = q; - if (ion_lev){ qp *= ion_lev[i]; } - UpdateMomentumBorisWithRadiationReaction( - ux[i], uy[i], uz[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - qp, m, dt); - } - ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ - amrex::ParallelFor(pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - Real qp = q; - if (ion_lev){ qp *= ion_lev[i]; } - UpdateMomentumBoris( - ux[i], uy[i], uz[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - qp, m, dt); - } - ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay){ - amrex::ParallelFor(pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - Real qp = q; - if (ion_lev){ qp *= ion_lev[i]; } - UpdateMomentumVay( - ux[i], uy[i], uz[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - qp, m, dt); - } - ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::HigueraCary){ - amrex::ParallelFor(pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumHigueraCary( ux[i], uy[i], uz[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - q, m, dt); - } - ); - } else { - amrex::Abort("Unknown particle pusher"); - } + // Loop over the particles and update their momentum + const amrex::Real q = this->charge; + const amrex::Real m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = 0._rt, Eyp = 0._rt, Ezp = 0._rt; + getExternalE(ip, Exp, Eyp, Ezp); + + amrex::ParticleReal Bxp = 0._rt, Byp = 0._rt, Bzp = 0._rt; + getExternalB(ip, Bxp, Byp, Bzp); + + // first gather E and B to the particle positions + doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, l_lower_order_in_v); + + if (do_crr) { + amrex::Real qp = q; + if (ion_lev) { qp *= ion_lev[ip]; } + UpdateMomentumBorisWithRadiationReaction(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else if (pusher_algo == ParticlePusherAlgo::Boris) { + amrex::Real qp = q; + if (ion_lev) { qp *= ion_lev[ip]; } + UpdateMomentumBoris( ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else if (pusher_algo == ParticlePusherAlgo::Vay) { + amrex::Real qp = q; + if (ion_lev){ qp *= ion_lev[ip]; } + UpdateMomentumVay( ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else if (pusher_algo == ParticlePusherAlgo::HigueraCary) { + amrex::Real qp = q; + if (ion_lev){ qp *= ion_lev[ip]; } + UpdateMomentumHigueraCary( ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else { + amrex::Abort("Unknown particle pusher"); + } + }); } } } @@ -1732,6 +1579,14 @@ PhysicalParticleContainer::GetParticleSlice ( #pragma omp parallel #endif { + // Temporary arrays to store copy_flag and copy_index + // for particles that cross the z-slice + // These arrays are defined before the WarpXParIter to prevent them + // from going out of scope after each iteration, while the kernels + // may still need access to them. + // Note that the destructor for WarpXParIter is synchronized. + amrex::Gpu::ManagedDeviceVector FlagForPartCopy; + amrex::Gpu::ManagedDeviceVector IndexForPartCopy; for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& box = pti.validbox(); @@ -1766,10 +1621,8 @@ PhysicalParticleContainer::GetParticleSlice ( Real uzfrm = -WarpX::gamma_boost*WarpX::beta_boost*PhysConst::c; Real inv_c2 = 1.0/PhysConst::c/PhysConst::c; - // temporary arrays to store copy_flag and copy_index - // for particles that cross the z-slice - amrex::Gpu::ManagedDeviceVector FlagForPartCopy(np); - amrex::Gpu::ManagedDeviceVector IndexForPartCopy(np); + FlagForPartCopy.resize(np); + IndexForPartCopy.resize(np); int* const AMREX_RESTRICT Flag = FlagForPartCopy.dataPtr(); int* const AMREX_RESTRICT IndexLocation = IndexForPartCopy.dataPtr(); @@ -1889,44 +1742,30 @@ PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) AddPlasma(lev, injection_box); } -/* \brief Gather fields from FArrayBox exfab, eyfab, ezfab, bxfab, byfab, - * bzfab into arrays of fields on particles Exp, Eyp, Ezp, Bxp, Byp, Bzp. - * \param Exp-Bzp: fields on particles. - * \param exfab-bzfab: FAB of electric and magnetic fields for particles in pti - * \param ngE: number of guard cells for E - * \param e_is_nodal: 0 if E is staggered, 1 if E is nodal - * \param offset: index of first particle for which fields are gathered - * \param np_to_gather: number of particles onto which fields are gathered - * \param lev: level on which particles are located - * \param gather_lev: level from which particles gather fields (lev-1) for - particles in buffers. +/* \brief Perform the field gather and particle push operations in one fused kernel + * */ void -PhysicalParticleContainer::FieldGather (WarpXParIter& pti, - RealVector& Exp, - RealVector& Eyp, - RealVector& Ezp, - RealVector& Bxp, - RealVector& Byp, - RealVector& Bzp, - amrex::FArrayBox const * exfab, - amrex::FArrayBox const * eyfab, - amrex::FArrayBox const * ezfab, - amrex::FArrayBox const * bxfab, - amrex::FArrayBox const * byfab, - amrex::FArrayBox const * bzfab, - const int ngE, const int /*e_is_nodal*/, - const long offset, - const long np_to_gather, - int lev, - int gather_lev) +PhysicalParticleContainer::PushPX (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int /*e_is_nodal*/, + const long offset, + const long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || (gather_lev==(lev )), "Gather buffers only work for lev-1"); // If no particles, do not do anything // If do_not_gather = 1 by user, do not do anything - if (np_to_gather == 0 || do_not_gather) return; + if (np_to_push == 0 || do_not_gather) return; // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); @@ -1945,6 +1784,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, box.grow(ngE); const auto getPosition = GetParticlePosition(pti, offset); + auto setPosition = SetParticlePosition(pti, offset); const auto getExternalE = GetExternalEField(pti, offset); const auto getExternalB = GetExternalBField(pti, offset); @@ -1958,63 +1798,107 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, const Dim3 lo = lbound(box); - // Depending on l_lower_in_v and WarpX::nox, call - // different versions of template function doGatherShapeN - if (WarpX::l_lower_order_in_v){ - if (WarpX::nox == 1){ - doGatherShapeN<1,1>(getPosition, getExternalE, getExternalB, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, - Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - np_to_gather, dx, - xyzmin, lo, WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 2){ - doGatherShapeN<2,1>(getPosition, getExternalE, getExternalB, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, - Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - np_to_gather, dx, - xyzmin, lo, WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 3){ - doGatherShapeN<3,1>(getPosition, getExternalE, getExternalB, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, - Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - np_to_gather, dx, - xyzmin, lo, WarpX::n_rz_azimuthal_modes); - } - } else { - if (WarpX::nox == 1){ - doGatherShapeN<1,0>(getPosition, getExternalE, getExternalB, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, - Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - np_to_gather, dx, - xyzmin, lo, WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 2){ - doGatherShapeN<2,0>(getPosition, getExternalE, getExternalB, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, - Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - np_to_gather, dx, - xyzmin, lo, WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 3){ - doGatherShapeN<3,0>(getPosition, getExternalE, getExternalB, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, - Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, - np_to_gather, dx, - xyzmin, lo, WarpX::n_rz_azimuthal_modes); - } + int l_lower_order_in_v = WarpX::l_lower_order_in_v; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab->array(); + amrex::Array4 const& ey_arr = eyfab->array(); + amrex::Array4 const& ez_arr = ezfab->array(); + amrex::Array4 const& bx_arr = bxfab->array(); + amrex::Array4 const& by_arr = byfab->array(); + amrex::Array4 const& bz_arr = bzfab->array(); + + amrex::IndexType const ex_type = exfab->box().ixType(); + amrex::IndexType const ey_type = eyfab->box().ixType(); + amrex::IndexType const ez_type = ezfab->box().ixType(); + amrex::IndexType const bx_type = bxfab->box().ixType(); + amrex::IndexType const by_type = byfab->box().ixType(); + amrex::IndexType const bz_type = bzfab->box().ixType(); + + auto& attribs = pti.GetAttribs(); + ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + + auto copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + int do_copy = (WarpX::do_back_transformed_diagnostics && + do_back_transformed_diagnostics && + (a_dt_type!=DtType::SecondHalf)); + + int* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization) { + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } -} + // Loop over the particles and update their momentum + const amrex::Real q = this->charge; + const amrex::Real m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; +#ifdef WARPX_QED + const auto do_sync = m_do_qed_quantum_sync; + amrex::Real t_chi_max = 0.0; + if (do_sync) t_chi_max = m_shr_p_qs_engine->get_ref_ctrl().chi_part_min; + + QuantumSynchrotronEvolveOpticalDepth evolve_opt; + amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; + const bool local_has_quantum_sync = has_quantum_sync(); + if (local_has_quantum_sync) { + evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); + p_optical_depth_QSR = pti.GetAttribs(particle_comps["optical_depth_QSR"]).dataPtr(); + } +#endif + + amrex::ParallelFor( np_to_push, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = 0._rt, Eyp = 0._rt, Ezp = 0._rt; + getExternalE(ip, Exp, Eyp, Ezp); + + amrex::ParticleReal Bxp = 0._rt, Byp = 0._rt, Bzp = 0._rt; + getExternalB(ip, Bxp, Byp, Bzp); + + // first gather E and B to the particle positions + doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, l_lower_order_in_v); + + scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + +#ifdef WARPX_WED + if (local_has_quantum_sync) { + const ParticleReal px = m * ux[ip]; + const ParticleReal py = m * uy[ip]; + const ParticleReal pz = m * uz[ip]; + + bool has_event_happened = evolve_opt(px, py, pz, + Exp, Eyp, Ezp, + Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } +#endif + + doParticlePush(getPosition, setPosition, copyAttribs, ip, + ux[ip+offset], uy[ip+offset], uz[ip+offset], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, do_copy, +#ifdef WARPX_QED + do_sync, + t_chi_max, +#endif + dt); + }); +} void PhysicalParticleContainer::InitIonizationModule () @@ -2066,16 +1950,26 @@ PhysicalParticleContainer::InitIonizationModule () } IonizationFilterFunc -PhysicalParticleContainer::getIonizationFunc () +PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, + int lev, + int ngE, + const amrex::FArrayBox& Ex, + const amrex::FArrayBox& Ey, + const amrex::FArrayBox& Ez, + const amrex::FArrayBox& Bx, + const amrex::FArrayBox& By, + const amrex::FArrayBox& Bz) { WARPX_PROFILE("PPC::getIonizationFunc"); - return IonizationFilterFunc{ionization_energies.dataPtr(), + return IonizationFilterFunc(pti, lev, ngE, Ex, Ey, Ez, Bx, By, Bz, + v_galilean, + ionization_energies.dataPtr(), adk_prefactor.dataPtr(), adk_exp_prefactor.dataPtr(), adk_power.dataPtr(), particle_icomps["ionization_level"], - ion_atomic_number}; + ion_atomic_number); } #ifdef WARPX_QED diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index 1f8a190ea3a..b9233d40967 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -17,9 +17,6 @@ /** \brief Functor that creates copies of the current particle * positions and momenta for later use. This is needed * by the back-transformed diagnostics. - * - * \param a_pti iterator to the tile containing the macroparticles - * \param a_tmp holder for the temporary particle data */ struct CopyParticleAttribs { @@ -39,25 +36,35 @@ struct CopyParticleAttribs amrex::ParticleReal* AMREX_RESTRICT uypold = nullptr; amrex::ParticleReal* AMREX_RESTRICT uzpold = nullptr; - CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data) noexcept + /** \brief Construct a new functor + * + * \param a_pti iterator to the tile containing the macroparticles + * \param a_tmp holder for the temporary particle data + * \param a_offset offset to apply when reading / writing particle data + * This is needed because when we use field gather buffers we don't + * always start at the particle with index 0. + */ + CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, + int a_offset = 0) noexcept { if (tmp_particle_data.size() == 0) return; auto& attribs = a_pti.GetAttribs(); - uxp = attribs[PIdx::ux].dataPtr(); - uyp = attribs[PIdx::uy].dataPtr(); - uzp = attribs[PIdx::uz].dataPtr(); + + uxp = attribs[PIdx::ux].dataPtr() + a_offset; + uyp = attribs[PIdx::uy].dataPtr() + a_offset; + uzp = attribs[PIdx::uz].dataPtr() + a_offset; const auto lev = a_pti.GetLevel(); const auto index = a_pti.GetPairIndex(); - xpold = tmp_particle_data[lev][index][TmpIdx::xold ].dataPtr(); - ypold = tmp_particle_data[lev][index][TmpIdx::yold ].dataPtr(); - zpold = tmp_particle_data[lev][index][TmpIdx::zold ].dataPtr(); - uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr(); - uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr(); - uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr(); - - m_get_position = GetParticlePosition(a_pti); + xpold = tmp_particle_data[lev][index][TmpIdx::xold ].dataPtr() + a_offset; + ypold = tmp_particle_data[lev][index][TmpIdx::yold ].dataPtr() + a_offset; + zpold = tmp_particle_data[lev][index][TmpIdx::zold ].dataPtr() + a_offset; + uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr() + a_offset; + uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr() + a_offset; + uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr() + a_offset; + + m_get_position = GetParticlePosition(a_pti, a_offset); } /** \brief copy the position and momentum of particle i to the diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index cc824ce00d5..068b67bedfb 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -50,6 +50,12 @@ public: const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + const amrex::MultiFab& Ex_avg, + const amrex::MultiFab& Ey_avg, + const amrex::MultiFab& Ez_avg, + const amrex::MultiFab& Bx_avg, + const amrex::MultiFab& By_avg, + const amrex::MultiFab& Bz_avg, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, @@ -68,7 +74,19 @@ public: amrex::Real dt, DtType a_dt_type=DtType::Full) override; - virtual void PushPX (WarpXParIter& pti, amrex::Real dt, DtType a_dt_type=DtType::Full) override; + virtual void PushPX (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int /*e_is_nodal*/, + const long offset, + const long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type=DtType::Full) override; virtual void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index a897dddaf24..110d304bb4d 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -20,6 +20,8 @@ #include "Pusher/UpdateMomentumBorisWithRadiationReaction.H" #include "Pusher/UpdateMomentumHigueraCary.H" #include "Pusher/GetAndSetPosition.H" +#include "Gather/ScaleFields.H" +#include "Gather/FieldGather.H" #include #include @@ -224,10 +226,20 @@ RigidInjectedParticleContainer::BoostandRemapParticles() } void -RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, Real dt, DtType a_dt_type) +RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int e_is_nodal, + const long offset, + const long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields /*scaleFields*/, + DtType a_dt_type) { - - // This wraps the momentum and position advance so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; @@ -243,12 +255,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, Real dt, DtType a_dt_ ParticleReal* const AMREX_RESTRICT ux = uxp.dataPtr(); ParticleReal* const AMREX_RESTRICT uy = uyp.dataPtr(); ParticleReal* const AMREX_RESTRICT uz = uzp.dataPtr(); - ParticleReal* const AMREX_RESTRICT Exp = attribs[PIdx::Ex].dataPtr(); - ParticleReal* const AMREX_RESTRICT Eyp = attribs[PIdx::Ey].dataPtr(); - ParticleReal* const AMREX_RESTRICT Ezp = attribs[PIdx::Ez].dataPtr(); - ParticleReal* const AMREX_RESTRICT Bxp = attribs[PIdx::Bx].dataPtr(); - ParticleReal* const AMREX_RESTRICT Byp = attribs[PIdx::By].dataPtr(); - ParticleReal* const AMREX_RESTRICT Bzp = attribs[PIdx::Bz].dataPtr(); if (!done_injecting_lev) { @@ -282,32 +288,15 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, Real dt, DtType a_dt_ uyp_save_ptr[i] = uy[i]; uzp_save_ptr[i] = uz[i]; }); - - // Scale the fields of particles about to cross the injection plane. - // This only approximates what should be happening. The particles - // should by advanced a fraction of a time step instead. - // Scaling the fields is much easier and may be good enough. - const Real v_boost = WarpX::beta_boost*PhysConst::c; - const Real z_plane_previous = zinject_plane_lev_previous; - const Real vz_ave_boosted = vzbeam_ave_boosted; - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - ParticleReal xp, yp, zp; - GetPosition(i, xp, yp, zp); - const Real dtscale = dt - (z_plane_previous - zp)/(vz_ave_boosted + v_boost); - if (0. < dtscale && dtscale < dt) { - Exp[i] *= dtscale; - Eyp[i] *= dtscale; - Ezp[i] *= dtscale; - Bxp[i] *= dtscale; - Byp[i] *= dtscale; - Bzp[i] *= dtscale; - } - } - ); } - PhysicalParticleContainer::PushPX(pti, dt, a_dt_type); + const bool do_scale = not done_injecting_lev; + const Real v_boost = WarpX::beta_boost*PhysConst::c; + PhysicalParticleContainer::PushPX(pti, exfab, eyfab, ezfab, bxfab, byfab, bzfab, + ngE, e_is_nodal, offset, np_to_push, lev, gather_lev, dt, + ScaleFields(do_scale, dt, zinject_plane_lev_previous, + vzbeam_ave_boosted, v_boost), + a_dt_type); if (!done_injecting_lev) { @@ -351,6 +340,8 @@ void RigidInjectedParticleContainer::Evolve (int lev, const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + const MultiFab& Ex_avg, const MultiFab& Ey_avg, const MultiFab& Ez_avg, + const MultiFab& Bx_avg, const MultiFab& By_avg, const MultiFab& Bz_avg, MultiFab& jx, MultiFab& jy, MultiFab& jz, MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, MultiFab* rho, MultiFab* crho, @@ -378,6 +369,8 @@ RigidInjectedParticleContainer::Evolve (int lev, PhysicalParticleContainer::Evolve (lev, Ex, Ey, Ez, Bx, By, Bz, + Ex_avg, Ey_avg, Ez_avg, + Bx_avg, By_avg, Bz_avg, jx, jy, jz, cjx, cjy, cjz, rho, crho, @@ -395,23 +388,16 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, if (do_not_push) return; + const std::array& dx = WarpX::CellSize(std::max(lev,0)); + #ifdef _OPENMP #pragma omp parallel #endif { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - auto& attribs = pti.GetAttribs(); - - auto& uxp = attribs[PIdx::ux]; - auto& uyp = attribs[PIdx::uy]; - auto& uzp = attribs[PIdx::uz]; - auto& Exp = attribs[PIdx::Ex]; - auto& Eyp = attribs[PIdx::Ey]; - auto& Ezp = attribs[PIdx::Ez]; - auto& Bxp = attribs[PIdx::Bx]; - auto& Byp = attribs[PIdx::By]; - auto& Bzp = attribs[PIdx::Bz]; + amrex::Box box = pti.tilebox(); + box.grow(Ex.nGrow()); const long np = pti.numParticles(); @@ -423,76 +409,108 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const FArrayBox& byfab = By[pti]; const FArrayBox& bzfab = Bz[pti]; - int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, - 0, np, lev, lev); + const auto getPosition = GetParticlePosition(pti); + auto setPosition = SetParticlePosition(pti); + + const auto getExternalE = GetExternalEField(pti); + const auto getExternalB = GetExternalBField(pti); + + const auto& xyzmin = WarpX::GetInstance().LowerCornerWithGalilean(box,v_galilean,lev); + + const Dim3 lo = lbound(box); + + int l_lower_order_in_v = WarpX::l_lower_order_in_v; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab.array(); + amrex::Array4 const& ey_arr = eyfab.array(); + amrex::Array4 const& ez_arr = ezfab.array(); + amrex::Array4 const& bx_arr = bxfab.array(); + amrex::Array4 const& by_arr = byfab.array(); + amrex::Array4 const& bz_arr = bzfab.array(); + + amrex::IndexType const ex_type = exfab.box().ixType(); + amrex::IndexType const ey_type = eyfab.box().ixType(); + amrex::IndexType const ez_type = ezfab.box().ixType(); + amrex::IndexType const bx_type = bxfab.box().ixType(); + amrex::IndexType const by_type = byfab.box().ixType(); + amrex::IndexType const bz_type = bzfab.box().ixType(); + + auto& attribs = pti.GetAttribs(); + auto& uxp = attribs[PIdx::ux]; + auto& uyp = attribs[PIdx::uy]; + auto& uzp = attribs[PIdx::uz]; + amrex::ParticleReal* const AMREX_RESTRICT uxpp = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uypp = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uzpp = attribs[PIdx::uz].dataPtr(); + + int* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization) { + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); + } // Save the position and momenta, making copies auto uxp_save = uxp; auto uyp_save = uyp; auto uzp_save = uzp; - // This wraps the momentum advance so that inheritors can modify the call. - // Extract pointers to the different particle quantities - const auto GetPosition = GetParticlePosition(pti); - ParticleReal* const AMREX_RESTRICT uxpp = uxp.dataPtr(); - ParticleReal* const AMREX_RESTRICT uypp = uyp.dataPtr(); - ParticleReal* const AMREX_RESTRICT uzpp = uzp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Expp = Exp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Eypp = Eyp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bypp = Byp.dataPtr(); - const ParticleReal* const AMREX_RESTRICT Bzpp = Bzp.dataPtr(); - // Loop over the particles and update their momentum - const Real q = this->charge; - const Real m = this->mass; - - //Assumes that all consistency checks have been done at initialization - if(do_classical_radiation_reaction){ - amrex::ParallelFor( - pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBorisWithRadiationReaction( - uxpp[i], uypp[i], uzpp[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - q, m, dt); - } - ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBoris( - uxpp[i], uypp[i], uzpp[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - q, m, dt); - } - ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumVay( - uxpp[i], uypp[i], uzpp[i], - Expp[i], Eypp[i], Ezpp[i], - Bxpp[i], Bypp[i], Bzpp[i], - q, m, dt); - } - ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::HigueraCary) { - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumHigueraCary( uxpp[i], uypp[i], uzpp[i], - Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); - } - ); - } else { - amrex::Abort("Unknown particle pusher"); - }; + const amrex::Real q = this->charge; + const amrex::Real m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = 0._rt, Eyp = 0._rt, Ezp = 0._rt; + getExternalE(ip, Exp, Eyp, Ezp); + + amrex::ParticleReal Bxp = 0._rt, Byp = 0._rt, Bzp = 0._rt; + getExternalB(ip, Bxp, Byp, Bzp); + + // first gather E and B to the particle positions + doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, l_lower_order_in_v); + + if (do_crr) { + amrex::Real qp = q; + if (ion_lev) { qp *= ion_lev[ip]; } + UpdateMomentumBorisWithRadiationReaction(uxpp[ip], uypp[ip], uzpp[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else if (pusher_algo == ParticlePusherAlgo::Boris) { + amrex::Real qp = q; + if (ion_lev) { qp *= ion_lev[ip]; } + UpdateMomentumBoris( uxpp[ip], uypp[ip], uzpp[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else if (pusher_algo == ParticlePusherAlgo::Vay) { + amrex::Real qp = q; + if (ion_lev){ qp *= ion_lev[ip]; } + UpdateMomentumVay( uxpp[ip], uypp[ip], uzpp[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else if (pusher_algo == ParticlePusherAlgo::HigueraCary) { + amrex::Real qp = q; + if (ion_lev){ qp *= ion_lev[ip]; } + UpdateMomentumHigueraCary( uxpp[ip], uypp[ip], uzpp[ip], + Exp, Eyp, Ezp, Bxp, + Byp, Bzp, qp, m, dt); + } else { + amrex::Abort("Unknown particle pusher"); + } + }); // Undo the push for particles not injected yet. // It is assumed that PushP will only be called on the first and last steps @@ -501,17 +519,16 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const ParticleReal* const AMREX_RESTRICT uy_save = uyp_save.dataPtr(); const ParticleReal* const AMREX_RESTRICT uz_save = uzp_save.dataPtr(); const ParticleReal zz = zinject_plane_levels[lev]; - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - ParticleReal xp, yp, zp; - GetPosition(i, xp, yp, zp); - if (zp <= zz) { - uxpp[i] = ux_save[i]; - uypp[i] = uy_save[i]; - uzpp[i] = uz_save[i]; - } - } - ); + amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) + { + ParticleReal xp, yp, zp; + getPosition(i, xp, yp, zp); + if (zp <= zz) { + uxpp[i] = ux_save[i]; + uypp[i] = uy_save[i]; + uzpp[i] = uz_save[i]; + } + }); } } } diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 3e793bcdb16..7e961b2ceed 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -17,7 +17,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE int compute_shape_factor(amrex::Real* const sx, amrex::Real xint) { return 0; -}; +} /** * Compute shape factor and return index of leftmost cell where diff --git a/Source/Particles/Sorting/CMakeLists.txt b/Source/Particles/Sorting/CMakeLists.txt new file mode 100644 index 00000000000..8b8ad199c21 --- /dev/null +++ b/Source/Particles/Sorting/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(WarpX + PRIVATE + Partition.cpp +) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 8bc4491d7bf..4c493b21fd6 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -32,7 +32,7 @@ struct PIdx { enum { // Particle Attributes stored in amrex::ParticleContainer's struct of array w = 0, // weight - ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, + ux, uy, uz, #ifdef WARPX_DIM_RZ theta, // RZ needs all three position components #endif @@ -65,14 +65,8 @@ namespace ParticleStringNames {"ux", PIdx::ux }, {"uy", PIdx::uy }, {"uz", PIdx::uz }, - {"Ex", PIdx::Ex }, - {"Ey", PIdx::Ey }, - {"Ez", PIdx::Ez }, - {"Bx", PIdx::Bx }, - {"By", PIdx::By }, - {"Bz", PIdx::Bz } #ifdef WARPX_DIM_RZ - ,{"theta", PIdx::theta} + {"theta", PIdx::theta} #endif }; } @@ -85,6 +79,8 @@ public: WarpXParIter (ContainerType& pc, int level); + WarpXParIter (ContainerType& pc, int level, amrex::MFItInfo& info); + const std::array& GetAttribs () const { return GetStructOfArrays().GetRealData(); } @@ -128,9 +124,7 @@ class MultiParticleContainer; * push. * * Note: many functions are pure virtual (meaning they MUST be defined in - * derived classes, e.g., Evolve) or empty function (meaning they - * do not do anything, e.g., FieldGather, meant to be overriden by derived - * function) or actual functions (e.g. CurrentDeposition). + * derived classes, e.g., Evolve) or actual functions (e.g. CurrentDeposition). */ class WarpXParticleContainer : public amrex::ParticleContainer<0,0,PIdx::nattribs> @@ -155,14 +149,6 @@ public: virtual void InitData () = 0; - virtual void FieldGatherES (const amrex::Vector, 3> >& E, - const amrex::Vector > > >& masks) {} - - virtual void FieldGather (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, - const amrex::MultiFab& By, const amrex::MultiFab& Bz) {} - /** * Evolve is the central WarpXParticleContainer function that advances * particles for a time dt (typically one timestep). It is a pure virtual @@ -171,6 +157,8 @@ public: virtual void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + const amrex::MultiFab& Ex_avg, const amrex::MultiFab& Ey_avg, const amrex::MultiFab& Ez_avg, + const amrex::MultiFab& Bx_avg, const amrex::MultiFab& By_avg, const amrex::MultiFab& Bz_avg, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, amrex::MultiFab* rho, amrex::MultiFab* crho, @@ -342,6 +330,8 @@ public: template bool AmIA () const noexcept {return (physical_species == PhysSpec);} + amrex::Array get_v_galilean () {return v_galilean;} + protected: amrex::Array v_galilean = {{0}}; std::map particle_comps; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 27d042f7451..90772129bad 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -26,7 +26,14 @@ using namespace amrex; WarpXParIter::WarpXParIter (ContainerType& pc, int level) - : ParIter(pc, level, MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) + : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) +{ +} + +WarpXParIter::WarpXParIter (ContainerType& pc, int level, MFItInfo& info) + : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + info.SetDynamic(WarpX::do_dynamic_scheduling)) { } @@ -34,9 +41,6 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) : ParticleContainer<0,0,PIdx::nattribs>(amr_core->GetParGDB()) , species_id(ispecies) { - for (unsigned int i = PIdx::Ex; i <= PIdx::Bz; ++i) { - communicate_real_comp[i] = false; // Don't need to communicate E and B. - } SetParticleSize(); ReadParameters(); @@ -45,12 +49,6 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) particle_comps["ux"] = PIdx::ux; particle_comps["uy"] = PIdx::uy; particle_comps["uz"] = PIdx::uz; - particle_comps["Ex"] = PIdx::Ex; - particle_comps["Ey"] = PIdx::Ey; - particle_comps["Ez"] = PIdx::Ez; - particle_comps["Bx"] = PIdx::Bx; - particle_comps["By"] = PIdx::By; - particle_comps["Bz"] = PIdx::Bz; #ifdef WARPX_DIM_RZ particle_comps["theta"] = PIdx::theta; #endif @@ -428,7 +426,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, tilebox.grow(ngRho); const Box tb = amrex::convert( tilebox, rho->ixType().toIntVect() ); - const int nc = (rho->nComp() == 1 ? 1 : rho->nComp()/2); + const int nc = WarpX::ncomps; #ifdef AMREX_USE_GPU // No tiling on GPU: rho_fab points to the full rho array. diff --git a/Source/Python/CMakeLists.txt b/Source/Python/CMakeLists.txt new file mode 100644 index 00000000000..00c9e02f18d --- /dev/null +++ b/Source/Python/CMakeLists.txt @@ -0,0 +1,5 @@ +target_sources(WarpX + PRIVATE + WarpX_py.cpp + WarpXWrappers.cpp +) diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp index e87a20a7a1b..495e07e54a8 100644 --- a/Source/Python/WarpXWrappers.cpp +++ b/Source/Python/WarpXWrappers.cpp @@ -213,6 +213,11 @@ extern "C" ConvertLabParamsToBoost(); } + void warpx_CheckGriddingForRZSpectral() + { + CheckGriddingForRZSpectral(); + } + amrex::Real warpx_getProbLo(int dir) { WarpX& warpx = WarpX::GetInstance(); diff --git a/Source/Python/WarpXWrappers.h b/Source/Python/WarpXWrappers.h index 4a649a56412..bf1b305f466 100644 --- a/Source/Python/WarpXWrappers.h +++ b/Source/Python/WarpXWrappers.h @@ -78,6 +78,8 @@ extern "C" { void warpx_ConvertLabParamsToBoost(); + void warpx_CheckGriddingForRZSpectral(); + amrex::Real warpx_getProbLo(int dir); amrex::Real warpx_getProbHi(int dir); diff --git a/Source/Utils/CMakeLists.txt b/Source/Utils/CMakeLists.txt new file mode 100644 index 00000000000..bd500e4e5eb --- /dev/null +++ b/Source/Utils/CMakeLists.txt @@ -0,0 +1,11 @@ +target_sources(WarpX + PRIVATE + CoarsenIO.cpp + CoarsenMR.cpp + Interpolate.cpp + IntervalsParser.cpp + WarpXAlgorithmSelection.cpp + WarpXMovingWindow.cpp + WarpXTagging.cpp + WarpXUtil.cpp +) diff --git a/Source/Utils/CoarsenIO.H b/Source/Utils/CoarsenIO.H index 2a1515e92c3..670fcb1300b 100644 --- a/Source/Utils/CoarsenIO.H +++ b/Source/Utils/CoarsenIO.H @@ -80,7 +80,7 @@ namespace CoarsenIO{ } } return c; - }; + } /** * \brief Loops over the boxes of the coarsened MultiFab \c mf_dst and fills diff --git a/Source/Utils/CoarsenMR.H b/Source/Utils/CoarsenMR.H index bb42bfaa9cf..4a64a62c853 100644 --- a/Source/Utils/CoarsenMR.H +++ b/Source/Utils/CoarsenMR.H @@ -108,7 +108,7 @@ namespace CoarsenMR{ } } return c; - }; + } /** * \brief Loops over the boxes of the coarsened MultiFab \c mf_dst and fills diff --git a/Source/Utils/Interpolate.cpp b/Source/Utils/Interpolate.cpp index a12fa7b106b..b5d713b7524 100644 --- a/Source/Utils/Interpolate.cpp +++ b/Source/Utils/Interpolate.cpp @@ -1,5 +1,5 @@ #include "Interpolate.H" -#include +#include "Interpolate_K.H" namespace Interpolate { @@ -64,84 +64,40 @@ namespace Interpolate interpolated_F[0].reset( new MultiFab(Fx_fp->boxArray(), dm, 1, ngrow) ); interpolated_F[1].reset( new MultiFab(Fy_fp->boxArray(), dm, 1, ngrow) ); interpolated_F[2].reset( new MultiFab(Fz_fp->boxArray(), dm, 1, ngrow) ); - for (int i=0; i<3; i++) interpolated_F[i]->setVal(0.); - // Loop through the boxes and interpolate the values from the _cp data - const int use_limiter = 0; + IntVect fx_type = interpolated_F[0]->ixType().toIntVect(); + IntVect fy_type = interpolated_F[1]->ixType().toIntVect(); + IntVect fz_type = interpolated_F[2]->ixType().toIntVect(); + #ifdef _OPENMP #pragma omp parallel #endif + for (MFIter mfi(*interpolated_F[0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - std::array ffab; // Temporary array ; contains interpolated fields - for (MFIter mfi(*interpolated_F[0]); mfi.isValid(); ++mfi) - { - Box ccbx = mfi.fabbox(); - ccbx.enclosedCells(); - ccbx.coarsen(r_ratio).refine(r_ratio); // so that ccbx is coarsenable + Box const& boxx = mfi.growntilebox(fx_type); + Box const& boxy = mfi.growntilebox(fy_type); + Box const& boxz = mfi.growntilebox(fz_type); - const FArrayBox& cxfab = (*Fx_cp)[mfi]; - const FArrayBox& cyfab = (*Fy_cp)[mfi]; - const FArrayBox& czfab = (*Fz_cp)[mfi]; - ffab[0].resize(amrex::convert(ccbx,(*Fx_fp)[mfi].box().type())); - ffab[1].resize(amrex::convert(ccbx,(*Fy_fp)[mfi].box().type())); - ffab[2].resize(amrex::convert(ccbx,(*Fz_fp)[mfi].box().type())); - - // - Face centered, in the same way as B on a Yee grid - if ( (*Fx_fp)[mfi].box().type() == IntVect{AMREX_D_DECL(1,0,0)} ){ -#if (AMREX_SPACEDIM == 3) - amrex_interp_div_free_bfield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(cyfab), - BL_TO_FORTRAN_ANYD(czfab), - dx, &r_ratio, &use_limiter); -#else - amrex_interp_div_free_bfield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(czfab), - dx, &r_ratio, &use_limiter); - amrex_interp_cc_bfield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(cyfab), - &r_ratio, &use_limiter); -#endif - // - Edge centered, in the same way as E on a Yee grid - } else if ( (*Fx_fp)[mfi].box().type() == IntVect{AMREX_D_DECL(0,1,1)} ){ -#if (AMREX_SPACEDIM == 3) - amrex_interp_efield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(cyfab), - BL_TO_FORTRAN_ANYD(czfab), - &r_ratio, &use_limiter); -#else - amrex_interp_efield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[0]), - BL_TO_FORTRAN_ANYD(ffab[2]), - BL_TO_FORTRAN_ANYD(cxfab), - BL_TO_FORTRAN_ANYD(czfab), - &r_ratio,&use_limiter); - amrex_interp_nd_efield(ccbx.loVect(), ccbx.hiVect(), - BL_TO_FORTRAN_ANYD(ffab[1]), - BL_TO_FORTRAN_ANYD(cyfab), - &r_ratio); -#endif - } else { - amrex::Abort("Unknown field staggering."); - } + Array4 const& fx = interpolated_F[0]->array(mfi); + Array4 const& fy = interpolated_F[1]->array(mfi); + Array4 const& fz = interpolated_F[2]->array(mfi); + Array4 const& cx = Fx_cp->const_array(mfi); + Array4 const& cy = Fy_cp->const_array(mfi); + Array4 const& cz = Fz_cp->const_array(mfi); - // Add temporary array to the returned structure - for (int i = 0; i < 3; ++i) { - const Box& bx = (*interpolated_F[i])[mfi].box(); - (*interpolated_F[i])[mfi].plus(ffab[i], bx, bx, 0, 0, 1); - } - } + amrex::ParallelFor(boxx, boxy, boxz, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + interp(j,k,l,fx,cx,r_ratio,fx_type); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + interp(j,k,l,fy,cy,r_ratio,fy_type); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + interp(j,k,l,fz,cz,r_ratio,fz_type); + }); } return interpolated_F; } diff --git a/Source/Utils/Interpolate_K.H b/Source/Utils/Interpolate_K.H new file mode 100644 index 00000000000..9c8e0a2b443 --- /dev/null +++ b/Source/Utils/Interpolate_K.H @@ -0,0 +1,47 @@ +#ifndef WARPX_INTERP_K_H_ +#define WARPX_INTERP_K_H_ + +#include + +namespace Interpolate { + +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void interp (int j, int k, int l, + amrex::Array4 const& fine, + amrex::Array4 const& crse, + int r_ratio, IntVect const& type) noexcept +{ + using amrex::Real; + Real const rr = 1.0_rt/static_cast(r_ratio); + + int const jg = amrex::coarsen(j,r_ratio); + Real const wx = static_cast(type[0]) * static_cast(j-jg*r_ratio) * rr; + Real const owx = 1.0_rt-wx; + + int const kg = amrex::coarsen(k,r_ratio); + Real const wy = static_cast(type[1]) * static_cast(k-kg*r_ratio) * rr; + Real const owy = 1.0_rt-wy; + +#if (AMREX_SPACEDIM == 2) + fine(j,k,l) = owx * owy * crse(jg ,kg ,0) + + owx * wy * crse(jg ,kg+1,0) + + wx * owy * crse(jg+1,kg ,0) + + wx * wy * crse(jg+1,kg+1,0); +#else + int const lg = amrex::coarsen(l,r_ratio); + Real const wz = static_cast(type[2]) * static_cast(l-lg*r_ratio) * rr; + Real const owz = 1.0_rt-wz; + fine(j,k,l) = owx * owy * owz * crse(jg ,kg ,lg ) + + wx * owy * owz * crse(jg+1,kg ,lg ) + + owx * wy * owz * crse(jg ,kg+1,lg ) + + wx * wy * owz * crse(jg+1,kg+1,lg ) + + owx * owy * wz * crse(jg ,kg ,lg+1) + + wx * owy * wz * crse(jg+1,kg ,lg+1) + + owx * wy * wz * crse(jg ,kg+1,lg+1) + + wx * wy * wz * crse(jg+1,kg+1,lg+1); +#endif +} + +} + +#endif diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 482bc1f2189..8435864b8f3 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -130,6 +130,10 @@ WarpX::MoveWindow (bool move_j) } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, E_external_grid[dim], use_Eparser, Efield_parser); + if (fft_do_time_averaging) { + shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, B_external_grid[dim], use_Bparser, Bfield_parser); + shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, E_external_grid[dim], use_Eparser, Efield_parser); + } if (move_j) { shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, ng_zero); } @@ -146,6 +150,12 @@ WarpX::MoveWindow (bool move_j) shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); + if (fft_do_time_averaging) { + shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, B_external_grid[dim], use_Bparser, Bfield_parser); + shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, E_external_grid[dim], use_Eparser, Efield_parser); + shiftMF(*Bfield_avg_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); + shiftMF(*Efield_avg_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); + } if (move_j) { shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero); } @@ -310,7 +320,7 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, AMREX_PARALLEL_FOR_4D ( outbox, nc, i, j, k, n, { srcfab(i,j,k,n) = external_field; - }); + }) } else if (useparser == true) { // index type of the src mf auto const& mf_IndexType = (tmpmf).ixType(); @@ -350,7 +360,7 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, AMREX_PARALLEL_FOR_4D ( dstBox, nc, i, j, k, n, { dstfab(i,j,k,n) = srcfab(i+shift.x,j+shift.y,k+shift.z,n); - }); + }) } } diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index 6cdc71a0c76..d3682be2ff2 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,11 @@ void ReadBoostedFrameParameters(amrex::Real& gamma_boost, amrex::Real& beta_boos void ConvertLabParamsToBoost(); +/** + * \brief Ensures that the blocks are setup correctly for the RZ spectral solver + */ +void CheckGriddingForRZSpectral(); + void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax); diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index e4080187a7e..c7949925c06 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -161,7 +161,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) #endif if ( (z_gridpoint >= zmin) && (z_gridpoint < zmax) ) { arr(i,j,k) = 0.; - }; + } } ); } @@ -213,6 +213,86 @@ WarpXParser makeParser (std::string const& parse_function, std::vector= the number of processors. + */ +void CheckGriddingForRZSpectral () +{ +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) + + int max_level; + Vector n_cell(AMREX_SPACEDIM, -1); + + ParmParse pp_amr("amr"); + + pp_amr.get("max_level",max_level); + pp_amr.getarr("n_cell",n_cell,0,AMREX_SPACEDIM); + + Vector blocking_factor_x(max_level+1); + Vector max_grid_size_x(max_level+1); + + // Set the radial block size to be equal to the radial grid size. + blocking_factor_x[0] = n_cell[0]; + max_grid_size_x[0] = n_cell[0]; + + for (int lev=1 ; lev <= max_level ; lev++) { + // For this to be correct, this needs to read in any user specified refinement ratios. + // But since that is messy and unlikely to be needed anytime soon, the ratio is + // fixed to 2 which will be the most likely value. + blocking_factor_x[lev] = blocking_factor_x[lev-1]*2; // refRatio(lev-1); + max_grid_size_x[lev] = max_grid_size_x[lev-1]*2; // refRatio(lev-1); + } + + // Note that any user input values for these parameters are discarded. + pp_amr.addarr("blocking_factor_x", blocking_factor_x); + pp_amr.addarr("max_grid_size_x", max_grid_size_x); + + // Adjust the longitudinal block sizes, making sure that there are + // more blocks than processors. + // The factor of 8 is there to make some room for higher order + // shape factors and filtering. + int nprocs = ParallelDescriptor::NProcs(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(n_cell[1] >= 8*nprocs, + "With RZ spectral, there must be at least eight z-cells per processor so that there can be at least one block per processor."); + + // Get the longitudinal blocking factor in case it was set by the user. + // If not set, use the default value of 8. + Vector bf; + pp_amr.queryarr("blocking_factor",bf); + pp_amr.queryarr("blocking_factor_y",bf); + bf.resize(std::max(static_cast(bf.size()),1),8); + + // Modify the default or any user input, making sure that the blocking factor + // is small enough so that there will be at least as many blocks as there are + // processors. Because of the ASSERT above, bf will never be less than 8. + while (n_cell[1] < nprocs*bf[0]) { + bf[0] /= 2; + } + pp_amr.addarr("blocking_factor_y", bf); + + // Get the longitudinal max grid size in case it was set by the user. + // If not set, use the default value of 128. + Vector mg; + pp_amr.queryarr("max_grid_size",mg); + pp_amr.queryarr("max_grid_size_y",mg); + mg.resize(std::max(static_cast(mg.size()),1),128); + + // Modify the default or any user input, making sure that the max grid size + // (of the coarsest level) is small enough so that there will be at least + // as many blocks as there are processors. + while (n_cell[1] < nprocs*mg[0]) { + mg[0] /= 2; + } + pp_amr.addarr("max_grid_size_y", mg); + +#endif +} + namespace WarpXUtilMsg{ void AlwaysAssert(bool is_expression_true, const std::string& msg = "ERROR!") diff --git a/Source/WarpX.H b/Source/WarpX.H index e15acbc7bf2..be02066e7a2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -79,11 +79,12 @@ public: WarpX (); ~WarpX (); - static std::string Version (); //! Version of WarpX executable - static std::string PicsarVersion (); //! Version of PICSAR dependency + static std::string Version (); //!< Version of WarpX executable + static std::string PicsarVersion (); //!< Version of PICSAR dependency int Verbose () const { return verbose; } + void InitData (); void Evolve (int numsteps = -1); @@ -136,10 +137,17 @@ public: static int em_solver_medium; static int macroscopic_solver_algo; - // Static public variable to switch on current correction in Fourier space - // (equation (19) of https://doi.org/10.1016/j.jcp.2013.03.010): can be set - // true by the user as an input parameter - bool do_current_correction = false; +#ifdef WARPX_USE_PSATD + // If true (overwritten by the user in the input file), the current correction + // defined in equation (19) of https://doi.org/10.1016/j.jcp.2013.03.010 is applied + bool current_correction = false; +#endif + +#ifdef WARPX_USE_PSATD + // If true (overwritten by the user in the input file), the update equation + // for E contains both J and rho (at times n and n+1) on the right hand side + bool update_with_rho = false; +#endif // div E cleaning static int do_dive_cleaning; @@ -260,6 +268,7 @@ public: amrex::Real time_of_last_gal_shift = 0; amrex::Array v_galilean = {{0}}; + static int num_mirrors; amrex::Vector mirror_z; amrex::Vector mirror_z_width; @@ -292,10 +301,10 @@ public: void EvolveB (int lev, PatchType patch_type, amrex::Real dt); void EvolveE (int lev, PatchType patch_type, amrex::Real dt); void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type); + void MacroscopicEvolveE ( amrex::Real dt); void MacroscopicEvolveE (int lev, amrex::Real dt); void MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real dt); - void FieldGather (); /** \brief apply QED correction on electric field * \param dt vector of time steps (for all levels) @@ -350,6 +359,22 @@ public: PML* GetPML (int lev); + /** Run the ionization module on all species */ + void doFieldIonization (); + /** Run the ionization module on all species at level lev + * \param lev level + */ + void doFieldIonization (int lev); + +#ifdef WARPX_QED + /** Run the QED module on all species */ + void doQEDEvents (); + /** Run the QED module on all species at level lev + * \param lev level + */ + void doQEDEvents (int lev); +#endif + void PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full); void PushParticlesandDepose ( amrex::Real cur_time); @@ -362,10 +387,16 @@ public: // Fill boundary cells including coarse/fine boundaries void FillBoundaryB (amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); void FillBoundaryE (amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); + void FillBoundaryB_avg (amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); + void FillBoundaryE_avg (amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); + void FillBoundaryF (amrex::IntVect ng); void FillBoundaryAux (amrex::IntVect ng); void FillBoundaryE (int lev, amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); void FillBoundaryB (int lev, amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); + void FillBoundaryE_avg (int lev, amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); + void FillBoundaryB_avg (int lev, amrex::IntVect ng, amrex::IntVect ng_extra_fine=amrex::IntVect::TheZeroVector()); + void FillBoundaryF (int lev, amrex::IntVect ng); void FillBoundaryAux (int lev, amrex::IntVect ng); @@ -405,6 +436,11 @@ public: std::array galilean_shift, int lev); static std::array UpperCorner (const amrex::Box& bx, int lev); + /* + /brief This computes the lower of the problem domain, taking into account any shift when using the Galilean algorithm. + */ + std::array LowerCornerWithGalilean (const amrex::Box& bx, const amrex::Array& v_galilean, int lev); + static amrex::IntVect RefRatio (int lev); static const amrex::iMultiFab* CurrentBufferMasks (int lev); @@ -455,8 +491,17 @@ public: * This function initializes the E and B fields on each level * using the parser and the user-defined function for the external fields. * The subroutine will parse the x_/y_z_external_grid_function and - * then, the B or E multifab is initialized based on the (x,y,z) position - * on the staggered yee-grid or cell-centered grid. + * then, the field multifab is initialized based on the (x,y,z) position + * on the staggered yee-grid or cell-centered grid, in the interior cells + * and guard cells. + * + * \param[in] mfx, x-component of the field to be initialized + * \param[in] mfy, y-component of the field to be initialized + * \param[in] mfz, z-component of the field to be initialized + * \param[in] xfield_parser, parser function to initialize x-field + * \param[in] yfield_parser, parser function to initialize y-field + * \param[in] zfield_parser, parser function to initialize z-field + * \param[in] lev, level of the Multifabs that is initialized */ void InitializeExternalFieldsOnGridUsingParser ( amrex::MultiFab *mfx, amrex::MultiFab *mfy, amrex::MultiFab *mfz, @@ -538,6 +583,9 @@ private: void FillBoundaryE (int lev, PatchType patch_type, amrex::IntVect ng); void FillBoundaryF (int lev, PatchType patch_type, amrex::IntVect ng); + void FillBoundaryB_avg (int lev, PatchType patch_type, amrex::IntVect ng); + void FillBoundaryE_avg (int lev, PatchType patch_type, amrex::IntVect ng); + void OneStep_nosub (amrex::Real t); void OneStep_sub1 (amrex::Real t); @@ -627,6 +675,8 @@ private: // Full solution amrex::Vector, 3 > > Efield_aux; amrex::Vector, 3 > > Bfield_aux; + amrex::Vector, 3 > > Efield_avg_aux; + amrex::Vector, 3 > > Bfield_avg_aux; // Fine patch amrex::Vector< std::unique_ptr > F_fp; @@ -634,7 +684,8 @@ private: amrex::Vector, 3 > > current_fp; amrex::Vector, 3 > > Efield_fp; amrex::Vector, 3 > > Bfield_fp; - + amrex::Vector, 3 > > Efield_avg_fp; + amrex::Vector, 3 > > Bfield_avg_fp; // store fine patch amrex::Vector, 3 > > current_store; @@ -644,6 +695,8 @@ private: amrex::Vector, 3 > > current_cp; amrex::Vector, 3 > > Efield_cp; amrex::Vector, 3 > > Bfield_cp; + amrex::Vector, 3 > > Efield_avg_cp; + amrex::Vector, 3 > > Bfield_avg_cp; // Copy of the coarse aux amrex::Vector, 3 > > Efield_cax; @@ -731,7 +784,7 @@ private: std::string restart_chkfile; - bool plot_rho = false; + bool plot_rho = false; amrex::VisMF::Header::Version plotfile_headerversion = amrex::VisMF::Header::Version_v1; amrex::VisMF::Header::Version slice_plotfile_headerversion = amrex::VisMF::Header::Version_v1; @@ -760,6 +813,7 @@ private: amrex::Vector, 3 > > Efield_slice; amrex::Vector, 3 > > Bfield_slice; + bool fft_do_time_averaging = false; bool fft_periodic_single_box = false; int nox_fft = 16; int noy_fft = 16; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 476474b9dc7..8c1dded6154 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -142,7 +142,6 @@ WarpX::ResetInstance () WarpX::WarpX () { m_instance = this; - ReadParameters(); BackwardCompatibility(); @@ -190,11 +189,16 @@ WarpX::WarpX () Efield_aux.resize(nlevs_max); Bfield_aux.resize(nlevs_max); + Efield_avg_aux.resize(nlevs_max); + Bfield_avg_aux.resize(nlevs_max); + F_fp.resize(nlevs_max); rho_fp.resize(nlevs_max); current_fp.resize(nlevs_max); Efield_fp.resize(nlevs_max); Bfield_fp.resize(nlevs_max); + Efield_avg_fp.resize(nlevs_max); + Bfield_avg_fp.resize(nlevs_max); current_store.resize(nlevs_max); @@ -203,6 +207,8 @@ WarpX::WarpX () current_cp.resize(nlevs_max); Efield_cp.resize(nlevs_max); Bfield_cp.resize(nlevs_max); + Efield_avg_cp.resize(nlevs_max); + Bfield_avg_cp.resize(nlevs_max); Efield_cax.resize(nlevs_max); Bfield_cax.resize(nlevs_max); @@ -554,6 +560,11 @@ WarpX::ReadParameters () // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1 pp.query("n_rz_azimuthal_modes", n_rz_azimuthal_modes); + +#if defined WARPX_DIM_RZ + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(Geom(0).isPeriodic(0) == 0, + "The problem must not be periodic in the radial direction"); +#endif #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) // Force do_nodal=true (that is, not staggered) and // use same shape factors in all directions, for gathering @@ -600,8 +611,10 @@ WarpX::ReadParameters () pp.query("nox", nox_fft); pp.query("noy", noy_fft); pp.query("noz", noz_fft); - pp.query("do_current_correction", do_current_correction); + pp.query("current_correction", current_correction); + pp.query("update_with_rho", update_with_rho); pp.query("v_galilean", v_galilean); + pp.query("do_time_averaging", fft_do_time_averaging); // Scale the velocity by the speed of light for (int i=0; i<3; i++) v_galilean[i] *= PhysConst::c; } @@ -854,7 +867,16 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm current_fp[lev][1].reset( new MultiFab(amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ)); current_fp[lev][2].reset( new MultiFab(amrex::convert(ba,jz_nodal_flag),dm,ncomps,ngJ)); - if (do_dive_cleaning || plot_rho) + + Bfield_avg_fp[lev][0].reset( new MultiFab(amrex::convert(ba,Bx_nodal_flag),dm,ncomps,ngE)); + Bfield_avg_fp[lev][1].reset( new MultiFab(amrex::convert(ba,By_nodal_flag),dm,ncomps,ngE)); + Bfield_avg_fp[lev][2].reset( new MultiFab(amrex::convert(ba,Bz_nodal_flag),dm,ncomps,ngE)); + + Efield_avg_fp[lev][0].reset( new MultiFab(amrex::convert(ba,Ex_nodal_flag),dm,ncomps,ngE)); + Efield_avg_fp[lev][1].reset( new MultiFab(amrex::convert(ba,Ey_nodal_flag),dm,ncomps,ngE)); + Efield_avg_fp[lev][2].reset( new MultiFab(amrex::convert(ba,Ez_nodal_flag),dm,ncomps,ngE)); + + if (do_dive_cleaning || (plot_rho && do_back_transformed_diagnostics)) { rho_fp[lev].reset(new MultiFab(amrex::convert(ba,rho_nodal_flag),dm,2*ncomps,ngRho)); } @@ -901,7 +923,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm bool const pml=false; spectral_solver_fp[lev].reset( new SpectralSolver( realspace_ba, dm, nox_fft, noy_fft, noz_fft, do_nodal, v_galilean, dx_vect, dt[lev], - pml, fft_periodic_single_box ) ); + pml, fft_periodic_single_box, update_with_rho ) ); # endif #endif m_fdtd_solver_fp[lev].reset( @@ -926,6 +948,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm for (int idir = 0; idir < 3; ++idir) { Efield_aux[lev][idir].reset(new MultiFab(*Efield_fp[lev][idir], amrex::make_alias, 0, ncomps)); Bfield_aux[lev][idir].reset(new MultiFab(*Bfield_fp[lev][idir], amrex::make_alias, 0, ncomps)); + + Efield_avg_aux[lev][idir].reset(new MultiFab(*Efield_avg_fp[lev][idir], amrex::make_alias, 0, ncomps)); + Bfield_avg_aux[lev][idir].reset(new MultiFab(*Bfield_avg_fp[lev][idir], amrex::make_alias, 0, ncomps)); } } else @@ -937,6 +962,16 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm Efield_aux[lev][0].reset( new MultiFab(amrex::convert(ba,Ex_nodal_flag),dm,ncomps,ngE)); Efield_aux[lev][1].reset( new MultiFab(amrex::convert(ba,Ey_nodal_flag),dm,ncomps,ngE)); Efield_aux[lev][2].reset( new MultiFab(amrex::convert(ba,Ez_nodal_flag),dm,ncomps,ngE)); + + + Bfield_avg_aux[lev][0].reset( new MultiFab(amrex::convert(ba,Bx_nodal_flag),dm,ncomps,ngE)); + Bfield_avg_aux[lev][1].reset( new MultiFab(amrex::convert(ba,By_nodal_flag),dm,ncomps,ngE)); + Bfield_avg_aux[lev][2].reset( new MultiFab(amrex::convert(ba,Bz_nodal_flag),dm,ncomps,ngE)); + + Efield_avg_aux[lev][0].reset( new MultiFab(amrex::convert(ba,Ex_nodal_flag),dm,ncomps,ngE)); + Efield_avg_aux[lev][1].reset( new MultiFab(amrex::convert(ba,Ey_nodal_flag),dm,ncomps,ngE)); + Efield_avg_aux[lev][2].reset( new MultiFab(amrex::convert(ba,Ez_nodal_flag),dm,ncomps,ngE)); + } // @@ -958,12 +993,22 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm Efield_cp[lev][1].reset( new MultiFab(amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngE)); Efield_cp[lev][2].reset( new MultiFab(amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngE)); + // Create the MultiFabs for B_avg + Bfield_avg_cp[lev][0].reset( new MultiFab(amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngE)); + Bfield_avg_cp[lev][1].reset( new MultiFab(amrex::convert(cba,By_nodal_flag),dm,ncomps,ngE)); + Bfield_avg_cp[lev][2].reset( new MultiFab(amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngE)); + + // Create the MultiFabs for E_avg + Efield_avg_cp[lev][0].reset( new MultiFab(amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngE)); + Efield_avg_cp[lev][1].reset( new MultiFab(amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngE)); + Efield_avg_cp[lev][2].reset( new MultiFab(amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngE)); + // Create the MultiFabs for the current current_cp[lev][0].reset( new MultiFab(amrex::convert(cba,jx_nodal_flag),dm,ncomps,ngJ)); current_cp[lev][1].reset( new MultiFab(amrex::convert(cba,jy_nodal_flag),dm,ncomps,ngJ)); current_cp[lev][2].reset( new MultiFab(amrex::convert(cba,jz_nodal_flag),dm,ncomps,ngJ)); - if (do_dive_cleaning || plot_rho){ + if (do_dive_cleaning || (plot_rho && do_back_transformed_diagnostics)) { rho_cp[lev].reset(new MultiFab(amrex::convert(cba,rho_nodal_flag),dm,2*ncomps,ngRho)); } if (do_dive_cleaning) @@ -992,7 +1037,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm # else realspace_ba.grow(ngE); // add guard cells spectral_solver_cp[lev].reset( new SpectralSolver( realspace_ba, dm, - nox_fft, noy_fft, noz_fft, do_nodal, v_galilean, cdx_vect, dt[lev] ) ); + nox_fft, noy_fft, noz_fft, do_nodal, v_galilean, cdx_vect, dt[lev], + pml, fft_periodic_single_box, update_with_rho ) ); # endif #endif m_fdtd_solver_cp[lev].reset( @@ -1102,6 +1148,15 @@ WarpX::UpperCorner(const Box& bx, int lev) #endif } +std::array +WarpX::LowerCornerWithGalilean (const Box& bx, const amrex::Array& v_galilean, int lev) +{ + amrex::Real cur_time = gett_new(lev); + amrex::Real time_shift = (cur_time - time_of_last_gal_shift); + amrex::Array galilean_shift = { v_galilean[0]*time_shift, v_galilean[1]*time_shift, v_galilean[2]*time_shift }; + return WarpX::LowerCorner(bx, galilean_shift, lev); +} + IntVect WarpX::RefRatio (int lev) { @@ -1361,13 +1416,3 @@ WarpX::PicsarVersion () return std::string("Unknown"); #endif } - -void -WarpX::FieldGather () -{ - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->FieldGather(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } -} diff --git a/Source/main.cpp b/Source/main.cpp index 56e84dd9a88..4bbdce2dace 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -34,8 +34,15 @@ int main(int argc, char* argv[]) amrex::Initialize(argc,argv); + // in Debug mode, we need a larger stack limit than usual bc of the parser. +#if defined(AMREX_USE_CUDA) && defined(AMREX_DEBUG) + AMREX_CUDA_SAFE_CALL(cudaDeviceSetLimit(cudaLimitStackSize, 20*1024)); +#endif + ConvertLabParamsToBoost(); + CheckGriddingForRZSpectral(); + WARPX_PROFILE_VAR("main()", pmain); const Real strt_total = amrex::second(); diff --git a/Tools/BatchScripts/batch_juwels.sh b/Tools/BatchScripts/batch_juwels.sh new file mode 100644 index 00000000000..60f10a138b9 --- /dev/null +++ b/Tools/BatchScripts/batch_juwels.sh @@ -0,0 +1,20 @@ +#!/bin/bash -l + +#SBATCH -A $proj +#SBATCH --partition=gpus +#SBATCH --nodes=2 +#SBATCH --ntasks=8 +#SBATCH --ntasks-per-node=4 +#SBATCH --gres=gpu:4 +#SBATCH --time=00:05:00 +#SBATCH --job-name=warpx +#SBATCH --output=warpx-%j-%N.txt +#SBATCH --error=warpx-%j-%N.err + +export OMP_NUM_THREADS=1 + +module load GCC +module load OpenMPI +module load CUDA + +srun -n 8 --cpu_bind=sockets $HOME/src/warpx/Bin/main3d.gnu.TPROF.MPI.CUDA.ex inputs diff --git a/Tools/BatchScripts/batch_summit.sh b/Tools/BatchScripts/batch_summit.sh index 6b9b4957b96..75e48455cd9 100644 --- a/Tools/BatchScripts/batch_summit.sh +++ b/Tools/BatchScripts/batch_summit.sh @@ -1,14 +1,19 @@ #!/bin/bash -# Copyright 2019 Maxence Thevenet +# Copyright 2019-2020 Maxence Thevenet, Axel Huebl # # This file is part of WarpX. # # License: BSD-3-Clause-LBNL +# +# Refs.: +# https://jsrunvisualizer.olcf.ornl.gov/?s4f0o11n6c7g1r11d1b1l0= +# https://docs.olcf.ornl.gov/systems/summit_user_guide.html#cuda-aware-mpi #BSUB -P #BSUB -W 00:10 #BSUB -nnodes 2 +#BSUB -alloc_flags smt4 #BSUB -J WarpX #BSUB -o WarpXo.%J #BSUB -e WarpXe.%J @@ -16,8 +21,5 @@ module load gcc module load cuda -omp=1 -export OMP_NUM_THREADS=${omp} - -num_nodes=$(( $(printf '%s\n' ${LSB_HOSTS} | sort -u | wc -l) - 1 )) -jsrun -n ${num_nodes} -a 6 -g 6 -c 6 --bind=packed:${omp} > output.txt +export OMP_NUM_THREADS=1 +jsrun -r 6 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs --smpiargs="-gpu" > output.txt diff --git a/Tools/BatchScripts/batch_summit_power9.sh b/Tools/BatchScripts/batch_summit_power9.sh new file mode 100644 index 00000000000..00816761e7f --- /dev/null +++ b/Tools/BatchScripts/batch_summit_power9.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2019-2020 Maxence Thevenet, Axel Huebl, Michael Rowan +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# Refs.: +# https://jsrunvisualizer.olcf.ornl.gov/?s1f0o121n2c21g0r11d1b1l0= + +#BSUB -P +#BSUB -W 00:10 +#BSUB -nnodes 1 +#BSUB -alloc_flags "smt1" +#BSUB -J WarpX +#BSUB -o WarpXo.%J +#BSUB -e WarpXe.%J + + +export OMP_NUM_THREADS=21 +jsrun -n 2 -a 1 -c 21 -r 2 -l CPU-CPU -d packed -b rs > output.txt diff --git a/Tools/LibEnsemble/requirements.txt b/Tools/LibEnsemble/requirements.txt index bd9820cb94f..a9b77134d57 100644 --- a/Tools/LibEnsemble/requirements.txt +++ b/Tools/LibEnsemble/requirements.txt @@ -1,6 +1,8 @@ matplotlib numpy scipy +pytest libensemble yt nlopt + diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py index b6126b5d673..e6fd35c1ae7 100644 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ b/Tools/LibEnsemble/run_libensemble_on_warpx.py @@ -50,7 +50,7 @@ # Import machine-specific run parameters if machine == 'local': machine_specs = all_machine_specs.local_specs -elif machine == 'aposmm': +elif machine == 'summit': machine_specs = all_machine_specs.summit_specs else: print("you shouldn' hit that") diff --git a/Tools/PerformanceTests/run_alltests.py b/Tools/PerformanceTests/run_alltests.py index 7b0a93350c3..fb13c670d99 100644 --- a/Tools/PerformanceTests/run_alltests.py +++ b/Tools/PerformanceTests/run_alltests.py @@ -346,4 +346,4 @@ def process_analysis(): if args.commit == True: os.system('git add ' + log_dir + log_file + ';'\ 'git commit -m "performance tests";'\ - 'git push -u origin master') + 'git push -u origin development') diff --git a/Tools/PerformanceTests/run_alltests_1node.py b/Tools/PerformanceTests/run_alltests_1node.py index 5745055e7d3..8a9685e0446 100644 --- a/Tools/PerformanceTests/run_alltests_1node.py +++ b/Tools/PerformanceTests/run_alltests_1node.py @@ -298,7 +298,7 @@ def process_analysis(): if do_commit == True: os.system('git add ' + log_dir + log_file + ';'\ 'git commit -m "performance tests";'\ - 'git push -u origin master') + 'git push -u origin development') # Plot file import numpy as np diff --git a/Tools/PostProcessing/Visualization.ipynb b/Tools/PostProcessing/Visualization.ipynb index 71f482afb43..8dd2f344161 100644 --- a/Tools/PostProcessing/Visualization.ipynb +++ b/Tools/PostProcessing/Visualization.ipynb @@ -6,11 +6,7 @@ "source": [ "# Overview\n", "\n", - "This a notebook that inspects the results of a WarpX simulation.\n", - "\n", - "# Instruction\n", - "\n", - "Enter the path of the data you wish to visualize below. Then execute the cells one by one, by selecting them with your mouse and typing `Shift + Enter`" + "This a notebook that inspects the results of a WarpX simulation." ] }, { @@ -34,6 +30,28 @@ "## Read data in the simulation frame" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Instruction\n", + "\n", + "Enter the path of the data you wish to visualize below. Then execute the cells one by one, by selecting them with your mouse and typing `Shift + Enter`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plotfile = './diags/plotfiles/plt00001'\n", + "field = 'Ex'\n", + "species = 'electron'\n", + "ds = yt.load( plotfile ) # Load the plotfile\n", + "# ds.field_list # Print all available quantities" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -47,12 +65,10 @@ "metadata": {}, "outputs": [], "source": [ - "ds = yt.load( '../Examples/Modules/RigidInjection/diags/plotfiles/plt00318/' ) # Create a dataset object\n", - "sl = yt.SlicePlot(ds, 2, 'Ex', aspect=.2) # Create a sliceplot object\n", - "sl.annotate_particles(width=(10.e-6, 'm'), p_size=2, ptype='beam', col='black')\n", + "sl = yt.SlicePlot(ds, 2, field, aspect=.2) # Create a sliceplot object\n", + "sl.annotate_particles(width=(10.e-6, 'm'), p_size=2, ptype=species, col='black')\n", "sl.annotate_grids() # Show grids\n", - "sl.show() # Show the plot\n", - "# sl.save('./toto.png')" + "sl.show() # Show the plot" ] }, { @@ -70,23 +86,20 @@ "source": [ "# Get field quantities\n", "all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions)\n", - "Bx = all_data_level_0['boxlib', 'Ex'].v.squeeze()\n", + "Bx = all_data_level_0['boxlib', field].v.squeeze()\n", "Dx = ds.domain_width/ds.domain_dimensions\n", "extent = [ds.domain_left_edge[ds.dimensionality-1], ds.domain_right_edge[ds.dimensionality-1],\n", " ds.domain_left_edge[0], ds.domain_right_edge[0] ]\n", "\n", "# Get particle quantities\n", "ad = ds.all_data()\n", - "x = ad['beam', 'particle_position_x'].v\n", - "z = ad['beam', 'particle_position_y'].v\n", + "x = ad[species, 'particle_position_x'].v\n", + "z = ad[species, 'particle_position_y'].v\n", "\n", "# Plot image\n", "plt.figure()\n", "plt.imshow(Bx, extent=extent)\n", - "plt.scatter(z,x,s=.1,c='k')\n", - "\n", - "# Print all available quantities\n", - "ds.field_list" + "plt.scatter(z,x,s=.1,c='k')" ] }, { @@ -113,9 +126,7 @@ "metadata": {}, "outputs": [], "source": [ - "species = 'beam'\n", "iteration = 1\n", - "field = 'Ex'\n", "\n", "snapshot = './lab_frame_data/' + 'snapshot' + str(iteration).zfill(5)\n", "header = './lab_frame_data/Header'\n", diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake new file mode 100644 index 00000000000..ff49c938bdf --- /dev/null +++ b/cmake/WarpXFunctions.cmake @@ -0,0 +1,237 @@ +# find the CCache tool and use it if found +# +macro(set_ccache) + find_program(CCACHE_PROGRAM ccache) + if(CCACHE_PROGRAM) + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + if(ENABLE_CUDA) + set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + endif() + endif() + mark_as_advanced(CCACHE_PROGRAM) +endmacro() + + +# set names and paths of temporary build directories +# the defaults in CMake are sub-ideal for historic reasons, lets make them more +# Unix-ish and portable. +# +macro(set_default_build_dirs) + if(NOT CMAKE_ARCHIVE_OUTPUT_DIRECTORY) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + CACHE PATH "Build directory for archives") + mark_as_advanced(CMAKE_ARCHIVE_OUTPUT_DIRECTORY) + endif() + if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + CACHE PATH "Build directory for libraries") + mark_as_advanced(CMAKE_LIBRARY_OUTPUT_DIRECTORY) + endif() + if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" + CACHE PATH "Build directory for binaries") + mark_as_advanced(CMAKE_RUNTIME_OUTPUT_DIRECTORY) + endif() +endmacro() + + +# set names and paths of install directories +# the defaults in CMake are sub-ideal for historic reasons, lets make them more +# Unix-ish and portable. +# +macro(set_default_install_dirs) + include(GNUInstallDirs) + if(NOT CMAKE_INSTALL_CMAKEDIR) + set(CMAKE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/WarpX" + CACHE PATH "CMake config package location for installed targets") + if(WIN32) + set(CMAKE_INSTALL_LIBDIR Lib + CACHE PATH "Object code libraries") + set_property(CACHE CMAKE_INSTALL_CMAKEDIR PROPERTY VALUE "cmake") + endif() + mark_as_advanced(CMAKE_INSTALL_CMAKEDIR) + endif() +endmacro() + + +# change the default CMAKE_BUILD_TYPE +# the default in CMake is Debug for historic reasons +# +macro(set_default_build_type default_build_type) + set(CMAKE_CONFIGURATION_TYPES "Release;Debug;MinSizeRel;RelWithDebInfo") + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE ${default_build_type} + CACHE STRING + "Choose the build type, e.g. Release, Debug, or RelWithDebInfo." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE + PROPERTY STRINGS ${CMAKE_CONFIGURATION_TYPES}) + endif() +endmacro() + +# Set CXX +# Note: this is a bit legacy and one should use CMake TOOLCHAINS instead. +# +macro(set_cxx_warnings) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # list(APPEND CMAKE_CXX_FLAGS "-fsanitize=address") # address, memory, undefined + # set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + # set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + # set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address") + + # note: might still need a + # export LD_PRELOAD=libclang_rt.asan.so + # or on Debian 9 with Clang 6.0 + # export LD_PRELOAD=/usr/lib/llvm-6.0/lib/clang/6.0.0/lib/linux/libclang_rt.asan-x86_64.so: + # /usr/lib/llvm-6.0/lib/clang/6.0.0/lib/linux/libclang_rt.ubsan_minimal-x86_64.so + # at runtime when used with symbol-hidden code (e.g. pybind11 module) + + #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code") + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code") + elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + # Warning C4503: "decorated name length exceeded, name was truncated" + # Symbols longer than 4096 chars are truncated (and hashed instead) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4503") + # Yes, you should build against the same C++ runtime and with same + # configuration (Debug/Release). MSVC does inconvenient choices for their + # developers, so be it. (Our Windows-users use conda-forge builds, which + # are consistent.) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd4251") + endif () +endmacro() + + +# Take an and expose it as INTERFACE target with +# WarpX::thirdparty:: naming and SYSTEM includes. +# +function(make_third_party_includes_system imported_target propagated_name) + add_library(WarpX::thirdparty::${propagated_name} INTERFACE IMPORTED) + target_link_libraries(WarpX::thirdparty::${propagated_name} INTERFACE ${imported_target}) + get_target_property(ALL_INCLUDES ${imported_target} INCLUDE_DIRECTORIES) + set_target_properties(WarpX::thirdparty::${propagated_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") + target_include_directories(WarpX::thirdparty::${propagated_name} SYSTEM INTERFACE ${ALL_INCLUDES}) +endfunction() + + +# Set a feature-based binary name for the WarpX executable and create a generic +# warpx symlink to it. Only sets options relevant for users (see summary). +# +function(set_warpx_binary_name) + set_target_properties(WarpX PROPERTIES OUTPUT_NAME "warpx") + if(WarpX_DIMS STREQUAL 3) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".3d") + elseif(WarpX_DIMS STREQUAL 2) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".2d") + elseif(WarpX_DIMS STREQUAL RZ) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".RZ") + endif() + + if(WarpX_MPI) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".MPI") + else() + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".NOMPI") + endif() + + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".${WarpX_COMPUTE}") + + if(WarpX_PRECISION STREQUAL "double") + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".DP") + else() + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".SP") + endif() + + if(WarpX_ASCENT) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".ASCENT") + endif() + + if(WarpX_OPENPMD) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".OMP") + endif() + + if(WarpX_PSATD) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".PSATD") + endif() + + if(WarpX_QED) + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".QED") + endif() + + if(CMAKE_BUILD_TYPE MATCHES "Debug") + set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".DEBUG") + endif() + + # alias to the latest build, because using the full name is often confusing + add_custom_command(TARGET WarpX POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + $ + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/warpx + ) +endfunction() + + +# Set an MPI_TEST_EXE variable for test runs which runs num_ranks +# ranks. On some systems, you might need to use the a specific +# mpiexec wrapper, e.g. on Summit (ORNL) pass the hint +# -DMPIEXEC_EXECUTABLE=$(which jsrun) to run ctest. +# +function(configure_mpiexec num_ranks) + # OpenMPI root guard: https://github.com/open-mpi/ompi/issues/4451 + if("$ENV{USER}" STREQUAL "root") + # calling even --help as root will abort and warn on stderr + execute_process(COMMAND ${MPIEXEC_EXECUTABLE} --help + ERROR_VARIABLE MPIEXEC_HELP_TEXT + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(${MPIEXEC_HELP_TEXT} MATCHES "^.*allow-run-as-root.*$") + set(MPI_ALLOW_ROOT --allow-run-as-root) + endif() + endif() + set(MPI_TEST_EXE + ${MPIEXEC_EXECUTABLE} + ${MPI_ALLOW_ROOT} + ${MPIEXEC_NUMPROC_FLAG} ${num_ranks} + PARENT_SCOPE + ) +endfunction() + + +# Prints a summary of WarpX options at the end of the CMake configuration +# +function(warpx_print_summary) + message("") + message("WarpX build configuration:") + message(" Version: ${WarpX_VERSION}") + message(" C++ Compiler: ${CMAKE_CXX_COMPILER_ID} " + "${CMAKE_CXX_COMPILER_VERSION} " + "${CMAKE_CXX_COMPILER_WRAPPER}") + message(" ${CMAKE_CXX_COMPILER}") + message("") + message(" Installation prefix: ${CMAKE_INSTALL_PREFIX}") + message(" bin: ${CMAKE_INSTALL_BINDIR}") + message(" lib: ${CMAKE_INSTALL_LIBDIR}") + message(" include: ${CMAKE_INSTALL_INCLUDEDIR}") + message(" cmake: ${CMAKE_INSTALL_CMAKEDIR}") + if(WarpX_HAVE_PYTHON) + message(" python: ${CMAKE_INSTALL_PYTHONDIR}") + endif() + message("") + message(" Build type: ${CMAKE_BUILD_TYPE}") + #if(BUILD_SHARED_LIBS) + # message(" Library: shared") + #else() + # message(" Library: static") + #endif() + #message(" Testing: ${BUILD_TESTING}") + #message(" Invasive Tests: ${WarpX_USE_INVASIVE_TESTS}") + #message(" Internal VERIFY: ${WarpX_USE_VERIFY}") + message(" Build options:") + message(" ASCENT: ${WarpX_ASCENT}") + message(" COMPUTE: ${WarpX_COMPUTE}") + message(" DIMS: ${WarpX_DIMS}") + message(" MPI: ${WarpX_MPI}") + message(" PSATD: ${WarpX_PSATD}") + message(" PRECISION: ${WarpX_PRECISION}") + message(" OPENPMD: ${WarpX_OPENPMD}") + message(" QED: ${WarpX_QED}") + message("") +endfunction() diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake new file mode 100644 index 00000000000..21290115466 --- /dev/null +++ b/cmake/dependencies/AMReX.cmake @@ -0,0 +1,156 @@ +macro(find_amrex) + if(WarpX_amrex_internal) + message(STATUS "Downloading AMReX ...") + include(FetchContent) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + + # see https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#customization-options + if(WarpX_ASCENT) + set(ENABLE_ASCENT ON CACHE INTERNAL "") + set(ENABLE_CONDUIT ON CACHE INTERNAL "") + endif() + + if("${CMAKE_BUILD_TYPE}" MATCHES "Debug") + set(ENABLE_ASSERTIONS ON CACHE BOOL "") + # note: floating-point exceptions can slow down debug runs a lot + set(ENABLE_FPE ON CACHE BOOL "") + else() + set(ENABLE_ASSERTIONS OFF CACHE BOOL "") + set(ENABLE_FPE OFF CACHE BOOL "") + endif() + + if(WarpX_COMPUTE STREQUAL CUDA) + set(ENABLE_ACC OFF CACHE INTERNAL "") + set(ENABLE_CUDA ON CACHE INTERNAL "") + set(ENABLE_DPCPP OFF CACHE BOOL "") + #set(ENABLE_HIP OFF CACHE BOOL "") + set(ENABLE_OMP OFF CACHE INTERNAL "") + elseif(WarpX_COMPUTE STREQUAL OMP) + set(ENABLE_ACC OFF CACHE INTERNAL "") + set(ENABLE_CUDA OFF CACHE INTERNAL "") + set(ENABLE_DPCPP OFF CACHE BOOL "") + #set(ENABLE_HIP OFF CACHE BOOL "") + set(ENABLE_OMP ON CACHE INTERNAL "") + elseif(WarpX_COMPUTE STREQUAL DPCPP) + set(ENABLE_ACC OFF CACHE INTERNAL "") + set(ENABLE_CUDA OFF CACHE INTERNAL "") + set(ENABLE_DPCPP ON CACHE BOOL "") + #set(ENABLE_HIP OFF CACHE BOOL "") + set(ENABLE_OMP OFF CACHE INTERNAL "") + else() + set(ENABLE_ACC OFF CACHE INTERNAL "") + set(ENABLE_CUDA OFF CACHE INTERNAL "") + set(ENABLE_DPCPP OFF CACHE BOOL "") + #set(ENABLE_HIP OFF CACHE BOOL "") + set(ENABLE_OMP OFF CACHE INTERNAL "") + endif() + + if(WarpX_MPI) + set(ENABLE_MPI ON CACHE INTERNAL "") + else() + set(ENABLE_MPI OFF CACHE INTERNAL "") + endif() + + if(WarpX_PRECISION STREQUAL "double") + set(ENABLE_DP ON CACHE INTERNAL "") + set(ENABLE_DP_PARTICLES ON CACHE INTERNAL "") + else() + set(ENABLE_DP OFF CACHE INTERNAL "") + set(ENABLE_DP_PARTICLES OFF CACHE INTERNAL "") + endif() + + set(ENABLE_FORTRAN OFF CACHE INTERNAL "") + set(ENABLE_FORTRAN_INTERFACES OFF CACHE INTERNAL "") + set(ENABLE_TUTORIALS OFF CACHE INTERNAL "") + set(ENABLE_PARTICLES ON CACHE INTERNAL "") + set(ENABLE_TINY_PROFILE ON CACHE BOOL "") + + # ENABLE_SENSEI_IN_SITU + # we'll need this for Python bindings + #set(ENABLE_PIC ON CACHE INTERNAL "") + + if(WarpX_DIMS STREQUAL RZ) + set(DIM 2 CACHE INTERNAL "") + else() + set(DIM ${WarpX_DIMS} CACHE INTERNAL "") + endif() + + FetchContent_Declare(fetchedamrex + GIT_REPOSITORY ${WarpX_amrex_repo} + GIT_TAG ${WarpX_amrex_branch} + BUILD_IN_SOURCE 0 + ) + FetchContent_GetProperties(fetchedamrex) + + if(NOT fetchedamrex_POPULATED) + FetchContent_Populate(fetchedamrex) + list(APPEND CMAKE_MODULE_PATH "${fetchedamrex_SOURCE_DIR}/Tools/CMake") + if(ENABLE_CUDA) + enable_language(CUDA) + include(AMReX_SetupCUDA) + endif() + add_subdirectory(${fetchedamrex_SOURCE_DIR} ${fetchedamrex_BINARY_DIR}) + endif() + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDAMREX) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDAMREX) + + # AMReX options not relevant to WarpX users + mark_as_advanced(AMREX_BUILD_DATETIME) + mark_as_advanced(DIM) + mark_as_advanced(ENABLE_ACC) + mark_as_advanced(ENABLE_ASSERTIONS) + mark_as_advanced(ENABLE_AMRDATA) + mark_as_advanced(ENABLE_BASE_PROFILE) # mutually exclusive to tiny profile + mark_as_advanced(ENABLE_CONDUIT) + mark_as_advanced(ENABLE_CUDA) + mark_as_advanced(ENABLE_DP) + mark_as_advanced(ENABLE_DP_PARTICLES) + mark_as_advanced(ENABLE_DPCPP) + mark_as_advanced(ENABLE_EB) + mark_as_advanced(ENABLE_FPE) + mark_as_advanced(ENABLE_FORTRAN) + mark_as_advanced(ENABLE_FORTRAN_INTERFACES) + mark_as_advanced(ENABLE_HDF5) # we do HDF5 I/O (and more) via openPMD-api + mark_as_advanced(ENABLE_LINEAR_SOLVERS) + mark_as_advanced(ENABLE_MEM_PROFILE) + mark_as_advanced(ENABLE_MPI) + mark_as_advanced(ENABLE_OMP) + mark_as_advanced(ENABLE_PIC) + mark_as_advanced(ENABLE_SENSEI_INSITU) + mark_as_advanced(ENABLE_TINY_PROFILE) + mark_as_advanced(TP_PROFILE) + mark_as_advanced(USE_XSDK_DEFAULTS) + + message(STATUS "AMReX: Using INTERNAL version '${AMREX_PKG_VERSION}' (${AMREX_GIT_VERSION})") + else() + # https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#importing-amrex-into-your-cmake-project + if(WarpX_ASCENT) + set(COMP_ASCENT ENABLE_ASCENT ENABLE_CONDUIT) + else() + set(COMP_ASCENT) + endif() + if(WarpX_DIMS STREQUAL RZ) + set(COMP_DIM 2D) + else() + set(COMP_DIM ${WarpX_DIMS}D) + endif() + + find_package(AMReX 20.05 CONFIG REQUIRED COMPONENTS ${COMP_ASCENT} ${COMP_DIM} PARTICLES DPARTICLES DP TINYP LSOLVERS FINTERFACES) + message(STATUS "AMReX: Found version '${AMReX_VERSION}'") + endif() +endmacro() + +set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" + CACHE STRING + "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") +set(WarpX_amrex_branch "development" + CACHE STRING + "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") + +find_amrex() diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake new file mode 100644 index 00000000000..cac75d997bf --- /dev/null +++ b/cmake/dependencies/PICSAR.cmake @@ -0,0 +1,55 @@ +function(find_picsar) + if(WarpX_picsar_internal) + message(STATUS "Downloading PICSAR ...") + include(FetchContent) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + + # FIXME no option to control WarpX_QED_TABLE_GEN / Boost trigger yet + + FetchContent_Declare(fetchedpicsar + GIT_REPOSITORY ${WarpX_picsar_repo} + GIT_TAG ${WarpX_picsar_branch} + BUILD_IN_SOURCE 0 + ) + FetchContent_GetProperties(fetchedpicsar) + + if(NOT fetchedpicsar_POPULATED) + FetchContent_Populate(fetchedpicsar) + add_subdirectory(${fetchedpicsar_SOURCE_DIR}/src/multi_physics ${fetchedpicsar_BINARY_DIR}) + endif() + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDPICSAR) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDPICSAR) + + # PICSAR options not relevant to WarpX users + mark_as_advanced(DIM) + mark_as_advanced(USE_XSDK_DEFAULTS) + + message(STATUS "PICSAR: Using INTERNAL version '${PICSAR_VERSION}'") + else() + # not supported by PICSAR + # find_package(PICSAR 20.05 CONFIG REQUIRED QED) + # message(STATUS "PICSAR: Found version '${PICSAR_VERSION}'") + message(FATAL_ERROR "PICSAR: Cannot be used as externally installed library yet.") + endif() +endfunction() + +if(WarpX_QED) + option(WarpX_picsar_internal "Download & build PICSAR" ON) + set(WarpX_picsar_repo "https://github.com/ECP-WarpX/picsar.git" + CACHE STRING + "Repository URI to pull and build PICSAR from if(WarpX_picsar_internal)") + set(WarpX_picsar_branch "development" + CACHE STRING + "Repository branch for WarpX_picsar_repo if(WarpX_picsar_internal)") + + cmake_dependent_option(WarpX_QED_TABLE_GEN "generate QED lookup tables (requires boost)" + ON "WarpX_QED" OFF) + + find_picsar() +endif() diff --git a/cmake/dependencies/openPMD.cmake b/cmake/dependencies/openPMD.cmake new file mode 100644 index 00000000000..562996d9097 --- /dev/null +++ b/cmake/dependencies/openPMD.cmake @@ -0,0 +1,71 @@ +function(find_openpmd) + if(WarpX_openpmd_internal) + message(STATUS "Downloading openPMD-api ...") + include(FetchContent) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + + # see https://openpmd-api.readthedocs.io/en/0.11.1-alpha/dev/buildoptions.html + set(openPMD_USE_MPI ${WarpX_MPI} CACHE INTERNAL "") + set(openPMD_USE_PYTHON OFF CACHE INTERNAL "") + set(BUILD_CLI_TOOLS OFF CACHE INTERNAL "") # FIXME + set(BUILD_EXAMPLES OFF CACHE INTERNAL "") # FIXME + set(BUILD_TESTING OFF CACHE INTERNAL "") # FIXME + set(openPMD_INSTALL ${BUILD_SHARED_LIBS} CACHE INTERNAL "") + + FetchContent_Declare(fetchedopenpmd + GIT_REPOSITORY ${WarpX_openpmd_repo} + GIT_TAG ${WarpX_openpmd_branch} + BUILD_IN_SOURCE 0 + ) + FetchContent_GetProperties(fetchedopenpmd) + + if(NOT fetchedopenpmd_POPULATED) + FetchContent_Populate(fetchedopenpmd) + add_subdirectory(${fetchedopenpmd_SOURCE_DIR} ${fetchedopenpmd_BINARY_DIR}) + endif() + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDOPENPMD) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDOPENPMD) + + # openPMD options not relevant to WarpX users + mark_as_advanced(openPMD_USE_INTERNAL_VARIANT) + mark_as_advanced(openPMD_USE_INTERNAL_CATCH) + mark_as_advanced(openPMD_USE_INTERNAL_PYBIND11) + mark_as_advanced(openPMD_USE_INTERNAL_JSON) + mark_as_advanced(openPMD_HAVE_PKGCONFIG) + mark_as_advanced(openPMD_USE_INVASIVE_TESTS) + mark_as_advanced(openPMD_USE_VERIFY) + mark_as_advanced(ADIOS2_DOR) + mark_as_advanced(ADIOS_CONFIG) + mark_as_advanced(HDF5_DIR) + mark_as_advanced(JSON_MultipleHeaders) + + message(STATUS "openPMD-api: Using INTERNAL version '${WarpX_openpmd_branch}'") + else() + if(WarpX_MPI) + set(COMPONENT_WMPI MPI) + else() + set(COMPONENT_WMPI NOMPI) + endif() + find_package(openPMD 0.11.1 CONFIG REQUIRED COMPONENTS ${COMPONENT_WMPI}) + message(STATUS "openPMD-api: Found version '${openPMD_VERSION}'") + endif() +endfunction() + +if(WarpX_OPENPMD) + option(WarpX_openpmd_internal "Download & build openPMD-api" ON) + set(WarpX_openpmd_repo "https://github.com/openPMD/openPMD-api.git" + CACHE STRING + "Repository URI to pull and build openPMD-api from if(WarpX_openpmd_internal)") + set(WarpX_openpmd_branch "dev" + CACHE STRING + "Repository branch for WarpX_openpmd_repo if(WarpX_openpmd_internal)") + + set(WarpX_HAVE_OPENPMD TRUE) + find_openpmd() +endif() diff --git a/run_test.sh b/run_test.sh index f6fa414778d..546452f8a22 100755 --- a/run_test.sh +++ b/run_test.sh @@ -50,9 +50,9 @@ cd test_dir git clone --branch development https://github.com/AMReX-Codes/amrex.git # Use QED brach for QED tests if [ "${WARPX_CI_QED}" = "TRUE" ]; then - git clone --branch QED https://bitbucket.org/berkeleylab/picsar.git + git clone --branch QED https://github.com/ECP-WarpX/picsar.git else - git clone --branch master https://bitbucket.org/berkeleylab/picsar.git + git clone --branch development https://github.com/ECP-WarpX/picsar.git fi # Clone the AMReX regression test utility @@ -69,8 +69,8 @@ cp -r Checksum ../../regression_testing/ cd ../../regression_testing/ # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then -python regtest.py ../rt-WarpX/travis-tests.ini --no_update all --source_git_hash=${WARPX_TEST_COMMIT} "${tests_run}" + python regtest.py ../rt-WarpX/travis-tests.ini --no_update all --source_git_hash=${WARPX_TEST_COMMIT} "${tests_run}" # run all tests (variables tests_arg and tests_run are empty) else -python regtest.py ../rt-WarpX/travis-tests.ini --no_update all --source_git_hash=${WARPX_TEST_COMMIT} + python regtest.py ../rt-WarpX/travis-tests.ini --no_update all --source_git_hash=${WARPX_TEST_COMMIT} fi