Vay Deposition: Add Cumulative Sums (ECP-WarpX#2958)

* Implement Cumulative Sum

* Implement Subtraction of Average

* Fix Bug, Remove Abort w/ Periodic Single Box

* Enforce Consistency w/ Periodic Single Box

* Cleaning

* Cleaning

* Fix Bugs

* Cleaning

* Compute Cumulative Sums Before Sync

* Always Loop Over Full Boxes

* MFIter without Tiling

* Cleaning

* Do Not Store Cumulative Sums

* Add Two-Particle Test (3D)

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Set 3D Velocity in CI Test

* Precompute Normalization Factors

* Add Two-Particle Test (2D)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Examples/Tests/VayDeposition/analysis.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+# Copyright 2019-2022
+#
+# This file is part of WarpX.
+#
+# License: BSD-3-Clause-LBNL
+
+import os
+import sys
+
+import numpy as np
+from scipy.constants import epsilon_0
+import yt
+
+yt.funcs.mylog.setLevel(50)
+
+sys.path.insert(1, '../../../../warpx/Regression/Checksum/')
+import checksumAPI
+
+# Plotfile data set
+fn = sys.argv[1]
+ds = yt.load(fn)
+
+# Check relative L-infinity spatial norm of rho/epsilon_0 - div(E)
+data = ds.covering_grid(
+    level=0,
+    left_edge=ds.domain_left_edge,
+    dims=ds.domain_dimensions)
+rho  = data[('boxlib','rho')].to_ndarray()
+divE = data[('boxlib','divE')].to_ndarray()
+error_rel = np.amax(np.abs(divE-rho/epsilon_0))/np.amax(np.abs(rho/epsilon_0))
+tolerance = 1e-3
+print("Error on charge conservation:")
+print("error_rel = {}".format(error_rel))
+print("tolerance = {}".format(tolerance))
+assert( error_rel < tolerance )
+
+# Checksum analysis
+test_name = os.path.split(os.getcwd())[1]
+checksumAPI.evaluate_checksum(test_name, fn)

Examples/Tests/VayDeposition/inputs_2d

@@ -0,0 +1,65 @@
+# algo
+algo.charge_deposition = standard
+algo.current_deposition = vay
+algo.maxwell_solver = psatd
+algo.particle_pusher = vay
+algo.particle_shape = 3
+
+# amr
+amr.max_level = 0
+amr.n_cell = 128 128
+
+# boundary
+boundary.field_hi = periodic periodic
+boundary.field_lo = periodic periodic
+
+# diag
+diag1.diag_type = Full
+diag1.intervals = 50
+diag1.fields_to_plot = By Ex Ez jx jz rho divE
+
+# diagnostics
+diagnostics.diags_names = diag1
+
+# electron
+electron.charge = -q_e
+electron.injection_style = "SingleParticle"
+electron.mass = 9e-22
+electron.single_particle_pos = 0. 0. 0.
+electron.single_particle_vel = 0.5 0. 0.5
+electron.single_particle_weight = 1.
+
+# geometry
+geometry.coord_sys = 0
+geometry.dims = 2
+geometry.prob_hi =  45.  45.
+geometry.prob_lo = -45. -45.
+
+# ion
+ion.charge = q_e
+ion.injection_style = "SingleParticle"
+ion.mass = 9e-22
+ion.single_particle_pos = 0. 0. 0.
+ion.single_particle_vel = -0.5 0. -0.5
+ion.single_particle_weight = 1.
+
+# max_step
+max_step = 50
+
+# particles
+particles.species_names = electron ion
+
+# psatd
+psatd.nox = 4
+psatd.noz = 4
+psatd.nx_guard = 20
+psatd.nz_guard = 20
+psatd.periodic_single_box_fft = 0
+psatd.update_with_rho = 0
+
+# warpx
+warpx.cfl = 0.9999
+warpx.do_nodal = 1
+warpx.serialize_initial_conditions = 1
+warpx.use_filter = 1
+warpx.verbose = 1

Examples/Tests/VayDeposition/inputs_3d

@@ -0,0 +1,67 @@
+# algo
+algo.charge_deposition = standard
+algo.current_deposition = vay
+algo.maxwell_solver = psatd
+algo.particle_pusher = vay
+algo.particle_shape = 3
+
+# amr
+amr.max_level = 0
+amr.n_cell = 64 64 64
+
+# boundary
+boundary.field_hi = periodic periodic periodic
+boundary.field_lo = periodic periodic periodic
+
+# diag
+diag1.diag_type = Full
+diag1.intervals = 25
+diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho divE
+
+# diagnostics
+diagnostics.diags_names = diag1
+
+# electron
+electron.charge = -q_e
+electron.injection_style = "SingleParticle"
+electron.mass = 9e-22
+electron.single_particle_pos = 0. 0. 0.
+electron.single_particle_vel = 0.5 0.5 0.5
+electron.single_particle_weight = 1.
+
+# geometry
+geometry.coord_sys = 0
+geometry.dims = 3
+geometry.prob_hi =  45.  45.  45.
+geometry.prob_lo = -45. -45. -45.
+
+# ion
+ion.charge = q_e
+ion.injection_style = "SingleParticle"
+ion.mass = 9e-22
+ion.single_particle_pos = 0. 0. 0.
+ion.single_particle_vel = -0.5 -0.5 -0.5
+ion.single_particle_weight = 1.
+
+# max_step
+max_step = 25
+
+# particles
+particles.species_names = electron ion
+
+# psatd
+psatd.nox = 4
+psatd.noy = 4
+psatd.noz = 4
+psatd.nx_guard = 20
+psatd.ny_guard = 20
+psatd.nz_guard = 20
+psatd.periodic_single_box_fft = 0
+psatd.update_with_rho = 0
+
+# warpx
+warpx.cfl = 0.9999
+warpx.do_nodal = 1
+warpx.serialize_initial_conditions = 1
+warpx.use_filter = 1
+warpx.verbose = 1

Regression/Checksum/benchmarks_json/VayDeposition2D.json

@@ -0,0 +1,31 @@
+{
+  "electron": {
+    "particle_cpu": 0.0,
+    "particle_id": 1.0,
+    "particle_momentum_x": 1.349066061e-13,
+    "particle_momentum_y": 0.0,
+    "particle_momentum_z": 1.349066061e-13,
+    "particle_position_x": 14.351043713724023,
+    "particle_position_y": 14.351043713724023,
+    "particle_weight": 1.0
+  },
+  "ion": {
+    "particle_cpu": 0.0,
+    "particle_id": 2.0,
+    "particle_momentum_x": 1.349066061e-13,
+    "particle_momentum_y": 0.0,
+    "particle_momentum_z": 1.349066061e-13,
+    "particle_position_x": 14.351043713724023,
+    "particle_position_y": 14.351043713724023,
+    "particle_weight": 1.0
+  },
+  "lev=0": {
+    "By": 4.8850423335524976e-15,
+    "Ex": 1.5748554237210878e-06,
+    "Ez": 1.5745640654762762e-06,
+    "divE": 7.364434433257197e-08,
+    "jx": 7.96477185276095e-11,
+    "jz": 7.971109000864354e-11,
+    "rho": 6.481496783075554e-19
+  }
+}

Regression/Checksum/benchmarks_json/VayDeposition3D.json

@@ -0,0 +1,37 @@
+{
+  "electron": {
+    "particle_cpu": 0.0,
+    "particle_id": 1.0,
+    "particle_momentum_x": 1.349066061e-13,
+    "particle_momentum_y": 1.349066061e-13,
+    "particle_momentum_z": 1.349066061e-13,
+    "particle_position_x": 13.286484722880228,
+    "particle_position_y": 13.286484722880228,
+    "particle_position_z": 13.286484722880228,
+    "particle_weight": 1.0
+  },
+  "ion": {
+    "particle_cpu": 0.0,
+    "particle_id": 2.0,
+    "particle_momentum_x": 1.349066061e-13,
+    "particle_momentum_y": 1.349066061e-13,
+    "particle_momentum_z": 1.349066061e-13,
+    "particle_position_x": 13.286484722880228,
+    "particle_position_y": 13.286484722880228,
+    "particle_position_z": 13.286484722880228,
+    "particle_weight": 1.0
+  },
+  "lev=0": {
+    "Bx": 8.481813081143231e-16,
+    "By": 8.481813081143229e-16,
+    "Bz": 8.481813081143229e-16,
+    "Ex": 3.0819459288113856e-07,
+    "Ey": 3.0819459288113856e-07,
+    "Ez": 3.0819459288113856e-07,
+    "divE": 1.317737929174611e-08,
+    "jx": 1.3111027684847054e-11,
+    "jy": 1.311102768484716e-11,
+    "jz": 1.3111027684847218e-11,
+    "rho": 1.1522660947689872e-19
+  }
+}

Regression/WarpX-tests.ini

@@ -3409,3 +3409,39 @@ compileTest = 0
 doVis = 0
 compareParticles = 1
 analysisRoutine = Examples/Tests/ion_stopping/analysis_ion_stopping.py
+
+[VayDeposition2D]
+buildDir = .
+inputFile = Examples/Tests/VayDeposition/inputs_2d
+runtime_params =
+dim = 2
+addToCompileString = USE_PSATD=TRUE
+cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON
+restartTest = 0
+useMPI = 1
+numprocs = 2
+useOMP = 1
+numthreads = 1
+compileTest = 0
+doVis = 0
+compareParticles = 1
+particleTypes = electron ion
+analysisRoutine = Examples/Tests/VayDeposition/analysis.py
+
+[VayDeposition3D]
+buildDir = .
+inputFile = Examples/Tests/VayDeposition/inputs_3d
+runtime_params =
+dim = 3
+addToCompileString = USE_PSATD=TRUE
+cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON
+restartTest = 0
+useMPI = 1
+numprocs = 2
+useOMP = 1
+numthreads = 1
+compileTest = 0
+doVis = 0
+compareParticles = 1
+particleTypes = electron ion
+analysisRoutine = Examples/Tests/VayDeposition/analysis.py