Merge branch 'development' into amrex_parser

ECP-WarpX · Jul 14, 2021 · e3ca37a · e3ca37a
2 parents 8f902b3 + 40e36e1
commit e3ca37a
Show file tree

Hide file tree

Showing 27 changed files with 224 additions and 143 deletions.
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -78,6 +78,7 @@ jobs:
           -DWarpX_openpmd_internal=OFF \
           -DWarpX_PRECISION=SINGLE     \
           -DWarpX_PSATD=ON             \
+          -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build_sp -j 2
 

diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst
@@ -1750,9 +1750,12 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
     divisor of ``blocking_factor``. If ``warpx.numprocs`` is used instead, the total number of cells in a given
     dimension must be a multiple of the ``coarsening_ratio`` multiplied by ``numprocs`` in that dimension.
 
-* ``<diag_name>.file_prefix`` (`string`) optional (default `diags/plotfiles/plt`)
+* ``<diag_name>.file_prefix`` (`string`) optional (default `diags/<diag_name>`)
     Root for output file names. Supports sub-directories.
 
+* ``<diag_name>.file_min_digits`` (`int`) optional (default `5`)
+    The minimum number of digits used for the iteration number appended to the diagnostic file names.
+
 * ``<diag_name>.diag_lo`` (list `float`, 1 per dimension) optional (default `-infinity -infinity -infinity`)
     Lower corner of the output fields (if smaller than ``warpx.dom_lo``, then set to ``warpx.dom_lo``). Currently, when the ``diag_lo`` is different from ``warpx.dom_lo``, particle output is disabled.
 

diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp
@@ -611,7 +611,7 @@ BTDiagnostics::Flush (int i_buffer)
     double const labtime = m_t_lab[i_buffer];
     m_flush_format->WriteToFile(
         m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
-        labtime, m_output_species, nlev_output, file_name,
+        labtime, m_output_species, nlev_output, file_name, m_file_min_digits,
         m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F,
         isBTD, i_buffer, m_geom_snapshot[i_buffer][0], isLastBTDFlush);
 

diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H
@@ -112,6 +112,8 @@ protected:
     std::string m_diag_name;
     /** Prefix for output directories */
     std::string m_file_prefix;
+    /** Minimum number of digits to iteration number in file name */
+    int m_file_min_digits = 5;
     /** Index of diagnostics in MultiDiagnostics::alldiags */
     int m_diag_index;
     /** Names of  each component requested by the user.

diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp
@@ -48,6 +48,7 @@ Diagnostics::BaseReadParameters ()
     amrex::ParmParse pp_diag_name(m_diag_name);
     m_file_prefix = "diags/" + m_diag_name;
     pp_diag_name.query("file_prefix", m_file_prefix);
+    pp_diag_name.query("file_min_digits", m_file_min_digits);
     pp_diag_name.query("format", m_format);
     pp_diag_name.query("dump_last_timestep", m_dump_last_timestep);
 

diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H
@@ -16,7 +16,7 @@ public:
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
-        const std::string prefix,
+        const std::string prefix, const int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
         bool plot_raw_rho, bool plot_raw_F,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H
@@ -33,7 +33,8 @@ public:
         const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
-        const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
+        const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
+        const std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
         bool plot_raw_rho, bool plot_raw_F,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp
@@ -12,7 +12,7 @@ FlushFormatAscent::WriteToFile (
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
-    const std::string prefix, bool plot_raw_fields,
+    const std::string prefix, int /*file_min_digits*/, bool plot_raw_fields,
     bool plot_raw_fields_guards, bool /*plot_raw_rho*/, bool plot_raw_F,
     bool /*isBTD*/, int /*snapshotID*/, const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const
 {

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H
@@ -20,7 +20,8 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile
         const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
-        const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
+        const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
+        const std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
         bool plot_raw_rho, bool plot_raw_F,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp
@@ -27,7 +27,8 @@ FlushFormatCheckpoint::WriteToFile (
         const amrex::Vector<amrex::MultiFab>& /*mf*/,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double /*time*/,
-        const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
+        const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
+        const std::string prefix, int file_min_digits,
         bool /*plot_raw_fields*/,
         bool /*plot_raw_fields_guards*/,
         bool /*plot_raw_rho*/, bool /*plot_raw_F*/,
@@ -42,7 +43,7 @@ FlushFormatCheckpoint::WriteToFile (
     VisMF::Header::Version current_version = VisMF::GetHeaderVersion();
     VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1);
 
-    const std::string& checkpointname = amrex::Concatenate(prefix,iteration[0]);
+    const std::string& checkpointname = amrex::Concatenate(prefix, iteration[0], file_min_digits);
 
     amrex::Print() << "  Writing checkpoint " << checkpointname << "\n";
 

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H
@@ -32,7 +32,8 @@ public:
         const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
-        const amrex::Vector<ParticleDiag>& particle_diags, int /*nlev*/, const std::string prefix,
+        const amrex::Vector<ParticleDiag>& particle_diags, int /*nlev*/,
+        const std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
         bool plot_raw_rho, bool plot_raw_F,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
@@ -93,7 +93,7 @@ FlushFormatOpenPMD::WriteToFile (
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int /*nlev*/,
-    const std::string prefix, bool plot_raw_fields,
+    const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F,
     bool isBTD, int snapshotID, const amrex::Geometry& full_BTD_snapshot,
     bool isLastBTDFlush) const
@@ -112,7 +112,7 @@ FlushFormatOpenPMD::WriteToFile (
         output_iteration = snapshotID;
 
     // Set step and output directory name.
-    m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, isBTD);
+    m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, file_min_digits, isBTD);
 
     // fields: only dumped for coarse level
     m_OpenPMDPlotWriter->WriteOpenPMDFieldsAll(

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H
@@ -27,7 +27,8 @@ public:
         const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
-        const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
+        const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
+        const std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
         bool plot_raw_rho, bool plot_raw_F,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
@@ -56,14 +56,14 @@ FlushFormatPlotfile::WriteToFile (
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
-    const std::string prefix, bool plot_raw_fields,
+    const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F,
     bool /*isBTD*/, int /*snapshotID*/, const amrex::Geometry& /*full_BTD_snapshot*/,
     bool /*isLastBTDFlush*/) const
 {
     WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()");
     auto & warpx = WarpX::GetInstance();
-    const std::string& filename = amrex::Concatenate(prefix, iteration[0]);
+    const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits);
     amrex::Print() << "  Writing plotfile " << filename << "\n";
 
     Vector<std::string> rfs;

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H
@@ -50,7 +50,8 @@ public:
         const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
-        const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
+        const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
+        const std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
         bool plot_raw_rho, bool plot_raw_F,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp
@@ -53,7 +53,7 @@ FlushFormatSensei::WriteToFile (
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
-    const std::string prefix, bool plot_raw_fields,
+    const std::string prefix, int /*file_min_digits*/, bool plot_raw_fields,
     bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F,
     bool /*isBTD*/, int /*snapshotID*/,
     const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const

diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp
@@ -118,7 +118,7 @@ FullDiagnostics::Flush ( int i_buffer )
 
     m_flush_format->WriteToFile(
         m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
-        warpx.gett_new(0), m_output_species, nlev_output, m_file_prefix,
+        warpx.gett_new(0), m_output_species, nlev_output, m_file_prefix, m_file_min_digits,
         m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F);
 
     FlushRaw();

diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H
@@ -118,7 +118,7 @@ public:
    * @note If an iteration has been written, then it will give a warning
    *
    */
-  void SetStep (int ts, const std::string& dirPrefix,
+  void SetStep (int ts, const std::string& dirPrefix, int file_min_digits,
                 bool isBTD=false);
 
   /** Close the step
@@ -263,6 +263,10 @@ private:
    */
   std::string m_dirPrefix;
 
+  /** This is the minimum number of digits in the step number that is used as the
+   * suffix for file names when doing file based encoding */
+  int m_file_min_digits;
+
   int m_MPIRank = 0;
   int m_MPISize = 1;
 

diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp
@@ -301,19 +301,22 @@ WarpXOpenPMDPlot::GetFileName (std::string& filepath)
   //
   // OpenPMD supports timestepped names
   //
-  if (m_Encoding == openPMD::IterationEncoding::fileBased)
-      filename = filename.append("_%06T");
+  if (m_Encoding == openPMD::IterationEncoding::fileBased) {
+      std::string fileSuffix = std::string("_%0") + std::to_string(m_file_min_digits) + std::string("T");
+      filename = filename.append(fileSuffix);
+  }
   filename.append(".").append(m_OpenPMDFileType);
   filepath.append(filename);
   return filename;
 }
 
-void WarpXOpenPMDPlot::SetStep (int ts, const std::string& dirPrefix,
+void WarpXOpenPMDPlot::SetStep (int ts, const std::string& dirPrefix, int file_min_digits,
                                 bool isBTD)
 {
     AMREX_ALWAYS_ASSERT_WITH_MESSAGE(ts >= 0 , "openPMD iterations are unsigned");
 
     m_dirPrefix = dirPrefix;
+    m_file_min_digits = file_min_digits;
 
     if( ! isBTD ) {
         if (m_CurrentStep >= ts) {

diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp
@@ -145,6 +145,10 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
         m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm,
                                                        {1,1,1}, // Not clear how many ghost cells we need yet
                                                        amrex::EBSupport::full);
+        ComputeEdgeLengths();
+        ComputeFaceAreas();
+        ScaleEdges();
+        ScaleAreas();
 #else
         m_field_factory[lev] = std::make_unique<FArrayBoxFactory>();
 #endif

diff --git a/Source/Particles/Collision/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCCCollision.H
@@ -12,20 +12,21 @@
 #include "MCCProcess.H"
 
 #include <AMReX_REAL.H>
+#include <AMReX_Vector.H>
 #include <AMReX_GpuContainers.H>
 
+#include <memory>
 #include <string>
 
-class BackgroundMCCCollision
+class BackgroundMCCCollision final
     : public CollisionBase
 {
 public:
-    template <typename T>
-    using VectorType = amrex::Gpu::ManagedVector<T>;
-
     BackgroundMCCCollision (std::string collision_name);
 
-    amrex::Real get_nu_max ( VectorType<MCCProcess*> const& mcc_processes );
+    virtual ~BackgroundMCCCollision () = default;
+
+    amrex::Real get_nu_max (amrex::Vector<MCCProcess> const& mcc_processes);
 
     /** Perform the collisions
      *
@@ -57,8 +58,10 @@ public:
 
 private:
 
-    VectorType< MCCProcess* > m_scattering_processes;
-    VectorType< MCCProcess* > m_ionization_processes;
+    amrex::Vector<MCCProcess> m_scattering_processes;
+    amrex::Vector<MCCProcess> m_ionization_processes;
+    amrex::Gpu::DeviceVector<MCCProcess::Executor> m_scattering_processes_exe;
+    amrex::Gpu::DeviceVector<MCCProcess::Executor> m_ionization_processes_exe;
 
     bool init_flag = false;
     bool ionization_flag = false;