Diagnostics: print when writing openPMD/Ascent/Sensei and BTD plotfil…

…e/openPMD (ECP-WarpX#3424) * print when writing openPMD, Ascent, Sensei * remove unnecesssary variables, include min_digits * print snapshot id when writing btd * fix ascent print * compile without warnings * still working on Ascent * still working Ascent * Print when BTD buffer is flushed * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * print when writing openPMD, Ascent, Sensei * remove unnecesssary variables, include min_digits * print snapshot id when writing btd * fix ascent print * compile without warnings * still working on Ascent * still working Ascent * Print when BTD buffer is flushed * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add warning message if BTD not full * adjust for CI, no in-situ vis BTD * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * revert to correct plotfile naming * change warning topic to BTD * nicer format for print statements * extend examples to fill BTD * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Reset Checksums of BTD_ReducedSliceDiag * Upgrade CI Test LaserAccelerationBoost * Apply suggestions from code review Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> * improve BTD warning notes * edit final step notes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix formatting Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edoardo Zoni <ezoni@lbl.gov> Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
dpgrote · Nov 22, 2022 · e6365a5 · e6365a5
1 parent f10e2bb
commit e6365a5
Show file tree

Hide file tree

Showing 23 changed files with 174 additions and 64 deletions.
diff --git a/Examples/Modules/RigidInjection/inputs_2d_BoostedFrame b/Examples/Modules/RigidInjection/inputs_2d_BoostedFrame
@@ -1,6 +1,6 @@
 # stop_time = 1.5e-13
 
-warpx.zmax_plasma_to_compute_max_step = 50.e-6
+warpx.zmax_plasma_to_compute_max_step = 56.e-6
 warpx.gamma_boost = 5.
 warpx.boost_direction = z
 

diff --git a/Examples/Modules/boosted_diags/inputs_3d_slice b/Examples/Modules/boosted_diags/inputs_3d_slice
@@ -1,4 +1,4 @@
-warpx.zmax_plasma_to_compute_max_step = 0.0031
+warpx.zmax_plasma_to_compute_max_step = 0.0055
 
 amr.n_cell =  32 32 64
 amr.max_grid_size = 64
@@ -126,7 +126,7 @@ diag1.write_species = 1
 
 diag2.diag_type = BackTransformed
 diag2.do_back_transformed_fields = 1
-diag2.intervals = 0:4:2, 1:3:2
+diag2.intervals = 0:3:2, 1:3:2
 diag2.dz_snapshots_lab = 0.001
 diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho
 diag2.format = openpmd

diff --git a/Examples/Physics_applications/laser_acceleration/inputs_2d_boost b/Examples/Physics_applications/laser_acceleration/inputs_2d_boost
@@ -44,9 +44,6 @@ algo.particle_shape = 3
 #################################
 warpx.gamma_boost = 10.
 warpx.boost_direction = z
-warpx.do_back_transformed_diagnostics = 1
-warpx.num_snapshots_lab = 7
-warpx.dt_snapshots_lab = 1.6678204759907604e-12
 
 #################################
 ############ PLASMA #############
@@ -123,6 +120,12 @@ laser1.wavelength = 0.81e-6         # The wavelength of the laser (in meters)
 
 # Diagnostics
 diagnostics.diags_names = diag1
-diag1.intervals = 100
-diag1.diag_type = Full
+
+diag1.diag_type = BackTransformed
+diag1.do_back_transformed_fields = 1
+diag1.num_snapshots_lab = 3
+diag1.dt_snapshots_lab = 1.6678204759907604e-12
 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho
+diag1.format = plotfile
+diag1.buffer_size = 32
+diag1.write_species = 1
diff --git a/Regression/Checksum/benchmarks_json/BTD_ReducedSliceDiag.json b/Regression/Checksum/benchmarks_json/BTD_ReducedSliceDiag.json
@@ -9,9 +9,9 @@
     "particle_weight": 62415.090744607616
   },
   "electrons": {
-    "particle_momentum_x": 9.678379481090198e-20,
-    "particle_momentum_y": 2.5764224165967887e-19,
-    "particle_momentum_z": 3.0774186320322946e-19,
+    "particle_momentum_x": 8.238176902434188e-20,
+    "particle_momentum_y": 2.362174160683283e-19,
+    "particle_momentum_z": 2.776294933800578e-19,
     "particle_position_x": 0.0025853613792120563,
     "particle_position_y": 0.0037728464704368673,
     "particle_position_z": 0.1901073014926564,
@@ -29,4 +29,4 @@
     "jz": 2.3459512897800802e+19,
     "rho": 74967092858.04996
   }
-}
+}
diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json
@@ -1,38 +1,38 @@
 {
   "beam": {
-    "particle_momentum_x": 4.40462890925364e-19,
-    "particle_momentum_y": 4.375137239523109e-19,
-    "particle_momentum_z": 2.6808820042118736e-18,
-    "particle_position_x": 0.0008126719835048633,
-    "particle_position_y": 0.3596147573048867,
+    "particle_momentum_x": 3.535284585563231e-19,
+    "particle_momentum_y": 4.403094613346061e-19,
+    "particle_momentum_z": 5.658013779496569e-17,
+    "particle_position_x": 0.008317876520240174,
+    "particle_position_y": 1.1704335094514386,
     "particle_weight": 62415090744.60765
   },
   "electrons": {
-    "particle_momentum_x": 6.2983486271535245e-27,
-    "particle_momentum_y": 2.561148643336908e-22,
-    "particle_momentum_z": 3.2606826949441258e-18,
-    "particle_position_x": 0.07200000000522772,
-    "particle_position_y": 0.040410946815682504,
-    "particle_weight": 7.205670325760674e+16
+    "particle_momentum_x": 2.2135959939611405e-23,
+    "particle_momentum_y": 2.822519730011994e-22,
+    "particle_momentum_z": 5.260625039372931e-22,
+    "particle_position_x": 0.010800577787577741,
+    "particle_position_y": 0.21115060628258137,
+    "particle_weight": 4.121554826246186e+16
   },
   "ions": {
-    "particle_momentum_x": 7.729986759772206e-30,
-    "particle_momentum_y": 2.561148659606118e-22,
-    "particle_momentum_z": 5.987111293060833e-15,
-    "particle_position_x": 0.072,
-    "particle_position_y": 0.04041094673206954,
-    "particle_weight": 7.205670325760674e+16
+    "particle_momentum_x": 6.248472008953412e-23,
+    "particle_momentum_y": 4.449200926395666e-22,
+    "particle_momentum_z": 5.768167708374496e-22,
+    "particle_position_x": 0.010800001678510793,
+    "particle_position_y": 0.21114947608115497,
+    "particle_weight": 4.121554826246186e+16
   },
   "lev=0": {
-    "Bx": 7184.641132345461,
-    "By": 16.4296392148614,
-    "Bz": 425.92136410328976,
-    "Ex": 4509329694.688026,
-    "Ey": 1984276461005.9053,
-    "Ez": 6310104620.90831,
-    "jx": 4752150599.756195,
-    "jy": 553920892529501.1,
-    "jz": 450362837764.43524,
-    "rho": 1569.6222852017581
+    "Bx": 4818965.7728866,
+    "By": 1716.1766615204324,
+    "Bz": 14516.299533155969,
+    "Ex": 2355477198959.7603,
+    "Ey": 1446115193004757.5,
+    "Ez": 21864073851126.54,
+    "jx": 1995525322686390.0,
+    "jy": 5.31264244145399e+16,
+    "jz": 2.0300854881306136e+16,
+    "rho": 67807419.3491986
   }
 }
diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini
@@ -2156,7 +2156,7 @@ analysisRoutine = Examples/analysis_default_regression.py
 [LaserAccelerationBoost]
 buildDir = .
 inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d_boost
-runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=64 512 max_step=20
+runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=64 512 max_step=300
 dim = 2
 addToCompileString =
 cmakeSetupOpts = -DWarpX_DIMS=2

diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp
@@ -111,7 +111,6 @@ void
 BTDPlotfileHeaderImpl::WriteHeader ()
 {
     if ( amrex::FileExists(m_Header_path) ) {
-        amrex::Print() << Utils::TextMsg::Info(" removing this file : " + m_Header_path);
         amrex::FileSystem::Remove(m_Header_path);
     }
     std::ofstream HeaderFile;
@@ -262,7 +261,6 @@ void
 BTDMultiFabHeaderImpl::WriteMultiFabHeader ()
 {
     if ( amrex::FileExists(m_Header_path) ) {
-        amrex::Print() << Utils::TextMsg::Info(" removing this file : " + m_Header_path);
         amrex::FileSystem::Remove(m_Header_path);
     }
     std::ofstream FabHeaderFile;

diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp
@@ -22,6 +22,8 @@
 #include "WarpX.H"
 
 #include <ablastr/utils/Communication.H>
+#include <ablastr/utils/SignalHandling.H>
+#include <ablastr/warn_manager/WarnManager.H>
 
 #include <AMReX.H>
 #include <AMReX_Algorithm.H>
@@ -123,6 +125,48 @@ void BTDiagnostics::DerivedInitData ()
     m_particles_buffer.resize(m_num_buffers);
     m_totalParticles_flushed_already.resize(m_num_buffers);
     m_totalParticles_in_buffer.resize(m_num_buffers);
+
+    // check that simulation can fill all BTD snapshots
+    const int lev = 0;
+    const amrex::Real dt_boosted_frame = warpx.getdt(lev);
+    const int moving_dir = warpx.moving_window_dir;
+    const amrex::Real Lz_lab = warpx.Geom(lev).ProbLength(moving_dir) / warpx.gamma_boost / (1._rt+warpx.beta_boost);
+    const int ref_ratio = 1;
+    const amrex::Real dz_snapshot_grid = dz_lab(dt_boosted_frame, ref_ratio);
+    // Need enough buffers so the snapshot length is longer than the lab frame length
+    // num_buffers * m_buffer_size * dz_snapshot_grid >= Lz
+    const int num_buffers = ceil(Lz_lab / m_buffer_size / dz_snapshot_grid);
+    const int final_snapshot_iteration = m_intervals.GetFinalIteration();
+
+    // the final snapshot starts filling when the
+    // right edge of the moving window intersects the final snapshot
+    // time of final snapshot : t_sn = t0 + i*dt_snapshot
+    // where t0 is the time of first BTD snapshot, t0 = zmax / c  * beta / (1-beta)
+    //
+    // the right edge of the moving window at the time of the final snapshot
+    // has space time coordinates
+    // time t_intersect = t_sn, position  z_intersect=zmax + c*t_sn
+    // the boosted time of this space time pair is
+    // t_intersect_boost = gamma * (t_intersect - beta * z_intersect_boost/c)
+    //                   = gamma * (t_sn * (1 - beta) - beta * zmax / c)
+    //                   = gamma * (zmax*beta/c + i*dt_snapshot*(1-beta) - beta*zmax/c)
+    //                   = gamma * i * dt_snapshot * (1-beta)
+    //                   = i * dt_snapshot / gamma / (1+beta)
+    //
+    // if j = final snapshot starting step, then we want to solve
+    // j dt_boosted_frame >= t_intersect_boost = i * dt_snapshot / gamma / (1+beta)
+    // j >= i / gamma / (1+beta) * dt_snapshot / dt_boosted_frame
+    const int final_snapshot_starting_step = ceil(final_snapshot_iteration / warpx.gamma_boost / (1._rt+warpx.beta_boost) * m_dt_snapshots_lab / dt_boosted_frame);
+    const int final_snapshot_fill_iteration = final_snapshot_starting_step + num_buffers * m_buffer_size - 1;
+    if (final_snapshot_fill_iteration > warpx.maxStep()) {
+        std::string warn_string =
+            "\nSimulation might not run long enough to fill all BTD snapshots.\n"
+            "Final step: " + std::to_string(warpx.maxStep()) + "\n"
+            "Last BTD snapshot fills around step: " + std::to_string(final_snapshot_fill_iteration);
+        ablastr::warn_manager::WMRecordWarning(
+            "BTD", warn_string,
+            ablastr::warn_manager::WarnPriority::low);
+    }
 }
 
 void
@@ -205,7 +249,6 @@ BTDiagnostics::ReadParameters ()
     bool particle_fields_to_plot_specified = pp_diag_name.queryarr("particle_fields_to_plot", m_pfield_varnames);
     WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!particle_fields_to_plot_specified, "particle_fields_to_plot is currently not supported for BackTransformed Diagnostics");
 
-
 }
 
 bool
@@ -810,7 +853,8 @@ BTDiagnostics::Flush (int i_buffer)
         m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
         labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits,
         m_plot_raw_fields, m_plot_raw_fields_guards,
-        use_pinned_pc, isBTD, i_buffer, m_geom_snapshot[i_buffer][0], isLastBTDFlush,
+        use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter[i_buffer],
+        m_max_buffer_multifabs[i_buffer], m_geom_snapshot[i_buffer][0], isLastBTDFlush,
         m_totalParticles_flushed_already[i_buffer]);
 
     for (int isp = 0; isp < m_particles_buffer.at(i_buffer).size(); ++isp) {

diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp
@@ -146,6 +146,8 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer)
     // This is not a backtransform diagnostics
     bool const isBTD = false;
     bool const isLastBTD = false;
+    int const bufferID = 0;
+    int const numBTDBuffers = 0;
     // The data being written out is saved in a pinned particle container
     bool const use_pinned_pc = true;
     const amrex::Geometry& geom = warpx.Geom(0); // For compatibility with `WriteToFile` ; not used
@@ -156,7 +158,8 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer)
     m_flush_format->WriteToFile(
         m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
         warpx.gett_new(0), m_output_species[i_buffer], nlev_output, file_prefix,
-        m_file_min_digits, false, false, use_pinned_pc, isBTD, warpx.getistep(0), geom,
+        m_file_min_digits, false, false, use_pinned_pc, isBTD,
+        warpx.getistep(0), bufferID, numBTDBuffers, geom,
         isLastBTD, m_totalParticles_flushed_already[i_buffer]);
 
     // Now that the data has been written out, clear out the buffer

diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H
@@ -22,6 +22,7 @@ public:
         bool plot_raw_fields_guards,
         const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
+        int bufferID = 1, int numBuffers = 1,
         const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const = 0;

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H
@@ -39,6 +39,7 @@ public:
         bool plot_raw_fields_guards,
         const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
+        int bufferID = 1, int numBuffers = 1,
         const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp
@@ -16,16 +16,24 @@ FlushFormatAscent::WriteToFile (
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
-    const std::string prefix, int /*file_min_digits*/, bool plot_raw_fields,
+    const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards,
     const bool /*use_pinned_pc*/,
-    bool /*isBTD*/, int /*snapshotID*/, const amrex::Geometry& /*full_BTD_snapshot*/,
+    bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/,
+    const amrex::Geometry& /*full_BTD_snapshot*/,
     bool /*isLastBTDFlush*/, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
 {
 #ifdef AMREX_USE_ASCENT
     WARPX_PROFILE("FlushFormatAscent::WriteToFile()");
     auto & warpx = WarpX::GetInstance();
 
+    WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
+        !isBTD,
+        "In-situ visualization is not currently supported for back-transformed diagnostics.");
+
+    const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits);
+    amrex::Print() << Utils::TextMsg::Info("Writing Ascent file " + filename);
+
     // wrap mesh data
     WARPX_PROFILE_VAR("FlushFormatAscent::WriteToFile::MultiLevelToBlueprint", prof_ascent_mesh_blueprint);
     conduit::Node bp_mesh;
@@ -59,7 +67,7 @@ FlushFormatAscent::WriteToFile (
 
 #else
     amrex::ignore_unused(varnames, mf, geom, iteration, time,
-        particle_diags, nlev);
+        particle_diags, nlev, file_min_digits, isBTD);
 #endif // AMREX_USE_ASCENT
     amrex::ignore_unused(prefix, plot_raw_fields, plot_raw_fields_guards);
 }

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H
@@ -26,6 +26,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile
         bool plot_raw_fields_guards,
         const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
+        int bufferID = 1, int numBuffers = 1,
         const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override final;

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp
@@ -37,6 +37,7 @@ FlushFormatCheckpoint::WriteToFile (
         bool /*plot_raw_fields_guards*/,
         const bool /*use_pinned_pc*/,
         bool /*isBTD*/, int /*snapshotID*/,
+        int /*bufferID*/, int /*numBuffers*/,
         const amrex::Geometry& /*full_BTD_snapshot*/,
         bool /*isLastBTDFlush*/, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
 {

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H
@@ -38,6 +38,7 @@ public:
         bool plot_raw_fields_guards,
         const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
+        int bufferID = 1, int numBuffers = 1,
         const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
@@ -114,10 +114,24 @@ FlushFormatOpenPMD::WriteToFile (
     const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards,
     const bool use_pinned_pc,
-    bool isBTD, int snapshotID, const amrex::Geometry& full_BTD_snapshot,
+    bool isBTD, int snapshotID, int bufferID, int numBuffers,
+    const amrex::Geometry& full_BTD_snapshot,
     bool isLastBTDFlush, const amrex::Vector<int>& totalParticlesFlushedAlready) const
 {
     WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()");
+    const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits);
+    if (!isBTD)
+    {
+      amrex::Print() << Utils::TextMsg::Info("Writing openPMD file " + filename);
+    } else
+    {
+      amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers)
+                         + " to snapshot " + std::to_string(snapshotID) +  " to openPMD BTD " + prefix);
+      if (isLastBTDFlush)
+      {
+        amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in openPMD BTD " + prefix);
+      }
+    }
 
     WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
         !plot_raw_fields && !plot_raw_fields_guards,

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H
@@ -33,6 +33,7 @@ public:
         bool plot_raw_fields_guards,
         const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
+        int bufferID = 1, int numBuffers = 1,
         const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;

diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
@@ -62,13 +62,25 @@ FlushFormatPlotfile::WriteToFile (
     const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards,
     const bool /*use_pinned_pc*/,
-    bool isBTD, int /*snapshotID*/, const amrex::Geometry& /*full_BTD_snapshot*/,
-    bool /*isLastBTDFlush*/, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
+    bool isBTD, int snapshotID,  int bufferID, int numBuffers,
+    const amrex::Geometry& /*full_BTD_snapshot*/,
+    bool isLastBTDFlush, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
 {
     WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()");
     auto & warpx = WarpX::GetInstance();
     const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits);
-    amrex::Print() << Utils::TextMsg::Info("Writing plotfile " + filename);
+    if (!isBTD)
+    {
+      amrex::Print() << Utils::TextMsg::Info("Writing plotfile " + filename);
+    } else
+    {
+      amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers)
+                        + " to snapshot " + std::to_string(snapshotID) +  " in plotfile BTD " + prefix );
+      if (isLastBTDFlush)
+      {
+        amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in plotfile BTD " + filename);
+      }
+    }
 
     Vector<std::string> rfs;
     VisMF::Header::Version current_version = VisMF::GetHeaderVersion();