UserWorkBeforeRestartOutput

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR1062]](https://github.com/parthenon-hpc-lab/parthenon/pull/1062) UserWorkBeforeRestartOutput #1062
 - [[PR 1043]](https://github.com/parthenon-hpc-lab/parthenon/pull/1043) Unify flux correction with boundary communication, make fluxes variables, allow fluxes for non-cell fields
 - [[PR 1060]](https://github.com/parthenon-hpc-lab/parthenon/pull/1060) Add the ability to request new MeshData/MeshBlockData objects by selecting variables by UID.
 - [[PR1039]](https://github.com/parthenon-hpc-lab/parthenon/pull/1039) Add ability to output custom coordinate positions for Visit/Paraview

doc/sphinx/src/README.rst

@@ -184,6 +184,7 @@ Mesh
 -  ``PostStepUserWorkInLoop``
 -  ``UserWorkAfterLoop``
 -  ``UserMeshWorkBeforeOutput``
+-  ``UserMeshWorkBeforeRestartOutput``
 
 MeshBlock
 ^^^^^^^^^
@@ -193,6 +194,7 @@ MeshBlock
 -  ``ProblemGenerator``
 -  ``PostInitialization``
 -  ``UserWorkBeforeOutput``
+-  ``UserWorkBeforeRestartOutput``
 
 To redefine these functions, the user sets the respective function
 pointers in the ApplicationInput member app_input of the

doc/sphinx/src/outputs.rst

@@ -143,6 +143,10 @@ selected as all the variables that have either the ``Independent`` or
 ``Restart`` ``Metadata`` flags specified. No other intervention is
 required by the developer.
 
+Additional user work at the per-mesh and per-meshblock levels can be performed
+immediately prior to restart files being written with the optional
+``UserMeshWorkBeforeRestart`` and ``UserWorkBeforeRestartOutput`` callbacks.
+
 Postprocessing/native analysis
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

src/CMakeLists.txt

@@ -185,6 +185,7 @@ add_library(parthenon
   outputs/output_utils.hpp
   outputs/outputs.cpp
   outputs/outputs.hpp
+  outputs/output_parameters.hpp
   outputs/parthenon_hdf5.cpp
   outputs/parthenon_hdf5_attributes.cpp
   outputs/parthenon_hdf5_attributes_read.cpp

src/application_input.hpp

@@ -22,6 +22,7 @@
 #include "bvals/boundary_conditions.hpp"
 #include "defs.hpp"
 #include "interface/state_descriptor.hpp"
+#include "outputs/output_parameters.hpp"
 #include "parameter_input.hpp"
 #include "parthenon_arrays.hpp"
 
@@ -30,41 +31,40 @@ namespace parthenon {
 struct ApplicationInput {
  public:
   // ParthenonManager functions
-  std::function<Packages_t(std::unique_ptr<ParameterInput> &)> ProcessPackages = nullptr;
+  std::function<Packages_t(std::unique_ptr<ParameterInput> &)> ProcessPackages;
 
   // Mesh functions
-  std::function<void(Mesh *, ParameterInput *)> InitUserMeshData = nullptr;
-  std::function<void(Mesh *, ParameterInput *, MeshData<Real> *)> MeshProblemGenerator =
-      nullptr;
-  std::function<void(Mesh *, ParameterInput *, MeshData<Real> *)> MeshPostInitialization =
-      nullptr;
+  std::function<void(Mesh *, ParameterInput *)> InitUserMeshData;
+  std::function<void(Mesh *, ParameterInput *, MeshData<Real> *)> MeshProblemGenerator;
+  std::function<void(Mesh *, ParameterInput *, MeshData<Real> *)> MeshPostInitialization;
 
-  std::function<void(Mesh *, ParameterInput *, SimTime &)> PreStepMeshUserWorkInLoop =
-      nullptr;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> PreStepMeshUserWorkInLoop;
   std::function<void(Mesh *, ParameterInput *, SimTime const &)>
-      PostStepMeshUserWorkInLoop = nullptr;
+      PostStepMeshUserWorkInLoop;
 
-  std::function<void(Mesh *, ParameterInput *, SimTime const &)>
-      UserMeshWorkBeforeOutput = nullptr;
+  std::function<void(Mesh *, ParameterInput *, SimTime const &)> UserMeshWorkBeforeOutput;
+  std::function<void(Mesh *, ParameterInput *, SimTime const &, OutputParameters *)>
+      UserMeshWorkBeforeRestartOutput;
 
+  std::function<void(Mesh *, ParameterInput *, SimTime const &)> PreStepDiagnosticsInLoop;
   std::function<void(Mesh *, ParameterInput *, SimTime const &)>
-      PreStepDiagnosticsInLoop = nullptr;
-  std::function<void(Mesh *, ParameterInput *, SimTime const &)>
-      PostStepDiagnosticsInLoop = nullptr;
+      PostStepDiagnosticsInLoop;
 
-  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkAfterLoop = nullptr;
-  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoop = nullptr;
-  BValFunc boundary_conditions[BOUNDARY_NFACES] = {nullptr};
-  SBValFunc swarm_boundary_conditions[BOUNDARY_NFACES] = {nullptr};
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkAfterLoop;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoop;
+  BValFunc boundary_conditions[BOUNDARY_NFACES];
+  SBValFunc swarm_boundary_conditions[BOUNDARY_NFACES];
 
   // MeshBlock functions
   std::function<std::unique_ptr<MeshBlockApplicationData>(MeshBlock *, ParameterInput *)>
-      InitApplicationMeshBlockData = nullptr;
-  std::function<void(MeshBlock *, ParameterInput *)> InitMeshBlockUserData = nullptr;
-  std::function<void(MeshBlock *, ParameterInput *)> ProblemGenerator = nullptr;
-  std::function<void(MeshBlock *, ParameterInput *)> PostInitialization = nullptr;
-  std::function<void(MeshBlock *, ParameterInput *)> MeshBlockUserWorkBeforeOutput =
-      nullptr;
+      InitApplicationMeshBlockData;
+  std::function<void(MeshBlock *, ParameterInput *)> InitMeshBlockUserData;
+  std::function<void(MeshBlock *, ParameterInput *)> ProblemGenerator;
+  std::function<void(MeshBlock *, ParameterInput *)> PostInitialization;
+  std::function<void(MeshBlock *, ParameterInput *, const SimTime &)>
+      MeshBlockUserWorkBeforeOutput;
+  std::function<void(MeshBlock *, ParameterInput *, const SimTime &, OutputParameters *)>
+      MeshBlockUserWorkBeforeRestartOutput;
 };
 
 } // namespace parthenon

src/driver/driver.cpp

@@ -96,17 +96,25 @@ DriverStatus EvolutionDriver::Execute() {
     while (tm.KeepGoing() && signal != OutputSignal::analysis) {
       if (Globals::my_rank == 0) OutputCycleDiagnostics();
 
-      pmesh->PreStepUserWorkInLoop(pmesh, pinput, tm);
-      pmesh->PreStepUserDiagnosticsInLoop(pmesh, pinput, tm);
+      if (pmesh->PreStepUserWorkInLoop) {
+        pmesh->PreStepUserWorkInLoop(pmesh, pinput, tm);
+      }
+      if (pmesh->PreStepUserDiagnosticsInLoop) {
+        pmesh->PreStepUserDiagnosticsInLoop(pmesh, pinput, tm);
+      }
 
       TaskListStatus status = Step();
       if (status != TaskListStatus::complete) {
         std::cerr << "Step failed to complete all tasks." << std::endl;
         return DriverStatus::failed;
       }
 
-      pmesh->PostStepUserWorkInLoop(pmesh, pinput, tm);
-      pmesh->PostStepUserDiagnosticsInLoop(pmesh, pinput, tm);
+      if (pmesh->PostStepUserWorkInLoop) {
+        pmesh->PostStepUserWorkInLoop(pmesh, pinput, tm);
+      }
+      if (pmesh->PostStepUserDiagnosticsInLoop) {
+        pmesh->PostStepUserDiagnosticsInLoop(pmesh, pinput, tm);
+      }
 
       tm.ncycle++;
       tm.time += tm.dt;
@@ -140,7 +148,9 @@ DriverStatus EvolutionDriver::Execute() {
       // ======================================================
   }   // Main t < tmax loop region
 
-  pmesh->UserWorkAfterLoop(pmesh, pinput, tm);
+  if (pmesh->UserWorkAfterLoop) {
+    pmesh->UserWorkAfterLoop(pmesh, pinput, tm);
+  }
 
   DriverStatus status = DriverStatus::complete;
   // Do *not* write the "final" output, if this is analysis run.

src/interface/params.cpp

@@ -94,6 +94,7 @@ void Params::WriteAllToHDF5(const std::string &prefix, const HDF5::H5G &group) c
   // strings
   WriteToHDF5AllParamsOfType<std::string>(prefix, group);
   WriteToHDF5AllParamsOfType<std::vector<std::string>>(prefix, group);
+  WriteToHDF5AllParamsOfType<std::vector<char>>(prefix, group);
 }
 
 void Params::ReadFromRestart(const std::string &prefix, const HDF5::H5G &group) {
@@ -109,6 +110,7 @@ void Params::ReadFromRestart(const std::string &prefix, const HDF5::H5G &group)
   // strings
   ReadFromHDF5AllParamsOfType<std::string>(prefix, group);
   ReadFromHDF5AllParamsOfType<std::vector<std::string>>(prefix, group);
+  ReadFromHDF5AllParamsOfType<std::vector<char>>(prefix, group);
 }
 
 #endif // ifdef ENABLE_HDF5

src/interface/state_descriptor.hpp

@@ -33,6 +33,7 @@
 #include "interface/swarm.hpp"
 #include "interface/var_id.hpp"
 #include "interface/variable.hpp"
+#include "outputs/output_parameters.hpp"
 #include "prolong_restrict/prolong_restrict.hpp"
 #include "utils/error_checking.hpp"
 
@@ -412,31 +413,41 @@ class StateDescriptor {
     if (UserWorkBeforeLoopMesh != nullptr) return UserWorkBeforeLoopMesh(pmesh, pin, tm);
   }
 
+  void UserWorkBeforeOutput(Mesh *pmesh, ParameterInput *pin, SimTime &tm) const {
+    if (UserWorkBeforeOutputMesh) return UserWorkBeforeOutputMesh(pmesh, pin, tm);
+  }
+
+  void UserWorkBeforeRestartOutput(Mesh *pmesh, ParameterInput *pin, SimTime &tm,
+                                   OutputParameters *pparams) const {
+    if (UserWorkBeforeRestartOutputMesh)
+      return UserWorkBeforeRestartOutputMesh(pmesh, pin, tm, pparams);
+  }
+
   std::vector<std::shared_ptr<AMRCriteria>> amr_criteria;
 
-  std::function<void(MeshBlockData<Real> *rc)> PreCommFillDerivedBlock = nullptr;
-  std::function<void(MeshData<Real> *rc)> PreCommFillDerivedMesh = nullptr;
-  std::function<void(MeshBlockData<Real> *rc)> PreFillDerivedBlock = nullptr;
-  std::function<void(MeshData<Real> *rc)> PreFillDerivedMesh = nullptr;
-  std::function<void(MeshBlockData<Real> *rc)> PostFillDerivedBlock = nullptr;
-  std::function<void(MeshData<Real> *rc)> PostFillDerivedMesh = nullptr;
-  std::function<void(MeshBlockData<Real> *rc)> FillDerivedBlock = nullptr;
-  std::function<void(MeshData<Real> *rc)> FillDerivedMesh = nullptr;
-  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoopMesh =
-      nullptr;
-
-  std::function<void(SimTime const &simtime, MeshData<Real> *rc)> PreStepDiagnosticsMesh =
-      nullptr;
-  std::function<void(SimTime const &simtime, MeshData<Real> *rc)>
-      PostStepDiagnosticsMesh = nullptr;
-
-  std::function<Real(MeshBlockData<Real> *rc)> EstimateTimestepBlock = nullptr;
-  std::function<Real(MeshData<Real> *rc)> EstimateTimestepMesh = nullptr;
-
-  std::function<AmrTag(MeshBlockData<Real> *rc)> CheckRefinementBlock = nullptr;
-
-  std::function<void(MeshData<Real> *rc)> InitNewlyAllocatedVarsMesh = nullptr;
-  std::function<void(MeshBlockData<Real> *rc)> InitNewlyAllocatedVarsBlock = nullptr;
+  std::function<void(MeshBlockData<Real> *rc)> PreCommFillDerivedBlock;
+  std::function<void(MeshData<Real> *rc)> PreCommFillDerivedMesh;
+  std::function<void(MeshBlockData<Real> *rc)> PreFillDerivedBlock;
+  std::function<void(MeshData<Real> *rc)> PreFillDerivedMesh;
+  std::function<void(MeshBlockData<Real> *rc)> PostFillDerivedBlock;
+  std::function<void(MeshData<Real> *rc)> PostFillDerivedMesh;
+  std::function<void(MeshBlockData<Real> *rc)> FillDerivedBlock;
+  std::function<void(MeshData<Real> *rc)> FillDerivedMesh;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoopMesh;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeOutputMesh;
+  std::function<void(Mesh *, ParameterInput *, SimTime &, OutputParameters *)>
+      UserWorkBeforeRestartOutputMesh;
+
+  std::function<void(SimTime const &simtime, MeshData<Real> *rc)> PreStepDiagnosticsMesh;
+  std::function<void(SimTime const &simtime, MeshData<Real> *rc)> PostStepDiagnosticsMesh;
+
+  std::function<Real(MeshBlockData<Real> *rc)> EstimateTimestepBlock;
+  std::function<Real(MeshData<Real> *rc)> EstimateTimestepMesh;
+
+  std::function<AmrTag(MeshBlockData<Real> *rc)> CheckRefinementBlock;
+
+  std::function<void(MeshData<Real> *rc)> InitNewlyAllocatedVarsMesh;
+  std::function<void(MeshBlockData<Real> *rc)> InitNewlyAllocatedVarsBlock;
 
   friend std::ostream &operator<<(std::ostream &os, const StateDescriptor &sd);
   std::array<std::vector<BValFunc>, BOUNDARY_NFACES> UserBoundaryFunctions;

src/mesh/mesh-amr_loadbalance.cpp

@@ -937,8 +937,11 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, ApplicationInput
           MPI_Waitall(send_reqs.size(), send_reqs.data(), MPI_STATUSES_IGNORE));
 #endif
     // init meshblock data
-    for (auto &pmb : block_list)
-      pmb->InitMeshBlockUserData(pmb.get(), pin);
+    for (auto &pmb : block_list) {
+      if (pmb->InitMeshBlockUserData) {
+        pmb->InitMeshBlockUserData(pmb.get(), pin);
+      }
+    }
 
     // Find the non-cell centered fields that are communicated
     Metadata::FlagCollection fc;

src/mesh/mesh.cpp

@@ -166,6 +166,9 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
   if (app_in->UserMeshWorkBeforeOutput != nullptr) {
     UserMeshWorkBeforeOutput = app_in->UserMeshWorkBeforeOutput;
   }
+  if (app_in->UserMeshWorkBeforeRestartOutput != nullptr) {
+    UserMeshWorkBeforeRestartOutput = app_in->UserMeshWorkBeforeRestartOutput;
+  }
   if (app_in->PreStepDiagnosticsInLoop != nullptr) {
     PreStepUserDiagnosticsInLoop = app_in->PreStepDiagnosticsInLoop;
   }
@@ -250,7 +253,9 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
     max_level = 63;
   }
 
-  InitUserMeshData(this, pin);
+  if (InitUserMeshData) {
+    InitUserMeshData(this, pin);
+  }
 
   if (multilevel) {
     if (block_size.nx(X1DIR) % 2 == 1 || (block_size.nx(X2DIR) % 2 == 1 && (ndim >= 2)) ||
@@ -584,7 +589,9 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr,
     max_level = 63;
   }
 
-  InitUserMeshData(this, pin);
+  if (InitUserMeshData) {
+    InitUserMeshData(this, pin);
+  }
 
   // Populate legacy logical locations
   auto lx123 = mesh_info.lx123;
@@ -901,11 +908,34 @@ void Mesh::EnrollBndryFncts_(ApplicationInput *app_in) {
 void Mesh::ApplyUserWorkBeforeOutput(Mesh *mesh, ParameterInput *pin,
                                      SimTime const &time) {
   // call Mesh version
-  mesh->UserMeshWorkBeforeOutput(mesh, pin, time);
+  if (mesh->UserMeshWorkBeforeOutput) {
+    mesh->UserMeshWorkBeforeOutput(mesh, pin, time);
+  }
 
   // call MeshBlock version
   for (auto &pmb : block_list) {
-    pmb->UserWorkBeforeOutput(pmb.get(), pin);
+    if (pmb->UserWorkBeforeOutput) {
+      pmb->UserWorkBeforeOutput(pmb.get(), pin, time);
+    }
+  }
+}
+
+//----------------------------------------------------------------------------------------
+// \!fn void Mesh::ApplyUserWorkBeforeRestartOutput
+// \brief Apply Mesh and Meshblock versions of UserWorkBeforeRestartOutput
+void Mesh::ApplyUserWorkBeforeRestartOutput(Mesh *mesh, ParameterInput *pin,
+                                            SimTime const &time,
+                                            OutputParameters *pparams) {
+  // call Mesh version
+  if (mesh->UserMeshWorkBeforeRestartOutput) {
+    mesh->UserMeshWorkBeforeRestartOutput(mesh, pin, time, pparams);
+  }
+
+  // call MeshBlock version
+  for (auto &pmb : block_list) {
+    if (pmb->UserWorkBeforeRestartOutput) {
+      pmb->UserWorkBeforeRestartOutput(pmb.get(), pin, time, pparams);
+    }
   }
 }
 
@@ -1076,7 +1106,9 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput *
     // init meshblock data
     for (int i = 0; i < nmb; ++i) {
       MeshBlock *pmb = block_list[i].get();
-      pmb->InitMeshBlockUserData(pmb, pin);
+      if (pmb->InitMeshBlockUserData) {
+        pmb->InitMeshBlockUserData(pmb, pin);
+      }
     }
 
     const int num_partitions = DefaultNumPartitions();
@@ -1103,6 +1135,9 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput *
       } else {
         for (int i = 0; i < nmb; ++i) {
           auto &pmb = block_list[i];
+          PARTHENON_REQUIRE(
+              pmb->ProblemGenerator,
+              "Neither mesh- nor meshblock-level ProblemGenerator defined!");
           pmb->ProblemGenerator(pmb.get(), pin);
         }
       }
@@ -1119,7 +1154,9 @@ void Mesh::Initialize(bool init_problem, ParameterInput *pin, ApplicationInput *
       } else {
         for (int i = 0; i < nmb; ++i) {
           auto &pmb = block_list[i];
-          pmb->PostInitialization(pmb.get(), pin);
+          if (pmb->PostInitialization) {
+            pmb->PostInitialization(pmb.get(), pin);
+          }
         }
       }