miscellanea

ornladios · Apr 22, 2024 · ce075a4 · ce075a4
1 parent 6f0ca3b
commit ce075a4
Show file tree

Hide file tree

Showing 12 changed files with 74 additions and 5 deletions.
diff --git a/bindings/C/adios2/c/adios2_c_engine.cpp b/bindings/C/adios2/c/adios2_c_engine.cpp
@@ -34,6 +34,9 @@ adios2::Mode adios2_ToMode(const adios2_mode mode, const std::string &hint)
     case adios2_mode_readRandomAccess:
         modeCpp = adios2::Mode::ReadRandomAccess;
         break;
+    case adios2_mode_readFlattenSteps:
+        modeCpp = adios2::Mode::ReadFlattenSteps;
+        break;
     case adios2_mode_deferred:
         modeCpp = adios2::Mode::Deferred;
         break;
@@ -63,6 +66,9 @@ adios2_mode adios2_fromMode(const adios2::Mode mode, const std::string &hint)
     case adios2::Mode::ReadRandomAccess:
         modeC = adios2_mode_readRandomAccess;
         break;
+    case adios2::Mode::ReadFlattenSteps:
+        modeC = adios2_mode_readFlattenSteps;
+        break;
     case adios2::Mode::Deferred:
         modeC = adios2_mode_deferred;
         break;

diff --git a/bindings/C/adios2/c/adios2_c_io.h b/bindings/C/adios2/c/adios2_c_io.h
@@ -329,8 +329,8 @@ adios2_error adios2_remove_all_attributes(adios2_io *io);
  * MPI Collective function as it calls MPI_Comm_dup
  * @param io engine owner
  * @param name unique engine identifier
- * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and
- * adios2_mode_readRandomAccess
+ * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append,
+ * adios2_mode_readRandomAccess and adios2_mode_readFlattenSteps
  * @return success: handler, failure: NULL
  */
 adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mode);
@@ -341,8 +341,8 @@ adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mo
  * MPI Collective function as it calls MPI_Comm_dup
  * @param io engine owner
  * @param name unique engine identifier
- * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and
- * adios2_mode_readRandomAccess
+ * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append,
+ * adios2_mode_readRandomAccess and adios2_mode_readFlattenSteps
  * @param comm communicator other than adios' handler comm. MPI only.
  * @return success: handler, failure: NULL
  */

diff --git a/bindings/C/adios2/c/adios2_c_io.tcc b/bindings/C/adios2/c/adios2_c_io.tcc
@@ -36,6 +36,10 @@ adios2::Mode adios2_ToOpenMode(const adios2_mode modeC)
         mode = adios2::Mode::ReadRandomAccess;
         break;
 
+    case adios2_mode_readFlattenSteps:
+        mode = adios2::Mode::ReadFlattenSteps;
+        break;
+
     default:
         break;
     }

diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h
@@ -103,6 +103,7 @@ typedef enum
     adios2_mode_read = 2,
     adios2_mode_append = 3,
     adios2_mode_readRandomAccess = 6,
+    adios2_mode_readFlattenSteps = 7,
 
     adios2_mode_deferred = 4,
     adios2_mode_sync = 5

diff --git a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp
@@ -89,6 +89,9 @@ adios2::Mode fstream::ToMode(const openmode mode) const noexcept
     case (openmode::in_random_access):
         modeCpp = adios2::Mode::ReadRandomAccess;
         break;
+    case (openmode::in_flatten_steps):
+        modeCpp = adios2::Mode::ReadFlattenSteps;
+        break;
     case (openmode::app):
         modeCpp = adios2::Mode::Append;
         break;

diff --git a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h
@@ -48,6 +48,7 @@ class fstream
         out,              //!< write
         in,               //!< read
         in_random_access, //!< read_random_access
+        in_flatten_steps, //!< flatten all input steps to 1
         app               //!< append, not yet supported
     };
 

diff --git a/bindings/Fortran/modules/adios2_parameters_mod.f90 b/bindings/Fortran/modules/adios2_parameters_mod.f90
@@ -56,6 +56,7 @@ module adios2_parameters_mod
    integer, parameter :: adios2_mode_read = 2
    integer, parameter :: adios2_mode_append = 3
    integer, parameter :: adios2_mode_readRandomAccess = 6
+   integer, parameter :: adios2_mode_readFlattenSteps = 7
 
    integer, parameter :: adios2_mode_deferred = 4
    integer, parameter :: adios2_mode_sync = 5

diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp
@@ -98,6 +98,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .value("Write", adios2::Mode::Write)
         .value("Read", adios2::Mode::Read)
         .value("ReadRandomAccess", adios2::Mode::ReadRandomAccess)
+        .value("ReadFlattenSteps", adios2::Mode::ReadFlattenSteps)
         .value("Append", adios2::Mode::Append)
         .value("Deferred", adios2::Mode::Deferred)
         .value("Sync", adios2::Mode::Sync)

diff --git a/docs/user_guide/source/components/anatomy.rst b/docs/user_guide/source/components/anatomy.rst
@@ -114,7 +114,10 @@ named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data ac
 current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at
 once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using
 `SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold
-multiple steps of the variable content.
+multiple steps of the variable content.  Note that ADIOS streaming
+engines (like SST, DataMan, etc.) do not support `ReadRandomAccess`
+mode.  Also newer file Engines like BP5 to not allow
+`BeginStep/EndStep` calls in `ReadRandomAccess` mode.
 
 .. code:: C++
 
@@ -134,3 +137,45 @@ multiple steps of the variable content.
     |   |--> IO goes out of scope
     |
     |--> ADIOS goes out of scope or adios2_finalize()
+
+Previously we explored how to read using the input mode `adios2::Mode::Read`. Nonetheless, ADIOS has another input mode
+named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data access only timestep by timestep using
+`BeginStep/EndStep`, but generally it is more memory efficient as ADIOS is only required to load metadata for the
+current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at
+once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using
+`SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold
+multiple steps of the variable content.  Note that ADIOS streaming
+engines (like SST, DataMan, etc.) do not support `ReadRandomAccess`
+mode.  Also newer file Engines like BP5 to not allow
+`BeginStep/EndStep` calls in `ReadRandomAccess` mode.
+
+.. code:: C++
+
+    ADIOS adios("config.xml", MPI_COMM_WORLD);
+    |
+    |   IO io = adios.DeclareIO(...);
+    |   |
+    |   |   Engine e = io.Open("InputFileName.bp", adios2::Mode::ReadRandomAccess);
+    |   |   |
+    |   |   |   Variable var = io.InquireVariable(...)
+    |   |   |   |   var.SetStepSelection()
+    |   |   |   |   e.Get(var, datapointer);
+    |   |   |   |
+    |   |   |
+    |   |   e.Close();
+    |   |
+    |   |--> IO goes out of scope
+    |
+    |--> ADIOS goes out of scope or adios2_finalize()
+    
+
+In addition to the two read modes discussed above, ADIOS has another
+input mode named `adios2::Mode::ReadFlattenSteps`. This is a highly
+specialized mode built that is unlikely to be of general utility, but
+we describe it for completeness.  In `ReadFlattenSteps` mode, ADIOS
+loads all the metadata in the file upon Open (just like
+`ReadRandomAccess` mode, but everything that was written appears that
+it was output on the same step, regardless of how many steps actually
+appear in the file.  This affects the operation of many reader-side
+ADIOS functions, including Steps(), BlocksInfo(), Get(), etc.
+
diff --git a/python/adios2/stream.py b/python/adios2/stream.py
@@ -37,6 +37,9 @@ def string_to_mode(mode: str) -> [bindings.Mode, bool]:
     elif mode == "rra":
         bmode = bindings.Mode.ReadRandomAccess
         read_mode = True
+    elif mode == "rfs":
+        bmode = bindings.Mode.ReadFlattenSteps
+        read_mode = True
     elif mode == "w":
         bmode = bindings.Mode.Write
     elif mode == "a":

diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp
@@ -177,6 +177,8 @@ static void OpenHandler(CManager cm, CMConnection conn, void *vevent, void *clie
     std::string strMode = "Streaming";
     if (open_msg->Mode == RemoteOpenRandomAccess)
         strMode = "RandomAccess";
+    if (open_msg->Mode == RemoteOpenFlattenSteps)
+        strMode = "FlattenSteps";
     std::cout << "Got an open request (mode " << strMode << ") for file " << open_msg->FileName
               << std::endl;
     AnonADIOSFile *f =

diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
@@ -91,6 +91,8 @@ set(CTEST_TEST_TIMEOUT 10)
 bp_gtest_add_tests_helper(WriteReadADIOS2 MPI_ALLOW)
 async_gtest_add_tests_helper(WriteReadADIOS2 MPI_ALLOW)
 
+gtest_add_tests_helper(WriteReadFlatten MPI_ALLOW BP Engine.BP. .BP5 WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" )
+
 bp_gtest_add_tests_helper(WriteReadADIOS2fstream MPI_ALLOW)
 bp_gtest_add_tests_helper(WriteReadADIOS2stdio MPI_ALLOW)
 bp_gtest_add_tests_helper(WriteReadAsStreamADIOS2 MPI_ALLOW)