Merge pull request #2348 from pnorbert/test-steps

Tests for reading variables with Steps
ornladios · Jun 26, 2020 · 7a3bb53 · 7a3bb53
2 parents 942438c + fba2d99
commit 7a3bb53
Show file tree

Hide file tree

Showing 10 changed files with 2,811 additions and 25 deletions.
diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -395,7 +395,13 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant,
                     " was found with an index file but md.0 "
                     "has not contained enough data within "
                     "the specified timeout of " +
-                    std::to_string(timeoutSeconds.count()) + " seconds.");
+                    std::to_string(timeoutSeconds.count()) +
+                    " seconds. index size = " +
+                    std::to_string(metadataIndexFileSize) +
+                    " metadata size = " + std::to_string(fileSize) +
+                    " expected size = " + std::to_string(expectedMinFileSize) +
+                    ". One reason could be if the reader finds old data while "
+                    "the writer is creating the new files.");
             }
         }
     }

diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc
@@ -69,7 +69,10 @@ BP3Deserializer::InitVariableBlockInfo(core::Variable<T> &variable,
     }
 
     auto itStep = std::next(indices.begin(), stepsStart);
+    // BlocksInfo() expects absolute step, stepsStart is relative
+    const size_t absStep = itStep->first;
 
+    // Check that we have enough steps from stepsStart
     for (size_t i = 0; i < stepsCount; ++i)
     {
         if (itStep == indices.end())
@@ -88,8 +91,11 @@ BP3Deserializer::InitVariableBlockInfo(core::Variable<T> &variable,
 
     if (variable.m_SelectionType == SelectionType::WriteBlock)
     {
+        // BlocksInfo() expects absolute step, stepsStart is relative
+        // BlocksInfo() adds +1 to match the step starting from 1
+        // but absStep already is the actual step in the map
         const std::vector<typename core::Variable<T>::Info> blocksInfo =
-            BlocksInfo(variable, stepsStart);
+            BlocksInfo(variable, absStep - 1);
 
         if (variable.m_BlockID >= blocksInfo.size())
         {
@@ -749,10 +755,8 @@ inline void BP3Deserializer::DefineVariableInEngineIO<std::string>(
         variable->m_ShapeID = ShapeID::GlobalArray;
         variable->m_SingleValue = true;
     }
-    /* Update variable's starting step, which equals to the min value in
-    the sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
@@ -924,11 +928,8 @@ void BP3Deserializer::DefineVariableInEngineIO(const ElementIndexHeader &header,
         // in metadata
         variable->m_SingleValue = true;
     }
-
-    /* Update variable's starting step, which equals to the min value in the
-     * sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
@@ -988,13 +989,14 @@ BP3Deserializer::GetSubFileInfo(const core::Variable<T> &variable) const
 
     const auto &buffer = m_Metadata.m_Buffer;
 
-    const size_t stepStart = variable.m_StepsStart + 1;
-    const size_t stepEnd = stepStart + variable.m_StepsCount; // exclusive
+    auto itStep = variable.m_AvailableStepBlockIndexOffsets.begin();
+    const size_t absStepStart = itStep->first; // absolute step in map
+    const size_t stepEnd = absStepStart + variable.m_StepsCount; // exclusive
 
     const Box<Dims> selectionBox = helper::StartEndBox(
         variable.m_Start, variable.m_Count, m_ReverseDimensions);
 
-    for (size_t step = stepStart; step < stepEnd; ++step)
+    for (size_t step = absStepStart; step < stepEnd; ++step)
     {
         auto itBlockStarts =
             variable.m_AvailableStepBlockIndexOffsets.find(step);

diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc
@@ -72,7 +72,10 @@ BP4Deserializer::InitVariableBlockInfo(core::Variable<T> &variable,
     }
 
     auto itStep = std::next(indices.begin(), stepsStart);
+    // BlocksInfo() expects absolute step, stepsStart is relative
+    const size_t absStep = itStep->first;
 
+    // Check that we have enough steps from stepsStart
     for (size_t i = 0; i < stepsCount; ++i)
     {
         if (itStep == indices.end())
@@ -91,8 +94,11 @@ BP4Deserializer::InitVariableBlockInfo(core::Variable<T> &variable,
 
     if (variable.m_SelectionType == SelectionType::WriteBlock)
     {
+        // BlocksInfo() expects absolute step, stepsStart is relative
+        // BlocksInfo() adds +1 to match the step starting from 1
+        // but absStep already is the actual step in the map
         const std::vector<typename core::Variable<T>::Info> blocksInfo =
-            BlocksInfo(variable, stepsStart);
+            BlocksInfo(variable, absStep - 1);
 
         if (variable.m_BlockID >= blocksInfo.size())
         {
@@ -758,10 +764,8 @@ inline void BP4Deserializer::DefineVariableInEngineIOPerStep<std::string>(
         variable->m_ShapeID = ShapeID::GlobalArray;
         variable->m_SingleValue = true;
     }
-    /* Update variable's starting step, which equals to the min value in the
-     * sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;
@@ -1010,10 +1014,8 @@ void BP4Deserializer::DefineVariableInEngineIOPerStep(
         // in metadata
         variable->m_SingleValue = true;
     }
-    /* Update variable's starting step, which equals to the min value in the
-     * sorted map minus one */
-    variable->m_StepsStart =
-        variable->m_AvailableStepBlockIndexOffsets.begin()->first - 1;
+    /* Update variable's starting step, which is always 0 */
+    variable->m_StepsStart = 0;
 
     // update variable Engine for read streaming functions
     variable->m_Engine = &engine;

diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
@@ -41,6 +41,8 @@ bp3_bp4_gtest_add_tests_helper(WriteReadBlockInfo MPI_ALLOW)
 bp3_bp4_gtest_add_tests_helper(WriteReadVariableSpan MPI_ALLOW)
 bp3_bp4_gtest_add_tests_helper(TimeAggregation MPI_ALLOW)
 bp3_bp4_gtest_add_tests_helper(NoXMLRecovery MPI_ALLOW)
+bp3_bp4_gtest_add_tests_helper(StepsFileGlobalArray MPI_ALLOW)
+bp3_bp4_gtest_add_tests_helper(StepsFileLocalArray MPI_ALLOW)
 
 if(NOT MSVC)
   bp3_bp4_gtest_add_tests_helper(BufferSize MPI_NONE)
@@ -59,4 +61,10 @@ gtest_add_tests_helper(WriteNull MPI_ALLOW BP Engine.BP. .BP3
 gtest_add_tests_helper(WriteAppendReadADIOS2 MPI_ALLOW BP Engine.BP. .BP4
   WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
 )
+gtest_add_tests_helper(StepsInSituGlobalArray MPI_ALLOW BP Engine.BP. .BP4
+  WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
+)
+gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .BP4
+  WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
+)