From 5b0d7ab8e4bd0582c68fc92b13c2728d3bd8d5f9 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 3 May 2022 02:27:15 -0400 Subject: [PATCH 1/6] added BlocksInfo API for VariableNT --- bindings/CXX11/adios2/cxx11/Engine.cpp | 75 ++++++++++++++++++++++++ bindings/CXX11/adios2/cxx11/Engine.h | 6 ++ bindings/CXX11/adios2/cxx11/VariableNT.h | 11 ++++ 3 files changed, 92 insertions(+) diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index 78a596bdc3..0df8227d16 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -223,6 +223,81 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) ADIOS2_FOREACH_PRIMITIVE_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation +std::vector Engine::BlocksInfo(const VariableNT &variable, + const size_t step) const +{ + std::vector ret; + if (variable.m_Variable->m_Type == DataType::Struct) + { + } +#define declare_type(T) \ + else if (variable.m_Variable->m_Type == helper::GetDataType()) \ + { \ + auto blocksInfoT = \ + BlocksInfo(Variable(reinterpret_cast *>( \ + variable.m_Variable)), \ + step); \ + for (const auto &b : blocksInfoT) \ + { \ + ret.emplace_back(); \ + auto &br = ret.back(); \ + br.Start = b.Start; \ + br.Count = b.Count; \ + br.WriterID = b.WriterID; \ + br.Step = b.Step; \ + br.IsReverseDims = b.IsReverseDims; \ + br.IsValue = b.IsValue; \ + } \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + else + { + helper::Throw("bindings::CXX11", "Engine", + "BlocksInfo", "invalid data type"); + } + return ret; +} + +std::map> +Engine::AllStepsBlocksInfo(const VariableNT &variable) const +{ + std::map> ret; + if (variable.m_Variable->m_Type == DataType::Struct) + { + } +#define declare_type(T) \ + else if (variable.m_Variable->m_Type == helper::GetDataType()) \ + { \ + auto blocksInfoT = AllStepsBlocksInfo(Variable( \ + reinterpret_cast *>(variable.m_Variable))); \ + for (const auto &bv : blocksInfoT) \ + { \ + auto &bvr = ret[bv.first]; \ + for (const auto &b : bv.second) \ + { \ + bvr.emplace_back(); \ + auto &br = bvr.back(); \ + br.Start = b.Start; \ + br.Count = b.Count; \ + br.WriterID = b.WriterID; \ + br.Step = b.Step; \ + br.IsReverseDims = b.IsReverseDims; \ + br.IsValue = b.IsValue; \ + } \ + } \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + else + { + helper::Throw("bindings::CXX11", "Engine", + "AllStepsBlocksInfo", + "invalid data type"); + } + return ret; +} + #define declare_template_instantiation(T) \ template void Engine::Put(Variable, const T *, const Mode); \ template void Engine::Put(const std::string &, const T *, const Mode); \ diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 7a55f61c20..c98d480666 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -430,6 +430,9 @@ class Engine std::map::Info>> AllStepsBlocksInfo(const Variable variable) const; + std::map> + AllStepsBlocksInfo(const VariableNT &variable) const; + /** * Extracts all available blocks information for a particular * variable and step. @@ -443,6 +446,9 @@ class Engine std::vector::Info> BlocksInfo(const Variable variable, const size_t step) const; + std::vector BlocksInfo(const VariableNT &variable, + const size_t step) const; + /** * Get the absolute steps of a variable in a file. This is for * information purposes only, because absolute steps cannot be used diff --git a/bindings/CXX11/adios2/cxx11/VariableNT.h b/bindings/CXX11/adios2/cxx11/VariableNT.h index 7fed658329..fe0af1e3db 100644 --- a/bindings/CXX11/adios2/cxx11/VariableNT.h +++ b/bindings/CXX11/adios2/cxx11/VariableNT.h @@ -230,6 +230,17 @@ class VariableNT std::complex DComplex; }; + struct Info + { + adios2::Dims Start; + adios2::Dims Count; + int WriterID = 0; + size_t BlockID = 0; + size_t Step = 0; + bool IsReverseDims = false; + bool IsValue = false; + }; + T Min(const size_t step = adios2::DefaultSizeT) const; T Max(const size_t step = adios2::DefaultSizeT) const; std::pair MinMax(const size_t step = adios2::DefaultSizeT) const; From 10d8d8675f184e95b5b261a8dbf5ef8c6a12c982 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 3 May 2022 04:35:07 -0400 Subject: [PATCH 2/6] added BlockInfo for struct variable in ssc --- bindings/CXX11/adios2/cxx11/Engine.cpp | 40 +++++++++++++++++++ source/adios2/core/Engine.cpp | 39 ++++++++++++++++++ source/adios2/core/Engine.h | 19 +++++++++ source/adios2/engine/ssc/SscReader.cpp | 6 +++ source/adios2/engine/ssc/SscReader.h | 2 + source/adios2/engine/ssc/SscReaderBase.h | 3 ++ source/adios2/engine/ssc/SscReaderGeneric.cpp | 29 ++++++++++++++ source/adios2/engine/ssc/SscReaderGeneric.h | 3 ++ source/adios2/engine/ssc/SscReaderGeneric.tcc | 4 +- source/adios2/engine/ssc/SscReaderNaive.cpp | 27 +++++++++++++ source/adios2/engine/ssc/SscReaderNaive.h | 3 ++ source/adios2/engine/ssc/SscReaderNaive.tcc | 4 +- 12 files changed, 173 insertions(+), 6 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index 0df8227d16..c4f7aaaf28 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -229,6 +229,24 @@ std::vector Engine::BlocksInfo(const VariableNT &variable, std::vector ret; if (variable.m_Variable->m_Type == DataType::Struct) { + adios2::helper::CheckForNullptr( + m_Engine, "for Engine in call to Engine::BlocksInfo"); + adios2::helper::CheckForNullptr( + variable.m_Variable, "for variable in call to Engine::BlocksInfo"); + auto blocksInfo = m_Engine->BlocksInfo( + *reinterpret_cast(variable.m_Variable), + step); + for (const auto &b : blocksInfo) + { + ret.emplace_back(); + auto &br = ret.back(); + br.Start = b.Start; + br.Count = b.Count; + br.WriterID = b.WriterID; + br.Step = b.Step; + br.IsReverseDims = b.IsReverseDims; + br.IsValue = b.IsValue; + } } #define declare_type(T) \ else if (variable.m_Variable->m_Type == helper::GetDataType()) \ @@ -265,6 +283,28 @@ Engine::AllStepsBlocksInfo(const VariableNT &variable) const std::map> ret; if (variable.m_Variable->m_Type == DataType::Struct) { + adios2::helper::CheckForNullptr( + m_Engine, "for Engine in call to Engine::AllStepsBlocksInfo"); + adios2::helper::CheckForNullptr( + variable.m_Variable, + "for variable in call to Engine::AllStepsBlocksInfo"); + auto blocksInfo = m_Engine->AllStepsBlocksInfo( + *reinterpret_cast(variable.m_Variable)); + for (const auto &bv : blocksInfo) + { + auto &bvr = ret[bv.first]; + for (const auto &b : bv.second) + { + bvr.emplace_back(); + auto &br = bvr.back(); + br.Start = b.Start; + br.Count = b.Count; + br.WriterID = b.WriterID; + br.Step = b.Step; + br.IsReverseDims = b.IsReverseDims; + br.IsValue = b.IsValue; + } + } } #define declare_type(T) \ else if (variable.m_Variable->m_Type == helper::GetDataType()) \ diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 4240b3cf3e..0effeee2b8 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -237,6 +237,27 @@ void Engine::DoGetStructDeferred(VariableStruct &, void *) ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type +std::map> +Engine::DoAllStepsBlocksInfo(const VariableStruct &variable) const +{ + ThrowUp("DoAllStepsBlocksInfo"); + return std::map>(); +} + +std::vector> +Engine::DoAllRelativeStepsBlocksInfo(const VariableStruct &variable) const +{ + ThrowUp("DoAllRelativeStepsBlocksInfo"); + return std::vector>(); +} + +std::vector +Engine::DoBlocksInfo(const VariableStruct &variable, const size_t step) const +{ + ThrowUp("DoBlocksInfo"); + return std::vector(); +} + #define declare_type(T, L) \ T *Engine::DoBufferData_##L(const int bufferIdx, \ const size_t payloadPosition, \ @@ -290,6 +311,24 @@ void Engine::CommonChecks(VariableBase &variable, const void *data, } } +std::map> +Engine::AllStepsBlocksInfo(const VariableStruct &variable) const +{ + return DoAllStepsBlocksInfo(variable); +} + +std::vector> +Engine::AllRelativeStepsBlocksInfo(const VariableStruct &variable) const +{ + return DoAllRelativeStepsBlocksInfo(variable); +} + +std::vector +Engine::BlocksInfo(const VariableStruct &variable, const size_t step) const +{ + return DoBlocksInfo(variable, step); +} + // PUBLIC TEMPLATE FUNCTIONS EXPANSION WITH SCOPED TYPES #define declare_template_instantiation(T) \ \ diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index f9b084ed26..abaa9c1ba1 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -30,6 +30,7 @@ #include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" #include "adios2/core/Variable.h" +#include "adios2/core/VariableStruct.h" #include "adios2/helper/adiosComm.h" namespace adios2 @@ -404,6 +405,9 @@ class Engine std::map::BPInfo>> AllStepsBlocksInfo(const Variable &variable) const; + std::map> + AllStepsBlocksInfo(const VariableStruct &variable) const; + /** * This function is internal, for public interface use * Variable::AllStepsBlocksInfo @@ -414,6 +418,9 @@ class Engine std::vector::BPInfo>> AllRelativeStepsBlocksInfo(const Variable &variable) const; + std::vector> + AllRelativeStepsBlocksInfo(const VariableStruct &variable) const; + /** * Extracts all available blocks information for a particular * variable and step. @@ -427,6 +434,9 @@ class Engine std::vector::BPInfo> BlocksInfo(const Variable &variable, const size_t step) const; + std::vector + BlocksInfo(const VariableStruct &variable, const size_t step) const; + /** * Get the absolute steps of a variable in a file. This is for * information purposes only, because absolute steps cannot be used @@ -572,6 +582,15 @@ class Engine ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type + virtual std::map> + DoAllStepsBlocksInfo(const VariableStruct &variable) const; + + virtual std::vector> + DoAllRelativeStepsBlocksInfo(const VariableStruct &variable) const; + + virtual std::vector + DoBlocksInfo(const VariableStruct &variable, const size_t step) const; + #define declare_type(T, L) \ virtual T *DoBufferData_##L(const int bufferIdx, \ const size_t payloadPosition, \ diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index ca23c59fba..29cc5001e8 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -121,6 +121,12 @@ void SscReader::DoClose(const int transportIndex) ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type +std::vector +SscReader::DoBlocksInfo(const VariableStruct &variable, const size_t step) const +{ + return m_EngineInstance->BlocksInfo(variable, step); +} + void SscReader::DoGetStructSync(VariableStruct &variable, void *data) { PERFSTUBS_SCOPED_TIMER_FUNC(); diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 7e87a2461a..0063712fca 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -45,6 +45,8 @@ class SscReader : public Engine void DoGetStructSync(VariableStruct &, void *) final; void DoGetStructDeferred(VariableStruct &, void *) final; + std::vector + DoBlocksInfo(const VariableStruct &variable, const size_t step) const final; void DoClose(const int transportIndex = -1) final; diff --git a/source/adios2/engine/ssc/SscReaderBase.h b/source/adios2/engine/ssc/SscReaderBase.h index fb5e6c8cb8..4dbf5175b0 100644 --- a/source/adios2/engine/ssc/SscReaderBase.h +++ b/source/adios2/engine/ssc/SscReaderBase.h @@ -46,6 +46,9 @@ class SscReaderBase ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type + virtual std::vector + BlocksInfo(const VariableStruct &variable, const size_t step) const = 0; + virtual void GetDeferred(VariableBase &, void *) = 0; protected: diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp index 8744f98125..df1b8ef4f6 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.cpp +++ b/source/adios2/engine/ssc/SscReaderGeneric.cpp @@ -570,6 +570,35 @@ void SscReaderGeneric::GetDeferred(VariableBase &variable, void *data) } } +std::vector +SscReaderGeneric::BlocksInfo(const VariableStruct &variable, + const size_t step) const +{ + std::vector ret; + + for (const auto &r : m_GlobalWritePattern) + { + for (auto &v : r) + { + if (v.name == variable.m_Name) + { + ret.emplace_back(); + auto &b = ret.back(); + b.Start = v.start; + b.Count = v.count; + b.Shape = v.shape; + b.Step = m_CurrentStep; + b.StepsStart = m_CurrentStep; + b.StepsCount = 1; + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) + { + b.IsReverseDims = true; + } + } + } + } + return ret; +} } } } diff --git a/source/adios2/engine/ssc/SscReaderGeneric.h b/source/adios2/engine/ssc/SscReaderGeneric.h index 5d65e740e7..f4c82adea0 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.h +++ b/source/adios2/engine/ssc/SscReaderGeneric.h @@ -45,6 +45,9 @@ class SscReaderGeneric : public SscReaderBase ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type + std::vector + BlocksInfo(const VariableStruct &variable, const size_t step) const final; + void GetDeferred(VariableBase &, void *) final; private: diff --git a/source/adios2/engine/ssc/SscReaderGeneric.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc index 5a7db33202..fc51523a40 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.tcc +++ b/source/adios2/engine/ssc/SscReaderGeneric.tcc @@ -46,9 +46,7 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, b.StepsCount = 1; if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { - std::reverse(b.Start.begin(), b.Start.end()); - std::reverse(b.Count.begin(), b.Count.end()); - std::reverse(b.Shape.begin(), b.Shape.end()); + b.IsReverseDims = true; } if (v.shapeId == ShapeID::GlobalValue || v.shapeId == ShapeID::LocalValue) diff --git a/source/adios2/engine/ssc/SscReaderNaive.cpp b/source/adios2/engine/ssc/SscReaderNaive.cpp index 7c8f37137f..c26b3860de 100644 --- a/source/adios2/engine/ssc/SscReaderNaive.cpp +++ b/source/adios2/engine/ssc/SscReaderNaive.cpp @@ -205,6 +205,33 @@ void SscReaderNaive::GetDeferred(VariableBase &variable, void *data) } } +std::vector +SscReaderNaive::BlocksInfo(const VariableStruct &variable, + const size_t step) const +{ + std::vector ret; + auto it = m_BlockMap.find(variable.m_Name); + if (it != m_BlockMap.end()) + { + for (const auto &v : it->second) + { + ret.emplace_back(); + auto &b = ret.back(); + b.Start = v.start; + b.Count = v.count; + b.Shape = v.shape; + b.Step = m_CurrentStep; + b.StepsStart = m_CurrentStep; + b.StepsCount = 1; + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) + { + b.IsReverseDims = true; + } + } + } + return ret; +} + } } } diff --git a/source/adios2/engine/ssc/SscReaderNaive.h b/source/adios2/engine/ssc/SscReaderNaive.h index 93b8c1670a..409f2acd03 100644 --- a/source/adios2/engine/ssc/SscReaderNaive.h +++ b/source/adios2/engine/ssc/SscReaderNaive.h @@ -45,6 +45,9 @@ class SscReaderNaive : public SscReaderBase ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type + std::vector + BlocksInfo(const VariableStruct &variable, const size_t step) const final; + void GetDeferred(VariableBase &, void *) final; private: diff --git a/source/adios2/engine/ssc/SscReaderNaive.tcc b/source/adios2/engine/ssc/SscReaderNaive.tcc index d118118e45..974deba27c 100644 --- a/source/adios2/engine/ssc/SscReaderNaive.tcc +++ b/source/adios2/engine/ssc/SscReaderNaive.tcc @@ -44,9 +44,7 @@ SscReaderNaive::BlocksInfoCommon(const Variable &variable, b.StepsCount = 1; if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { - std::reverse(b.Start.begin(), b.Start.end()); - std::reverse(b.Count.begin(), b.Count.end()); - std::reverse(b.Shape.begin(), b.Shape.end()); + b.IsReverseDims = true; } if (v.shapeId == ShapeID::GlobalValue || v.shapeId == ShapeID::LocalValue) From 0520d14e6a72a7fd8825231d755a9129b1bd9012 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 3 May 2022 13:31:41 -0400 Subject: [PATCH 3/6] renamed BlocksInfo functions for struct type to avoid partial overload error --- bindings/CXX11/adios2/cxx11/Engine.cpp | 4 ++-- source/adios2/core/Engine.cpp | 20 +++++++++++--------- source/adios2/core/Engine.h | 12 ++++++------ source/adios2/engine/ssc/SscReader.cpp | 3 ++- source/adios2/engine/ssc/SscReader.h | 3 ++- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index c4f7aaaf28..2b7c420432 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -233,7 +233,7 @@ std::vector Engine::BlocksInfo(const VariableNT &variable, m_Engine, "for Engine in call to Engine::BlocksInfo"); adios2::helper::CheckForNullptr( variable.m_Variable, "for variable in call to Engine::BlocksInfo"); - auto blocksInfo = m_Engine->BlocksInfo( + auto blocksInfo = m_Engine->BlocksInfoStruct( *reinterpret_cast(variable.m_Variable), step); for (const auto &b : blocksInfo) @@ -288,7 +288,7 @@ Engine::AllStepsBlocksInfo(const VariableNT &variable) const adios2::helper::CheckForNullptr( variable.m_Variable, "for variable in call to Engine::AllStepsBlocksInfo"); - auto blocksInfo = m_Engine->AllStepsBlocksInfo( + auto blocksInfo = m_Engine->AllStepsBlocksInfoStruct( *reinterpret_cast(variable.m_Variable)); for (const auto &bv : blocksInfo) { diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 0effeee2b8..47315ea3a9 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -238,21 +238,22 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type std::map> -Engine::DoAllStepsBlocksInfo(const VariableStruct &variable) const +Engine::DoAllStepsBlocksInfoStruct(const VariableStruct &variable) const { ThrowUp("DoAllStepsBlocksInfo"); return std::map>(); } std::vector> -Engine::DoAllRelativeStepsBlocksInfo(const VariableStruct &variable) const +Engine::DoAllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const { ThrowUp("DoAllRelativeStepsBlocksInfo"); return std::vector>(); } std::vector -Engine::DoBlocksInfo(const VariableStruct &variable, const size_t step) const +Engine::DoBlocksInfoStruct(const VariableStruct &variable, + const size_t step) const { ThrowUp("DoBlocksInfo"); return std::vector(); @@ -312,21 +313,22 @@ void Engine::CommonChecks(VariableBase &variable, const void *data, } std::map> -Engine::AllStepsBlocksInfo(const VariableStruct &variable) const +Engine::AllStepsBlocksInfoStruct(const VariableStruct &variable) const { - return DoAllStepsBlocksInfo(variable); + return DoAllStepsBlocksInfoStruct(variable); } std::vector> -Engine::AllRelativeStepsBlocksInfo(const VariableStruct &variable) const +Engine::AllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const { - return DoAllRelativeStepsBlocksInfo(variable); + return DoAllRelativeStepsBlocksInfoStruct(variable); } std::vector -Engine::BlocksInfo(const VariableStruct &variable, const size_t step) const +Engine::BlocksInfoStruct(const VariableStruct &variable, + const size_t step) const { - return DoBlocksInfo(variable, step); + return DoBlocksInfoStruct(variable, step); } // PUBLIC TEMPLATE FUNCTIONS EXPANSION WITH SCOPED TYPES diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index abaa9c1ba1..fa0f5b28ca 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -406,7 +406,7 @@ class Engine AllStepsBlocksInfo(const Variable &variable) const; std::map> - AllStepsBlocksInfo(const VariableStruct &variable) const; + AllStepsBlocksInfoStruct(const VariableStruct &variable) const; /** * This function is internal, for public interface use @@ -419,7 +419,7 @@ class Engine AllRelativeStepsBlocksInfo(const Variable &variable) const; std::vector> - AllRelativeStepsBlocksInfo(const VariableStruct &variable) const; + AllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const; /** * Extracts all available blocks information for a particular @@ -435,7 +435,7 @@ class Engine BlocksInfo(const Variable &variable, const size_t step) const; std::vector - BlocksInfo(const VariableStruct &variable, const size_t step) const; + BlocksInfoStruct(const VariableStruct &variable, const size_t step) const; /** * Get the absolute steps of a variable in a file. This is for @@ -583,13 +583,13 @@ class Engine #undef declare_type virtual std::map> - DoAllStepsBlocksInfo(const VariableStruct &variable) const; + DoAllStepsBlocksInfoStruct(const VariableStruct &variable) const; virtual std::vector> - DoAllRelativeStepsBlocksInfo(const VariableStruct &variable) const; + DoAllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const; virtual std::vector - DoBlocksInfo(const VariableStruct &variable, const size_t step) const; + DoBlocksInfoStruct(const VariableStruct &variable, const size_t step) const; #define declare_type(T, L) \ virtual T *DoBufferData_##L(const int bufferIdx, \ diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 29cc5001e8..e33f08db25 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -122,7 +122,8 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type std::vector -SscReader::DoBlocksInfo(const VariableStruct &variable, const size_t step) const +SscReader::DoBlocksInfoStruct(const VariableStruct &variable, + const size_t step) const { return m_EngineInstance->BlocksInfo(variable, step); } diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 0063712fca..aa4e150ad3 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -46,7 +46,8 @@ class SscReader : public Engine void DoGetStructSync(VariableStruct &, void *) final; void DoGetStructDeferred(VariableStruct &, void *) final; std::vector - DoBlocksInfo(const VariableStruct &variable, const size_t step) const final; + DoBlocksInfoStruct(const VariableStruct &variable, + const size_t step) const final; void DoClose(const int transportIndex = -1) final; From 7e3a855e4deeee33ecddd783143647494ca7e276 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 3 May 2022 18:02:59 -0400 Subject: [PATCH 4/6] added test for ssc struct BlocksInfo --- source/adios2/engine/ssc/SscReaderGeneric.cpp | 9 ++++++--- source/adios2/engine/ssc/SscReaderGeneric.tcc | 9 ++++++--- source/adios2/engine/ssc/SscReaderNaive.cpp | 3 +++ source/adios2/engine/ssc/SscReaderNaive.tcc | 3 +++ testing/adios2/engine/ssc/TestSscStruct.cpp | 17 ++++++++++------- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp index df1b8ef4f6..c94f7346de 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.cpp +++ b/source/adios2/engine/ssc/SscReaderGeneric.cpp @@ -575,10 +575,10 @@ SscReaderGeneric::BlocksInfo(const VariableStruct &variable, const size_t step) const { std::vector ret; - - for (const auto &r : m_GlobalWritePattern) + size_t blockID = 0; + for (size_t i = 0; i < m_GlobalWritePattern.size(); ++i) { - for (auto &v : r) + for (auto &v : m_GlobalWritePattern[i]) { if (v.name == variable.m_Name) { @@ -590,10 +590,13 @@ SscReaderGeneric::BlocksInfo(const VariableStruct &variable, b.Step = m_CurrentStep; b.StepsStart = m_CurrentStep; b.StepsCount = 1; + b.WriterID = i; + b.BlockID = blockID; if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { b.IsReverseDims = true; } + ++blockID; } } } diff --git a/source/adios2/engine/ssc/SscReaderGeneric.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc index fc51523a40..31df8f6ffc 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.tcc +++ b/source/adios2/engine/ssc/SscReaderGeneric.tcc @@ -29,10 +29,10 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, const size_t step) const { std::vector::BPInfo> ret; - - for (const auto &r : m_GlobalWritePattern) + size_t blockID = 0; + for (size_t i = 0; i < m_GlobalWritePattern.size(); ++i) { - for (auto &v : r) + for (auto &v : m_GlobalWritePattern[i]) { if (v.name == variable.m_Name) { @@ -44,6 +44,8 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, b.Step = m_CurrentStep; b.StepsStart = m_CurrentStep; b.StepsCount = 1; + b.WriterID = i; + b.BlockID = blockID; if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { b.IsReverseDims = true; @@ -66,6 +68,7 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, v.bufferCount); } } + ++blockID; } } } diff --git a/source/adios2/engine/ssc/SscReaderNaive.cpp b/source/adios2/engine/ssc/SscReaderNaive.cpp index c26b3860de..2abcad0e45 100644 --- a/source/adios2/engine/ssc/SscReaderNaive.cpp +++ b/source/adios2/engine/ssc/SscReaderNaive.cpp @@ -210,6 +210,7 @@ SscReaderNaive::BlocksInfo(const VariableStruct &variable, const size_t step) const { std::vector ret; + size_t blockID = 0; auto it = m_BlockMap.find(variable.m_Name); if (it != m_BlockMap.end()) { @@ -223,10 +224,12 @@ SscReaderNaive::BlocksInfo(const VariableStruct &variable, b.Step = m_CurrentStep; b.StepsStart = m_CurrentStep; b.StepsCount = 1; + b.BlockID = blockID; if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { b.IsReverseDims = true; } + ++blockID; } } return ret; diff --git a/source/adios2/engine/ssc/SscReaderNaive.tcc b/source/adios2/engine/ssc/SscReaderNaive.tcc index 974deba27c..e24f69098c 100644 --- a/source/adios2/engine/ssc/SscReaderNaive.tcc +++ b/source/adios2/engine/ssc/SscReaderNaive.tcc @@ -29,6 +29,7 @@ SscReaderNaive::BlocksInfoCommon(const Variable &variable, const size_t step) const { std::vector::BPInfo> ret; + size_t blockID = 0; auto it = m_BlockMap.find(variable.m_Name); if (it != m_BlockMap.end()) { @@ -42,6 +43,7 @@ SscReaderNaive::BlocksInfoCommon(const Variable &variable, b.Step = m_CurrentStep; b.StepsStart = m_CurrentStep; b.StepsCount = 1; + b.BlockID = blockID; if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { b.IsReverseDims = true; @@ -53,6 +55,7 @@ SscReaderNaive::BlocksInfoCommon(const Variable &variable, std::memcpy(reinterpret_cast(&b.Value), v.value.data(), v.value.size()); } + ++blockID; } } return ret; diff --git a/testing/adios2/engine/ssc/TestSscStruct.cpp b/testing/adios2/engine/ssc/TestSscStruct.cpp index 504a3976ef..e02caf664d 100644 --- a/testing/adios2/engine/ssc/TestSscStruct.cpp +++ b/testing/adios2/engine/ssc/TestSscStruct.cpp @@ -13,6 +13,8 @@ using namespace adios2; int mpiRank = 0; int mpiSize = 1; +int worldRank = 0; +int worldSize = 2; MPI_Comm mpiComm; class SscEngineTest : public ::testing::Test @@ -262,6 +264,14 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, ASSERT_EQ(myParticles[i].b[2], 20); ASSERT_EQ(myParticles[i].b[3], 30); } + auto structBlocks = engine.BlocksInfo(varStruct, currentStep); + ASSERT_EQ(structBlocks.size(), worldSize - mpiSize); + for (size_t i = 0; i < structBlocks.size(); ++i) + { + ASSERT_FALSE(structBlocks[i].IsValue); + ASSERT_FALSE(structBlocks[i].IsReverseDims); + ASSERT_EQ(structBlocks[i].Step, currentStep); + } ASSERT_EQ(i, currentStep); engine.EndStep(); @@ -284,9 +294,6 @@ TEST_F(SscEngineTest, TestSscStruct) { std::string filename = "TestSscStruct"; adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); int mpiGroup = worldRank / (worldSize / 2); MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); MPI_Comm_rank(mpiComm, &mpiRank); @@ -310,9 +317,6 @@ TEST_F(SscEngineTest, TestSscStruct) std::string filename = "TestSscStructNaive"; adios2::Params engineParams = {{"Verbose", "0"}, {"EngineMode", "naive"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); int mpiGroup = worldRank / (worldSize / 2); MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); MPI_Comm_rank(mpiComm, &mpiRank); @@ -337,7 +341,6 @@ TEST_F(SscEngineTest, TestSscStruct) int main(int argc, char **argv) { MPI_Init(&argc, &argv); - int worldRank, worldSize; MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); MPI_Comm_size(MPI_COMM_WORLD, &worldSize); ::testing::InitGoogleTest(&argc, argv); From 6fc75c8a1aa20d2d17e42ff6d852092f7e39da94 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 3 May 2022 18:14:44 -0400 Subject: [PATCH 5/6] fix warnings --- source/adios2/engine/ssc/SscReaderGeneric.cpp | 2 +- source/adios2/engine/ssc/SscReaderGeneric.tcc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp index c94f7346de..1df8cf56bc 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.cpp +++ b/source/adios2/engine/ssc/SscReaderGeneric.cpp @@ -576,7 +576,7 @@ SscReaderGeneric::BlocksInfo(const VariableStruct &variable, { std::vector ret; size_t blockID = 0; - for (size_t i = 0; i < m_GlobalWritePattern.size(); ++i) + for (int i = 0; i < m_GlobalWritePattern.size(); ++i) { for (auto &v : m_GlobalWritePattern[i]) { diff --git a/source/adios2/engine/ssc/SscReaderGeneric.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc index 31df8f6ffc..dcdefc50be 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.tcc +++ b/source/adios2/engine/ssc/SscReaderGeneric.tcc @@ -30,7 +30,7 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, { std::vector::BPInfo> ret; size_t blockID = 0; - for (size_t i = 0; i < m_GlobalWritePattern.size(); ++i) + for (int i = 0; i < m_GlobalWritePattern.size(); ++i) { for (auto &v : m_GlobalWritePattern[i]) { From 7a1096c40133333a8db160fed0fb0058cb956d4a Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 3 May 2022 18:35:15 -0400 Subject: [PATCH 6/6] fix more warnings --- source/adios2/engine/ssc/SscReaderGeneric.cpp | 2 +- source/adios2/engine/ssc/SscReaderGeneric.tcc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp index 1df8cf56bc..4250826d5b 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.cpp +++ b/source/adios2/engine/ssc/SscReaderGeneric.cpp @@ -576,7 +576,7 @@ SscReaderGeneric::BlocksInfo(const VariableStruct &variable, { std::vector ret; size_t blockID = 0; - for (int i = 0; i < m_GlobalWritePattern.size(); ++i) + for (int i = 0; i < static_cast(m_GlobalWritePattern.size()); ++i) { for (auto &v : m_GlobalWritePattern[i]) { diff --git a/source/adios2/engine/ssc/SscReaderGeneric.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc index dcdefc50be..cdaae5a54c 100644 --- a/source/adios2/engine/ssc/SscReaderGeneric.tcc +++ b/source/adios2/engine/ssc/SscReaderGeneric.tcc @@ -30,7 +30,7 @@ SscReaderGeneric::BlocksInfoCommon(const Variable &variable, { std::vector::BPInfo> ret; size_t blockID = 0; - for (int i = 0; i < m_GlobalWritePattern.size(); ++i) + for (int i = 0; i < static_cast(m_GlobalWritePattern.size()); ++i) { for (auto &v : m_GlobalWritePattern[i]) {