added support to query BP5 files (#3809)

* added support to query BP5 files * size_type fix * updated python test and added to ctest * moved QueryWorker to the beginstep/endstep loop because bp5 metadata loading pattern deprecate bp3 engine test and replaced with bp5 engine test * clang-format * clang-format * removed some comments * flake8 * correct previous commit. Wrong files changed * furthur formatting * removed comments * added new doc file * Polished query.rst * more touch ups * more clean ups * subblock calculated by the helper function is relative to block so added block start before return to user * clang-format fix * clang-format fix
ornladios · Sep 20, 2023 · d5dcb98 · d5dcb98
1 parent 1487ad8
commit d5dcb98
Show file tree

Hide file tree

Showing 11 changed files with 299 additions and 145 deletions.
diff --git a/bindings/CXX11/adios2/cxx11/Query.cpp b/bindings/CXX11/adios2/cxx11/Query.cpp
@@ -14,7 +14,13 @@ QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader)
     delete m;
 }
 
-void QueryWorker::GetResultCoverage(adios2::Box<adios2::Dims> &outputSelection,
+void QueryWorker::GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks)
+{
+    adios2::Box<adios2::Dims> empty;
+    GetResultCoverage(empty, touched_blocks);
+}
+
+void QueryWorker::GetResultCoverage(const adios2::Box<adios2::Dims> &outputSelection,
                                     std::vector<adios2::Box<adios2::Dims>> &touched_blocks)
 {
     if (m_Worker)

diff --git a/bindings/CXX11/adios2/cxx11/Query.h b/bindings/CXX11/adios2/cxx11/Query.h
@@ -26,9 +26,15 @@ class Worker;
 class QueryWorker
 {
 public:
+    // configFile has query, can be either xml or json
     QueryWorker(const std::string &configFile, adios2::Engine &engine);
 
-    void GetResultCoverage(adios2::Box<adios2::Dims> &,
+    // touched_blocks is a list of regions specified by (start, count),
+    // that contains data that satisfies the query file
+    void GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks);
+
+    // supply output bound for the results
+    void GetResultCoverage(const adios2::Box<adios2::Dims> &,
                            std::vector<adios2::Box<adios2::Dims>> &touched_blocks);
 
 private:

diff --git a/bindings/Python/py11Query.cpp b/bindings/Python/py11Query.cpp
@@ -30,9 +30,8 @@ Query::operator bool() const noexcept { return (m_QueryWorker == nullptr) ? fals
 
 std::vector<Box<Dims>> Query::GetResult()
 {
-    // std::cout<<"Do something"<<std::endl;
-    adios2::Box<adios2::Dims> empty; // look into all data
     std::vector<Box<Dims>> touched_blocks;
+    adios2::Box<adios2::Dims> empty;
     m_QueryWorker->GetResultCoverage(empty, touched_blocks);
     return touched_blocks;
 }

diff --git a/bindings/Python/py11Query.h b/bindings/Python/py11Query.h
@@ -34,8 +34,6 @@ class Query
     explicit operator bool() const noexcept;
 
     std::vector<Box<Dims>> GetResult();
-    // const Box< Dims > & refinedSelectionIfAny,
-    // std::vector< Box< Dims > > &touched_blocks
 
 private:
     Query(adios2::query::Worker *qw);

diff --git a/docs/user_guide/source/advanced/query.rst b/docs/user_guide/source/advanced/query.rst
@@ -0,0 +1,93 @@
+#################
+ADIOS2 query API
+#################
+
+The query API in ADIOS2 allows a client to pass a query in XML or json format,
+and get back a list of blocks or subblocks that contains hits. 
+Both BP4 and BP5 engines are supported.  
+
+
+The interface
+=============
+User is expected to pass a query file (configFile), and init a read engine (engine)
+to construct a query and evaluate using the engine.
+(note that the engine and query should be using the same ADIOS IO)
+
+.. code-block:: c++
+
+    class QueryWorker
+    {
+    public:
+        // configFile has query, can be either xml or json
+        QueryWorker(const std::string &configFile, adios2::Engine &engine);
+
+	// touched_blocks is a list of regions specified by (start, count),
+	// that contains data that satisfies the query file
+        void GetResultCoverage(std::vector<adios2::Box<adios2::Dims>> &touched_blocks);
+    ... 
+    }
+
+A Sample Compound Query  
+----------------------
+
+This query targets a 1D variable "doubleV", data of interest is (x  > 6.6) or (x < -0.17) or (2.8 < x < 2.9) 
+In addition, this query also specied an output region [start=5,count=80]. 
+
+
+.. code-block:: xml
+
+	<adios-query>
+  	  <io name="query">
+   	  <var name="doubleV">
+      	    <boundingbox  start="5" count="80"/>
+            <op value="OR">
+              <range  compare="GT" value="6.6"/>
+              <range  compare="LT" value="-0.17"/>
+              <op value="AND">
+                 <range  compare="LT" value="2.9"/>
+                 <range  compare="GT" value="2.8"/>
+              </op>
+            </op>
+          </var>
+          </io>
+        </adios-query>
+		
+
+Code EXAMPLES:
+==============
+C++:
+----
+.. code-block:: c++
+
+    while (reader.BeginStep() == adios2::StepStatus::OK)
+    {
+        adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader);
+        w.GetResultCoverage(touched_blocks);
+
+        std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size()
+                  << std::endl;
+    }
+
+
+The Full C++ example is here:
+    https://github.com/ornladios/ADIOS2/blob/master/examples/query/test.cpp
+
+
+Python:
+-------
+
+.. code-block:: python
+	
+	while (reader.BeginStep() == adios2.StepStatus.OK):
+        # say only rank 0 wants to process result
+        var = [queryIO.InquireVariable("T")]
+
+        if (rank == 0):
+            touched_blocks = w.GetResult()
+            doAnalysis(reader, touched_blocks, var)
+
+Full python example is here:
+	https://github.com/ornladios/ADIOS2/blob/master/testing/adios2/bindings/python/TestQuery.py
+
+	This example generates data, the query file (in xml) and runs the query, all in python.
+
diff --git a/examples/query/test.cpp b/examples/query/test.cpp
@@ -12,6 +12,36 @@
 
 // #include "adios2/toolkit/query/Worker.h"
 
+void queryWithStreaming(adios2::IO &queryIO, std::string &dataFileName, std::string &queryFile)
+{
+    adios2::Engine reader = queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD);
+    // adios2::QueryWorker* worker = NULL;
+    queryIO.SetParameter("StreamReader", "true");
+    std::vector<adios2::Box<adios2::Dims>> touched_blocks;
+
+    while (reader.BeginStep() == adios2::StepStatus::OK)
+    {
+        adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader);
+        w.GetResultCoverage(touched_blocks);
+
+        std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size()
+                  << std::endl;
+        for (auto n : touched_blocks)
+        {
+            std::ostringstream startStr;
+            std::ostringstream countStr;
+            for (size_t k = 0; k < n.first.size(); k++)
+            {
+                startStr << n.first[k] << " ";
+                countStr << n.second[k] << " ";
+            }
+            std::cout << "\t[" << startStr.str() << "]  [" << countStr.str() << "]" << std::endl;
+        }
+        reader.EndStep();
+    }
+    reader.Close();
+}
+
 int main(int argc, char *argv[])
 {
     int provided;
@@ -47,43 +77,16 @@ int main(int argc, char *argv[])
         adios2::ADIOS ad = adios2::ADIOS(configFileName, MPI_COMM_WORLD);
 
         adios2::IO queryIO = ad.DeclareIO("query");
-        adios2::Engine reader = queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD);
-#ifdef NEVER
-        adios2::QueryWorker w = adios2::QueryWorker(configFileName, reader);
-#else
+
         std::string queryFile = configFileName;
         if (argc > 3)
         {
             queryFile = argv[3];
         }
         std::cout << "Testing query file  ..." << queryFile << std::endl;
-        adios2::QueryWorker w = adios2::QueryWorker(queryFile, reader);
-#endif
-        std::vector<adios2::Box<adios2::Dims>> touched_blocks;
 
-        while (reader.BeginStep() == adios2::StepStatus::OK)
-        {
-            adios2::Box<adios2::Dims> empty;
-            w.GetResultCoverage(empty, touched_blocks);
-            // adios2::Box<adios2::Dims> tt({10,10}, {12,12});
-            // w.GetResultCoverage(tt, touched_blocks);
-            std::cout << " ... now can read out touched blocks ... size=" << touched_blocks.size()
-                      << std::endl;
-            for (auto n : touched_blocks)
-            {
-                std::ostringstream startStr;
-                std::ostringstream countStr;
-                for (size_t k = 0; k < n.first.size(); k++)
-                {
-                    startStr << n.first[k] << " ";
-                    countStr << n.second[k] << " ";
-                }
-                std::cout << "\t[" << startStr.str() << "]  [" << countStr.str() << "]"
-                          << std::endl;
-            }
-            reader.EndStep();
-        }
-        reader.Close();
+        queryWithStreaming(queryIO, dataFileName, queryFile);
+
         return 0;
     }
     catch (std::exception &e)

diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h
@@ -22,28 +22,67 @@ class BlockIndex
     };
 
 public:
-    BlockIndex<T>(adios2::core::Variable<T> &var, adios2::core::IO &io,
+    BlockIndex<T>(adios2::core::Variable<T> *var, adios2::core::IO &io,
                   adios2::core::Engine &reader)
-    : m_Var(var), m_IdxIO(io), m_IdxReader(reader)
+    : m_VarPtr(var), m_IdxIO(io), m_IdxReader(reader)
     {
     }
 
     void Generate(std::string &fromBPFile, const adios2::Params &inputs) {}
 
     void Evaluate(const QueryVar &query, std::vector<adios2::Box<adios2::Dims>> &resultSubBlocks)
     {
-        RunBP4Stat(query, resultSubBlocks);
+        if (m_IdxReader.m_EngineType.find("5") != std::string::npos) // a bp5 reader
+            RunBP5Stat(query, resultSubBlocks);
+        else
+            RunBP4Stat(query, resultSubBlocks);
+    }
+
+    void RunBP5Stat(const QueryVar &query, std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
+    {
+        size_t currStep = m_IdxReader.CurrentStep();
+        adios2::Dims currShape = m_VarPtr->Shape();
+        if (!query.IsSelectionValid(currShape))
+            return;
+
+        auto MinBlocksInfo = m_IdxReader.MinBlocksInfo(*m_VarPtr, currStep);
+        if (!MinBlocksInfo)
+        { // no info, can't do anything
+            return;
+        }
+        for (auto &blockInfo : MinBlocksInfo->BlocksInfo)
+        {
+            Dims ss(MinBlocksInfo->Dims);
+            Dims cc(MinBlocksInfo->Dims);
+            for (std::vector<int>::size_type i = 0; i < ss.size(); i++)
+            {
+                ss[i] = blockInfo.Start[i];
+                cc[i] = blockInfo.Count[i];
+            }
+            if (!query.TouchSelection(ss, cc))
+                continue;
+
+            T bmin = *(T *)&blockInfo.MinMax.MinUnion;
+            T bmax = *(T *)&blockInfo.MinMax.MaxUnion;
+            bool isHit = query.m_RangeTree.CheckInterval(bmin, bmax);
+            if (isHit)
+            {
+                adios2::Box<adios2::Dims> box = {ss, cc};
+                hitBlocks.push_back(box);
+            }
+        }
+        delete MinBlocksInfo;
     }
 
     void RunBP4Stat(const QueryVar &query, std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
     {
         size_t currStep = m_IdxReader.CurrentStep();
-        adios2::Dims currShape = m_Var.Shape();
+        adios2::Dims currShape = m_VarPtr->Shape();
         if (!query.IsSelectionValid(currShape))
             return;
 
         std::vector<typename adios2::core::Variable<T>::BPInfo> varBlocksInfo =
-            m_IdxReader.BlocksInfo(m_Var, currStep);
+            m_IdxReader.BlocksInfo(*m_VarPtr, currStep);
 
         for (auto &blockInfo : varBlocksInfo)
         {
@@ -62,6 +101,10 @@ class BlockIndex
                     {
                         adios2::Box<adios2::Dims> currSubBlock =
                             adios2::helper::GetSubBlock(blockInfo.Count, blockInfo.SubBlockInfo, i);
+                        for (size_t d = 0; d < blockInfo.Count.size(); ++d)
+                        {
+                            currSubBlock.first[d] += blockInfo.Start[d];
+                        }
                         if (!query.TouchSelection(currSubBlock.first, currSubBlock.second))
                             continue;
                         hitBlocks.push_back(currSubBlock);
@@ -80,40 +123,12 @@ class BlockIndex
         }
     }
 
-    /*
-    void RunDefaultBPStat(const QueryVar &query,
-                          std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
-    {
-        size_t currStep = m_IdxReader.CurrentStep();
-        adios2::Dims currShape = m_Var.Shape();
-        if (!query.IsSelectionValid(currShape))
-            return;
-
-        std::vector<typename adios2::core::Variable<T>::BPInfo> varBlocksInfo =
-            m_IdxReader.BlocksInfo(m_Var, currStep);
-
-        for (auto &blockInfo : varBlocksInfo)
-        {
-            if (!query.TouchSelection(blockInfo.Start, blockInfo.Count))
-                continue;
-
-            T min = blockInfo.Min;
-            T max = blockInfo.Max;
-
-            // std::cout<<" min: "<<min<<"  max: "<<max<<std::endl;
-            bool isHit = query.m_RangeTree.CheckInterval(min, max);
-            if (isHit)
-            {
-                adios2::Box<adios2::Dims> box = {blockInfo.Start,
-                                                 blockInfo.Count};
-                hitBlocks.push_back(box);
-            }
-        }
-    }
-    */
-
     Tree m_Content;
-    adios2::core::Variable<T> m_Var;
+
+    // can not be unique_ptr as it changes with bp5 through steps
+    // as BP5Deserializer::SetupForStep calls io.RemoveVariables()
+    // must use ptr as bp5 associates ptrs with blockinfo, see MinBlocksInfo() in bp5
+    adios2::core::Variable<T> *m_VarPtr;
 
 private:
     //

diff --git a/source/adios2/toolkit/query/Query.cpp b/source/adios2/toolkit/query/Query.cpp
@@ -310,7 +310,7 @@ void QueryVar::BlockIndexEvaluate(adios2::core::IO &io, adios2::core::Engine &re
     if (varType == adios2::helper::GetDataType<T>())                                               \
     {                                                                                              \
         core::Variable<T> *var = io.InquireVariable<T>(m_VarName);                                 \
-        BlockIndex<T> idx(*var, io, reader);                                                       \
+        BlockIndex<T> idx(var, io, reader);                                                        \
         idx.Evaluate(*this, touchedBlocks);                                                        \
     }
     // ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_type) //skip complex types

diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt
@@ -25,6 +25,7 @@ if(ADIOS2_HAVE_MPI)
   add_python_mpi_test(BPBlocksInfo)
   add_python_mpi_test(BPChangingShapeHighLevelAPI)
   add_python_mpi_test(NullEngine)
+  add_python_mpi_test(Query)	
   # Currently hangs in H5Fclose for unknown reasons
   #if(ADIOS2_HAVE_HDF5)
   #  add_python_mpi_test(BPWriteTypesHighLevelAPI_HDF5)