Merge branch 'release_29' into b3503-dataman-1D-python-arrays-fin

ornladios · Mar 20, 2023 · 41a7fa7 · 41a7fa7
2 parents a0d4586 + 2ccf622
commit 41a7fa7
Show file tree

Hide file tree

Showing 49 changed files with 710 additions and 78 deletions.
diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml
@@ -376,17 +376,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        code: [examples, lammps, scorpio, tau]
+        code: [examples, lammps, tau]
         include:
           - code: examples
             repo: ornladios/ADIOS2-Examples
             ref: master
           - code: lammps
-            repo: lammps/lammps
-            ref: patch_10Feb2021
-          - code: scorpio
-            repo: E3SM-Project/scorpio
-            ref: scorpio-v1.2.1
+            repo: pnorbert/lammps
+            ref: fix-deprecated-adios-init
           - code: tau
             repo: ornladios/ADIOS2-Examples
             ref: master

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -173,11 +173,19 @@ if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA)
     set(CMAKE_CUDA_STANDARD 11)
     set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
     if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-        # Mininum common non-deprecated architecture
-        set(CMAKE_CUDA_ARCHITECTURES 52)
+        if(DEFINED Kokkos_CUDA_ARCHITECTURES)
+            set(CMAKE_CUDA_ARCHITECTURES Kokkos_CUDA_ARCHITECTURES)
+        else()
+            # Mininum common non-deprecated architecture
+            set(CMAKE_CUDA_ARCHITECTURES 52)
+        endif()
     endif()
 endif()
 
+if(NOT DEFINED CMAKE_HIP_ARCHITECTURES AND DEFINED Kokkos_HIP_ARCHITECTURES)
+    set(CMAKE_HIP_ARCHITECTURES Kokkos_HIP_ARCHITECTURES)
+endif()
+
 if(ADIOS2_HAVE_MPI)
   if(MPIEXEC_MAX_NUMPROCS LESS 4 AND "$ENV{OMPI_MCA_rmaps_base_oversubscribe}")
     message(STATUS "OpenMPI oversubscribe detected: raising MPIEXEC_MAX_NUMPROCS to 4 for testing")

diff --git a/bindings/Fortran/modules/adios2_parameters_mod.f90 b/bindings/Fortran/modules/adios2_parameters_mod.f90
@@ -55,6 +55,8 @@ module adios2_parameters_mod
     integer, parameter :: adios2_mode_write = 1
     integer, parameter :: adios2_mode_read = 2
     integer, parameter :: adios2_mode_append = 3
+    integer, parameter :: adios2_mode_readRandomAccess = 6
+
     integer, parameter :: adios2_mode_deferred = 4
     integer, parameter :: adios2_mode_sync = 5
 

diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp
@@ -129,6 +129,12 @@ void Engine::Get(Variable variable, pybind11::array &array, const Mode launch)
 #define declare_type(T)                                                        \
     else if (type == helper::GetDataType<T>())                                 \
     {                                                                          \
+        if (!array.dtype().is(pybind11::dtype::of<T>()))                       \
+        {                                                                      \
+            throw std::invalid_argument(                                       \
+                "In ADIOS2 Get - Type mismatch between Python buffer and "     \
+                "incoming data.");                                             \
+        }                                                                      \
         m_Engine->Get(                                                         \
             *dynamic_cast<core::Variable<T> *>(variable.m_VariableBase),       \
             reinterpret_cast<T *>(const_cast<void *>(array.data())), launch);  \

diff --git a/docs/README.md b/docs/README.md
@@ -4,7 +4,7 @@
 
 This user guide is hosted in readthedocs: https://adios2.readthedocs.io/en/latest/
 
-To generate the User Guide under docs/user_guide/build/html format from the Sphinx source files:
+To generate the User Guide under docs/user_guide/build/html format from the Sphinx source files, using pip:
 
 ```bash
 $ cd ADIOS2/docs
@@ -14,3 +14,23 @@ docs$ pip3 install -r requirements.txt
 user_guide$ cd user_guide
 user_guide$ make html
 ```
+
+Or using conda:
+
+```bash
+$ cd ADIOS2/docs
+docs$ conda env create -f environment.yml
+docs$ conda activate adios2-python-docs
+user_guide$ cd user_guide
+user_guide$ make html
+```
+
+# Updating dependencies
+
+Read the Docs uses only the environment.yml file, so if you make changes to dependencies in requirements.txt, they will not take effect in RTD.
+The requirements.txt is provided only for building locally for those who don't have conda installed.
+If you make changes to the conda environment, you should update requirements.txt as well, either manually, or by running the following command (with the conda adios2-python-docs environment activated):
+
+```bash
+docs$ pip list --format=freeze > requirements.txt
+```
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,44 @@
+alabaster==0.7.12
+Babel==2.11.0
+blockdiag==3.0.0
+breathe==4.33.0
+brotlipy==0.7.0
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==2.1.1
+colorama==0.4.6
+cryptography==39.0.1
+docutils==0.17
+funcparserlib==1.0.1
+idna==3.4
+imagesize==1.4.1
+importlib-metadata==4.11.4
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+mpi4py==3.1.3
+numpy==1.22.0
+packaging==22.0
+Pillow==9.4.0
+pip==22.3.1
+pycparser==2.21
+Pygments==2.14.0
+pyOpenSSL==23.0.0
+PySocks==1.7.1
+pytz==2022.7
+requests==2.28.1
+setuptools==65.6.3
+snowballstemmer==2.2.0
+Sphinx==4.5.0
+sphinx-rtd-theme==1.0.0
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-blockdiag==3.0.0
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+typing_extensions==4.4.0
+urllib3==1.26.13
+webcolors==1.12
+wheel==0.38.4
+zipp==3.11.0
diff --git a/docs/user_guide/source/advanced/plugins.rst b/docs/user_guide/source/advanced/plugins.rst
@@ -131,6 +131,35 @@ If ``ADIOS2_PLUGIN_PATH`` is not set, and a path is not specified when loading y
 .. note::
     The ``ADIOS2_PLUGIN_PATH`` environment variable can contain multiple paths, which must be separated with a ``:``.
 
+
+When building on Windows, you will likely need to explicitly export the Create and Destroy symbols for your plugin, as symbols are invisible by default on Windows.
+To do this in a portable way across platforms, you can add something similar to the following lines to your CMakeLists.txt:
+
+.. code-block:: cmake
+
+    include(GenerateExportHeader)
+    generate_export_header(PluginEngineWrite BASE_NAME plugin_engine_write)
+    target_include_directories(PluginEngineWrite PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+        $<INSTALL_INTERFACE:include>)
+
+
+Then in your plugin header, you'll need to ``#include "plugin_engine_write_export.h"``. Then edit your function defintions as follows:
+
+.. code-block:: c++
+
+    extern "C" {
+
+    PLUGIN_ENGINE_WRITE_EXPORT adios2::plugin::ExampleWritePlugin *
+        EngineCreate(adios2::core::IO &io, const std::string &name,
+                 const adios2::Mode mode, adios2::helper::Comm comm);
+
+    PLUGIN_ENGINE_WRITE_EXPORT void
+        EngineDestroy(adios2::plugin::ExampleWritePlugin * obj);
+
+    }
+
+
 ***********************************
 Using Your Plugin in an Application
 ***********************************

diff --git a/docs/user_guide/source/api_full/fortran.rst b/docs/user_guide/source/api_full/fortran.rst
@@ -552,7 +552,7 @@ ADIOS2 Fortran bindings handlers are mapped 1-to-1 to the ADIOS2 components desc
 
    .. code-block:: fortran
    
-      subroutine adios2_remove_variable(io, ierr)
+      subroutine adios2_remove_all_variables(io, ierr)
         
       ! WHERE:
       

diff --git a/docs/user_guide/source/engines/sst.rst b/docs/user_guide/source/engines/sst.rst
@@ -291,6 +291,7 @@ BeginStep timeouts) and writer-side rules (like queue limit behavior) apply.
  DataTransport                   string              **default varies by platform**, UCX, MPI, RDMA, WAN
  WANDataTransport                string              **sockets**, enet, ib
  ControlTransport                string              **TCP**, Scalable
+ MarshalMethod                   string              **BP5**, BP, FFS
  NetworkInterface                string              **NULL**
  ControlInterface                string              **NULL**
  DataInterface                   string              **NULL**

diff --git a/examples/plugins/engine/CMakeLists.txt b/examples/plugins/engine/CMakeLists.txt
@@ -3,15 +3,26 @@
 # accompanying file Copyright.txt for details.
 #------------------------------------------------------------------------------#
 
+include(GenerateExportHeader)
 add_library(PluginEngineWrite
   ExampleWritePlugin.cpp
 )
 target_link_libraries(PluginEngineWrite adios2::cxx11 adios2_core)
+generate_export_header(PluginEngineWrite BASE_NAME plugin_engine_write)
+target_include_directories(PluginEngineWrite PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:include>
+)
 
 add_library(PluginEngineRead
   ExampleReadPlugin.cpp
 )
 target_link_libraries(PluginEngineRead adios2::cxx11 adios2_core)
+generate_export_header(PluginEngineRead BASE_NAME plugin_engine_read)
+target_include_directories(PluginEngineRead PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+  $<INSTALL_INTERFACE:include>
+)
 
 add_executable(examplePluginEngine_write
   examplePluginEngine_write.cpp

diff --git a/examples/plugins/engine/ExampleReadPlugin.h b/examples/plugins/engine/ExampleReadPlugin.h
@@ -15,6 +15,8 @@
 #ifndef EXAMPLEREADPLUGIN_H_
 #define EXAMPLEREADPLUGIN_H_
 
+#include "plugin_engine_read_export.h"
+
 #include <fstream>
 #include <string>
 
@@ -80,11 +82,11 @@ class ExampleReadPlugin : public PluginEngineInterface
 
 extern "C" {
 
-adios2::plugin::ExampleReadPlugin *EngineCreate(adios2::core::IO &io,
-                                                const std::string &name,
-                                                const adios2::Mode mode,
-                                                adios2::helper::Comm comm);
-void EngineDestroy(adios2::plugin::ExampleReadPlugin *obj);
+PLUGIN_ENGINE_READ_EXPORT adios2::plugin::ExampleReadPlugin *
+EngineCreate(adios2::core::IO &io, const std::string &name,
+             const adios2::Mode mode, adios2::helper::Comm comm);
+PLUGIN_ENGINE_READ_EXPORT void
+EngineDestroy(adios2::plugin::ExampleReadPlugin *obj);
 }
 
 #endif /* EXAMPLEREADPLUGIN_H_ */
diff --git a/examples/plugins/engine/ExampleWritePlugin.h b/examples/plugins/engine/ExampleWritePlugin.h
@@ -14,6 +14,8 @@
 #ifndef EXAMPLEWRITEPLUGIN_H_
 #define EXAMPLEWRITEPLUGIN_H_
 
+#include "plugin_engine_write_export.h"
+
 #include <fstream>
 #include <memory>
 #include <string>
@@ -81,11 +83,11 @@ class ExampleWritePlugin : public PluginEngineInterface
 
 extern "C" {
 
-adios2::plugin::ExampleWritePlugin *EngineCreate(adios2::core::IO &io,
-                                                 const std::string &name,
-                                                 const adios2::Mode mode,
-                                                 adios2::helper::Comm comm);
-void EngineDestroy(adios2::plugin::ExampleWritePlugin *obj);
+PLUGIN_ENGINE_WRITE_EXPORT adios2::plugin::ExampleWritePlugin *
+EngineCreate(adios2::core::IO &io, const std::string &name,
+             const adios2::Mode mode, adios2::helper::Comm comm);
+PLUGIN_ENGINE_WRITE_EXPORT void
+EngineDestroy(adios2::plugin::ExampleWritePlugin *obj);
 }
 
 #endif /* EXAMPLEWRITEPLUGIN_H_ */
diff --git a/examples/plugins/engine/examplePluginEngine_read.cpp b/examples/plugins/engine/examplePluginEngine_read.cpp
@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
 
     std::vector<float> myFloats = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
 
+    bool success = false;
     try
     {
         /** ADIOS class factory of IO class objects */
@@ -76,6 +77,7 @@ int main(int argc, char *argv[])
             adios2::Params params;
             params["PluginName"] = "ReadPlugin";
             params["PluginLibrary"] = "PluginEngineRead";
+            params["verbose"] = "5";
             io.SetParameters(params);
         }
         adios2::Engine reader = io.Open("TestPlugin", adios2::Mode::Read);
@@ -108,6 +110,7 @@ int main(int argc, char *argv[])
 
         /** Engine becomes unreachable after this*/
         reader.Close();
+        success = true;
     }
     catch (std::invalid_argument &e)
     {
@@ -125,5 +128,5 @@ int main(int argc, char *argv[])
         std::cout << e.what() << "\n";
     }
 
-    return 0;
+    return success ? EXIT_SUCCESS : EXIT_FAILURE;
 }
diff --git a/examples/plugins/engine/examplePluginEngine_write.cpp b/examples/plugins/engine/examplePluginEngine_write.cpp
@@ -52,6 +52,7 @@ int main(int argc, char *argv[])
     std::vector<float> myFloats = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
     const std::size_t Nx = myFloats.size();
 
+    bool success = false;
     try
     {
         /** ADIOS class factory of IO class objects */
@@ -72,6 +73,7 @@ int main(int argc, char *argv[])
             adios2::Params params;
             params["PluginName"] = "WritePlugin";
             params["PluginLibrary"] = "PluginEngineWrite";
+            params["verbose"] = "5";
             io.SetParameters(params);
         }
         adios2::Engine writer = io.Open("TestPlugin", adios2::Mode::Write);
@@ -83,10 +85,12 @@ int main(int argc, char *argv[])
         else
         {
             writer.Put<float>(var, myFloats.data());
+            writer.PerformPuts();
         }
 
         /** Engine becomes unreachable after this*/
         writer.Close();
+        success = true;
     }
     catch (std::invalid_argument &e)
     {
@@ -104,5 +108,5 @@ int main(int argc, char *argv[])
         std::cout << e.what() << "\n";
     }
 
-    return 0;
+    return success ? EXIT_SUCCESS : EXIT_FAILURE;
 }
diff --git a/examples/plugins/operator/examplePluginOperator_read.cpp b/examples/plugins/operator/examplePluginOperator_read.cpp
@@ -40,6 +40,7 @@ int main(int argc, char *argv[])
         0.0001,
     };
 
+    bool success = false;
     try
     {
         /** ADIOS class factory of IO class objects */
@@ -85,6 +86,7 @@ int main(int argc, char *argv[])
 
         /** Engine becomes unreachable after this*/
         reader.Close();
+        success = true;
     }
     catch (std::invalid_argument &e)
     {
@@ -102,5 +104,5 @@ int main(int argc, char *argv[])
         std::cout << e.what() << "\n";
     }
 
-    return 0;
+    return success ? EXIT_SUCCESS : EXIT_FAILURE;
 }
diff --git a/examples/plugins/operator/examplePluginOperator_write.cpp b/examples/plugins/operator/examplePluginOperator_write.cpp
@@ -41,6 +41,7 @@ int main(int argc, char *argv[])
     };
     const std::size_t Nx = myDoubles.size();
 
+    bool success = false;
     try
     {
         /** ADIOS class factory of IO class objects */
@@ -73,6 +74,7 @@ int main(int argc, char *argv[])
 
         /** Engine becomes unreachable after this*/
         writer.Close();
+        success = true;
     }
     catch (std::invalid_argument &e)
     {
@@ -90,5 +92,5 @@ int main(int argc, char *argv[])
         std::cout << e.what() << "\n";
     }
 
-    return 0;
+    return success ? EXIT_SUCCESS : EXIT_FAILURE;
 }
diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp
@@ -98,6 +98,8 @@ Dims Operator::ConvertDims(const Dims &dimensions, const DataType type,
     return ret;
 }
 
+size_t Operator::GetHeaderSize() const { return 0; }
+
 // PRIVATE
 void Operator::CheckCallbackType(const std::string type) const
 {