python: rewrite Python high level API in python

ornladios · Dec 26, 2023 · 433f0ff · 433f0ff
1 parent d650742
commit 433f0ff
Show file tree

Hide file tree

Showing 75 changed files with 3,258 additions and 2,296 deletions.
diff --git a/flake8.cfg → .flake8 b/flake8.cfg → .flake8
diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml
@@ -90,9 +90,15 @@ jobs:
       - name: CXX
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-clang-format.sh
-      - name: Python
+      - name: Python flake8
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-flake8.sh
+      - name: Python pylint
+        working-directory: source
+        run: ../gha/scripts/ci/scripts/run-pylint.sh
+      - name: Python black
+        working-directory: source
+        run: ../gha/scripts/ci/scripts/run-black.sh
       - name: Shell
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-shellcheck.sh

diff --git a/.gitignore b/.gitignore
@@ -40,3 +40,7 @@ docs/pyvenv.cfg
 # Visual Studio
 .vs/
 CMakeSettings.json
+
+# Python wheels stuff
+
+*.egg-info/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -328,6 +328,13 @@ add_subdirectory(source)
 #------------------------------------------------------------------------------#
 add_subdirectory(bindings)
 
+#------------------------------------------------------------------------------#
+# Language Libraries
+#------------------------------------------------------------------------------#
+if(ADIOS2_HAVE_Python)
+  add_subdirectory(python)
+endif()
+
 #------------------------------------------------------------------------------#
 # Plugins
 #------------------------------------------------------------------------------#
@@ -341,6 +348,7 @@ if(ADIOS2_BUILD_EXAMPLES)
   add_subdirectory(examples)
 endif()
 
+
 #------------------------------------------------------------------------------#
 # Testing
 #------------------------------------------------------------------------------#

diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt
@@ -7,14 +7,12 @@ Python_add_library(adios2_py MODULE
   py11Engine.cpp
   py11Operator.cpp
   py11Query.cpp
-  py11File.cpp py11File.tcc
   py11glue.cpp
 )
-target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2${ADIOS2_LIBRARY_SUFFIX}")
+target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2_bindings${ADIOS2_LIBRARY_SUFFIX}")
 if(ADIOS2_HAVE_MPI)
   target_sources(adios2_py PRIVATE
     py11ADIOSMPI.cpp
-    py11FileMPI.cpp
     py11IOMPI.cpp
   )
   set(maybe_adios2_cxx11_mpi adios2_cxx11_mpi)
@@ -29,44 +27,42 @@ target_link_libraries(adios2_py PRIVATE
   ${maybe_adios2_cxx11_mpi} adios2_cxx11
   ${maybe_adios2_core_mpi} adios2_core
   adios2::thirdparty::pybind11
-  ${maybe_mpi4py} Python::NumPy
+  ${maybe_mpi4py}
   )
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in
-  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
+  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
   @ONLY
 )
 
 set_target_properties(adios2_py PROPERTIES
   CXX_VISIBILITY_PRESET hidden
-  OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}
-  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
+  OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX}
+  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
 )
 
-string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2")
-if(CMAKE_SYSTEM_NAME MATCHES "Linux")
-  if (NOT ADIOS2_USE_PIP)
-    set_target_properties(adios2_py PROPERTIES
-      INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
-    )
-  endif()
-endif()
-
-set(install_location ${CMAKE_INSTALL_PYTHONDIR})
+set(install_location "${CMAKE_INSTALL_PYTHONDIR}/adios2")
 if (ADIOS2_USE_PIP)
   set(install_location ${CMAKE_INSTALL_LIBDIR})
 endif()
 
+string(REGEX REPLACE "[^/]+" ".." relative_base "${install_location}/bindings")
+if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+  set_property(TARGET adios2_py APPEND PROPERTY
+    INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
+  )
+endif()
+
 install(TARGETS adios2_py
-  DESTINATION ${install_location}
+  DESTINATION ${install_location}/bindings
   COMPONENT adios2_python-python
 )
-install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
-  DESTINATION ${install_location}
+install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
+  DESTINATION ${install_location}/bindings
   COMPONENT adios2_python-python
 )
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test

diff --git a/bindings/Python/__init__.py.in b/bindings/Python/__init__.py.in
@@ -1,3 +1,3 @@
-from .adios2@ADIOS2_LIBRARY_SUFFIX@ import *
+from .adios2_bindings@ADIOS2_LIBRARY_SUFFIX@ import *
 
 __version__ = "@ADIOS2_VERSION@"
diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp
@@ -171,6 +171,13 @@ void Engine::EndStep()
     m_Engine->EndStep();
 }
 
+
+bool Engine::BetweenStepPairs() const
+{
+    helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::EndStep");
+    return m_Engine->BetweenStepPairs();
+}
+
 void Engine::Flush(const int transportIndex)
 {
     helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Flush");

diff --git a/bindings/Python/py11Engine.h b/bindings/Python/py11Engine.h
@@ -60,6 +60,12 @@ class Engine
 
     void EndStep();
 
+    /**
+     * Returns current status information for each engine.
+     * @return if between BeginStep/EndStep() pair
+     */
+    bool BetweenStepPairs() const;
+
     void Flush(const int transportIndex = -1);
 
     void Close(const int transportIndex = -1);