From 5e6831f2e0d5b27cf0d6ac1e09e497aec94f04fb Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 21 Nov 2023 19:44:20 -0500 Subject: [PATCH] python: rewrite Python high level API in python --- flake8.cfg => .flake8 | 0 .gitignore | 4 + CMakeLists.txt | 2 + bindings/Python/CMakeLists.txt | 26 +- bindings/Python/__init__.py.in | 2 +- bindings/Python/py11Variable.cpp | 6 + bindings/Python/py11Variable.h | 2 + bindings/Python/py11glue.cpp | 35 +-- cmake/DetectOptions.cmake | 4 +- pyproject.toml | 43 ++++ python/CMakeLists.txt | 13 + python/adios2/__init__.py | 3 + python/adios2/adios.py | 36 +++ python/adios2/attribute.py | 20 ++ python/adios2/engine.py | 66 +++++ python/adios2/file.py | 325 +++++++++++++++++++++++++ python/adios2/io.py | 110 +++++++++ python/adios2/operator.py | 12 + python/adios2/variable.py | 65 +++++ scripts/ci/scripts/run-flake8.sh | 2 +- testing/adios2/CMakeLists.txt | 4 + testing/adios2/python/CMakeLists.txt | 12 + testing/adios2/python/TestADIOS.py | 56 +++++ testing/adios2/python/TestAttribute.py | 34 +++ testing/adios2/python/TestEngine.py | 91 +++++++ testing/adios2/python/TestFile.py | 21 ++ testing/adios2/python/TestIO.py | 102 ++++++++ testing/adios2/python/TestOperator.py | 38 +++ testing/adios2/python/TestVariable.py | 66 +++++ 29 files changed, 1166 insertions(+), 34 deletions(-) rename flake8.cfg => .flake8 (100%) create mode 100644 pyproject.toml create mode 100644 python/CMakeLists.txt create mode 100644 python/adios2/__init__.py create mode 100644 python/adios2/adios.py create mode 100644 python/adios2/attribute.py create mode 100644 python/adios2/engine.py create mode 100644 python/adios2/file.py create mode 100644 python/adios2/io.py create mode 100644 python/adios2/operator.py create mode 100644 python/adios2/variable.py create mode 100644 testing/adios2/python/CMakeLists.txt create mode 100644 testing/adios2/python/TestADIOS.py create mode 100644 testing/adios2/python/TestAttribute.py create mode 100644 testing/adios2/python/TestEngine.py create mode 100644 testing/adios2/python/TestFile.py create mode 100644 testing/adios2/python/TestIO.py create mode 100644 testing/adios2/python/TestOperator.py create mode 100644 testing/adios2/python/TestVariable.py diff --git a/flake8.cfg b/.flake8 similarity index 100% rename from flake8.cfg rename to .flake8 diff --git a/.gitignore b/.gitignore index 81ff18108b..35a8d5ab87 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,7 @@ docs/pyvenv.cfg # Visual Studio .vs/ CMakeSettings.json + +# Python wheels stuff + +*.egg-info/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a1b21d87c..f662fc4aee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -326,6 +326,8 @@ if(ADIOS2_BUILD_EXAMPLES) add_subdirectory(examples) endif() +add_subdirectory(python) + #------------------------------------------------------------------------------# # Testing #------------------------------------------------------------------------------# diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 5c6121e8f6..423a805995 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -10,7 +10,7 @@ Python_add_library(adios2_py MODULE py11File.cpp py11File.tcc py11glue.cpp ) -target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2${ADIOS2_LIBRARY_SUFFIX}") +target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2_bindings${ADIOS2_LIBRARY_SUFFIX}") if(ADIOS2_HAVE_MPI) target_sources(adios2_py PRIVATE py11ADIOSMPI.cpp @@ -29,25 +29,25 @@ target_link_libraries(adios2_py PRIVATE ${maybe_adios2_cxx11_mpi} adios2_cxx11 ${maybe_adios2_core_mpi} adios2_core adios2::thirdparty::pybind11 - ${maybe_mpi4py} Python::NumPy + ${maybe_mpi4py} ) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in - ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py + ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py @ONLY ) set_target_properties(adios2_py PROPERTIES CXX_VISIBILITY_PRESET hidden - OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX} - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 + OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX} + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings ) -string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") +string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings") if(CMAKE_SYSTEM_NAME MATCHES "Linux") set_target_properties(adios2_py PROPERTIES INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" @@ -55,10 +55,10 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") endif() install(TARGETS adios2_py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings COMPONENT adios2_python-python ) -install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 +install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings COMPONENT adios2_python-python ) diff --git a/bindings/Python/__init__.py.in b/bindings/Python/__init__.py.in index 01932e43ce..fb6c0ad344 100644 --- a/bindings/Python/__init__.py.in +++ b/bindings/Python/__init__.py.in @@ -1,3 +1,3 @@ -from .adios2@ADIOS2_LIBRARY_SUFFIX@ import * +from .adios2_bindings@ADIOS2_LIBRARY_SUFFIX@ import * __version__ = "@ADIOS2_VERSION@" diff --git a/bindings/Python/py11Variable.cpp b/bindings/Python/py11Variable.cpp index 2a57ef6ba2..80e2b7939c 100644 --- a/bindings/Python/py11Variable.cpp +++ b/bindings/Python/py11Variable.cpp @@ -189,6 +189,12 @@ size_t Variable::BlockID() const return m_VariableBase->m_BlockID; } +size_t Variable::SingleValue() const +{ + helper::CheckForNullptr(m_VariableBase, "in call to Variable::SingleValue"); + return m_VariableBase->m_SingleValue; +} + // size_t Variable::AddOperation(const Operator op, const Params ¶meters) {} // std::vector Variable::Operations() const {} diff --git a/bindings/Python/py11Variable.h b/bindings/Python/py11Variable.h index d04d1fc898..68da6f22f5 100644 --- a/bindings/Python/py11Variable.h +++ b/bindings/Python/py11Variable.h @@ -95,6 +95,8 @@ class Variable size_t BlockID() const; + size_t SingleValue() const; + /** * EXPERIMENTAL: Adds operation and parameters to current Variable object * @param op operator to be added diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 128c52d941..a735a83fe7 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -360,6 +360,23 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("GetResult", &adios2::py11::Query::GetResult) .def("GetBlockIDs", &adios2::py11::Query::GetBlockIDs); + pybind11::class_(m, "Operator") + // Python 2 + .def("__nonzero__", + [](const adios2::py11::Operator &op) { + const bool opBool = op ? true : false; + return opBool; + }) + // Python 3 + .def("__bool__", + [](const adios2::py11::Operator &op) { + const bool opBool = op ? true : false; + return opBool; + }) + .def("Type", &adios2::py11::Operator::Type) + .def("SetParameter", &adios2::py11::Operator::SetParameter) + .def("Parameters", &adios2::py11::Operator::Parameters); + pybind11::class_(m, "Variable") // Python 2 .def("__nonzero__", @@ -389,6 +406,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("Steps", &adios2::py11::Variable::Steps) .def("StepsStart", &adios2::py11::Variable::StepsStart) .def("BlockID", &adios2::py11::Variable::BlockID) + .def("SingleValue", &adios2::py11::Variable::SingleValue) .def("AddOperation", &adios2::py11::Variable::AddOperation) .def("Operations", &adios2::py11::Variable::Operations); @@ -482,23 +500,6 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("BlocksInfo", &adios2::py11::Engine::BlocksInfo); - pybind11::class_(m, "Operator") - // Python 2 - .def("__nonzero__", - [](const adios2::py11::Operator &op) { - const bool opBool = op ? true : false; - return opBool; - }) - // Python 3 - .def("__bool__", - [](const adios2::py11::Operator &op) { - const bool opBool = op ? true : false; - return opBool; - }) - .def("Type", &adios2::py11::Operator::Type) - .def("SetParameter", &adios2::py11::Operator::SetParameter) - .def("Parameters", &adios2::py11::Operator::Parameters); - pybind11::class_(m, "File") .def("__repr__", [](const adios2::py11::File &stream) { diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index efecb57624..05b3d25f56 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -396,12 +396,12 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() if(ADIOS2_USE_Python STREQUAL AUTO) - find_package(Python 3 COMPONENTS Interpreter Development NumPy) + find_package(Python 3 COMPONENTS Interpreter Development) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) endif() elseif(ADIOS2_USE_Python) - find_package(Python 3 REQUIRED COMPONENTS Interpreter Development NumPy) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development) if(ADIOS2_HAVE_MPI) find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h) endif() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..dd56276b7a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name="adios2" +version="0.0.1" +description="The Adaptable IO System" +authors = [ + {name = "Vicente Adolfo Bolea Sanchez", email = "vicente.bolea@kitware.com"}, +] + +keywords = [ + "Python", + "Web", + "Application", + "Framework", +] +classifiers = [ + "License :: Other/Proprietary License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +dependencies = [ + "numpy", +] + +[project.optional-dependencies] +dev = [ + "pip>=21.3", + "pytest", + "setuptools", + 'black', +] + +[tool.setuptools] +packages = [ "adios2"] + + +[tool.black] +line-length = 99 +target-version = ['py38', 'py39', 'py310'] +include = 'python/adios2/.*.py|testing/adios2/python/.*.py' diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000000..2c555163fd --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,13 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +add_custom_target(python_api ALL COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_SOURCE_DIR}/adios2 + ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2) + +install(DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/ + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/ + COMPONENT adios2_python-python +) diff --git a/python/adios2/__init__.py b/python/adios2/__init__.py new file mode 100644 index 0000000000..7b1a515f8f --- /dev/null +++ b/python/adios2/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/python3 + +import adios2.bindings diff --git a/python/adios2/adios.py b/python/adios2/adios.py new file mode 100644 index 0000000000..2e7ad7aab2 --- /dev/null +++ b/python/adios2/adios.py @@ -0,0 +1,36 @@ +from adios2.io import IO +from adios2.operator import Operator + +import adios2.bindings as bindings + + +class ADIOS: + def __init__(self): + self._impl = bindings.ADIOS() + self._operators = {} + self._ios = {} + + def DeclareIO(self, name): + self._ios[name] = IO(self._impl, name) + return self._ios[name] + + def AtIO(self, name): + return self._ios[name] + + def RemoveIO(self, name): + del self._ios[name] + self._impl.RemoveIO(name) + + def RemoveAllIOs(self): + self._ios = {} + self._impl.RemoveAllIOs() + + def DefineOperator(self, name, type, parameters={}): + self._operators[name] = Operator(self._impl, name, type, parameters) + return self._operators[name] + + def InquireOperator(self, name): + return self._operators[name] + + def FlushAll(self): + self._impl.FlushAll() diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py new file mode 100644 index 0000000000..0a1277822a --- /dev/null +++ b/python/adios2/attribute.py @@ -0,0 +1,20 @@ +import adios2.bindings + + +class Attribute: + def __init__(self, io, name, *args, **kwargs): + self._impl = io.DefineAttribute(name, *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, Attribute): + return self.Name() == other.Name() + return False + + def Name(self): + return self._impl.Name() + + def Data(self): + return self._impl.Data() + + def DataString(self): + return self._impl.DataString() diff --git a/python/adios2/engine.py b/python/adios2/engine.py new file mode 100644 index 0000000000..5f836845bb --- /dev/null +++ b/python/adios2/engine.py @@ -0,0 +1,66 @@ +import numpy as np +import adios2.bindings as bindings + +from adios2.attribute import Attribute +from adios2.variable import Variable + + +class Engine: + def __init__(self, io, name, mode): + self._impl = io.Open(name, mode) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.Close() + + def Close(self, transportIndex=-1): + self._impl.Close(transportIndex) + + def Steps(self): + return self._impl.Steps() + + def CurrentStep(self): + return self._impl.CurrentStep() + + def BeginStep(self, *args, **kwargs): + return self._impl.BeginStep(*args, **kwargs) + + def EndStep(self): + self._impl.EndStep() + + def BlocksInfo(self, name, step): + return self._impl.BlocksInfo(name, step) + + def Put(self, variable, content, mode=bindings.Mode.Sync): + if isinstance(content, np.ndarray): + self._impl.Put(variable._impl, content, mode) + else: + self._impl.Put(variable._impl, content) + self.PerformDataWrite() + + def PerformPuts(self): + self._impl.PerformPuts() + + def PerformDataWrite(self): + self._impl.PerformDataWrite() + + def Get(self, variable, content=None, mode=bindings.Mode.Sync): + if isinstance(content, np.ndarray): + self._impl.Get(variable._impl, content, mode) + return None + else: + return self._impl.Get(variable._impl, mode) + + def PerformGets(self): + self._impl.PerformGets() + + def LockReaderSelections(self): + self._impl.LockReaderSelections() + + def LockWriterDefinitions(self): + self._impl.LockWriterDefinitions() + + def Flush(self, transportIndex=-1): + self._impl.Flush(transportIndex) diff --git a/python/adios2/file.py b/python/adios2/file.py new file mode 100644 index 0000000000..b23ee5678c --- /dev/null +++ b/python/adios2/file.py @@ -0,0 +1,325 @@ +from adios2.adios import ADIOS +import adios2.bindings as bindings + +import numpy as np + + +def open(name, mode, engine_type="BPFile", config_file=None): + """File object open""" + return File(name, mode, engine_type) + + +def typeAdiosToNumPy(name): + translation = { + "int8_t": np.int8, + "uint8_t": np.uint8, + "int16_t": np.int16, + "uint16_t": np.uint16, + "int32_t": np.int32, + "uint32_t": np.uint32, + "int64_t": np.int64, + "uint64_t": np.uint64, + "float": np.float32, + "double": np.float64, + } + + return translation[name] + + +class File: + def __init__(self, path, mode="r", engine_type="BPFile", config_file=None): + self._io_name = f"file:{path}:engine_type:{engine_type}" + self._adios = ADIOS() + self._io = self._adios.DeclareIO(self._io_name) + self._mode = mode + + if mode == "r": + real_mode = bindings.Mode.Read + elif mode == "w": + real_mode = bindings.Mode.Write + else: + raise ValueError() + + self._engine = self._io.Open(path, real_mode) + self.index = 0 + self.max_steps = 0 + + def __repr__(self): + return f"" + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __iter__(self): + return self + + def __next__(self): + if self.index > 0: + self.end_step() + if self.index == self.max_steps: + self.index = 0 + self.max_steps = 0 + raise StopIteration + + self.index += 1 + self.begin_step() + return self + + def set_parameters(self, **kwargs): + """ + Sets parameters using a dictionary. + Removes any previous parameter. + + Parameters + parameters + input key/value parameters + + value + parameter value + """ + self._io.SetParameters(**kwargs) + + def set_transport(self, transport, parameters={}): + """ + + Adds a transport and its parameters to current IO. Must be + supported by current engine type. + + Parameters + type + must be a supported transport type for current engine. + + parameters + acceptable parameters for a particular transport + CAN'T use the keywords "Transport" or "transport" in key + + Returns + transport_index + handler to added transport + """ + self._io.AddTransport(transport, parameters) + + def available_variables(self): + """ + + Returns a 2-level dictionary with variable information. + Read mode only. + + Parameters + keys + list of variable information keys to be extracted (case insensitive) + keys=['AvailableStepsCount','Type','Max','Min','SingleValue','Shape'] + keys=['Name'] returns only the variable names as 1st-level keys + leave empty to return all possible keys + + Returns + variables dictionary + key + variable name + value + variable information dictionary + """ + return self._io.AvailableVariables() + + def available_attributes(self): + """ + Returns a 2-level dictionary with attribute information. + Read mode only. + + Returns + attributes dictionary + key + attribute name + value + attribute information dictionary + """ + return self._io.AvailableAttributes() + + def define_variable(self, name): + return self._io.DefineVariable(name) + + def add_operator(self, name): + pass + + def write( + self, + name, + content, + shape=[], + start=[], + count=[], + ): + """ + writes a self-describing array (numpy) variable + + Parameters + name + variable name + + content + variable data values + + shape + variable global MPI dimensions. + + start + variable offset for current MPI rank. + + count + variable dimension for current MPI rank. + + """ + variable = self._io.InquireVariable(name) + + if not variable: + if isinstance(content, str): + variable = self.define_variable(name) + elif isinstance(content, np.ndarray): + variable = self._io.DefineVariable(name, content) + elif isinstance(content, list): + content_np = np.array(content) + variable = self._io.DefineVariable(name, content_np) + else: + raise ValueError + + if isinstance(content, list): + content_np = np.array(content) + self._engine.Put(variable, content_np) + else: + self._engine.Put(variable, content) + + def read(self, name): + """ + Random access read allowed to select steps, + only valid with File Engines + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + + step_start + variable step start + + step_count + variable number of steps to read from step_start + + block_id + required for local array variables + + Returns + array + resulting array from selection + """ + variable = self._io.InquireVariable(name) + if not variable: + raise ValueError() + + if variable.Type() == "string" and variable.SingleValue() is True: + return self._engine.Get(variable) + + dtype = typeAdiosToNumPy(variable.Type()) + + output = np.zeros((variable.SelectionSize(),), dtype=dtype) + self._engine.Get(variable, output) + return output + + def write_attribute(name, content, variable_name="", separator="/", end_step=False): + """ + writes a self-describing single value array (numpy) variable + + Parameters + name + attribute name + + array + attribute numpy array data + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name ("var/attr") + Not used if variable_name is empty + + end_step + end current step, begin next step and flush + (default = false). + """ + pass + + def read_attribute(name, variable_name, separator="/"): + """ + Reads a numpy based attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + array + resulting array attribute data + """ + pass + + def begin_step(self): + self._engine.BeginStep() + + def end_step(self): + """ + Write mode: advances to the next step. Convenient when declaring + variable attributes as advancing to the next step is not attached + to any variable. + + Read mode: in streaming mode releases the current step (no effect + in file based engines) + """ + self._engine.EndStep() + + def close(self): + """ + Closes file, thus becoming unreachable. + Not required if using open in a with-as statement. + Required in all other cases per-open to avoid resource leaks. + """ + self._engine.Close() + self._engine = None + self._io.FlushAll() + self._io = None + self._adios.FlushAll() + self._adios = None + + def current_step(self): + """ + Inspect current step when using for-in loops, read mode only + + Returns + current step + """ + return self._engine.CurrentStep() + + def steps(self, num_steps=0): + if num_steps > 0: + self.max_steps = num_steps + else: + self.max_steps = self._engine.Steps() + + self.index = 0 + return self diff --git a/python/adios2/io.py b/python/adios2/io.py new file mode 100644 index 0000000000..eb5fe4fe7b --- /dev/null +++ b/python/adios2/io.py @@ -0,0 +1,110 @@ +import numpy as np +import adios2.bindings as bindings +from adios2.attribute import Attribute +from adios2.variable import Variable +from adios2.engine import Engine + + +class IO: + def __init__(self, adios, name): + self._impl = adios.DeclareIO(name) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.FlushAll() + + def DefineAttribute( + self, + name, + content=None, + variable_name="", + separator="/", + ): + return Attribute(self._impl, name, content, variable_name, separator) + + def InquireAttribute(self, name): + attr = None + attr_impl = self._impl.InquireAttribute(name) + if attr_impl: + attr = Attribute.__new__(Attribute) + attr._impl = attr_impl + return attr + + def AvailableAttributes(self): + attributes = {} + for name, attr in self._impl.AvailableAttributes(): + attributes[name] = Attribute(attr) + return attributes + + def RemoveAttribute(self, name): + self._impl.RemoveAttribute(name) + + def RemoveAllAttributes(self): + self._impl.RemoveAllAttributes() + + def DefineVariable( + self, + name, + content=None, + shape=None, + start=[0, 0], + count=None, + isConstantDims=bindings.ConstantDims, + ): + if isinstance(content, np.ndarray): + if not shape or not count: + shape = list(content.shape) + if len(shape) == 1: + shape = [shape[0], 1] + count = shape + + print(f"s:{shape} c:{count}") + return Variable(self._impl, name, content, shape, start, count, isConstantDims) + else: + return Variable(self._impl, name) + + def InquireVariable(self, name): + var = None + var_impl = self._impl.InquireVariable(name) + if var_impl: + var = Variable.__new__(Variable) + var._impl = var_impl + return var + + def AvailableVariables(self): + variables = {} + for name, attr in self._impl.AvailableVariables().items(): + variables[name] = attr + return variables + + def RemoveVariable(self, name): + self._impl.RemoveVariable(name) + + def RemoveAllVariables(self): + self._impl.RemoveAllVariables() + + def Open(self, name, mode): + return Engine(self._impl, name, mode) + + def SetEngine(self, name): + self._impl.SetEngine(name) + + def EngineType(self): + return self._impl.EngineType() + + def AddTransport(self, type, parameters=dict()): + self._impl.AddTransport(type, parameters) + + def Parameters(self): + return self._impl.Parameters() + + def SetParameter(self, key, value): + self._impl.SetParameter(key, value) + + def SetParameters(self, parameters): + self._impl.SetParameters(parameters) + + def FlushAll(self): + pass diff --git a/python/adios2/operator.py b/python/adios2/operator.py new file mode 100644 index 0000000000..f217904991 --- /dev/null +++ b/python/adios2/operator.py @@ -0,0 +1,12 @@ +import adios2.bindings + + +class Operator: + def __init__(self, adios_instance, name, type, parameters={}): + self._impl = adios_instance.DefineOperator(name, type, parameters) + + def GetParameters(self): + return self._impl.Parameters() + + def SetParameter(self, key, value): + self._impl.SetParameter(key, value) diff --git a/python/adios2/variable.py b/python/adios2/variable.py new file mode 100644 index 0000000000..64eb68b8c3 --- /dev/null +++ b/python/adios2/variable.py @@ -0,0 +1,65 @@ +import adios2.bindings + + +class Variable: + def __init__(self, io, name, *args, **kwargs): + self._impl = io.DefineVariable(name, *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, Variable): + return self.Name() == other.Name() + return False + + def BlockID(self): + return self._impl.BlockID() + + def Count(self): + return self._impl.Count() + + def SelectionSize(self): + return self._impl.SelectionSize() + + def SetBlockSelection(self, block_id): + self._impl.SetBlockSelection(block_id) + + def SetSelection(self, selection): + self._impl.SetSelection(selection) + + def SetShape(self, shape): + self._impl.SetShape(shape) + + def SetStepSelection(self, step_selection): + self._impl.SetStepSelection(step_selection) + + def Shape(self, step=0): + return self._impl.Shape(step) + + def ShapeID(self): + return self._impl.ShapeID() + + def Type(self): + return self._impl.Type() + + def SingleValue(self): + return bool(self._impl.SingleValue()) + + def Sizeof(self): + return self._impl.Sizeof() + + def Start(self): + return self._impl.Start() + + def Steps(self): + return self._impl.Steps() + + def StepsStart(self): + return self._impl.StepsStart() + + def Name(self): + return self._impl.Name() + + def AddOperation(self, op, params={}): + return self._impl.AddOperation(op._impl, params) + + def Operations(self): + return self._impl.Operations() diff --git a/scripts/ci/scripts/run-flake8.sh b/scripts/ci/scripts/run-flake8.sh index 1c5a1ad354..bacfb5894a 100755 --- a/scripts/ci/scripts/run-flake8.sh +++ b/scripts/ci/scripts/run-flake8.sh @@ -11,4 +11,4 @@ then cd ${SOURCE_DIR} fi -exec flake8 --config=flake8.cfg . +exec flake8 . diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt index dc1ede6bca..6e94ac1102 100644 --- a/testing/adios2/CMakeLists.txt +++ b/testing/adios2/CMakeLists.txt @@ -16,3 +16,7 @@ add_subdirectory(backward_compatibility) if (ADIOS2_HAVE_Derived_Variable) add_subdirectory(derived) endif() + +if (ADIOS2_HAVE_Python) +add_subdirectory(python) +endif() diff --git a/testing/adios2/python/CMakeLists.txt b/testing/adios2/python/CMakeLists.txt new file mode 100644 index 0000000000..7ef180e169 --- /dev/null +++ b/testing/adios2/python/CMakeLists.txt @@ -0,0 +1,12 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +python_add_test(NAME Api.Python.ADIOS SCRIPT TestADIOS.py) +python_add_test(NAME Api.Python.Engine SCRIPT TestEngine.py) +python_add_test(NAME Api.Python.IO SCRIPT TestIO.py) +python_add_test(NAME Api.Python.Operator SCRIPT TestOperator.py) +python_add_test(NAME Api.Python.Variable SCRIPT TestVariable.py) +python_add_test(NAME Api.Python.Attribute SCRIPT TestAttribute.py) +python_add_test(NAME Api.Python.File SCRIPT TestFile.py) diff --git a/testing/adios2/python/TestADIOS.py b/testing/adios2/python/TestADIOS.py new file mode 100644 index 0000000000..a7ddea2b26 --- /dev/null +++ b/testing/adios2/python/TestADIOS.py @@ -0,0 +1,56 @@ +from adios2.adios import ADIOS +import unittest + + +class TestADIOS(unittest.TestCase): + def test_define_operator(self): + adios = ADIOS() + op = adios.DefineOperator("op", "null") + self.assertNotEqual(op, None) + + def test_inquiry_operator(self): + adios = ADIOS() + op1 = adios.DefineOperator("op1", "null") + op2 = adios.DefineOperator("op2", "null") + op_x = adios.InquireOperator("op2") + self.assertNotEqual(op1, op2) + self.assertEqual(op2, op_x) + self.assertNotEqual(op1, op_x) + with self.assertRaises(KeyError): + adios.InquireOperator("NonExisting") + + def test_declare_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + self.assertNotEqual(writer, None) + + def test_at_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + reader = adios.DeclareIO("BPReader") + x = adios.AtIO("BPReader") + self.assertNotEqual(writer, reader) + self.assertEqual(reader, x) + self.assertNotEqual(writer, x) + with self.assertRaises(KeyError): + adios.AtIO("NonExisting") + + def test_remove_io(self): + adios = ADIOS() + adios.DeclareIO("BPWriter") + adios.RemoveIO("BPWriter") + with self.assertRaises(KeyError): + adios.AtIO("BPWriter") + + def test_remove_all_io(self): + adios = ADIOS() + adios.DeclareIO("BPWriter") + adios.DeclareIO("BPReader") + adios.RemoveAllIOs() + with self.assertRaises(KeyError): + adios.AtIO("BPWriter") + adios.AtIO("BPReader") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestAttribute.py b/testing/adios2/python/TestAttribute.py new file mode 100644 index 0000000000..9e5c286058 --- /dev/null +++ b/testing/adios2/python/TestAttribute.py @@ -0,0 +1,34 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestAttribute(unittest.TestCase): + def test_create_write(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + def test_create_reader(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + def test_create_write_ndarray(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + arr = np.array([2023, 11, 22]) + ts = writer.DefineAttribute("timestamp", arr) + self.assertEqual(ts.Name(), "timestamp") + self.assertTrue(np.array_equal(ts.Data(), [2023, 11, 22])) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestEngine.py b/testing/adios2/python/TestEngine.py new file mode 100644 index 0000000000..5def65186f --- /dev/null +++ b/testing/adios2/python/TestEngine.py @@ -0,0 +1,91 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestEngine(unittest.TestCase): + def test_close(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + with writer.Open("pythontestengine.bp", bindings.Mode.Write): + pass + + def test_put(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + engine.Put(pressure, "35PSI") + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + def test_get(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + temps = writer.DefineVariable( + name="temps", content=np.empty([4], dtype=np.int64), shape=[1, 4], count=[1, 4] + ) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + engine.Put(pressure, "35PSI") + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + pressure = reader.InquireVariable("pressure") + temps = reader.InquireVariable("temps") + pressure_reading = engine.Get(pressure) + temps_reading = np.empty([4], dtype=np.int64) + engine.Get(temps, temps_reading) + engine.EndStep() + self.assertEqual(pressure_reading, "35PSI") + self.assertTrue(np.array_equal(temps_reading, np.array([35, 40, 30, 45]))) + + def test_steps(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + for step in range(0, 10): + engine.BeginStep() + engine.Put(pressure, f"{step}PSI") + engine.EndStep() + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + for i in range(0, engine.Steps()): + engine.BeginStep() + pressure = reader.InquireVariable("pressure") + pressure_reading = engine.Get(pressure) + self.assertEqual(pressure_reading, f"{i}PSI") + engine.EndStep() + + def test_blockinfo(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + info = engine.BlocksInfo("temps", 0) + engine.EndStep() + + self.assertIsNot(info, None) + self.assertTrue(info[0]["Start"], "0,0") + self.assertTrue(info[0]["Count"], "0,0") + self.assertTrue(info[0]["WriterID"], "0") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestFile.py b/testing/adios2/python/TestFile.py new file mode 100644 index 0000000000..9546c6d797 --- /dev/null +++ b/testing/adios2/python/TestFile.py @@ -0,0 +1,21 @@ +from adios2 import file +from random import randint + +import unittest + + +class TestFile(unittest.TestCase): + def test_basic(self): + with file.File("pythonfiletest.bp", "w") as f: + for _ in f.steps(10): + f.write("Outlook", "Good") + f.write("temp", [randint(15, 35), randint(15, 35), randint(15, 35)]) + + with file.File("pythonfiletest.bp", "r") as f: + for _ in f.steps(): + for var_name in f.available_variables(): + print(f"var:{var_name} {f.read(var_name)}") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestIO.py b/testing/adios2/python/TestIO.py new file mode 100644 index 0000000000..5e77f5d130 --- /dev/null +++ b/testing/adios2/python/TestIO.py @@ -0,0 +1,102 @@ +from adios2.adios import ADIOS +import adios2.bindings as bindings + +import unittest + + +class TestIO(unittest.TestCase): + def test_io_empty(self): + adios = ADIOS() + adios.DeclareIO("BPWriter") + + def test_io_define_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertIsNot(ts, None) + + def test_io_inquire_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + coords = writer.DefineAttribute("coords", "43N74W") + x = writer.InquireAttribute("coords") + self.assertNotEqual(ts, coords) + self.assertNotEqual(ts, x) + self.assertEqual(coords, x) + + def test_available_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineAttribute("timestamp", "20231122") + writer.InquireAttribute("timestamp") + self.assertIs(writer.InquireAttribute("coords"), None) + + def test_remove_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineAttribute("timestamp", "20231122") + writer.RemoveAttribute("timestamp") + self.assertIs(writer.InquireAttribute("timestamp"), None) + + def test_remove_all_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineAttribute("timestamp", "20231122") + writer.DefineAttribute("coords", "43N74W") + writer.RemoveAllAttributes() + self.assertIs(writer.InquireAttribute("timestamp"), None) + self.assertIs(writer.InquireAttribute("coords"), None) + + def test_io_define_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + self.assertNotEqual(temp, None) + + def test_io_inquire_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + presure = writer.DefineVariable("pressure") + x = writer.InquireVariable("pressure") + self.assertNotEqual(temp, presure) + self.assertNotEqual(temp, x) + self.assertEqual(presure, x) + + def test_available_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineVariable("temp") + writer.InquireVariable("temp") + self.assertIs(writer.InquireAttribute("pressure"), None) + + def test_remove_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineVariable("temp") + writer.RemoveVariable("temp") + self.assertIs(writer.InquireAttribute("temp"), None) + + def test_remove_all_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineVariable("temp") + writer.DefineVariable("pressure") + writer.RemoveAllVariables() + self.assertIs(writer.InquireAttribute("pressure"), None) + self.assertIs(writer.InquireAttribute("temp"), None) + + def test_open_engine(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.SetEngine("BPFile") + writer.SetParameter("threads", "2") + writer.SetParameters({"AsyncOpen": "On", "MaxOpenFilesAtOnce": "512"}) + writer.AddTransport("File", {"Library": "POSIX"}) + engine = writer.Open("pythontest.bp", bindings.Mode.Write) + self.assertNotEqual(engine, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestOperator.py b/testing/adios2/python/TestOperator.py new file mode 100644 index 0000000000..4e64592659 --- /dev/null +++ b/testing/adios2/python/TestOperator.py @@ -0,0 +1,38 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestOperator(unittest.TestCase): + def test_operator_basic(self): + adios = ADIOS() + op1 = adios.DefineOperator("noop", "null") + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + temps.AddOperation(op1) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + op2 = adios.DefineOperator("noop2", "null") + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + temps.AddOperation(op2) + engine.EndStep() + + def test_operator_params(self): + adios = ADIOS() + op = adios.DefineOperator("noop", "null") + op.SetParameter("speed", "best") + op.GetParameters() + self.assertTrue("speed" in op.GetParameters()) + self.assertEqual(op.GetParameters()["speed"], "best") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestVariable.py b/testing/adios2/python/TestVariable.py new file mode 100644 index 0000000000..9ce0a6d34e --- /dev/null +++ b/testing/adios2/python/TestVariable.py @@ -0,0 +1,66 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestVariable(unittest.TestCase): + def test_create_write(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + self.assertEqual(temps.Name(), "temps") + self.assertEqual(temps.BlockID(), 0) + self.assertEqual(temps.Count(), [4, 1]) + self.assertEqual(temps.Shape(), [4, 1]) + self.assertEqual(temps.Sizeof(), 8) + self.assertEqual(temps.Steps(), 1) + self.assertEqual(temps.StepsStart(), 0) + + def test_create_reader(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + engine.EndStep() + + self.assertEqual(temps.Name(), "temps") + self.assertEqual(temps.BlockID(), 0) + self.assertEqual(temps.Count(), [4, 1]) + self.assertEqual(temps.Sizeof(), 8) + self.assertEqual(temps.Steps(), 1) + self.assertEqual(temps.StepsStart(), 0) + + def test_operators(self): + adios = ADIOS() + op1 = adios.DefineOperator("noop", "null") + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + temps.AddOperation(op1) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + op2 = adios.DefineOperator("noop2", "null") + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + temps.AddOperation(op2) + engine.EndStep() + + +if __name__ == "__main__": + unittest.main()