From 686c6e397aff3a746a407a1ce363ec26542bdc8c Mon Sep 17 00:00:00 2001 From: Sasha Rahlin Date: Thu, 26 Sep 2024 13:41:28 -0500 Subject: [PATCH] Separate shared libraries and python modules Each of the sub-libraries (core, calibration, dfmux, gcp, maps) are separated into two parts: A shared library called `libspt3g-`, which is installed to `$CMAKE_INSTALL_PREFIX/lib` and contains all of the C++ structures defined in that library. A shared module called `_lib`, which is added as an extension module to the `spt3g` python package, and whose namespace is internally imported into the `spt3g.` python module. This allows linking the shared libraries against each other, while treating the modules as independent endpoints that just expose the appropriate components to python, thus removing the need for the `dload` and `load_pybindings` infrastructure. Libraries link against the `Python::Module`` cmake target and use dynamic lookup to avoid linking against `libpython`. Compiled executables then link against `libpython` explicitly. This PR is the last step toward producing a portable package that can be installed using standard python tools like `pip`. --- .github/workflows/cmake.yml | 9 ++- CMakeLists.txt | 28 +++++--- README.rst | 17 ++++- calibration/CMakeLists.txt | 5 +- calibration/python/__init__.py | 3 +- calibration/tests/imports.py | 7 ++ cmake/Spt3gIncludes.cmake | 48 ++++++++++--- cmake/env-shell.sh.in | 2 +- core/CMakeLists.txt | 16 ++--- core/include/core/pybindings.h | 37 ++-------- core/python/__init__.py | 3 +- core/python/load_pybindings.py | 30 -------- core/src/dload.c | 122 --------------------------------- core/tests/imports.py | 8 +++ dfmux/CMakeLists.txt | 6 +- dfmux/python/__init__.py | 3 +- dfmux/tests/imports.py | 7 ++ examples/CMakeLists.txt | 2 +- gcp/CMakeLists.txt | 6 +- gcp/python/__init__.py | 3 +- gcp/src/python.cxx | 7 +- gcp/tests/imports.py | 7 ++ maps/CMakeLists.txt | 4 +- maps/python/__init__.py | 3 +- maps/src/python.cxx | 3 + maps/tests/imports.py | 7 ++ 26 files changed, 158 insertions(+), 235 deletions(-) create mode 100644 calibration/tests/imports.py delete mode 100644 core/python/load_pybindings.py delete mode 100644 core/src/dload.c create mode 100644 core/tests/imports.py create mode 100644 dfmux/tests/imports.py create mode 100644 gcp/tests/imports.py create mode 100644 maps/tests/imports.py diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index fc9c9f3d..7a83815c 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -61,7 +61,14 @@ jobs: working-directory: ${{runner.workspace}}/${{matrix.toolset}} shell: bash # Execute the build. You can specify a specific target with "--target " - run: cmake --build . --config $BUILD_TYPE + run: cmake --build . --config $BUILD_TYPE --parallel 4 + + - name: Upload + uses: actions/upload-artifact@v4 + if: runner.environment == 'github-hosted' + with: + name: cmake-${{matrix.os}}-${{matrix.toolset}} + path: ${{runner.workspace}}/${{matrix.toolset}} - name: Test working-directory: ${{runner.workspace}}/${{matrix.toolset}} diff --git a/CMakeLists.txt b/CMakeLists.txt index f0b92384..c9859222 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,21 +23,20 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) # Convenience variables -set(SPT3G_LIBRARY_DIR ${CMAKE_BINARY_DIR}/spt3g) +set(SPT3G_MODULE_DIR ${CMAKE_BINARY_DIR}/spt3g) set(SPT3G_INCLUDE_INSTALL_DIR "include/spt3g") include(Spt3gIncludes) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${SPT3G_LIBRARY_DIR}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -# Sneak in a namespace-like prefix for all libraries -set(CMAKE_SHARED_LIBRARY_PREFIX "libspt3g-") - # Raise every warning by default # (use target-specific options to disable particular warnings) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + # Find all the Boost and Python libraries set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) @@ -60,10 +59,20 @@ if (SPT3G_VERSION) endif() target_compile_features(spt3g INTERFACE cxx_std_11) -target_include_directories(spt3g INTERFACE ${Boost_INCLUDE_DIR} ${Python_INCLUDE_DIRS}) -target_link_libraries(spt3g INTERFACE Threads::Threads ${Boost_LIBRARIES} ${Python_LIBRARIES}) +target_include_directories(spt3g INTERFACE ${Boost_INCLUDE_DIR}) +target_link_libraries(spt3g INTERFACE Threads::Threads ${Boost_LIBRARIES}) target_include_directories(spt3g INTERFACE $) +# Python bindings +if(TARGET Python::Module) + target_link_libraries(spt3g INTERFACE Python::Module) +else() + target_include_directories(spt3g INTERFACE ${Python_INCLUDE_DIRS}) + if(APPLE) + target_link_options(spt3g INTERFACE "LINKER:-undefined,dynamic_lookup") + endif() +endif() + # Work around yet more bugs in GCC 4.4, this time with C++ 11 support # Also increase maximum number of arguments in python bindings target_compile_definitions(spt3g INTERFACE -DBOOST_PYTHON_MAX_ARITY=20 @@ -81,7 +90,8 @@ configure_file(${CMAKE_SOURCE_DIR}/cmake/env-shell.sh.in ${CMAKE_BINARY_DIR}/env # Set up python importability execute_process(COMMAND mkdir -p ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) execute_process(COMMAND mkdir -p ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) -execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/__init__.py) +execute_process(COMMAND mkdir -p ${SPT3G_MODULE_DIR}) +execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${SPT3G_MODULE_DIR}/__init__.py) set(BUILD_PROJECTS "${BUILD_PROJECTS}" CACHE STRING "The subset of available projects to actually build") if(NOT "${BUILD_PROJECTS}" STREQUAL "") @@ -112,7 +122,7 @@ if(APPLE) set(CMAKE_MACOSX_RPATH TRUE) set(CMAKE_SKIP_BUILD_RPATH FALSE) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) - set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${PYTHON_MODULE_DIR}/spt3g") + set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/lib) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) endif(APPLE) diff --git a/README.rst b/README.rst index 107d5552..44317c49 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ Minimum versions: - GCC >= 5.0 or clang >= 3.4 - Boost >= 1.48 - - cmake >= 3.5 + - cmake >= 3.12 - Python >= 2.7 (although pre-Python-3 support is best-effort) On Ubuntu/Debian, you can install the non-Python dependencies, including the optional ones, by doing: @@ -67,6 +67,21 @@ To build: cmake .. make +This will collect all of the necessary python tools into the ``build/spt3g`` directory, which can then be imported in python. To set the appropriate python environment *without* installing the python package, use the shell script in ``build/env_shell.sh`` to run commands that know where to find the ``spt3g`` package and libraries: + +.. code-block:: shell + + ./env-shell.sh python my_script.py # to run a python script + ./env-shell.sh ipython # to start an ipython session + +Alternatively, for users that only use a single build environment, set the following environment variables (e.g. in your ``.bash_profile`` file): + +.. code-block:: shell + + export SPT3G_SOFTWARE_BUILD_PATH=path/to/spt3g_software/build + export PYTHONPATH=$SPT3G_SOFTWARE_BUILD_PATH:$PYTHONPATH + export LD_LIBRARY_PATH=$SPT3G_SOFTWARE_BUILD_PATH/lib:$LD_LIBRARY_PATH + export PATH=$SPT3G_SOFTWARE_BUILD_PATH/bin:$PATH To build the documentation in the build directory type: diff --git a/calibration/CMakeLists.txt b/calibration/CMakeLists.txt index ed448c90..9f9c9355 100644 --- a/calibration/CMakeLists.txt +++ b/calibration/CMakeLists.txt @@ -1,8 +1,11 @@ add_spt3g_library(calibration SHARED - src/BoloProperties.cxx src/PointingProperties.cxx src/python.cxx + src/BoloProperties.cxx src/PointingProperties.cxx ) target_link_libraries(calibration PUBLIC core) link_python_dir() +add_spt3g_module(calibration src/python.cxx) + +add_spt3g_test(imports) add_spt3g_test(can_modify_bolo_props_in_map SLOWTEST) add_spt3g_test(band_format) diff --git a/calibration/python/__init__.py b/calibration/python/__init__.py index 0ab3840e..c36face7 100644 --- a/calibration/python/__init__.py +++ b/calibration/python/__init__.py @@ -1,5 +1,4 @@ -from spt3g.core.load_pybindings import load_pybindings -load_pybindings(__name__, __path__) +from .._libcalibration import * from . import build_cal_frames diff --git a/calibration/tests/imports.py b/calibration/tests/imports.py new file mode 100644 index 00000000..201c7e5a --- /dev/null +++ b/calibration/tests/imports.py @@ -0,0 +1,7 @@ +import spt3g._libcalibration +for k in dir(spt3g._libcalibration): + print(k, getattr(spt3g._libcalibration, k)) +from spt3g._libcalibration import BolometerProperties +import spt3g.calibration +from spt3g import calibration +from spt3g.calibration import BolometerProperties diff --git a/cmake/Spt3gIncludes.cmake b/cmake/Spt3gIncludes.cmake index 53cdea53..9d272d9f 100644 --- a/cmake/Spt3gIncludes.cmake +++ b/cmake/Spt3gIncludes.cmake @@ -1,11 +1,11 @@ # Convenience macros macro(link_python_dir) - execute_process(COMMAND mkdir -p ${SPT3G_LIBRARY_DIR}) + execute_process(COMMAND mkdir -p ${SPT3G_MODULE_DIR}) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/python) - execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR}/python ${SPT3G_LIBRARY_DIR}/${PROJECT}) + execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR}/python ${SPT3G_MODULE_DIR}/${PROJECT}) list(APPEND SPT3G_PYTHON_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/python) else(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/python) - execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR} ${SPT3G_LIBRARY_DIR}/${PROJECT}) + execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR} ${SPT3G_MODULE_DIR}/${PROJECT}) list(APPEND SPT3G_PYTHON_DIRS ${CMAKE_CURRENT_SOURCE_DIR}) endif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/python) set(SPT3G_PYTHON_DIRS ${SPT3G_PYTHON_DIRS} PARENT_SCOPE) @@ -26,25 +26,55 @@ macro(add_spt3g_program prog_name) execute_process(COMMAND ln -fsn ${prog_in} ${prog_out}) endmacro(add_spt3g_program prog_name) +macro(add_spt3g_executable prog_name) + add_executable(${prog_name} ${ARGN}) + if(TARGET Python::Python) + target_link_libraries(${prog_name} Python::Python) + else() + target_link_libraries(${prog_name} ${Python_LIBRARIES}) + endif() +endmacro(add_spt3g_executable prog_name) + macro(add_spt3g_library lib_name) add_library(${lib_name} ${ARGN}) + set_target_properties(${lib_name} PROPERTIES PREFIX "libspt3g-") if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(${lib_name} PUBLIC $) endif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/include) + # remove old libraries to avoid linking against the wrong one + file(REMOVE ${SPT3G_MODULE_DIR}/libspt3g-${lib_name}.so) + file(REMOVE ${SPT3G_MODULE_DIR}/libspt3g-${lib_name}.dylib) list(APPEND SPT3G_LIBRARIES ${lib_name}) set(SPT3G_LIBRARIES ${SPT3G_LIBRARIES} PARENT_SCOPE) install(TARGETS ${lib_name} EXPORT ${PROJECT_NAME}Config LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") - install(TARGETS ${lib_name} DESTINATION ${PYTHON_MODULE_DIR}/spt3g) endmacro(add_spt3g_library lib_name) +macro(add_spt3g_module lib_name) + set(mod_name "_lib${lib_name}") + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.17) + Python_add_library(${mod_name} MODULE WITH_SOABI ${ARGN}) + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + target_link_options(${mod_name} PUBLIC "LINKER:--no-as-needed") + endif() + else() + add_library(${mod_name} SHARED ${ARGN}) + set_target_properties(${mod_name} PROPERTIES PREFIX "" SUFFIX ".so") + target_include_directories(${mod_name} PRIVATE ${Python_INCLUDE_DIRS}) + target_link_libraries(${mod_name} PUBLIC ${Python_LIBRARIES}) + endif() + target_link_libraries(${mod_name} PUBLIC ${lib_name}) + set_target_properties(${mod_name} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${SPT3G_MODULE_DIR}) + install(TARGETS ${mod_name} DESTINATION ${PYTHON_MODULE_DIR}/spt3g) +endmacro(add_spt3g_module lib_name) + macro(add_spt3g_test test_name) add_test(${PROJECT}/${test_name} ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/${test_name}.py) set(extra_macro_args ${ARGN}) list(LENGTH extra_macro_args num_extra_args) set_tests_properties(${PROJECT}/${test_name} PROPERTIES ENVIRONMENT - "PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH};PYTHONPATH=${CMAKE_BINARY_DIR}:$ENV{PYTHONPATH};LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/spt3g:$ENV{LD_LIBRARY_PATH}") + "PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH};PYTHONPATH=${CMAKE_BINARY_DIR}:$ENV{PYTHONPATH};LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}") if (${num_extra_args} GREATER 0) list(GET extra_macro_args 0 test_labels) set_tests_properties(${PROJECT}/${test_name} PROPERTIES LABELS ${test_labels}) @@ -72,10 +102,10 @@ macro(add_spt3g_test_program test_name) file(WRITE ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx "#include \nG3TEST_MAIN_IMPL\n") endif(NOT EXISTS ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx) - add_executable(${PROJECT}-${test_name} - ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx - ${ADD_TEST_PROGRAM_SOURCE_FILES} - ) + add_spt3g_executable(${PROJECT}-${test_name} + ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx + ${ADD_TEST_PROGRAM_SOURCE_FILES} + ) target_include_directories(${PROJECT}-${test_name} PRIVATE ${CMAKE_SOURCE_DIR}/cmake) foreach(USED_PROJECT ${ADD_TEST_PROGRAM_USE_PROJECTS}) diff --git a/cmake/env-shell.sh.in b/cmake/env-shell.sh.in index ca0908f1..1a07b328 100755 --- a/cmake/env-shell.sh.in +++ b/cmake/env-shell.sh.in @@ -22,7 +22,7 @@ if [ "$SPT3G_SOFTWARE_PATH" != "@CMAKE_SOURCE_DIR@" ]; then # Add binaries to PATH, relative to this script to allow files to be movable SPT3G_BUILD_ROOT=$(cd `dirname $0`; pwd) export PATH=${SPT3G_BUILD_ROOT}/bin:$PATH - export LD_LIBRARY_PATH=${SPT3G_BUILD_ROOT}/spt3g:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${SPT3G_BUILD_ROOT}/lib:$LD_LIBRARY_PATH # And python bits... export PYTHONPATH=${SPT3G_BUILD_ROOT}:$PYTHONPATH diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index e1df7cd0..4870bb2c 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -1,13 +1,3 @@ -# Build a minimal library with a constant name which can be loaded as a python -# module with a plain `import`, which can then load other modules which are -# native libraries with normal names. -add_library(dload SHARED src/dload.c) -set_target_properties(dload PROPERTIES PREFIX "") -set_target_properties(dload PROPERTIES SUFFIX ".so") -install(TARGETS dload DESTINATION ${PYTHON_MODULE_DIR}/spt3g) -target_include_directories(dload PRIVATE ${Python_INCLUDE_DIRS}) -target_link_libraries(dload PUBLIC ${Python_LIBRARIES}) - # OS X has a broken implementatin of pthreads. if(APPLE) set(CORE_EXTRA_SRCS src/ApplePthreadBarrier.cxx) @@ -17,7 +7,7 @@ endif(APPLE) add_spt3g_library(core SHARED src/G3EventBuilder.cxx src/G3Frame.cxx src/G3TimeStamp.cxx - src/G3Pipeline.cxx src/G3Writer.cxx src/G3Reader.cxx src/python.cxx + src/G3Pipeline.cxx src/G3Writer.cxx src/G3Reader.cxx src/G3InfiniteSource.cxx src/G3Logging.cxx src/G3PrintfLogger.cxx src/G3Data.cxx src/G3Vector.cxx src/G3Map.cxx src/G3Timestream.cxx src/G3Timesample.cxx @@ -31,6 +21,9 @@ add_spt3g_library(core SHARED # Link dependencies target_link_libraries(core PUBLIC spt3g) +# add python bindings +add_spt3g_module(core src/python.cxx) + # Make core includes available without directories target_include_directories(core PUBLIC $ @@ -64,6 +57,7 @@ add_spt3g_program(bin/spt3g-inspect) add_spt3g_program(bin/gen-analysis-doc) #Tests +add_spt3g_test(imports) add_spt3g_test(copycons) add_spt3g_test(framepickle) add_spt3g_test(pipeline) diff --git a/core/include/core/pybindings.h b/core/include/core/pybindings.h index 93f68ae3..cd943632 100644 --- a/core/include/core/pybindings.h +++ b/core/include/core/pybindings.h @@ -6,8 +6,6 @@ #include #include -#include -#include #include #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push @@ -172,36 +170,11 @@ class G3ModuleRegistrator { .def(boost::python::init()) \ .def_pickle(g3frameobject_picklesuite()) -// Declare a python module with a name and the name of its enclosing package scope. -// name should be be a bare token, while pkg should be a string literal, e.g.: -// SPT3G_PYTHON_MODULE_2(foo, "spt3g.bar") -// for a package whose fully qualified name will be spt3g.bar.foo -#define SPT3G_PYTHON_MODULE_2(name, pkg) \ -BOOST_PYTHON_MODULE(name) { \ - namespace bp = boost::python; \ - auto mod = bp::scope(); \ - std::string package_prefix = pkg; \ - std::string full_name = package_prefix + "." + bp::extract(mod.attr("__name__"))(); \ - mod.attr("__name__") = full_name; \ - mod.attr("__package__") = package_prefix; \ - void BOOST_PP_CAT(spt3g_init_module_, name)(); \ - BOOST_PP_CAT(spt3g_init_module_, name)(); \ - if(PY_MAJOR_VERSION < 3){ \ - Py_INCREF(mod.ptr()); \ - PyDict_SetItemString(PyImport_GetModuleDict(),full_name.c_str(),mod.ptr()); \ - } \ -} \ -void BOOST_PP_CAT(spt3g_init_module_, name)() - -// Declare a python module with the given name, assuming that the enclosing package -// is the default "spt3g". -#define SPT3G_PYTHON_MODULE_1(name) SPT3G_PYTHON_MODULE_2(name, "spt3g") - -// Declare a python module with a name and optionally the name of its enclosing package scope. -// name should be be a bare token, while if provided the enclosing package name should be a -// string literal. -// If the enclosing package name is not specified, it will default to "spt3g". -#define SPT3G_PYTHON_MODULE(...) BOOST_PP_OVERLOAD(SPT3G_PYTHON_MODULE_,__VA_ARGS__)(__VA_ARGS__) +// Declare a python module with a name that is a bare token: +// SPT3G_PYTHON_MODULE(foo) +// for a package whose name will be _libfoo +#define SPT3G_PYTHON_MODULE(name) \ +BOOST_PYTHON_MODULE(BOOST_PP_CAT(_lib, name)) // Python runtime context to simplify acquiring or releasing the GIL as necessary. // To use, simply construct the context object where necessary, e.g. diff --git a/core/python/__init__.py b/core/python/__init__.py index 0f1a28d3..86e4b6de 100644 --- a/core/python/__init__.py +++ b/core/python/__init__.py @@ -1,5 +1,4 @@ -from .load_pybindings import load_pybindings -load_pybindings(__name__, __path__) +from .._libcore import * from .g3logging import log_trace, log_debug, log_info, log_notice, log_warn, log_error, log_fatal, set_log_level diff --git a/core/python/load_pybindings.py b/core/python/load_pybindings.py deleted file mode 100644 index 9e5a687f..00000000 --- a/core/python/load_pybindings.py +++ /dev/null @@ -1,30 +0,0 @@ -import platform,sys - -if platform.system().startswith('freebsd') or platform.system().startswith('FreeBSD'): - # C++ modules are extremely fragile when loaded with RTLD_LOCAL, - # which is what Python uses on FreeBSD by default, and maybe other - # systems. Convince it to use RTLD_GLOBAL. - - # See thread by Abrahams et al: - # http://mail.python.org/pipermail/python-dev/2002-May/024074.html - sys.setdlopenflags(0x102) - -lib_prefix = "libspt3g-" - -if platform.system().startswith('Darwin'): - # OSX compatibility requires .dylib suffix - lib_suffix = ".dylib" -else: - lib_suffix = ".so" - -def load_pybindings(name, path): - import os - mod = sys.modules[name] - p = os.path.split(path[0]) - from spt3g import dload - m = dload.load_dynamic(name, p[1], p[0] + "/" + lib_prefix + p[1] + lib_suffix) - sys.modules[name] = mod # Don't override Python mod with C++ - - for (k,v) in m.__dict__.items(): - if not k.startswith("_"): - mod.__dict__[k] = v diff --git a/core/src/dload.c b/core/src/dload.c deleted file mode 100644 index 9f2e8a6d..00000000 --- a/core/src/dload.c +++ /dev/null @@ -1,122 +0,0 @@ -#include - -#include - -static PyObject* load_dynamic_library(PyObject* mod, PyObject* args, PyObject* kwds){ - static const char* kwlist[] = {"full_name", "name", "path", NULL}; - char* full_name=NULL; - char* name=NULL; - char* path=NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "sss", (char**)kwlist, &full_name, &name, &path)) - return Py_None; - - void* h = dlopen(path, RTLD_NOW | RTLD_GLOBAL); - char* errmsg = dlerror(); - - if(h == NULL || errmsg != NULL){ - char err_buf[1024]; - snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': %s", name, path, - (errmsg == NULL ? "dlopen: unknown error" : errmsg)); - PyErr_SetString(PyExc_RuntimeError, err_buf); - return Py_None; - } - PyObject* (*init)(void); - char initName[256]; - int print_res = 0; -#if PY_MAJOR_VERSION >= 3 - print_res = snprintf(initName, 256, "PyInit_%s", name); -#else - print_res = snprintf(initName, 256, "init%s", name); -#endif - if(print_res<0){ - char err_buf[1024]; - snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': " - "module init function name too long", name, path); - PyErr_SetString(PyExc_RuntimeError, err_buf); - return Py_None; - } - - *(void **) (&init) = dlsym(h,initName); - errmsg = dlerror(); - if(errmsg != NULL){ - char err_buf[1024]; - snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': %s", name, path, - (errmsg == NULL ? "dlsym: unknown error" : errmsg)); - printf("%s\n", err_buf); - PyErr_SetString(PyExc_RuntimeError, err_buf); - return Py_None; - } - -#if PY_MAJOR_VERSION >= 3 - PyObject* module = (*init)(); - PyObject_SetAttrString(module, "__file__", PyUnicode_FromString(path)); -#else - (*init)(); - PyObject* module = PyDict_GetItemString(PyImport_GetModuleDict(), full_name); - if(nmod==NULL){ - char err_buf[1024]; - snprintf(err_buf, 1024, "dynamic loading error: %s not in global module dict", full_name); - PyErr_SetString(PyExc_RuntimeError, err_buf); - return Py_None; - } - PyObject_SetAttrString(module, "__file__", PyString_FromString(path)); -#endif - return module; -} - -static PyMethodDef spt3g_dload_methods[] = { - {"load_dynamic", (PyCFunction)&load_dynamic_library, METH_VARARGS | METH_KEYWORDS, - "\n" - "Load a compiled extension module. No restriction is placed on the relationship\n" - "between the name of the module and the name of the dynamic library file which\n" - "implements it; e.g. module `foo` need not reside in `foo.so`.\n" - "\n" - "Arguments\n" - "---------\n" - "full_name : str\n" - " The module's fully qualified name\n" - "name : str\n" - " The module's name\n" - "path : str\n" - " The path to the dynamic library which implements the module\n" - "\n" - "Returns\n" - "-------\n" - "The loaded module\n" - }, - {NULL, NULL, 0, NULL} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "spt3g_dload", - "A beach-head module for loading other compiled spt3g modules", - 0, - spt3g_dload_methods, - NULL, - NULL, - NULL, - NULL, -}; -#endif - -PyMODINIT_FUNC -#if PY_MAJOR_VERSION >= 3 -PyInit_dload(void){ -#else -initdload(void){ -#endif - PyObject* module; - -#if PY_MAJOR_VERSION >= 3 - module = PyModule_Create(&moduledef); - PyObject_SetAttrString(module, "__version__", PyUnicode_FromString("1.0.0")); - return module; -#else - module = Py_InitModule3("spt3g_dload", spt3g_dload_methods, - "A beach-head module for loading other compiled spt3g modules"); - PyObject_SetAttrString(module, "__version__", PyString_FromString("1.0.0")); - PyDict_SetItemString(PyImport_GetModuleDict(),_Py_PackageContext,module); -#endif -} \ No newline at end of file diff --git a/core/tests/imports.py b/core/tests/imports.py new file mode 100644 index 00000000..240a466b --- /dev/null +++ b/core/tests/imports.py @@ -0,0 +1,8 @@ +import spt3g +import spt3g._libcore +for k in dir(spt3g._libcore): + print(k, getattr(spt3g._libcore, k)) +from spt3g._libcore import BoolVector +import spt3g.core +from spt3g import core +from spt3g.core import BoolVector diff --git a/dfmux/CMakeLists.txt b/dfmux/CMakeLists.txt index df9b8a63..92174ffa 100644 --- a/dfmux/CMakeLists.txt +++ b/dfmux/CMakeLists.txt @@ -8,11 +8,13 @@ endif() add_spt3g_library(dfmux SHARED src/DfMuxBuilder.cxx src/DfMuxCollector.cxx src/DfMuxSample.cxx src/LegacyDfMuxCollector.cxx src/HardwareMap.cxx src/DfMuxCollator.cxx - src/Housekeeping.cxx src/python.cxx + src/Housekeeping.cxx ${DFMUX_LIB_EXTRA_SRC} ) target_link_libraries(dfmux PUBLIC core) +add_spt3g_module(dfmux src/python.cxx) + if (NetCDF_FOUND) target_link_libraries(dfmux PUBLIC NetCDF::NetCDF) add_spt3g_program(bin/ledgerman.py ledgerman) @@ -20,3 +22,5 @@ if (NetCDF_FOUND) endif() link_python_dir() + +add_spt3g_test(imports) diff --git a/dfmux/python/__init__.py b/dfmux/python/__init__.py index 339540bc..84b5eedd 100644 --- a/dfmux/python/__init__.py +++ b/dfmux/python/__init__.py @@ -1,5 +1,4 @@ -from spt3g.core.load_pybindings import load_pybindings -load_pybindings(__name__, __path__) +from .._libdfmux import * from .HardwareMapTools import ( GenerateFakeHardwareMap, diff --git a/dfmux/tests/imports.py b/dfmux/tests/imports.py new file mode 100644 index 00000000..54e4a775 --- /dev/null +++ b/dfmux/tests/imports.py @@ -0,0 +1,7 @@ +import spt3g._libdfmux +for k in dir(spt3g._libdfmux): + print(k, getattr(spt3g._libdfmux, k)) +from spt3g._libdfmux import DfMuxSample +import spt3g.dfmux +from spt3g import dfmux +from spt3g.dfmux import DfMuxSample diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2e588a9e..add87b5a 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,2 +1,2 @@ -add_executable(cppexample cppexample.cxx) +add_spt3g_executable(cppexample cppexample.cxx) target_link_libraries(cppexample core) diff --git a/gcp/CMakeLists.txt b/gcp/CMakeLists.txt index 3c1001a0..3675e5d8 100644 --- a/gcp/CMakeLists.txt +++ b/gcp/CMakeLists.txt @@ -1,10 +1,14 @@ add_spt3g_library(gcp SHARED - src/ARCFileReader.cxx src/python.cxx + src/ARCFileReader.cxx src/ACUStatus.cxx src/TrackerStatus.cxx src/TrackerPointing.cxx src/GCPMuxDataDecoder.cxx src/GCPLogger.cxx ) target_link_libraries(gcp PUBLIC core dfmux) link_python_dir() +add_spt3g_module(gcp src/python.cxx) + add_spt3g_program(bin/spt3g-arc-dump) add_spt3g_program(bin/spt3g-arc-verify) + +add_spt3g_test(imports) diff --git a/gcp/python/__init__.py b/gcp/python/__init__.py index caa02500..2e32054c 100644 --- a/gcp/python/__init__.py +++ b/gcp/python/__init__.py @@ -1,5 +1,4 @@ -from spt3g.core.load_pybindings import load_pybindings -load_pybindings(__name__, __path__) +from .._libgcp import * from .ARCExtractor import UnpackACUData, UnpackTrackerData, DecryptFeatureBit, ARCExtract, ARCExtractMinimal from .ARCHKExtractor import UnpackSPTpolHKData diff --git a/gcp/src/python.cxx b/gcp/src/python.cxx index 0de74490..c74388ac 100644 --- a/gcp/src/python.cxx +++ b/gcp/src/python.cxx @@ -1,15 +1,14 @@ #include -#include #include -#include -#include - namespace bp = boost::python; SPT3G_PYTHON_MODULE(gcp) { + // Python bindings dependencies bp::import("spt3g.core"); + bp::import("spt3g.dfmux"); + bp::docstring_options docopts(true, true, false); // Supported Experiments diff --git a/gcp/tests/imports.py b/gcp/tests/imports.py new file mode 100644 index 00000000..ff13b356 --- /dev/null +++ b/gcp/tests/imports.py @@ -0,0 +1,7 @@ +import spt3g._libgcp +for k in dir(spt3g._libgcp): + print(k, getattr(spt3g._libgcp, k)) +from spt3g._libgcp import ACUStatus +import spt3g.gcp +from spt3g import gcp +from spt3g.gcp import ACUStatus diff --git a/maps/CMakeLists.txt b/maps/CMakeLists.txt index 797c377f..d3d7ba7b 100644 --- a/maps/CMakeLists.txt +++ b/maps/CMakeLists.txt @@ -16,7 +16,6 @@ add_spt3g_library(maps SHARED src/mapdata.cxx src/maputils.cxx src/pointing.cxx - src/python.cxx ) target_link_libraries(maps PUBLIC core calibration) @@ -27,10 +26,13 @@ if(OpenMP_FOUND) set(OpenMP_FOUND ${OpenMP_FOUND} PARENT_SCOPE) endif() +add_spt3g_module(maps src/python.cxx) + link_python_dir() add_spt3g_program(bin/spt3g-coadd-maps) +add_spt3g_test(imports) add_spt3g_test(quatangtest) add_spt3g_test(transtest SLOWTEST) add_spt3g_test(flatsky_maps) diff --git a/maps/python/__init__.py b/maps/python/__init__.py index 0673a70e..43c28885 100644 --- a/maps/python/__init__.py +++ b/maps/python/__init__.py @@ -1,5 +1,4 @@ -from spt3g.core.load_pybindings import load_pybindings -load_pybindings(__name__, __path__) +from .._libmaps import * # Just run this, no symbols we need though from .skymapaddons import * diff --git a/maps/src/python.cxx b/maps/src/python.cxx index 6d5a3f05..cb855491 100644 --- a/maps/src/python.cxx +++ b/maps/src/python.cxx @@ -4,7 +4,10 @@ namespace bp = boost::python; SPT3G_PYTHON_MODULE(maps) { + // Python bindings dependencies bp::import("spt3g.core"); + bp::import("spt3g.calibration"); + bp::docstring_options docopts(true, true, false); G3ModuleRegistrator::CallRegistrarsFor("maps"); } diff --git a/maps/tests/imports.py b/maps/tests/imports.py new file mode 100644 index 00000000..ebd1b3a5 --- /dev/null +++ b/maps/tests/imports.py @@ -0,0 +1,7 @@ +import spt3g._libmaps +for k in dir(spt3g._libmaps): + print(k, getattr(spt3g._libmaps, k)) +from spt3g._libmaps import FlatSkyMap +import spt3g.maps +from spt3g import maps +from spt3g.maps import FlatSkyMap