Separate shared libraries and python modules

.github/workflows/cmake.yml

@@ -61,7 +61,14 @@ jobs:
       working-directory: ${{runner.workspace}}/${{matrix.toolset}}
       shell: bash
       # Execute the build.  You can specify a specific target with "--target <NAME>"
-      run: cmake --build . --config $BUILD_TYPE
+      run: cmake --build . --config $BUILD_TYPE --parallel 4
+
+    - name: Upload
+      uses: actions/upload-artifact@v4
+      if: runner.environment == 'github-hosted'
+      with:
+        name: cmake-${{matrix.os}}-${{matrix.toolset}}
+        path: ${{runner.workspace}}/${{matrix.toolset}}
 
     - name: Test
       working-directory: ${{runner.workspace}}/${{matrix.toolset}}

CMakeLists.txt

@@ -23,21 +23,20 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 # Convenience variables
-set(SPT3G_LIBRARY_DIR ${CMAKE_BINARY_DIR}/spt3g)
+set(SPT3G_MODULE_DIR ${CMAKE_BINARY_DIR}/spt3g)
 set(SPT3G_INCLUDE_INSTALL_DIR "include/spt3g")
 
 include(Spt3gIncludes)
 
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${SPT3G_LIBRARY_DIR})
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
-# Sneak in a namespace-like prefix for all libraries
-set(CMAKE_SHARED_LIBRARY_PREFIX "libspt3g-")
-
 # Raise every warning by default
 # (use target-specific options to disable particular warnings)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
 
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 # Find all the Boost and Python libraries
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
@@ -60,10 +59,20 @@ if (SPT3G_VERSION)
 endif()
 
 target_compile_features(spt3g INTERFACE cxx_std_11)
-target_include_directories(spt3g INTERFACE ${Boost_INCLUDE_DIR} ${Python_INCLUDE_DIRS})
-target_link_libraries(spt3g INTERFACE Threads::Threads ${Boost_LIBRARIES} ${Python_LIBRARIES})
+target_include_directories(spt3g INTERFACE ${Boost_INCLUDE_DIR})
+target_link_libraries(spt3g INTERFACE Threads::Threads ${Boost_LIBRARIES})
 target_include_directories(spt3g INTERFACE $<INSTALL_INTERFACE:${SPT3G_INCLUDE_INSTALL_DIR}>)
 
+# Python bindings
+if(TARGET Python::Module)
+	target_link_libraries(spt3g INTERFACE Python::Module)
+else()
+	target_include_directories(spt3g INTERFACE ${Python_INCLUDE_DIRS})
+	if(APPLE)
+		target_link_options(spt3g INTERFACE "LINKER:-undefined,dynamic_lookup")
+	endif()
+endif()
+
 # Work around yet more bugs in GCC 4.4, this time with C++ 11 support
 # Also increase maximum number of arguments in python bindings
 target_compile_definitions(spt3g INTERFACE -DBOOST_PYTHON_MAX_ARITY=20
@@ -81,7 +90,8 @@ configure_file(${CMAKE_SOURCE_DIR}/cmake/env-shell.sh.in ${CMAKE_BINARY_DIR}/env
 # Set up python importability
 execute_process(COMMAND mkdir -p ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 execute_process(COMMAND mkdir -p ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
-execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/__init__.py)
+execute_process(COMMAND mkdir -p ${SPT3G_MODULE_DIR})
+execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${SPT3G_MODULE_DIR}/__init__.py)
 
 set(BUILD_PROJECTS "${BUILD_PROJECTS}" CACHE STRING "The subset of available projects to actually build")
 if(NOT "${BUILD_PROJECTS}" STREQUAL "")
@@ -112,7 +122,7 @@ if(APPLE)
 	set(CMAKE_MACOSX_RPATH TRUE)
 	set(CMAKE_SKIP_BUILD_RPATH FALSE)
 	set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${PYTHON_MODULE_DIR}/spt3g")
+	set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/lib)
 	set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 endif(APPLE)
 

README.rst

@@ -21,7 +21,7 @@ Minimum versions:
 
 	- GCC >= 5.0 or clang >= 3.4
 	- Boost >= 1.48
-	- cmake >= 3.5
+	- cmake >= 3.12
 	- Python >= 2.7 (although pre-Python-3 support is best-effort)
 
 On Ubuntu/Debian, you can install the non-Python dependencies, including the optional ones, by doing:
@@ -67,6 +67,21 @@ To build:
 	cmake ..
 	make
 
+This will collect all of the necessary python tools into the ``build/spt3g`` directory, which can then be imported in python.  To set the appropriate python environment *without* installing the python package, use the shell script in ``build/env_shell.sh`` to run commands that know where to find the ``spt3g`` package and libraries:
+
+.. code-block:: shell
+
+	./env-shell.sh python my_script.py  # to run a python script
+	./env-shell.sh ipython  # to start an ipython session
+
+Alternatively, for users that only use a single build environment, set the following environment variables (e.g. in your ``.bash_profile`` file):
+
+.. code-block:: shell
+
+	export SPT3G_SOFTWARE_BUILD_PATH=path/to/spt3g_software/build
+	export PYTHONPATH=$SPT3G_SOFTWARE_BUILD_PATH:$PYTHONPATH
+	export LD_LIBRARY_PATH=$SPT3G_SOFTWARE_BUILD_PATH/lib:$LD_LIBRARY_PATH
+	export PATH=$SPT3G_SOFTWARE_BUILD_PATH/bin:$PATH
 
 To build the documentation in the build directory type:
 

calibration/CMakeLists.txt

@@ -1,8 +1,11 @@
 add_spt3g_library(calibration SHARED
-	src/BoloProperties.cxx src/PointingProperties.cxx src/python.cxx
+	src/BoloProperties.cxx src/PointingProperties.cxx
 )
 target_link_libraries(calibration PUBLIC core)
 link_python_dir()
 
+add_spt3g_module(calibration src/python.cxx)
+
+add_spt3g_test(imports)
 add_spt3g_test(can_modify_bolo_props_in_map SLOWTEST)
 add_spt3g_test(band_format)

calibration/python/__init__.py

@@ -1,5 +1,4 @@
-from spt3g.core.load_pybindings import load_pybindings
-load_pybindings(__name__, __path__)
+from .._libcalibration import *
 
 from . import build_cal_frames
 

calibration/tests/imports.py

@@ -0,0 +1,7 @@
+import spt3g._libcalibration
+for k in dir(spt3g._libcalibration):
+    print(k, getattr(spt3g._libcalibration, k))
+from spt3g._libcalibration import BolometerProperties
+import spt3g.calibration
+from spt3g import calibration
+from spt3g.calibration import BolometerProperties

cmake/Spt3gIncludes.cmake

@@ -1,11 +1,11 @@
 # Convenience macros
 macro(link_python_dir)
-	execute_process(COMMAND mkdir -p ${SPT3G_LIBRARY_DIR})
+	execute_process(COMMAND mkdir -p ${SPT3G_MODULE_DIR})
 	if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/python)
-		execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR}/python ${SPT3G_LIBRARY_DIR}/${PROJECT})
+		execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR}/python ${SPT3G_MODULE_DIR}/${PROJECT})
 		list(APPEND SPT3G_PYTHON_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/python)
 	else(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/python)
-		execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR} ${SPT3G_LIBRARY_DIR}/${PROJECT})
+		execute_process(COMMAND ln -fsn ${CMAKE_CURRENT_SOURCE_DIR} ${SPT3G_MODULE_DIR}/${PROJECT})
 		list(APPEND SPT3G_PYTHON_DIRS ${CMAKE_CURRENT_SOURCE_DIR})
 	endif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/python)
 	set(SPT3G_PYTHON_DIRS ${SPT3G_PYTHON_DIRS} PARENT_SCOPE)
@@ -26,25 +26,55 @@ macro(add_spt3g_program prog_name)
 	execute_process(COMMAND ln -fsn ${prog_in} ${prog_out})
 endmacro(add_spt3g_program prog_name)
 
+macro(add_spt3g_executable prog_name)
+	add_executable(${prog_name} ${ARGN})
+	if(TARGET Python::Python)
+		target_link_libraries(${prog_name} Python::Python)
+	else()
+		target_link_libraries(${prog_name} ${Python_LIBRARIES})
+	endif()
+endmacro(add_spt3g_executable prog_name)
+
 macro(add_spt3g_library lib_name)
 	add_library(${lib_name} ${ARGN})
+	set_target_properties(${lib_name} PROPERTIES PREFIX "libspt3g-")
 	if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/include)
 		target_include_directories(${lib_name} PUBLIC
 			$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
 	endif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/include)
+	# remove old libraries to avoid linking against the wrong one
+	file(REMOVE ${SPT3G_MODULE_DIR}/libspt3g-${lib_name}.so)
+	file(REMOVE ${SPT3G_MODULE_DIR}/libspt3g-${lib_name}.dylib)
 	list(APPEND SPT3G_LIBRARIES ${lib_name})
 	set(SPT3G_LIBRARIES ${SPT3G_LIBRARIES} PARENT_SCOPE)
 	install(TARGETS ${lib_name} EXPORT ${PROJECT_NAME}Config LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-	install(TARGETS ${lib_name} DESTINATION ${PYTHON_MODULE_DIR}/spt3g)
 endmacro(add_spt3g_library lib_name)
 
+macro(add_spt3g_module lib_name)
+	set(mod_name "_lib${lib_name}")
+	if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.17)
+		Python_add_library(${mod_name} MODULE WITH_SOABI ${ARGN})
+		if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
+			target_link_options(${mod_name} PUBLIC "LINKER:--no-as-needed")
+		endif()
+	else()
+		add_library(${mod_name} SHARED ${ARGN})
+		set_target_properties(${mod_name} PROPERTIES PREFIX "" SUFFIX ".so")
+		target_include_directories(${mod_name} PRIVATE ${Python_INCLUDE_DIRS})
+		target_link_libraries(${mod_name} PUBLIC ${Python_LIBRARIES})
+	endif()
+	target_link_libraries(${mod_name} PUBLIC ${lib_name})
+	set_target_properties(${mod_name} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${SPT3G_MODULE_DIR})
+	install(TARGETS ${mod_name} DESTINATION ${PYTHON_MODULE_DIR}/spt3g)
+endmacro(add_spt3g_module lib_name)
+
 macro(add_spt3g_test test_name)
 	add_test(${PROJECT}/${test_name} ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/${test_name}.py)
 
 	set(extra_macro_args ${ARGN})
 	list(LENGTH extra_macro_args num_extra_args)
 	set_tests_properties(${PROJECT}/${test_name} PROPERTIES ENVIRONMENT
-		"PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH};PYTHONPATH=${CMAKE_BINARY_DIR}:$ENV{PYTHONPATH};LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/spt3g:$ENV{LD_LIBRARY_PATH}")
+		"PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH};PYTHONPATH=${CMAKE_BINARY_DIR}:$ENV{PYTHONPATH};LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}")
 	if (${num_extra_args} GREATER 0)
 		list(GET extra_macro_args 0 test_labels)
 		set_tests_properties(${PROJECT}/${test_name} PROPERTIES LABELS ${test_labels})
@@ -72,10 +102,10 @@ macro(add_spt3g_test_program test_name)
 		file(WRITE ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx "#include <G3Test.h>\nG3TEST_MAIN_IMPL\n")
 	endif(NOT EXISTS ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx)
 
-	add_executable(${PROJECT}-${test_name}
-	               ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx
-	               ${ADD_TEST_PROGRAM_SOURCE_FILES}
-	               )
+	add_spt3g_executable(${PROJECT}-${test_name}
+	                     ${PROJECT_BINARY_DIR}/Spt3gTestMain.cxx
+	                     ${ADD_TEST_PROGRAM_SOURCE_FILES}
+	                     )
 	target_include_directories(${PROJECT}-${test_name} PRIVATE ${CMAKE_SOURCE_DIR}/cmake)
 
 	foreach(USED_PROJECT ${ADD_TEST_PROGRAM_USE_PROJECTS})

cmake/env-shell.sh.in

@@ -22,7 +22,7 @@ if [ "$SPT3G_SOFTWARE_PATH" != "@CMAKE_SOURCE_DIR@" ]; then
 	# Add binaries to PATH, relative to this script to allow files to be movable
 	SPT3G_BUILD_ROOT=$(cd `dirname $0`; pwd)
 	export PATH=${SPT3G_BUILD_ROOT}/bin:$PATH
-	export LD_LIBRARY_PATH=${SPT3G_BUILD_ROOT}/spt3g:$LD_LIBRARY_PATH
+	export LD_LIBRARY_PATH=${SPT3G_BUILD_ROOT}/lib:$LD_LIBRARY_PATH
 
 	# And python bits...
 	export PYTHONPATH=${SPT3G_BUILD_ROOT}:$PYTHONPATH

core/CMakeLists.txt

@@ -1,13 +1,3 @@
-# Build a minimal library with a constant name which can be loaded as a python
-# module with a plain `import`, which can then load other modules which are
-# native libraries with normal names.
-add_library(dload SHARED src/dload.c)
-set_target_properties(dload PROPERTIES PREFIX "")
-set_target_properties(dload PROPERTIES SUFFIX ".so")
-install(TARGETS dload DESTINATION ${PYTHON_MODULE_DIR}/spt3g)
-target_include_directories(dload PRIVATE ${Python_INCLUDE_DIRS})
-target_link_libraries(dload PUBLIC ${Python_LIBRARIES})
-
 # OS X has a broken implementatin of pthreads.
 if(APPLE)
 	set(CORE_EXTRA_SRCS src/ApplePthreadBarrier.cxx)
@@ -17,7 +7,7 @@ endif(APPLE)
 
 add_spt3g_library(core SHARED
 	src/G3EventBuilder.cxx src/G3Frame.cxx src/G3TimeStamp.cxx
-	src/G3Pipeline.cxx src/G3Writer.cxx src/G3Reader.cxx src/python.cxx
+	src/G3Pipeline.cxx src/G3Writer.cxx src/G3Reader.cxx
 	src/G3InfiniteSource.cxx src/G3Logging.cxx src/G3PrintfLogger.cxx
 	src/G3Data.cxx src/G3Vector.cxx src/G3Map.cxx src/G3Timestream.cxx
         src/G3Timesample.cxx
@@ -31,6 +21,9 @@ add_spt3g_library(core SHARED
 # Link dependencies
 target_link_libraries(core PUBLIC spt3g)
 
+# add python bindings
+add_spt3g_module(core src/python.cxx)
+
 # Make core includes available without directories
 target_include_directories(core PUBLIC
 	$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/core>
@@ -64,6 +57,7 @@ add_spt3g_program(bin/spt3g-inspect)
 add_spt3g_program(bin/gen-analysis-doc)
 
 #Tests
+add_spt3g_test(imports)
 add_spt3g_test(copycons)
 add_spt3g_test(framepickle)
 add_spt3g_test(pipeline)

core/include/core/pybindings.h

@@ -6,8 +6,6 @@
 #include <G3Logging.h>
 
 #include <boost/preprocessor/cat.hpp>
-#include <boost/preprocessor/facilities/overload.hpp>
-#include <boost/preprocessor/stringize.hpp>
 #include <boost/python.hpp>
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic push
@@ -172,36 +170,11 @@ class G3ModuleRegistrator {
 	    .def(boost::python::init<const T &>()) \
 	    .def_pickle(g3frameobject_picklesuite<T>())
 
-// Declare a python module with a name and the name of its enclosing package scope.
-// name should be be a bare token, while pkg should be a string literal, e.g.:
-//     SPT3G_PYTHON_MODULE_2(foo, "spt3g.bar")
-// for a package whose fully qualified name will be spt3g.bar.foo
-#define SPT3G_PYTHON_MODULE_2(name, pkg) \
-BOOST_PYTHON_MODULE(name) { \
-	namespace bp = boost::python; \
-	auto mod = bp::scope(); \
-	std::string package_prefix = pkg; \
-	std::string full_name = package_prefix + "." + bp::extract<std::string>(mod.attr("__name__"))(); \
-	mod.attr("__name__") = full_name; \
-	mod.attr("__package__") = package_prefix; \
-	void BOOST_PP_CAT(spt3g_init_module_, name)(); \
-	BOOST_PP_CAT(spt3g_init_module_, name)(); \
-	if(PY_MAJOR_VERSION < 3){ \
-		Py_INCREF(mod.ptr()); \
-		PyDict_SetItemString(PyImport_GetModuleDict(),full_name.c_str(),mod.ptr()); \
-	} \
-} \
-void BOOST_PP_CAT(spt3g_init_module_, name)()
-
-// Declare a python module with the given name, assuming that the enclosing package 
-// is the default "spt3g".
-#define SPT3G_PYTHON_MODULE_1(name) SPT3G_PYTHON_MODULE_2(name, "spt3g")
-
-// Declare a python module with a name and optionally the name of its enclosing package scope.
-// name should be be a bare token, while if provided the enclosing package name should be a
-// string literal.
-// If the enclosing package name is not specified, it will default to "spt3g".
-#define SPT3G_PYTHON_MODULE(...) BOOST_PP_OVERLOAD(SPT3G_PYTHON_MODULE_,__VA_ARGS__)(__VA_ARGS__)
+// Declare a python module with a name that is a bare token:
+//     SPT3G_PYTHON_MODULE(foo)
+// for a package whose name will be _libfoo
+#define SPT3G_PYTHON_MODULE(name) \
+BOOST_PYTHON_MODULE(BOOST_PP_CAT(_lib, name))
 
 // Python runtime context to simplify acquiring or releasing the GIL as necessary.
 // To use, simply construct the context object where necessary, e.g.

core/python/__init__.py

@@ -1,5 +1,4 @@
-from .load_pybindings import load_pybindings
-load_pybindings(__name__, __path__)
+from .._libcore import *
 
 from .g3logging import log_trace, log_debug, log_info, log_notice, log_warn, log_error, log_fatal, set_log_level