Use the same build directory for pip and cmake

.github/workflows/wheels.yml

@@ -85,8 +85,8 @@ jobs:
             DYLD_LIBRARY_PATH=$PWD/wheel/deps/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
           CIBW_BUILD_VERBOSITY: 1
           CIBW_ENVIRONMENT: >
-            CMAKE_BUILD_PARALLEL_LEVEL=4 BUILD_DIR=$PWD/wheel/build CMAKE_PREFIX_PATH=$PWD/wheel/deps
-          CIBW_TEST_COMMAND: ctest --test-dir {package}/wheel/build --output-on-failure
+            CMAKE_BUILD_PARALLEL_LEVEL=4 CMAKE_PREFIX_PATH=$PWD/wheel/deps
+          CIBW_TEST_COMMAND: ctest --test-dir {package}/build --output-on-failure
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4

CMakeLists.txt

@@ -97,7 +97,7 @@ configure_file(${CMAKE_SOURCE_DIR}/cmake/env-shell.sh.in ${CMAKE_BINARY_DIR}/env
 execute_process(COMMAND mkdir -p ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 execute_process(COMMAND mkdir -p ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 execute_process(COMMAND mkdir -p ${SPT3G_MODULE_DIR})
-execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${SPT3G_MODULE_DIR}/__init__.py)
+execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/package/__init__.py ${SPT3G_MODULE_DIR}/__init__.py)
 
 set(BUILD_PROJECTS "${BUILD_PROJECTS}" CACHE STRING "The subset of available projects to actually build")
 if(NOT "${BUILD_PROJECTS}" STREQUAL "")
@@ -157,9 +157,7 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.17)
 endif(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.17)
 
 # Target for version string
-if(SPT3G_VERSION_FILE)
-	execute_process(COMMAND ln -fsn ${SPT3G_VERSION_FILE} ${SPT3G_MODULE_DIR}/version.py)
-else()
+if(NOT PIP_SPT3G_VERSION_FILE)
 	add_custom_target(version ALL
 		COMMAND sh ${CMAKE_SOURCE_DIR}/cmake/getvers.sh ${CMAKE_SOURCE_DIR} ${SPT3G_MODULE_DIR}/version.py
 		BYPRODUCTS ${SPT3G_MODULE_DIR}/version.py

README.rst

@@ -129,7 +129,8 @@ To install the package from the github repo, run ``pip`` as usual (this may take
 	cd spt3g_software
 	CMAKE_BUILD_PARALLEL_LEVEL=4 pip install -v .
 
-For development builds, ensure that the ``BUILD_DIR`` environment variable is set, so that the shared libraries remain accessible for running the python library and for incrementally rebuilding the package when changes are made.  Use the ``--editable`` option to assemble the python package from the appropriate compiled extensions and python directories:
+By default this will create a directory called ``build`` in the repo and run the ``cmake`` build from there.  The build directory location can be changed by setting the ``BUILD_DIR`` environment variable, but keep in mind that ``pip`` requires that the build directory must be a path inside the repo file tree.
+For development builds, use the ``--editable`` option to assemble the python package from the appropriate compiled extensions and python directories:
 
 .. code-block:: shell
 
@@ -145,7 +146,7 @@ To pass arguments to the cmake build system, use the ``CMAKE_ARGS`` environment
 	cd spt3g_software
 	CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_MODULE_PATH=/usr/local/share/cmake" pip install -v --prefix=/usr/local .
 
-To run the test suite on the compiled package, you must have ``cmake``, and in particular the ``ctest`` utility, available on your path.  You must also know the location of the build directory where the cmake build was assembled (e.g. the value of ``$BUILD_DIR`` above or the ``build/temp`` directory generated by ``pip``).
+To run the test suite on the compiled package, you must have ``cmake``, and in particular the ``ctest`` utility, available on your path.  You must also know the location of the build directory where the cmake build was assembled (e.g. the value of ``$BUILD_DIR`` above).
 
 .. code-block:: shell
 

cmake/package/.gitignore

@@ -0,0 +1 @@
+version.py

pyproject.toml

@@ -41,7 +41,7 @@ else:
 upstream_branch = 'UNKNOWN VCS'
 upstream_url = 'UNKNOWN VCS'
 """
-version_file = "./wheel/spt3g/version.py"
+version_file = "./cmake/package/version.py"
 version_scheme = "no-guess-dev"
 fallback_version = "0.0"
 

setup.py

@@ -1,5 +1,6 @@
-import json
+import logging
 import os
+import re
 import subprocess
 import sys
 import sysconfig
@@ -11,15 +12,18 @@
 from setuptools.command.install_scripts import install_scripts
 
 
-# A CMakeExtension needs a sourcedir instead of a file list.
+# A CMakeExtension does not need a source list
 class CMakeExtension(Extension):
-    def __init__(self, name, sourcedir=""):
+    def __init__(self, name):
         super().__init__(name, sources=[])
-        self.sourcedir = os.fspath(Path(sourcedir).resolve())
 
 
-class CMakeBuild(build_ext):
-    def build_extension(self, ext):
+class CMakeBuildExt(build_ext):
+    def initialize_options(self):
+        super().initialize_options()
+        self.source_dir = Path(".").resolve()
+        self.build_dir = Path(os.getenv("BUILD_DIR", "./build")).resolve()
+
         cmake_args = []
 
         # Adding CMake arguments set as environment variable
@@ -41,43 +45,36 @@ def build_extension(self, ext):
 
         # pass version to C++ code
         if not any(["SPT3G_VERSION" in a for a in cmake_args]):
-            sys.path.insert(0, str(Path(ext.sourcedir) / "wheel"))
-            from spt3g import version as spt3g_version
-            sys.path.pop(0)
-            cmake_args += [f"-DSPT3G_VERSION_FILE={spt3g_version.__file__}"]
-            vtup = spt3g_version.version_tuple
-            if len(vtup) == 2:
-                v = "{}.{}".format(*vtup)
-                cmake_args += [f"-DSPT3G_VERSION={repr(v)}"]
-
-        # ensure that build directory isn't removed on completion, so that
-        # shared libraries are accessible
-        if self.editable_mode:
-            if "BUILD_DIR" not in os.environ:
-                raise RuntimeError(
-                    "BUILD_DIR environment variable required in editable mode"
-                )
-
-        if "BUILD_DIR" in os.environ:
-            build_temp = Path(os.getenv("BUILD_DIR"))
-        else:
-            build_temp = Path(self.build_temp)
-        if not build_temp.exists():
-            build_temp.mkdir(parents=True)
-        self.build_dir = build_temp
+            cmake_args += [f"-DPIP_SPT3G_VERSION_FILE=ON"]
+            version = self.distribution.metadata.version
+            if re.match(r"^[0-9]+\.[0-9]+$", version):
+                cmake_args += [f"-DSPT3G_VERSION={repr(version)}"]
+
+        self.cmake_args = cmake_args
+        self.cmake_done = False
 
-        if ext.name.endswith("core"):
+    def build_extension(self, ext):
+        if not self.cmake_done:
             # build once
-            self.announce(f"Building library in {build_temp}")
+            self.announce(f"Building library in {self.build_dir}", logging.INFO)
             subprocess.run(
-                ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True
+                ["cmake", self.source_dir, *self.cmake_args], cwd=self.build_dir, check=True
             )
-            subprocess.run(["cmake", "--build", "."], cwd=build_temp, check=True)
+            subprocess.run(["cmake", "--build", "."], cwd=self.build_dir, check=True)
+
+            # update package directory
+            if self.editable_mode:
+                pkgdir = self.build_dir.relative_to(self.source_dir)
+                self.distribution.package_dir["spt3g"] = pkgdir
+
+            # update version file
+            self.copy_file(self.source_dir / "cmake/package/version.py", self.build_dir)
 
             # trigger script installer for all python scripts
             self.distribution.scripts = [
-                str(f) for f in build_temp.glob("bin/*") if f.is_symlink()
+                str(f) for f in self.build_dir.glob("bin/*") if f.is_symlink()
             ]
+            self.cmake_done = True
 
         # add modules
         spt3g_lib = Path(self.build_lib) / "spt3g"
@@ -86,8 +83,8 @@ def build_extension(self, ext):
 
         libname = ext.name.split(".")[-1]
         libfiles = [
-            build_temp / "spt3g" / self.get_ext_filename(libname),
-            build_temp / "spt3g" / f"{libname}.so",
+            self.build_dir / "spt3g" / self.get_ext_filename(libname),
+            self.build_dir / "spt3g" / f"{libname}.so",
         ]
         for f in libfiles:
             if f.exists():
@@ -101,7 +98,7 @@ class CMakeInstallScripts(install_scripts):
     def run(self):
         super().run()
 
-        self.announce("Installing scripts")
+        self.announce("Installing cmake programs", logging.INFO)
         install_dir = Path(self.install_dir)
         if not install_dir.exists():
             install_dir.mkdir(parents=True)
@@ -117,35 +114,28 @@ def run(self):
         if "CIBUILDWHEEL" in os.environ:
             return
 
-        self.announce("Installing libraries and headers")
+        self.announce("Installing cmake libraries and headers", logging.INFO)
         build_dir = self.get_finalized_command("build_ext").build_dir
         subprocess.run(["make", "install"], cwd=build_dir, check=True)
 
 
-# create package
-pkgdir = Path("./wheel/spt3g")
-pkgdir.mkdir(parents=True, exist_ok=True)
-initpy = pkgdir / "__init__.py"
-if not initpy.is_symlink():
-    initpy.symlink_to(Path("./cmake/init.py").resolve())
-
 # gather libraries
-clibs = ["core"]
-pdirs = {"spt3g": str(pkgdir)}
+clibs = []
+pdirs = {"spt3g": "cmake/package"}
 
 for d in sorted(Path("./").glob("*/CMakeLists.txt")):
     lib = d.parent.name
-    if (d.parent / "src").exists() and lib not in clibs:
-        clibs.append(lib)
+    if (d.parent / "src").exists():
+        clibs.append(f"spt3g._lib{lib}")
     if (d.parent / "python").exists():
         pdirs[f"spt3g.{lib}"] = d.parent / "python"
     elif (d.parent / "__init__.py").exists():
         pdirs[f"spt3g.{lib}"] = d.parent
 
 setup(
-    ext_modules=[CMakeExtension(f"spt3g._lib{lib}") for lib in clibs],
+    ext_modules=[CMakeExtension(lib) for lib in clibs],
     cmdclass={
-        "build_ext": CMakeBuild,
+        "build_ext": CMakeBuildExt,
         "install_scripts": CMakeInstallScripts,
         "install": CMakeInstall,
     },

wheel/.gitignore

@@ -1,2 +1 @@
-spt3g
 deps