Trac #31377: ./configure --enable-editable

This configure switch will install sagelib in "develop" ("editable",
"in-place") mode instead of using sagelib's custom incremental build
system.

This will clutter the source directory with build artifacts (which are
`.gitignore`'d, of course) but it has the benefit that for changes to
Python files, one does not need to run `./sage -b`; restarting Sage is
enough.

It may also have benefits in certain develop environments that get
confused by sagelib's nonstandard build system.

This ticket is based on a subset of the changes in #30371, which
developed a version of `setup.py` suitable for the in-place build. This
version is `src/setup.py` and distinct from
`build/pkgs/sagelib/src/setup.py`.
The configure switch switches to this version of the build system.

To test:
{{{
./bootstrap
./configure --enable-editable
make build
}}}
Then use and test as normal. Verify that `local/lib/...site-packages/`
no longer contains a copy of `sage` - instead there is an "egg-link"
back to the source directory.

Switching to a standard build system (getting rid of the sage-specific
"installation cleaner") will also simplify the next step of the
modularization effort (#29705).

URL: https://trac.sagemath.org/31377
Reported by: mkoeppe
Ticket author(s): Tobias Diez, Matthias Koeppe
Reviewer(s): Matthias Koeppe, Tobias Diez, Jonathan Kliem

.gitignore

@@ -118,6 +118,7 @@ src/sage/modular/arithgroup/farey_symbol.h
 !src/sage/stats/distributions/dgs_bern.c
 !src/sage/stats/distributions/dgs_gauss_dp.c
 !src/sage/stats/distributions/dgs_gauss_mp.c
+/src/cython_debug
 
 # Temporary build files
 build/temp.*/

build/bin/sage-build-env-config.in

@@ -106,3 +106,5 @@ export SAGE_FREETYPE_PREFIX="@SAGE_FREETYPE_PREFIX@"
 export SAGE_SUITESPARSE_PREFIX="@SAGE_SUITESPARSE_PREFIX@"
 
 export SAGE_CONFIGURE_FFLAS_FFPACK="@SAGE_CONFIGURE_FFLAS_FFPACK@"
+
+export SAGE_EDITABLE="@SAGE_EDITABLE@"

build/make/Makefile.in

@@ -380,9 +380,16 @@ clean:
 	rm -rf "$(SAGE_LOCAL)/var/tmp/sage/build"
 
 # "c_lib", ".cython_version", "build" in $(SAGE_SRC) are from old sage versions
+# Cleaning .so files (and .c and .cpp files associated with .pyx files) is for editable installs.
+# Also cython_debug is for editable installs.
 sagelib-clean:
 	@echo "Deleting Sage library build artifacts..."
-	(cd "$(SAGE_SRC)" && rm -rf c_lib .cython_version; rm -rf build; find . -name '*.pyc' | xargs rm -f; rm -rf sage/ext/interpreters) && (cd "$(SAGE_ROOT)/build/pkgs/sagelib/src/" && rm -rf build)
+	(cd "$(SAGE_SRC)" && \
+	 rm -rf c_lib .cython_version cython_debug; \
+	 rm -rf build; find . -name '*.pyc' -o -name "*.so" | xargs rm -f; \
+	 rm -f $$(find . -name "*.pyx" | sed 's/\(.*\)[.]pyx$$/\1.c \1.cpp/'); \
+	 rm -rf sage/ext/interpreters) \
+	&& (cd "$(SAGE_ROOT)/build/pkgs/sagelib/src/" && rm -rf build)
 
 build-clean: clean doc-clean sagelib-clean
 

build/pkgs/sagelib/spkg-configure.m4

@@ -0,0 +1,4 @@
+AC_ARG_ENABLE([editable],
+              [AS_HELP_STRING([--enable-editable],
+                              [use an editable install of the Sage library])],
+              [AC_SUBST([SAGE_EDITABLE], [yes])])

build/pkgs/sagelib/spkg-install

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
-cd src
+if [ "$SAGE_EDITABLE" = yes ]; then
+    cd "$SAGE_SRC"
+else
+    cd src
+fi
 ## All sagelib-building is done by setup.py.
 ## This is so that sagelib can be installed by standard Python procedures,
 ## such as "./setup.py install" or "pip install ."
@@ -39,7 +43,20 @@ export SAGE_SHARE=/doesnotexist
 # spec, which includes setting a symlink to the installed documentation.
 # export SAGE_DOC=/doesnotexist
 
-time "$PYTHON" -u setup.py --no-user-cfg build install || exit 1
+SITEPACKAGESDIR=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')
+if [ "$SAGE_EDITABLE" = yes ]; then
+    # In an incremental build, we may need to uninstall old versions installed by distutils
+    # under the old distribution name "sage" (before #30912, which switched to setuptools
+    # and renamed the distribution to "sagemath-standard").  There is no clean way to uninstall
+    # them, so we just use rm.
+    (cd "$SITEPACKAGESDIR" && rm -rf sage sage_setup sage-[1-9]*.egg-info sage-[1-9]*.dist-info)
+    time python3 -m pip install --verbose --no-deps --no-index --no-build-isolation --isolated --editable . || exit 1
+else
+    # Likewise, we should remove the egg-link that may have been installed previously.
+    (cd "$SITEPACKAGESDIR" && rm -f sagemath-standard.egg-link)
+    time python3 -u setup.py --no-user-cfg build install || exit 1
+fi
+
 if [ "$UNAME" = "CYGWIN" ]; then
     sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null;
 fi

src/sage/graphs/base/boost_graph.pxd

@@ -1,4 +1,5 @@
 # distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
 
 #*****************************************************************************
 #       Copyright (C) 2015 Michele Borassi michele.borassi@imtlucca.it

src/sage/libs/m4ri.pxd

@@ -1,4 +1,5 @@
-# distutils: extra_compile_args = -std=c99
+# distutils: extra_compile_args = -std=c++11
+# distutils: language = c++
 
 cdef extern from "m4ri/m4ri.h":
     ctypedef int rci_t

src/sage/libs/singular/decl.pxd

@@ -3,6 +3,7 @@
 # distutils: libraries = SINGULAR_LIBRARIES
 # distutils: library_dirs = SINGULAR_LIBDIR
 # distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
 
 """
 Declarations of Singular's C/C++ Functions

src/sage/repl/interpreter.py

@@ -80,7 +80,7 @@
     ZeroDivisionError...Traceback (most recent call last)
     <ipython-input-...> in <module>...
     ----> 1 Integer(1)/Integer(0)
-    .../sage/rings/integer.pyx in sage.rings.integer.Integer...div... (.../cythonized/sage/rings/integer.c:...)()
+    .../sage/rings/integer.pyx in sage.rings.integer.Integer...div...
     ...
     -> ...                  raise ZeroDivisionError("rational division by zero")
        ....:            x = <Rational> Rational.__new__(Rational)

src/sage/rings/finite_rings/element_givaro.pxd

@@ -1,4 +1,4 @@
-# distutils: extra_compile_args = GIVARO_CFLAGS
+# distutils: extra_compile_args = GIVARO_CFLAGS -std=c++11
 # distutils: include_dirs = GIVARO_INCDIR
 
 from libcpp.vector cimport vector

src/sage_setup/autogen/__init__.py

@@ -1,5 +1,6 @@
 import os
-
+from . import interpreters
+from sage.env import SAGE_SRC
 
 def autogen_all():
     """
@@ -8,8 +9,6 @@ def autogen_all():
     Return a list of sub-packages that should be appended to the list
     of packages built/installed by setup.py.
     """
-
-    from . import interpreters
-    interpreters.rebuild(os.path.join("sage", "ext", "interpreters"))
+    interpreters.rebuild(os.path.join(SAGE_SRC, "sage", "ext", "interpreters"))
 
     return ['sage.ext.interpreters']

src/sage_setup/clean.py

@@ -78,7 +78,8 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module
     after installation, there are no stale files::
 
         sage: from sage.env import SAGE_SRC, SAGE_LIB, SAGE_ROOT
-        sage: cythonized_dir = os.path.join(SAGE_ROOT, "build", "pkgs", "sagelib", "src", "build", "cythonized")
+        sage: from sage_setup.find import _cythonized_dir
+        sage: cythonized_dir = _cythonized_dir(SAGE_SRC)
         sage: from sage_setup.find import find_python_sources, find_extra_files
         sage: python_packages, python_modules, cython_modules = find_python_sources(
         ....:     SAGE_SRC, ['sage', 'sage_setup'])

src/sage_setup/command/sage_build_ext_minimal.py

@@ -0,0 +1,35 @@
+import os
+import multiprocessing
+from setuptools.command.build_ext import build_ext
+
+
+class sage_build_ext_minimal(build_ext):
+    """
+    In contrast to :func:`~sage_setup.sage_build_ext.sage_build_ext`, this build extension is designed
+    to be used in combination with Cython's cythonize function.
+    Thus, we only take care of some options and letting Cython do the main work.
+    """
+
+    def initialize_options(self):
+        build_ext.initialize_options(self)
+        self.parallel = self.get_default_number_build_jobs()
+
+    @staticmethod
+    def get_default_number_build_jobs() -> int:
+        """
+        Get number of parallel build jobs used by default, i.e. unless explicitly
+        set by the --parallel command line argument of setup.py.
+
+        First, the environment variable `SAGE_NUM_THREADS` is checked.
+        If that is unset, return the number of processors on the system,
+        with a maximum of 10 (to prevent overloading the system if there a lot of CPUs).
+
+        OUTPUT:
+            number of parallel jobs that should be run
+        """
+        try:
+            cpu_count = len(os.sched_getaffinity(0))
+        except AttributeError:
+            cpu_count = multiprocessing.cpu_count()
+        cpu_count = min(cpu_count, 10)
+        return int(os.environ.get("SAGE_NUM_THREADS", cpu_count))

src/sage_setup/extensions.py

@@ -0,0 +1,10 @@
+
+def create_extension(template, kwds):
+    from Cython.Build.Dependencies import default_create_extension
+    from sage.env import sage_include_directories
+
+    # Add numpy and source folder to the include search path used by the compiler
+    # This is a workaround for https://github.com/cython/cython/issues/1480
+    include_dirs = kwds.get('include_dirs', []) + sage_include_directories(use_sources=True)
+    kwds['include_dirs'] = include_dirs
+    return default_create_extension(template, kwds)

src/sage_setup/find.py

@@ -173,6 +173,92 @@ def is_in_distributions(filename):
         os.chdir(cwd)
     return python_packages, python_modules, cython_modules
 
+def filter_cython_sources(src_dir, distributions):
+    """
+    Find all Cython modules in the given source directory that belong to the
+    given distributions.
+
+    INPUT:
+
+    - ``src_dir`` -- root directory for the sources
+
+    - ``distributions`` -- a sequence or set of strings: only find modules whose
+      ``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
+      directive in the module source file) is an element of
+      ``distributions``.
+
+    OUTPUT: List of absolute paths to Cython files (``*.pyx``).
+
+    EXAMPLES::
+
+        sage: from sage.env import SAGE_SRC
+        sage: from sage_setup.find import filter_cython_sources
+        sage: cython_modules = filter_cython_sources(SAGE_SRC, ["sage-tdlib"])
+
+    Cython module relying on tdlib::
+
+        sage: any(f.endswith('sage/graphs/graph_decompositions/tdlib.pyx') for f in cython_modules)
+        True
+
+    Cython module not relying on tdlib::
+
+        sage: any(f.endswith('sage/structure/sage_object.pyx') for f in cython_modules)
+        False
+
+    Benchmarking::
+
+        sage: timeit('filter_cython_sources(SAGE_SRC, ["sage-tdlib"])', # random output
+        ....:        number=1, repeat=1)
+        1 loops, best of 1: 850 ms per loop
+    """
+    files: list[str] = []
+
+    for dirpath, dirnames, filenames in os.walk(src_dir):
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            base, ext = os.path.splitext(filename)
+            if ext == '.pyx' and read_distribution(filepath) in distributions:
+                files.append(filepath)
+
+    return files
+
+def _cythonized_dir(src_dir=None, editable_install=None):
+    """
+    Return the path where Cython-generated files are placed by the build system.
+
+    INPUT:
+
+    - ``src_dir`` -- string or path (default: the value of ``SAGE_SRC``).  The
+      root directory for the sources.
+
+    - ``editable_install`` -- boolean (default: determined from the existing
+      installation). Whether this is an editable install of the Sage library.
+
+    EXAMPLES::
+
+        sage: from sage_setup.find import _cythonized_dir
+        sage: from sage.env import SAGE_SRC
+        sage: _cythonized_dir(SAGE_SRC)  # optional - build
+        PosixPath('...')
+        sage: _cythonized_dir(SAGE_SRC, editable_install=False)
+        PosixPath('.../build/cythonized')
+
+    """
+    from importlib import import_module
+    from pathlib import Path
+    from sage.env import SAGE_ROOT, SAGE_SRC
+    if editable_install is None:
+        if src_dir is None:
+            src_dir = SAGE_SRC
+        src_dir = Path(src_dir)
+        sage = import_module('sage')
+        d = Path(sage.__file__).resolve().parent.parent
+        editable_install = d == src_dir.resolve()
+    if editable_install:
+        # Editable install: Cython generates files in the source tree
+        return src_dir
+    else:
+        return Path(SAGE_ROOT) / "build" / "pkgs" / "sagelib" / "src" / "build" / "cythonized"
 
 def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[]):
     """
@@ -208,9 +294,9 @@ def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[]):
 
     EXAMPLES::
 
-        sage: from sage_setup.find import find_extra_files
+        sage: from sage_setup.find import find_extra_files, _cythonized_dir
         sage: from sage.env import SAGE_SRC, SAGE_ROOT
-        sage: cythonized_dir = os.path.join(SAGE_ROOT, "build", "pkgs", "sagelib", "src", "build", "cythonized")
+        sage: cythonized_dir = _cythonized_dir(SAGE_SRC)
         sage: extras = find_extra_files(SAGE_SRC, ["sage"], cythonized_dir)
         sage: extras["sage/libs/mpfr"]
         [...sage/libs/mpfr/types.pxd...]
@@ -269,8 +355,7 @@ def installed_files_by_module(site_packages, modules=('sage',)):
 
     EXAMPLES::
 
-        sage: from site import getsitepackages
-        sage: site_packages = getsitepackages()[0]
+        sage: site_packages = os.path.dirname(os.path.dirname(sage.__file__))
         sage: from sage_setup.find import installed_files_by_module
         sage: files_by_module = installed_files_by_module(site_packages)
         sage: from sage.misc.sageinspect import loadable_module_extension