Trac #29847: Install sage_setup with setuptools as a prerequisite for…

… building/installing sagelib

sage_setup will be used by the build system of various distribution
packages (#29705).

So we make it a regular distribution package itself, with sources in
`pkgs/sage-setup/` (#31577) and install it with `pip`.

The actual Python source files are not moved because `sage_setup` takes
part in Sage doctesting (see #31577).

`sagelib`'s `setup.py` will no longer install `sage_setup`.

`sage-setup` will become a `build-system requires` of `sagemath-
standard` (declared in `pyproject.toml`)

Follow-up:
 - #29924: Do not fail if `sage.env`, `sage.*` are not available

------
References:

python - pip ignores dependency_links in setup.py - Stack Overflow
https://stackoverflow.com/questions/12518499/pip-ignores-dependency-
links-in-setup-py

dependency_links flag ignored when package exists on PyPI · Issue #987 ·
pypa/setuptools
pypa/setuptools#987

Suggest alternatives for --process-dependency-links · Issue #6162 ·
pypa/pip
pypa/pip#6162

PEP 508 -- Dependency specification for Python Software Packages |
Python.org
https://www.python.org/dev/peps/pep-0508/#pep440

PEP 440 -- Version Identification and Dependency Specification |
Python.org
https://www.python.org/dev/peps/pep-0440/#local-version-segments

URL: https://trac.sagemath.org/29847
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Dima Pasechnik

.github/workflows/tox-gcc_spkg.yml

@@ -27,7 +27,7 @@ on:
     # Allow to run manually
 
 env:
-  TARGETS_PRE: sagelib-build-deps
+  TARGETS_PRE: all-sage-local
   TARGETS: build doc-html
   TARGETS_OPTIONAL: ptest
 

.github/workflows/tox.yml

@@ -27,7 +27,7 @@ on:
     # Allow to run manually
 
 env:
-  TARGETS_PRE: sagelib-build-deps
+  TARGETS_PRE: all-sage-local
   TARGETS: build doc-html
   TARGETS_OPTIONAL: ptest
 

build/bin/write-dockerfile.sh

@@ -230,7 +230,7 @@ ARG USE_MAKEFLAGS="-k V=0"
 ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!gfan,!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!networkx,!rpy2,!symengine_py,!sage_sws2rst"
 #:make:
-ARG TARGETS_PRE="sagelib-build-deps"
+ARG TARGETS_PRE="all-sage-local"
 $RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS_PRE} $ENDRUN
 
 FROM with-targets-pre as with-targets

build/make/Makefile.in

@@ -379,6 +379,9 @@ sagelib-clean:
 sage_docbuild-clean:
 	(cd "$(SAGE_ROOT)/build/pkgs/sage_docbuild/src" && rm -rf build)
 
+sage_setup-clean:
+	(cd "$(SAGE_ROOT)/build/pkgs/sage_setup/src" && rm -rf build)
+
 build-clean: clean doc-clean sagelib-clean sage_docbuild-clean
 
 # Special target for cleaning up a broken GCC install detected by configure
@@ -652,6 +655,8 @@ $(1)-$(4)-uninstall:
 
 $(1)-uninstall: $(1)-$(4)-uninstall
 
+$(1)-clean: $(1)-uninstall
+
 .PHONY: $(1) $(1)-uninstall $(1)-build-deps $(1)-no-deps $(1)-clean
 endef
 

build/pkgs/sage_conf/install-requires.txt

@@ -0,0 +1 @@
+sage-conf

build/pkgs/sage_setup/SPKG.rst

@@ -0,0 +1 @@
+src/README.rst

build/pkgs/sage_setup/dependencies

@@ -0,0 +1,5 @@
+$(PYTHON) cython pkgconfig | $(PYTHON_TOOLCHAIN)
+
+----------
+All lines of this file are ignored except the first.
+It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/sage_setup/install-requires.txt

@@ -0,0 +1 @@
+sage-setup

build/pkgs/sage_setup/package-version.txt

@@ -0,0 +1 @@
+../sagelib/package-version.txt

build/pkgs/sage_setup/spkg-install

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# From sage-spkg.
+# For type=script packages, the build rule in build/make/Makefile sources
+# sage-env but not sage-dist-helpers.
+lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
+source "$lib"
+if [ $? -ne 0 ]; then
+    echo >&2 "Error: failed to source $lib"
+    echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
+    exit 1
+fi
+cd src && sdh_pip_install .

build/pkgs/sage_setup/spkg-src

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Script to prepare an sdist tarball for sage_setup
+# This script is not used during build.
+#
+# HOW TO MAKE THE TARBALL:
+# ./sage --sh build/pkgs/sage_setup/spkg-src
+
+if [ -z "$SAGE_ROOT" ] ; then
+    echo >&2 "Error - SAGE_ROOT undefined ... exiting"
+    echo >&2 "Maybe run 'sage -sh'?"
+    exit 1
+fi
+
+# Exit on failure
+set -e
+
+cd build/pkgs/sage_setup
+
+cd src
+python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

build/pkgs/sage_setup/src

@@ -0,0 +1 @@
+../../../pkgs/sage-setup

build/pkgs/sage_setup/type

@@ -0,0 +1 @@
+standard

build/pkgs/sagelib/dependencies

@@ -1,4 +1,4 @@
-FORCE $(SCRIPTS) arb boost_cropped $(BLAS) brial cliquer cypari cysignals cython ecl eclib ecm flint libgd gap giac givaro glpk gmpy2 gsl iml jinja2 jupyter_core lcalc lrcalc libbraiding libhomfly libpng linbox m4ri m4rie memory_allocator mpc mpfi mpfr $(MP_LIBRARY) ntl numpy pari pip pkgconfig planarity ppl pplpy pycygwin pynac $(PYTHON) ratpoints rw sage_conf singular symmetrica zn_poly $(PCFILES)
+FORCE $(SCRIPTS) arb boost_cropped $(BLAS) brial cliquer cypari cysignals cython ecl eclib ecm flint libgd gap giac givaro glpk gmpy2 gsl iml jinja2 jupyter_core lcalc lrcalc libbraiding libhomfly libpng linbox m4ri m4rie memory_allocator mpc mpfi mpfr $(MP_LIBRARY) ntl numpy pari pip pkgconfig planarity ppl pplpy pycygwin pynac $(PYTHON) ratpoints rw sage_conf singular symmetrica zn_poly $(PCFILES) | $(PYTHON_TOOLCHAIN) sage_setup
 
 ----------
 All lines of this file are ignored except the first.

pkgs/README.rst

@@ -0,0 +1 @@
+This directory contains source trees of independent Python distribution packages.

pkgs/sage-setup/README.rst

@@ -0,0 +1,4 @@
+sage-setup: Build system of the SageMath library
+================================================
+
+This is the build system of the Sage library, based on setuptools.

pkgs/sage-setup/VERSION.txt

@@ -0,0 +1 @@
+../../src/VERSION.txt

pkgs/sage-setup/requirements.txt

@@ -0,0 +1,2 @@
+Cython
+pkgconfig

pkgs/sage-setup/setup.cfg

@@ -0,0 +1,37 @@
+[metadata]
+name = sage-setup
+version = file: VERSION.txt
+description = Sage: Open Source Mathematics Software: Build system of the Sage library
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+license = GNU General Public License (GPL) v2 or later
+author = The Sage Developers
+author_email = sage-support@googlegroups.com
+url = https://www.sagemath.org
+
+classifiers =
+    Development Status :: 6 - Mature
+    Intended Audience :: Education
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)
+    Operating System :: POSIX
+    Operating System :: MacOS :: MacOS X
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: Implementation :: CPython
+    Topic :: Scientific/Engineering :: Mathematics
+
+[options]
+packages =
+    sage_setup
+    sage_setup.autogen
+    sage_setup.autogen.interpreters
+    sage_setup.autogen.interpreters.specs
+    sage_setup.command
+
+python_requires = >=3.7, <3.10
+
+install_requires =
+    pkgconfig

pkgs/sage-setup/setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup()

pkgs/sage-setup/tox.ini

@@ -0,0 +1,30 @@
+# First pip-install tox:
+#
+#   ./sage -pip install tox
+#
+# To build and test in the tox environment:
+#
+# ./sage -sh -c '(cd pkgs/sage-setup && tox)'
+#
+# To test interactively:
+#
+# pkgs/sage-setup/.tox/python/bin/python
+#
+[tox]
+
+[testenv]
+deps = -rrequirements.txt
+
+setenv =
+    # Sage scripts such as sage-runtests like to use $HOME/.sage
+    HOME={envdir}
+
+whitelist_externals =
+    bash
+
+commands =
+    # Beware of the treacherous non-src layout.
+    python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage_setup; import sage_setup.find'
+
+# TODO: Test importing sage_setup.library_order -- when that can handle missing pkgconfig libraries...
+# TODO: Test more modules -- when the dependency on sage.env has been removed...

pkgs/sagemath-standard/setup.py

@@ -83,7 +83,7 @@
 
 from sage_setup.find import find_python_sources
 python_packages, python_modules, cython_modules = find_python_sources(
-    SAGE_SRC, ['sage', 'sage_setup'], distributions=distributions)
+    SAGE_SRC, ['sage'], distributions=distributions)
 
 log.debug('python_packages = {0}'.format(python_packages))
 

src/MANIFEST.in

@@ -7,3 +7,4 @@ prune .tox
 
 graft   sage/libs/gap/test
 prune sage/ext/interpreters   # In particular, __init__.py must not be present in the distribution; or sage_setup.autogen.interpreters.rebuild will not generate the code
+prune sage_setup

src/doc/en/developer/portability_testing.rst

@@ -358,7 +358,7 @@ Finally, to build and test...::
   ENV MAKE="make -j${NUMPROC}"
   ARG USE_MAKEFLAGS="-k"
   RUN make ${USE_MAKEFLAGS} base-toolchain
-  ARG TARGETS_PRE="sagelib-build-deps"
+  ARG TARGETS_PRE="all-sage-local"
   RUN make ${USE_MAKEFLAGS} ${TARGETS_PRE}
   ADD src src
   ARG TARGETS="build ptest"
@@ -462,7 +462,7 @@ might not work on all platforms, ``surf``, which was marked as
   Step 2/28 : FROM ${BASE_IMAGE}
    ---> 549b9b86cb8d
   ...
-  Step 24/28 : ARG TARGETS_PRE="sagelib-build-deps"
+  Step 24/28 : ARG TARGETS_PRE="all-sage-local"
    ---> Running in 17d0ddb5ad7b
   Removing intermediate container 17d0ddb5ad7b
    ---> 7b51411520c3
@@ -637,7 +637,7 @@ for a non-silent build (``make V=1``), use::
   [mkoeppe@sage sage]$ EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"V=1\"" \
     tox -e docker-ubuntu-bionic-standard
 
-By default, tox uses ``TARGETS_PRE=sagelib-build-deps`` and
+By default, tox uses ``TARGETS_PRE=all-sage-local`` and
 ``TARGETS=build``, leading to a complete build of Sage without the
 documentation.  If you pass positional arguments to tox (separated
 from tox options by ``--``), then both ``TARGETS_PRE`` and ``TARGETS``

src/pyproject.toml.m4

@@ -8,6 +8,7 @@ requires = [
     esyscmd(`sage-get-system-packages install-requires-toml \
         setuptools     \
         wheel          \
+        sage_setup     \
         cypari         \
         cysignals      \
         cython         \

src/sage_setup/clean.py

@@ -103,7 +103,7 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module
     CEXTMOD_EXTS = get_extensions('extension')
     INIT_FILES = tuple('__init__' + x for x in PYMOD_EXTS)
 
-    module_files = installed_files_by_module(site_packages, ['sage', 'sage_setup'])
+    module_files = installed_files_by_module(site_packages, ['sage'])
 
     for mod in python_packages:
         try:

src/setup.cfg.m4

@@ -6,7 +6,7 @@ description = Sage: Open Source Mathematics Software: Standard Python Library
 long_description = file: README.rst
 long_description_content_type = text/x-rst
 license = GNU General Public License (GPL) v2 or later
-license_file = LICENSE.txt
+license_files = LICENSE.txt
 author = The Sage Developers
 author_email = sage-support@googlegroups.com
 url = https://www.sagemath.org

src/setup.py

@@ -85,7 +85,7 @@
 
 log.debug(f"files_to_exclude = {files_to_exclude}")
 
-python_packages = find_namespace_packages(where=SAGE_SRC, include=['sage', 'sage_setup', 'sage.*', 'sage_setup.*'])
+python_packages = find_namespace_packages(where=SAGE_SRC, include=['sage', 'sage.*'])
 log.debug(f"python_packages = {python_packages}")
 
 log.info(f"Discovered Python/Cython sources, time: {(time.time() - t):.2f} seconds.")

tox.ini

@@ -555,7 +555,7 @@ commands =
     docker:              --build-arg BASE_IMAGE={env:FULL_BASE_IMAGE_AND_TAG} \
     docker-conda:        --build-arg USE_CONDARC="{env:CONDARC}" \
     docker:              --build-arg BOOTSTRAP="{env:BOOTSTRAP}" \
-    docker:              --build-arg TARGETS_PRE="$(if test -n "$TARGETS_PRE"; then echo $TARGETS_PRE; else echo {posargs:sagelib-build-deps}; fi)" \
+    docker:              --build-arg TARGETS_PRE="$(if test -n "$TARGETS_PRE"; then echo $TARGETS_PRE; else echo {posargs:all-sage-local}; fi)" \
     docker:              --build-arg TARGETS="{posargs:build}" \
     docker:              --build-arg TARGETS_OPTIONAL="{env:TARGETS_OPTIONAL:ptest}" \
     docker:              {env:EXTRA_DOCKER_BUILD_ARGS:}; status=$?; \