From 0295c8f09cb6e01f60694084cfe758152041b68b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 3 Jun 2020 15:44:11 -0700 Subject: [PATCH 01/19] src/module_list.py: Move options for Extensions in sage.graphs.graph_decompositions to distutils directives --- src/module_list.py | 4 +--- src/sage/graphs/graph_decompositions/rankwidth.pyx | 1 + src/sage/graphs/graph_decompositions/tdlib.pyx | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/module_list.py b/src/module_list.py index 64937e841c5..a314b24c558 100644 --- a/src/module_list.py +++ b/src/module_list.py @@ -381,8 +381,7 @@ def uname_specific(name, value, alternative): sources = ['sage/graphs/strongly_regular_db.pyx']), Extension('sage.graphs.graph_decompositions.rankwidth', - sources = ['sage/graphs/graph_decompositions/rankwidth.pyx'], - libraries=['rw']), + sources = ['sage/graphs/graph_decompositions/rankwidth.pyx']), Extension('sage.graphs.graph_decompositions.bandwidth', sources = ['sage/graphs/graph_decompositions/bandwidth.pyx']), @@ -392,7 +391,6 @@ def uname_specific(name, value, alternative): OptionalExtension('sage.graphs.graph_decompositions.tdlib', sources = ['sage/graphs/graph_decompositions/tdlib.pyx'], - language="c++", package = 'tdlib'), Extension('sage.graphs.graph_decompositions.clique_separators', diff --git a/src/sage/graphs/graph_decompositions/rankwidth.pyx b/src/sage/graphs/graph_decompositions/rankwidth.pyx index a5c3b4d917f..c21e259a3b5 100644 --- a/src/sage/graphs/graph_decompositions/rankwidth.pyx +++ b/src/sage/graphs/graph_decompositions/rankwidth.pyx @@ -1,4 +1,5 @@ # cython: binding=True +# distutils: libraries = rw r""" Rank Decompositions of graphs diff --git a/src/sage/graphs/graph_decompositions/tdlib.pyx b/src/sage/graphs/graph_decompositions/tdlib.pyx index 5ffc28685a2..9bfe49b323d 100644 --- a/src/sage/graphs/graph_decompositions/tdlib.pyx +++ b/src/sage/graphs/graph_decompositions/tdlib.pyx @@ -1,3 +1,7 @@ +# distutils: language = c++ +# distutils: libraries = tdlib +# sage_setup: distribution = sage-tdlib + r""" Interface with TdLib (algorithms for tree decompositions) From b582789cc861ab8231236e33de8c1e400b1b864a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 3 Jun 2020 15:46:19 -0700 Subject: [PATCH 02/19] src/sage_setup/find.py: Filter by directive 'sage_setup: distribution = PKG', find Cython modules --- src/sage_setup/find.py | 121 +++++++++++++++++++++++++++++++++++------ src/setup.py | 4 +- 2 files changed, 105 insertions(+), 20 deletions(-) diff --git a/src/sage_setup/find.py b/src/sage_setup/find.py index 12914419fe3..93cfe65eddb 100644 --- a/src/sage_setup/find.py +++ b/src/sage_setup/find.py @@ -19,10 +19,48 @@ from collections import defaultdict +def read_distribution(src_file): + """ + Parse ``src_file`` for a ``# sage_setup: distribution = PKG`` directive. + + INPUT: + + - ``src_file`` -- file name of a Python or Cython source file -def find_python_sources(src_dir, modules=['sage']): + OUTPUT: + + - a string, the name of the distribution package (``PKG``); or the empty + string if no directive was found. + + EXAMPLES:: + + sage: from sage.env import SAGE_SRC + sage: from sage_setup.find import read_distribution + sage: read_distribution(os.path.join(SAGE_SRC, 'sage', 'graphs', 'graph_decompositions', 'tdlib.pyx')) + 'sage-tdlib' + sage: read_distribution(os.path.join(SAGE_SRC, 'sage', 'graphs', 'graph_decompositions', 'modular_decomposition.py')) + '' """ - Find all Python packages and Python modules in the sources. + from Cython.Utils import open_source_file + with open_source_file(src_file, error_handling='ignore') as fh: + for line in fh: + # Adapted from Cython's Build/Dependencies.py + line = line.lstrip() + if not line: + continue + if line[0] != '#': + break + line = line[1:].lstrip() + kind = "sage_setup:" + if line.startswith(kind): + key, _, value = [s.strip() for s in line[len(kind):].partition('=')] + if key == "distribution": + return value + return '' + +def find_python_sources(src_dir, modules=['sage'], distributions=None): + """ + Find all Python packages and Python/Cython modules in the sources. INPUT: @@ -31,12 +69,20 @@ def find_python_sources(src_dir, modules=['sage']): - ``modules`` -- (default: ``['sage']``) sequence of strings: the top-level directories in ``src_dir`` to be considered - OUTPUT: Pair consisting of + - ``distributions`` -- (default: ``None``) if not ``None``, + should be a sequence or set of strings: only find modules whose + ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` + directive in the module source file) is an element of + ``distributions``. + + OUTPUT: Triple consisting of - the list of package names (corresponding to directories with ``__init__.py``), - - module names (corresponding to other ``*.py`` files). + - Python module names (corresponding to other ``*.py`` files). + + - Cython extensions (corresponding to ``*.pyx`` files). Both use dot as separator. @@ -44,44 +90,83 @@ def find_python_sources(src_dir, modules=['sage']): sage: from sage.env import SAGE_SRC sage: from sage_setup.find import find_python_sources - sage: py_packages, py_modules = find_python_sources(SAGE_SRC) - sage: examples = ['sage.structure', 'sage.structure.formal_sum', - ....: 'sage.structure.sage_object', 'sage.doctest.tests'] - sage: [m in py_packages for m in examples] - [True, False, False, False] - sage: [m in py_modules for m in examples] - [False, True, False, False] + sage: py_packages, py_modules, cy_modules = find_python_sources(SAGE_SRC) + + Ordinary package (with ``__init__.py``):: + + sage: ['sage.structure' in L for L in (py_packages, py_modules)] + [True, False] + + Python module in an ordinary package:: + + sage: ['sage.structure.formal_sum' in L for L in (py_packages, py_modules)] + [False, True] + + Cython module in an ordinary package:: + + sage: ['sage.structure.sage_object' in L for L in (py_packages, py_modules)] + [False, False] + + Subdirectory without any Python files:: + + sage: ['sage.doctest.tests' in L for L in (py_packages, py_modules)] + [False, False] + + Filtering by distribution (distutils package):: + + sage: find_python_sources(SAGE_SRC, distributions=['sage-tdlib']) + ([], [], []) + + Benchmarking:: sage: timeit('find_python_sources(SAGE_SRC)', # random output ....: number=1, repeat=1) 1 loops, best of 1: 18.8 ms per loop sage: find_python_sources(SAGE_SRC, modules=['sage_setup']) - (['sage_setup', ...], [...'sage_setup.find'...]) + (['sage_setup', ...], [...'sage_setup.find'...], []) """ + from distutils.extension import Extension + PYMOD_EXT = get_extensions('source')[0] INIT_FILE = '__init__' + PYMOD_EXT python_packages = [] python_modules = [] + cython_modules = [] cwd = os.getcwd() try: os.chdir(src_dir) for module in modules: for dirpath, dirnames, filenames in os.walk(module): - if INIT_FILE not in filenames: + package = dirpath.replace(os.path.sep, '.') + if INIT_FILE in filenames: + # Ordinary package. + if distributions is None or '' in distributions: + python_packages.append(package) + else: continue - dirpath = dirpath.replace(os.path.sep, '.') - python_packages.append(dirpath) + + def is_in_distributions(filename): + if distributions is None: + return True + distribution = read_distribution(os.path.join(dirpath, filename)) + return distribution in distributions + for filename in filenames: base, ext = os.path.splitext(filename) if ext == PYMOD_EXT and base != '__init__': - python_modules.append(dirpath + '.' + base) + if is_in_distributions(filename): + python_modules.append(package + '.' + base) + if ext == '.pyx': + if is_in_distributions(filename): + cython_modules.append(Extension(package + '.' + base, + sources=[os.path.join(dirpath, filename)])) + finally: os.chdir(cwd) - return python_packages, python_modules - + return python_packages, python_modules, cython_modules def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[]): """ diff --git a/src/setup.py b/src/setup.py index 72c59ace715..ad81313b48b 100755 --- a/src/setup.py +++ b/src/setup.py @@ -63,7 +63,7 @@ print("Discovering Python/Cython source code....") t = time.time() from sage_setup.find import find_python_sources -python_packages, python_modules = find_python_sources( +python_packages, python_modules, cython_modules = find_python_sources( SAGE_SRC, ['sage', 'sage_setup']) log.debug('python_packages = {0}'.format(python_packages)) @@ -117,4 +117,4 @@ build_cython=sage_build_cython, build_ext=sage_build_ext, install=sage_install), - ext_modules = ext_modules) + ext_modules = ext_modules + cython_modules) From ae70c81c7a3a8c2713330cb407bbc08dac27bb5e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 3 Jun 2020 14:40:02 -0700 Subject: [PATCH 03/19] src/sage_setup/optional_extension.py (is_package_installed_and_updated): Factor out from OptionalExtension --- src/sage_setup/optional_extension.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/sage_setup/optional_extension.py b/src/sage_setup/optional_extension.py index 546191fa809..2f7e8f733ac 100644 --- a/src/sage_setup/optional_extension.py +++ b/src/sage_setup/optional_extension.py @@ -40,6 +40,16 @@ class CythonizeExtension(Extension): """ skip_build = True +def is_package_installed_and_updated(pkg): + from sage.misc.package import is_package_installed + try: + pkginfo = all_packages[pkg] + except KeyError: + # Might be an installed old-style package + condition = is_package_installed(pkg) + else: + condition = (pkginfo["installed_version"] == pkginfo["remote_version"]) + return condition def OptionalExtension(*args, **kwds): """ @@ -76,15 +86,7 @@ def OptionalExtension(*args, **kwds): condition = kwds.pop("condition") except KeyError: pkg = kwds.pop("package") - from sage.misc.package import is_package_installed - try: - pkginfo = all_packages[pkg] - except KeyError: - # Might be an installed old-style package - condition = is_package_installed(pkg) - else: - condition = (pkginfo["installed_version"] == pkginfo["remote_version"]) - + condition = is_package_installed_and_updated(pkg) if condition: return Extension(*args, **kwds) else: From ec7e9c599aebbec9b8a9227399cf6b8793dccd72 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 3 Jun 2020 14:41:25 -0700 Subject: [PATCH 04/19] src/setup.py: Remove use of module_list.py; filter by distributions --- src/setup.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/setup.py b/src/setup.py index ad81313b48b..f3399fd4c63 100755 --- a/src/setup.py +++ b/src/setup.py @@ -22,15 +22,6 @@ # This import allows instancemethods to be pickable import sage_setup.fpickle_setup -######################################################### -### List of Extensions -### -### The list of extensions resides in module_list.py in -### the same directory as this file -######################################################### - -from module_list import ext_modules - ######################################################### ### Configuration ######################################################### @@ -62,9 +53,23 @@ # TODO: This should be quiet by default print("Discovering Python/Cython source code....") t = time.time() + +distributions = [''] + +from sage_setup.optional_extension import is_package_installed_and_updated + +optional_packages_with_extensions = ['mcqd', 'bliss', 'tdlib', 'primecount', + 'coxeter3', 'fes', 'sirocco', 'meataxe'] + +distributions += ['sage-{}'.format(pkg) + for pkg in optional_packages_with_extensions + if is_package_installed_and_updated(pkg)] + +log.warn('distributions = {0}'.format(distributions)) + from sage_setup.find import find_python_sources python_packages, python_modules, cython_modules = find_python_sources( - SAGE_SRC, ['sage', 'sage_setup']) + SAGE_SRC, ['sage', 'sage_setup'], distributions=distributions) log.debug('python_packages = {0}'.format(python_packages)) @@ -117,4 +122,4 @@ build_cython=sage_build_cython, build_ext=sage_build_ext, install=sage_install), - ext_modules = ext_modules + cython_modules) + ext_modules = cython_modules) From 2d866a3bab2a1d55bc85fcae4f30d384498bcc12 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 3 Jun 2020 19:26:49 -0700 Subject: [PATCH 05/19] src/sage/graphs/graph_decompositions/tdlib.pyx: Fixup --- src/sage/graphs/graph_decompositions/tdlib.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sage/graphs/graph_decompositions/tdlib.pyx b/src/sage/graphs/graph_decompositions/tdlib.pyx index 9bfe49b323d..e90e0aba357 100644 --- a/src/sage/graphs/graph_decompositions/tdlib.pyx +++ b/src/sage/graphs/graph_decompositions/tdlib.pyx @@ -1,5 +1,4 @@ # distutils: language = c++ -# distutils: libraries = tdlib # sage_setup: distribution = sage-tdlib r""" From 7f8850adc52d8798852646b550f4f9c88b30b468 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 7 Jun 2020 11:53:46 -0700 Subject: [PATCH 06/19] src/module_list.py: Document that it is obsolete --- src/module_list.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/module_list.py b/src/module_list.py index 828478798ed..ea6e2349234 100644 --- a/src/module_list.py +++ b/src/module_list.py @@ -1,3 +1,11 @@ +######################################################### +### +### OBSOLETE FILE - DO NOT ADD TO IT +### +### See https://trac.sagemath.org/ticket/29701 +### +######################################################### + import os from distutils.extension import Extension from sage.env import SAGE_LOCAL From cdf501f96008303331b78fa398aa039a080c23fb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 24 Jun 2020 23:10:15 -0700 Subject: [PATCH 07/19] build/pkgs/sagelib/spkg-src: New --- build/pkgs/sagelib/spkg-src | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 build/pkgs/sagelib/spkg-src diff --git a/build/pkgs/sagelib/spkg-src b/build/pkgs/sagelib/spkg-src new file mode 100644 index 00000000000..d8e308186b5 --- /dev/null +++ b/build/pkgs/sagelib/spkg-src @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Script to prepare an sdist tarball for sagelib +# This script is not used during build. +# +# HOW TO MAKE THE TARBALL: +# ./sage --sh build/pkgs/sagelib/spkg-src + +if [ -z "$SAGE_ROOT" ] ; then + echo >&2 "Error - SAGE_ROOT undefined ... exiting" + echo >&2 "Maybe run 'sage -sh'?" + exit 1 +fi + +# Exit on failure +set -e + +cd build/pkgs/sagelib + +cd src +sage-python23 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES" From ef4f5b8b6536243e494faa80148cf0bbb0b348b4 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sat, 16 May 2020 22:41:26 -0700 Subject: [PATCH 08/19] Move src/MANIFEST.in to build/pkgs/sagelib/MANIFEST.in, fix up --- build/pkgs/sagelib/src/MANIFEST.in | 8 +++++++ src/MANIFEST.in | 38 ------------------------------ 2 files changed, 8 insertions(+), 38 deletions(-) create mode 100644 build/pkgs/sagelib/src/MANIFEST.in delete mode 100644 src/MANIFEST.in diff --git a/build/pkgs/sagelib/src/MANIFEST.in b/build/pkgs/sagelib/src/MANIFEST.in new file mode 100644 index 00000000000..182574a572a --- /dev/null +++ b/build/pkgs/sagelib/src/MANIFEST.in @@ -0,0 +1,8 @@ +global-include *.c *.cc *.cpp *.h *.hh *.hpp *.inc *.py *.pyx *.pxd *.pxi *.rst *.txt *.tex + +include MANIFEST.in + +prune .tox + +graft sage/libs/gap/test +recursive-exclude sage/ext/interpreters *.c *.h *.pyx *.pxd diff --git a/src/MANIFEST.in b/src/MANIFEST.in deleted file mode 100644 index ecc887652cb..00000000000 --- a/src/MANIFEST.in +++ /dev/null @@ -1,38 +0,0 @@ -global-include *.c *.cc *.cpp *.h *.hh *.hpp *.inc *.py *.pyx *.pxd *.pxi *.rst *.txt *.tex - -include MANIFEST.in - -graft sage/libs/gap/test -recursive-include sage -recursive-exclude sage/ext/interpreters *.c *.h *.pyx *.pxd -exclude sage/modular/arithgroup/farey_symbol.h -exclude sage/rings/real_mpfi.h -exclude sage/symbolic/pynac.h - -graft doc/common/static -graft doc/common/themes -include doc/common/python.inv -include doc/common/update-python-inv.sh -include doc/en/introspect/static/empty -include doc/en/introspect/templates/layout.html -include doc/en/website/static/pdf.png -include doc/en/website/templates/index.html -include doc/en/a_tour_of_sage/eigen_plot.png -include doc/en/a_tour_of_sage/sin_plot.png -include doc/en/bordeaux_2008/birch.png -include doc/en/bordeaux_2008/modpcurve.png -include doc/en/developer/sagenb/branch_dropdown.png -include doc/en/developer/sagenb/forking_button.png -include doc/en/developer/sagenb/pull_button.png -include doc/fr/a_tour_of_sage/eigen_plot.png -include doc/fr/a_tour_of_sage/sin_plot.png -include doc/tr/a_tour_of_sage/eigen_plot.png -include doc/tr/a_tour_of_sage/sin_plot.png -include doc/pt/a_tour_of_sage/eigen_plot.png -include doc/pt/a_tour_of_sage/sin_plot.png -graft doc/en/reference/*/media -graft doc/en/thematic_tutorials/media -graft doc/en/prep/media -prune doc/en/reference/*/sage -prune doc/en/reference/*/sagenb -prune doc/output From 233f6baf8f2d701104232601ad6fb7362fef7071 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 10:39:32 -0700 Subject: [PATCH 09/19] build/pkgs/sagelib/src/requirements.txt, tox.ini: New --- build/pkgs/sagelib/src/requirements.txt | 12 ++++++++++ build/pkgs/sagelib/src/tox.ini | 32 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 build/pkgs/sagelib/src/requirements.txt create mode 100644 build/pkgs/sagelib/src/tox.ini diff --git a/build/pkgs/sagelib/src/requirements.txt b/build/pkgs/sagelib/src/requirements.txt new file mode 100644 index 00000000000..cd2008a2274 --- /dev/null +++ b/build/pkgs/sagelib/src/requirements.txt @@ -0,0 +1,12 @@ +#sage_conf +sage_setup @ file:///Users/mkoeppe/s/sage/sage-rebasing/worktree-algebraic-2018-spring/upstream/sage-setup-0.0.0.tar.gz +six # use of six should be removed from sage_setup +Cython==0.29.17 +pkgconfig +cysignals +gmpy2==2.1.0b1 + +numpy # already needed by sage.env +jinja2 # sage_setup.autogen.interpreters + +cypari2 # but building bdist_wheel of cypari2 fails with recent pip... https://github.com/sagemath/cypari2/issues/93 diff --git a/build/pkgs/sagelib/src/tox.ini b/build/pkgs/sagelib/src/tox.ini new file mode 100644 index 00000000000..0a00d2cfdf3 --- /dev/null +++ b/build/pkgs/sagelib/src/tox.ini @@ -0,0 +1,32 @@ +# First pip-install tox: +# +# ./sage -pip install tox +# +# To build and test in the tox environment: +# +# ./sage -sh -c '(cd build/pkgs/sagelib/src && tox)' +# +# To test interactively: +# +# build/pkgs/sagelib/src/.tox/python/bin/python +# +[tox] + +[testenv] +deps = -rrequirements.txt + +## passenv = +## PATH +## CC +## CXX + +setenv = + # Sage scripts such as sage-runtests like to use $HOME/.sage + HOME={envdir} + +whitelist_externals = + bash + +commands = + # Beware of the treacherous non-src layout. "./sage/" shadows the install sage package. + python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.all' From 177cd4c40a9f74a547f605375ad501aa986c39ec Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 11:25:39 -0700 Subject: [PATCH 10/19] build/pkgs/sagelib/src/MANIFEST.in: Prune sage/ext/interpreters --- build/pkgs/sagelib/src/MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/pkgs/sagelib/src/MANIFEST.in b/build/pkgs/sagelib/src/MANIFEST.in index 182574a572a..7123d149222 100644 --- a/build/pkgs/sagelib/src/MANIFEST.in +++ b/build/pkgs/sagelib/src/MANIFEST.in @@ -5,4 +5,4 @@ include MANIFEST.in prune .tox graft sage/libs/gap/test -recursive-exclude sage/ext/interpreters *.c *.h *.pyx *.pxd +prune sage/ext/interpreters # In particular, __init__.py must not be present in the distribution; or sage_setup.autogen.interpreters.rebuild will not generate the code From 07b29fbe8af49c018f4b6453fe31225f7a9bc53c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 11:26:12 -0700 Subject: [PATCH 11/19] build/pkgs/sagelib/src/requirements.txt: Add packages required for runtime --- build/pkgs/sagelib/src/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build/pkgs/sagelib/src/requirements.txt b/build/pkgs/sagelib/src/requirements.txt index cd2008a2274..c75cebeed89 100644 --- a/build/pkgs/sagelib/src/requirements.txt +++ b/build/pkgs/sagelib/src/requirements.txt @@ -10,3 +10,8 @@ numpy # already needed by sage.env jinja2 # sage_setup.autogen.interpreters cypari2 # but building bdist_wheel of cypari2 fails with recent pip... https://github.com/sagemath/cypari2/issues/93 + +########## Runtime + +psutil +pexpect From ad3370590f559c4a13132896d31054b939ffaea5 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 11:26:42 -0700 Subject: [PATCH 12/19] Suggest tox -v -v --- build/pkgs/sagelib/src/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/pkgs/sagelib/src/tox.ini b/build/pkgs/sagelib/src/tox.ini index 0a00d2cfdf3..ee3c3df1d7d 100644 --- a/build/pkgs/sagelib/src/tox.ini +++ b/build/pkgs/sagelib/src/tox.ini @@ -4,7 +4,7 @@ # # To build and test in the tox environment: # -# ./sage -sh -c '(cd build/pkgs/sagelib/src && tox)' +# ./sage -sh -c '(cd build/pkgs/sagelib/src && tox -v -v)' # # To test interactively: # From 481e2128aa39dd1f9961955e51575d6bebb5031b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 23 Jun 2020 19:31:30 -0700 Subject: [PATCH 13/19] .gitignore: Ignore generated files in build/pkgs/*/src --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index d91c194b1b1..756cedd3bab 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,11 @@ gitlab-build-docker.log /src/bin/sage-env-config /build/bin/sage-build-env-config +/build/pkgs/*/src/build +/build/pkgs/*/src/dist +/build/pkgs/*/src/MANIFEST +/build/pkgs/*/src/*.egg-info + ####################### # tox generated files # ####################### From 9955a3ff69870ff844e5ebb45cf47c4b6f214263 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 11:46:06 -0700 Subject: [PATCH 14/19] Build sagelib from build/pkgs/sagelib/src --- .gitignore | 1 + build/pkgs/sagelib/spkg-install | 2 +- build/pkgs/sagelib/src/bin | 1 + build/pkgs/sagelib/src/requirements.txt | 2 +- build/pkgs/sagelib/src/sage | 1 + build/pkgs/sagelib/src/sage_setup | 1 + build/pkgs/sagelib/src/setup.py | 1 + 7 files changed, 7 insertions(+), 2 deletions(-) create mode 120000 build/pkgs/sagelib/src/bin create mode 120000 build/pkgs/sagelib/src/sage create mode 120000 build/pkgs/sagelib/src/sage_setup create mode 120000 build/pkgs/sagelib/src/setup.py diff --git a/.gitignore b/.gitignore index 756cedd3bab..35b0d843d7a 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ gitlab-build-docker.log /build/pkgs/*/src/dist /build/pkgs/*/src/MANIFEST /build/pkgs/*/src/*.egg-info +/build/pkgs/*/src/.tox ####################### # tox generated files # diff --git a/build/pkgs/sagelib/spkg-install b/build/pkgs/sagelib/spkg-install index 0fa42a7e912..0848bc307e3 100755 --- a/build/pkgs/sagelib/spkg-install +++ b/build/pkgs/sagelib/spkg-install @@ -1,5 +1,5 @@ #!/usr/bin/env bash -cd "$SAGE_SRC" +cd src ## All sagelib-building is done by setup.py. ## This is so that sagelib can be installed by standard Python procedures, ## such as "./setup.py install" or "pip install ." diff --git a/build/pkgs/sagelib/src/bin b/build/pkgs/sagelib/src/bin new file mode 120000 index 00000000000..459186de7e0 --- /dev/null +++ b/build/pkgs/sagelib/src/bin @@ -0,0 +1 @@ +../../../../src/bin \ No newline at end of file diff --git a/build/pkgs/sagelib/src/requirements.txt b/build/pkgs/sagelib/src/requirements.txt index c75cebeed89..ba1dd3d54c2 100644 --- a/build/pkgs/sagelib/src/requirements.txt +++ b/build/pkgs/sagelib/src/requirements.txt @@ -1,5 +1,5 @@ #sage_conf -sage_setup @ file:///Users/mkoeppe/s/sage/sage-rebasing/worktree-algebraic-2018-spring/upstream/sage-setup-0.0.0.tar.gz +#sage_setup six # use of six should be removed from sage_setup Cython==0.29.17 pkgconfig diff --git a/build/pkgs/sagelib/src/sage b/build/pkgs/sagelib/src/sage new file mode 120000 index 00000000000..c4f2d5df534 --- /dev/null +++ b/build/pkgs/sagelib/src/sage @@ -0,0 +1 @@ +../../../../src/sage \ No newline at end of file diff --git a/build/pkgs/sagelib/src/sage_setup b/build/pkgs/sagelib/src/sage_setup new file mode 120000 index 00000000000..a2cca6f09f7 --- /dev/null +++ b/build/pkgs/sagelib/src/sage_setup @@ -0,0 +1 @@ +../../../../src/sage_setup \ No newline at end of file diff --git a/build/pkgs/sagelib/src/setup.py b/build/pkgs/sagelib/src/setup.py new file mode 120000 index 00000000000..884bbad9470 --- /dev/null +++ b/build/pkgs/sagelib/src/setup.py @@ -0,0 +1 @@ +../../../../src/setup.py \ No newline at end of file From 516358b66059dbf458ed7c970ff05923b539b71c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 13:48:30 -0700 Subject: [PATCH 15/19] build/pkgs/sagelib/src/tox.ini: Pass SAGE_LOCAL --- build/pkgs/sagelib/src/tox.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build/pkgs/sagelib/src/tox.ini b/build/pkgs/sagelib/src/tox.ini index ee3c3df1d7d..e48f7eb0b01 100644 --- a/build/pkgs/sagelib/src/tox.ini +++ b/build/pkgs/sagelib/src/tox.ini @@ -15,10 +15,8 @@ [testenv] deps = -rrequirements.txt -## passenv = -## PATH -## CC -## CXX +passenv = + SAGE_LOCAL setenv = # Sage scripts such as sage-runtests like to use $HOME/.sage From d62da1559ca1448b07962ea69fcde212cfea75f0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 14:51:40 -0700 Subject: [PATCH 16/19] sage_setup.command.sage_build: Add the extensions to the distribution --- src/sage_setup/command/sage_build.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sage_setup/command/sage_build.py b/src/sage_setup/command/sage_build.py index 4b1ac6fdf12..963d4b207d2 100644 --- a/src/sage_setup/command/sage_build.py +++ b/src/sage_setup/command/sage_build.py @@ -12,10 +12,21 @@ def run_autogen(self): see :trac:`22106`. """ from sage_setup.autogen import autogen_all + from sage_setup.find import find_python_sources + from sage.env import SAGE_SRC + log.info("Generating auto-generated sources") - for pkg in autogen_all(): + + pkgs = autogen_all() + python_packages, python_modules, cython_modules = find_python_sources( + SAGE_SRC, [ pkg.replace('.', '/') for pkg in pkgs]) + + for pkg in python_packages: if pkg not in self.distribution.packages: - self.distribution.packages.append(pkg) + self.distribution.packages.append(pkg) + + for cython_module in cython_modules: + self.distribution.ext_modules.append(cython_module) def run(self): self.run_autogen() From eaced098339d5c7655244667b3dd1c90f5fd7f9e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 15:52:19 -0700 Subject: [PATCH 17/19] build/pkgs/sagelib: Add remaining Python dependencies for sage.all --- build/pkgs/sagelib/src/requirements.txt | 2 ++ build/pkgs/sagelib/src/tox.ini | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/build/pkgs/sagelib/src/requirements.txt b/build/pkgs/sagelib/src/requirements.txt index ba1dd3d54c2..1e5647c8608 100644 --- a/build/pkgs/sagelib/src/requirements.txt +++ b/build/pkgs/sagelib/src/requirements.txt @@ -15,3 +15,5 @@ cypari2 # but building bdist_wheel of cypari2 fails with recent pip... ht psutil pexpect +pplpy +ipython<=5.8 diff --git a/build/pkgs/sagelib/src/tox.ini b/build/pkgs/sagelib/src/tox.ini index e48f7eb0b01..d58eb823aa8 100644 --- a/build/pkgs/sagelib/src/tox.ini +++ b/build/pkgs/sagelib/src/tox.ini @@ -27,4 +27,9 @@ whitelist_externals = commands = # Beware of the treacherous non-src layout. "./sage/" shadows the install sage package. - python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.all' + python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.all; print(sage.all.__file__)' + + # FIXME: The following loads sage-env, which loads the wrong Python. + sage -c 'import sys; print("sys.path =", sys.path); import sage.all; print(sage.all.__file__)' + + #sage -t --all From b8e366ec56a9d3527c81df9d8d4206afc2562be4 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 28 Jun 2020 17:11:11 -0700 Subject: [PATCH 18/19] build/pkgs/sagelib/spkg-src: chmod +x --- build/pkgs/sagelib/spkg-src | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build/pkgs/sagelib/spkg-src diff --git a/build/pkgs/sagelib/spkg-src b/build/pkgs/sagelib/spkg-src old mode 100644 new mode 100755 From 06a360974cc56699a85f291494a493fed041dbc9 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 29 Jun 2020 12:19:45 -0700 Subject: [PATCH 19/19] sage_setup.find.find_python_sources: Add benchmark doctest --- src/sage_setup/find.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sage_setup/find.py b/src/sage_setup/find.py index 93cfe65eddb..0dd54fc1f9b 100644 --- a/src/sage_setup/find.py +++ b/src/sage_setup/find.py @@ -121,7 +121,11 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None): sage: timeit('find_python_sources(SAGE_SRC)', # random output ....: number=1, repeat=1) - 1 loops, best of 1: 18.8 ms per loop + 1 loops, best of 1: 30 ms per loop + + sage: timeit('find_python_sources(SAGE_SRC, distributions=[""])', # random output + ....: number=1, repeat=1) + 1 loops, best of 1: 850 ms per loop sage: find_python_sources(SAGE_SRC, modules=['sage_setup']) (['sage_setup', ...], [...'sage_setup.find'...], [])