From 13216c1318cf1c05ef4b48b35a7b312fd27f6c2f Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sun, 8 Nov 2020 20:20:39 +0100 Subject: [PATCH 1/9] Fix multiarch for shared libraries --- src/sage/env.py | 86 +++++++++++++++-------------- src/sage/libs/gap/util.pyx | 4 +- src/sage/libs/singular/singular.pyx | 16 ++---- 3 files changed, 54 insertions(+), 52 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index d654e98b6a5..5c74d560b4b 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -29,6 +29,7 @@ # **************************************************************************** from __future__ import absolute_import +from typing import Optional import sage import glob @@ -37,6 +38,7 @@ import sys import sysconfig from . import version +from pathlib import Path # All variables set by var() appear in this SAGE_ENV dict and also @@ -204,7 +206,7 @@ def var(key, *fallbacks, **kwds): var('SAGE_IMPORTALL', 'yes') -def _get_shared_lib_filename(libname, *additional_libnames): +def _get_shared_lib_path(libname, *additional_libnames) -> Optional[Path]: """ Return the full path to a shared library file installed in ``$SAGE_LOCAL/lib`` or the directories associated with the @@ -228,73 +230,75 @@ def _get_shared_lib_filename(libname, *additional_libnames): sage: import sys sage: from fnmatch import fnmatch - sage: from sage.env import _get_shared_lib_filename - sage: lib_filename = _get_shared_lib_filename("Singular", - ....: "singular-Singular") + sage: from sage.env import _get_shared_lib_path + sage: lib_filename = _get_shared_lib_path("Singular", "singular-Singular") sage: if sys.platform == 'cygwin': ....: pattern = "*/cygSingular-*.dll" ....: elif sys.platform == 'darwin': ....: pattern = "*/libSingular.dylib" ....: else: ....: pattern = "*/lib*Singular.so" - sage: fnmatch(lib_filename, pattern) + sage: fnmatch(str(lib_filename), pattern) True - sage: _get_shared_lib_filename("an_absurd_lib") is None + sage: _get_shared_lib_path("an_absurd_lib") is None True """ for libname in (libname,) + additional_libnames: + search_directories: list[Path] = [] + patterns: list[str] = [] if sys.platform == 'cygwin': - # Later down we take the last matching DLL found, so search - # SAGE_LOCAL second so that it takes precedence - bindirs = [ - sysconfig.get_config_var('BINDIR'), - os.path.join(SAGE_LOCAL, 'bin') + # Later down we take the first matching DLL found, so search + # SAGE_LOCAL first so that it takes precedence + search_directories = [ + get_sage_local() / 'bin', + Path(sysconfig.get_config_var('BINDIR')), ] - pats = ['cyg{}.dll'.format(libname), 'cyg{}-*.dll'.format(libname)] - filenames = [] - for bindir in bindirs: - for pat in pats: - filenames += glob.glob(os.path.join(bindir, pat)) - - # Note: This is not very robust, since if there are multi DLL - # versions for the same library this just selects one more or less - # at arbitrary. However, practically speaking, on Cygwin, there + # Note: The following is not very robust, since if there are multible + # versions for the same library this just selects one more or less + # at arbitrary. However, practically speaking, on Cygwin, there # will only ever be one version - if filenames: - return filenames[-1] + patterns = [f'cyg{libname}.dll', f'cyg{libname}-*.dll'] else: if sys.platform == 'darwin': ext = 'dylib' else: ext = 'so' - libdirs = [ - os.path.join(SAGE_LOCAL, 'lib'), - sysconfig.get_config_var('LIBDIR') - ] - multilib = sysconfig.get_config_var('MULTILIB') - if multilib: - libdirs.insert(1, os.path.join(libdirs[0], multilib)) + search_directories = [get_sage_local() / 'lib'] + if (libdir_str := sysconfig.get_config_var('LIBDIR')) is not None: + libdir = Path(libdir_str) + search_directories.append(libdir) + + if (multiarchlib := sysconfig.get_config_var('MULTIARCH')) is not None: + search_directories.append(libdir / multiarchlib), + + patterns = [f'lib{libname}.{ext}'] - for libdir in libdirs: - basename = 'lib{}.{}'.format(libname, ext) - filename = os.path.join(libdir, basename) - if os.path.exists(filename): - return filename + for directory in search_directories: + for pattern in patterns: + path = next(directory.glob(pattern), None) + if path is not None: + return path.resolve() # Just return None if no files were found return None +def get_sage_local() -> Path: + return Path(SAGE_LOCAL) -# locate singular shared object -# On Debian it's libsingular-Singular so try that as well -SINGULAR_SO = _get_shared_lib_filename('Singular', 'singular-Singular') -var('SINGULAR_SO', SINGULAR_SO) +def get_singular_lib_path() -> Optional[Path]: + """ + Return the location of the singular shared object. + """ + # On Debian it's libsingular-Singular so try that as well + return _get_shared_lib_path('Singular', 'singular-Singular') -# locate libgap shared object -GAP_SO= _get_shared_lib_filename('gap','') -var('GAP_SO', GAP_SO) +def get_gap_lib_path() -> Optional[Path]: + """ + Return the location of the libgap shared object. + """ + return _get_shared_lib_path('gap', '') # post process if ' ' in DOT_SAGE: diff --git a/src/sage/libs/gap/util.pyx b/src/sage/libs/gap/util.pyx index 7f5723b6e9e..ac052da85af 100644 --- a/src/sage/libs/gap/util.pyx +++ b/src/sage/libs/gap/util.pyx @@ -227,8 +227,10 @@ cdef initialize(): # global symbol table # Note: we could use RTLD_NOLOAD and avoid the subsequent dlclose() but # this isn't portable + cdef void* handle - libgapname = str_to_bytes(sage.env.GAP_SO) + from sage.env import get_gap_lib_path + libgapname = str_to_bytes(str(get_gap_lib_path())) handle = dlopen(libgapname, RTLD_NOW | RTLD_GLOBAL) if handle is NULL: raise RuntimeError( diff --git a/src/sage/libs/singular/singular.pyx b/src/sage/libs/singular/singular.pyx index 89d3c509456..78d83f12d49 100644 --- a/src/sage/libs/singular/singular.pyx +++ b/src/sage/libs/singular/singular.pyx @@ -768,23 +768,19 @@ cdef init_libsingular(): cdef void *handle = NULL - from sage.env import SINGULAR_SO - if not SINGULAR_SO or not os.path.exists(SINGULAR_SO): + from sage.env import get_singular_lib_path + singular_path = get_singular_lib_path() + if singular_path is None: raise RuntimeError( - "libSingular not found--a working Singular install in $SAGE_LOCAL " + "libSingular not found--a working Singular install " "is required for Sage to work") - lib = SINGULAR_SO - - if not os.path.exists(lib): - raise ImportError("cannot locate Singular library ({})".format(lib)) - - lib = str_to_bytes(lib, FS_ENCODING, "surrogateescape") + lib = str_to_bytes(str(singular_path), FS_ENCODING, "surrogateescape") handle = dlopen(lib, RTLD_GLOBAL|RTLD_LAZY) if not handle: err = dlerror() - raise ImportError("cannot load Singular library ({})".format(err)) + raise ImportError(f"cannot load Singular library from {singular_path} ({err})") # load SINGULAR siInit(lib) From d9f36dcd0a828fea90c1e2998d6d87352744bd0b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 12 Nov 2020 10:47:04 +0100 Subject: [PATCH 2/9] Don't use Python 3.8 syntax --- src/sage/env.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index 5c74d560b4b..993271769e9 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -266,11 +266,13 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[Path]: ext = 'so' search_directories = [get_sage_local() / 'lib'] - if (libdir_str := sysconfig.get_config_var('LIBDIR')) is not None: + libdir = sysconfig.get_config_var('LIBDIR') + if libdir is not None: libdir = Path(libdir_str) search_directories.append(libdir) - if (multiarchlib := sysconfig.get_config_var('MULTIARCH')) is not None: + multiarchlib = sysconfig.get_config_var('MULTIARCH') + if multiarchlib is not None: search_directories.append(libdir / multiarchlib), patterns = [f'lib{libname}.{ext}'] From 94e20c704da01082ace2da5a192ee158e091bc78 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 12 Nov 2020 10:56:58 +0100 Subject: [PATCH 3/9] Revert some of the changes --- src/sage/env.py | 28 ++++++++++++---------------- src/sage/libs/gap/util.pyx | 3 +-- src/sage/libs/singular/singular.pyx | 9 ++++----- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index 993271769e9..44c41fa5cf5 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -206,7 +206,7 @@ def var(key, *fallbacks, **kwds): var('SAGE_IMPORTALL', 'yes') -def _get_shared_lib_path(libname, *additional_libnames) -> Optional[Path]: +def _get_shared_lib_path(libname, *additional_libnames) -> Optional[str]: """ Return the full path to a shared library file installed in ``$SAGE_LOCAL/lib`` or the directories associated with the @@ -251,7 +251,7 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[Path]: # Later down we take the first matching DLL found, so search # SAGE_LOCAL first so that it takes precedence search_directories = [ - get_sage_local() / 'bin', + _get_sage_local() / 'bin', Path(sysconfig.get_config_var('BINDIR')), ] # Note: The following is not very robust, since if there are multible @@ -265,7 +265,7 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[Path]: else: ext = 'so' - search_directories = [get_sage_local() / 'lib'] + search_directories = [_get_sage_local() / 'lib'] libdir = sysconfig.get_config_var('LIBDIR') if libdir is not None: libdir = Path(libdir_str) @@ -281,26 +281,22 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[Path]: for pattern in patterns: path = next(directory.glob(pattern), None) if path is not None: - return path.resolve() + return str(path.resolve()) # Just return None if no files were found return None -def get_sage_local() -> Path: +def _get_sage_local() -> Path: return Path(SAGE_LOCAL) -def get_singular_lib_path() -> Optional[Path]: - """ - Return the location of the singular shared object. - """ - # On Debian it's libsingular-Singular so try that as well - return _get_shared_lib_path('Singular', 'singular-Singular') +# locate singular shared object +# On Debian it's libsingular-Singular so try that as well +SINGULAR_SO = _get_shared_lib_filename('Singular', 'singular-Singular') +var('SINGULAR_SO', SINGULAR_SO) -def get_gap_lib_path() -> Optional[Path]: - """ - Return the location of the libgap shared object. - """ - return _get_shared_lib_path('gap', '') +# locate libgap shared object +GAP_SO = _get_shared_lib_filename('gap','') +var('GAP_SO', GAP_SO) # post process if ' ' in DOT_SAGE: diff --git a/src/sage/libs/gap/util.pyx b/src/sage/libs/gap/util.pyx index ac052da85af..b4e868ea32f 100644 --- a/src/sage/libs/gap/util.pyx +++ b/src/sage/libs/gap/util.pyx @@ -229,8 +229,7 @@ cdef initialize(): # this isn't portable cdef void* handle - from sage.env import get_gap_lib_path - libgapname = str_to_bytes(str(get_gap_lib_path())) + libgapname = str_to_bytes(sage.env.GAP_SO) handle = dlopen(libgapname, RTLD_NOW | RTLD_GLOBAL) if handle is NULL: raise RuntimeError( diff --git a/src/sage/libs/singular/singular.pyx b/src/sage/libs/singular/singular.pyx index 78d83f12d49..f5440ba944b 100644 --- a/src/sage/libs/singular/singular.pyx +++ b/src/sage/libs/singular/singular.pyx @@ -768,19 +768,18 @@ cdef init_libsingular(): cdef void *handle = NULL - from sage.env import get_singular_lib_path - singular_path = get_singular_lib_path() - if singular_path is None: + from sage.env import SINGULAR_SO + if not SINGULAR_SO or not os.path.exists(SINGULAR_SO): raise RuntimeError( "libSingular not found--a working Singular install " "is required for Sage to work") - lib = str_to_bytes(str(singular_path), FS_ENCODING, "surrogateescape") + lib = str_to_bytes(str(SINGULAR_SO), FS_ENCODING, "surrogateescape") handle = dlopen(lib, RTLD_GLOBAL|RTLD_LAZY) if not handle: err = dlerror() - raise ImportError(f"cannot load Singular library from {singular_path} ({err})") + raise ImportError(f"cannot load Singular library from {SINGULAR_SO} ({err})") # load SINGULAR siInit(lib) From 6dd6e5cc427b056efc3a4a65d59234493a95f04c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 12 Nov 2020 12:01:00 +0100 Subject: [PATCH 4/9] Fix compilation --- src/sage/env.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index 44c41fa5cf5..96588fe156a 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -268,7 +268,7 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[str]: search_directories = [_get_sage_local() / 'lib'] libdir = sysconfig.get_config_var('LIBDIR') if libdir is not None: - libdir = Path(libdir_str) + libdir = Path(libdir) search_directories.append(libdir) multiarchlib = sysconfig.get_config_var('MULTIARCH') @@ -291,11 +291,11 @@ def _get_sage_local() -> Path: # locate singular shared object # On Debian it's libsingular-Singular so try that as well -SINGULAR_SO = _get_shared_lib_filename('Singular', 'singular-Singular') +SINGULAR_SO = _get_shared_lib_path('Singular', 'singular-Singular') var('SINGULAR_SO', SINGULAR_SO) # locate libgap shared object -GAP_SO = _get_shared_lib_filename('gap','') +GAP_SO = _get_shared_lib_path('gap','') var('GAP_SO', GAP_SO) # post process From eceefb3713d7dc87d0e81cc44b21d150a07d4ee1 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 12 Nov 2020 12:02:50 +0100 Subject: [PATCH 5/9] Remove string wrap --- src/sage/libs/singular/singular.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sage/libs/singular/singular.pyx b/src/sage/libs/singular/singular.pyx index f5440ba944b..fc01a6f3061 100644 --- a/src/sage/libs/singular/singular.pyx +++ b/src/sage/libs/singular/singular.pyx @@ -774,7 +774,7 @@ cdef init_libsingular(): "libSingular not found--a working Singular install " "is required for Sage to work") - lib = str_to_bytes(str(SINGULAR_SO), FS_ENCODING, "surrogateescape") + lib = str_to_bytes(SINGULAR_SO, FS_ENCODING, "surrogateescape") handle = dlopen(lib, RTLD_GLOBAL|RTLD_LAZY) if not handle: From d345bff9596c5e513aba8076a6c531fc9a366092 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 12 Nov 2020 14:47:04 +0100 Subject: [PATCH 6/9] Fix test --- src/sage/env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index 96588fe156a..fb81769f3ba 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -235,9 +235,9 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[str]: sage: if sys.platform == 'cygwin': ....: pattern = "*/cygSingular-*.dll" ....: elif sys.platform == 'darwin': - ....: pattern = "*/libSingular.dylib" + ....: pattern = "*/libSingular-*.dylib" ....: else: - ....: pattern = "*/lib*Singular.so" + ....: pattern = "*/lib*Singular-*.so" sage: fnmatch(str(lib_filename), pattern) True sage: _get_shared_lib_path("an_absurd_lib") is None From c47c4bf7352b13764a0ec1aa497a6f24f3913ad9 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 12 Nov 2020 23:57:46 +0100 Subject: [PATCH 7/9] Correct indent --- src/sage/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sage/env.py b/src/sage/env.py index fb81769f3ba..7581240f0b5 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -255,7 +255,7 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[str]: Path(sysconfig.get_config_var('BINDIR')), ] # Note: The following is not very robust, since if there are multible - # versions for the same library this just selects one more or less + # versions for the same library this just selects one more or less # at arbitrary. However, practically speaking, on Cygwin, there # will only ever be one version patterns = [f'cyg{libname}.dll', f'cyg{libname}-*.dll'] From 090e6f10f1135062d1f65a1f1fa7a366e51ac388 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Tue, 8 Dec 2020 11:03:15 +0100 Subject: [PATCH 8/9] Simplify code --- src/sage/env.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index a99cb3311e1..0a7de2aa8b5 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -28,7 +28,7 @@ # https://www.gnu.org/licenses/ # **************************************************************************** -from typing import Optional +from typing import List, Optional import sage import os @@ -210,7 +210,7 @@ def var(key, *fallbacks, **kwds): var('SAGE_IMPORTALL', 'yes') -def _get_shared_lib_path(libname, *additional_libnames) -> Optional[str]: +def _get_shared_lib_path(*libnames: str) -> Optional[str]: """ Return the full path to a shared library file installed in ``$SAGE_LOCAL/lib`` or the directories associated with the @@ -248,9 +248,9 @@ def _get_shared_lib_path(libname, *additional_libnames) -> Optional[str]: True """ - for libname in (libname,) + additional_libnames: - search_directories: list[Path] = [] - patterns: list[str] = [] + for libname in libnames: + search_directories: List[Path] = [] + patterns: List[str] = [] if sys.platform == 'cygwin': # Later down we take the first matching DLL found, so search # SAGE_LOCAL first so that it takes precedence @@ -299,7 +299,7 @@ def _get_sage_local() -> Path: var('SINGULAR_SO', SINGULAR_SO) # locate libgap shared object -GAP_SO = _get_shared_lib_path('gap','') +GAP_SO = _get_shared_lib_path('gap') var('GAP_SO', GAP_SO) # post process From fa4556ac2815bffa2d7866d94b3e061ba787d816 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 18 Dec 2020 13:25:31 +0100 Subject: [PATCH 9/9] Remove _get_sage_local --- src/sage/env.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/sage/env.py b/src/sage/env.py index 0a7de2aa8b5..e21a86c70b4 100644 --- a/src/sage/env.py +++ b/src/sage/env.py @@ -228,7 +228,7 @@ def _get_shared_lib_path(*libnames: str) -> Optional[str]: For distributions like Debian that use a multiarch layout, we also try the multiarch lib paths (i.e. ``/usr/lib//``). - This returns ``None`` if the file does not exist. + This returns ``None`` if no matching library file could be found. EXAMPLES:: @@ -255,7 +255,7 @@ def _get_shared_lib_path(*libnames: str) -> Optional[str]: # Later down we take the first matching DLL found, so search # SAGE_LOCAL first so that it takes precedence search_directories = [ - _get_sage_local() / 'bin', + Path(SAGE_LOCAL) / 'bin', Path(sysconfig.get_config_var('BINDIR')), ] # Note: The following is not very robust, since if there are multible @@ -269,7 +269,7 @@ def _get_shared_lib_path(*libnames: str) -> Optional[str]: else: ext = 'so' - search_directories = [_get_sage_local() / 'lib'] + search_directories = [Path(SAGE_LOCAL) / 'lib'] libdir = sysconfig.get_config_var('LIBDIR') if libdir is not None: libdir = Path(libdir) @@ -290,9 +290,6 @@ def _get_shared_lib_path(*libnames: str) -> Optional[str]: # Just return None if no files were found return None -def _get_sage_local() -> Path: - return Path(SAGE_LOCAL) - # locate singular shared object # On Debian it's libsingular-Singular so try that as well SINGULAR_SO = _get_shared_lib_path('Singular', 'singular-Singular')