Upgrade from datalad core as of 0.18.1-24-gabc454d4f

[yarikoptic]

not yet sure if correctly resolved merge conflicts... might need to redo -- I might also check it on datalad-extension-template but ideally it should also get adopted by datalad-core itself.

Here is the diff I see from core after merge conflict resolution via `git diff HEAD^2 `

diff --git a/setup.py b/setup.py
index b845f5e..3864500 100644
--- a/setup.py
+++ b/setup.py
@@ -8,19 +8,14 @@
 
 import datetime
 import os
-import platform
-import sys
-from os import (
-    linesep,
-    makedirs,
+from os.path import (
+    dirname,
+    join as opj,
+    sep as pathsep,
+    splitext,
 )
-from os.path import dirname
-from os.path import join as opj
-from os.path import sep as pathsep
-from os.path import splitext
-
 import setuptools
-from genericpath import exists
+import versioneer
 from packaging.version import Version
 from setuptools import (
     Command,
@@ -65,17 +60,25 @@ class BuildManPage(Command):
     description = 'Generate man page from an ArgumentParser instance.'
 
     user_options = [
-        ('manpath=', None, 'output path for manpages'),
-        ('rstpath=', None, 'output path for RST files'),
+        ('manpath=', None,
+         'output path for manpages (relative paths are relative to the '
+         'datalad package)'),
+        ('rstpath=', None,
+         'output path for RST files (relative paths are relative to the '
+         'datalad package)'),
         ('parser=', None, 'module path to an ArgumentParser instance'
          '(e.g. mymod:func, where func is a method or function which return'
          'a dict with one or more arparse.ArgumentParser instances.'),
+        ('cmdsuite=', None, 'module path to an extension command suite '
+         '(e.g. mymod:command_suite) to limit the build to the contained '
+         'commands.'),
     ]
 
     def initialize_options(self):
         self.manpath = opj('build', 'man')
         self.rstpath = opj('docs', 'source', 'generated', 'man')
         self.parser = 'datalad.cli.parser:setup_parser'
+        self.cmdsuite = None
 
     def finalize_options(self):
         if self.manpath is None:
@@ -84,8 +87,6 @@ class BuildManPage(Command):
             raise DistutilsOptionError('\'rstpath\' option is required')
         if self.parser is None:
             raise DistutilsOptionError('\'parser\' option is required')
-        self.manpath = _path_rel2file(self.manpath)
-        self.rstpath = _path_rel2file(self.rstpath)
         mod_name, func_name = self.parser.split(':')
         fromlist = mod_name.split('.')
         try:
@@ -94,10 +95,18 @@ class BuildManPage(Command):
                 ['datalad'],
                 formatter_class=fmt.ManPageFormatter,
                 return_subparsers=True,
-                help_ignore_extensions=True)
+                # ignore extensions only for the main package to avoid pollution
+                # with all extension commands that happen to be installed
+                help_ignore_extensions=self.distribution.get_name() == 'datalad')
 
         except ImportError as err:
             raise err
+        if self.cmdsuite:
+            mod_name, suite_name = self.cmdsuite.split(':')
+            mod = __import__(mod_name, fromlist=mod_name.split('.'))
+            suite = getattr(mod, suite_name)
+            self.cmdlist = [c[2] if len(c) > 2 else c[1].replace('_', '-').lower()
+                            for c in suite[1]]
 
         self.announce('Writing man page(s) to %s' % self.manpath)
         self._today = datetime.date.today()
@@ -147,9 +156,12 @@ class BuildManPage(Command):
         #appname = self._parser.prog
         appname = 'datalad'
 
+        cfg = read_configuration(
+            opj(dirname(dirname(__file__)), 'setup.cfg'))['metadata']
+
         sections = {
             'Authors': """{0} is developed by {1} <{2}>.""".format(
-                appname, dist.get_author(), dist.get_author_email()),
+                appname, cfg['author'], cfg['author_email']),
         }
 
         for cls, opath, ext in ((fmt.ManPageFormatter, self.manpath, '1'),
@@ -157,6 +169,8 @@ class BuildManPage(Command):
             if not os.path.exists(opath):
                 os.makedirs(opath)
             for cmdname in getattr(self, 'cmdline_names', list(self._parser)):
+                if hasattr(self, 'cmdlist') and cmdname not in self.cmdlist:
+                    continue
                 p = self._parser[cmdname]
                 cmdname = "{0}{1}".format(
                     'datalad ' if cmdname != 'datalad' else '',
@@ -164,7 +178,7 @@ class BuildManPage(Command):
                 format = cls(
                     cmdname,
                     ext_sections=sections,
-                    version=get_version(getattr(self, 'mod_name', appname)))
+                    version=versioneer.get_version())
                 formatted = format.format_man_page(p)
                 with open(opj(opath, '{0}.{1}'.format(
                         cmdname.replace(' ', '-'),
@@ -190,8 +204,6 @@ class BuildRSTExamplesFromScripts(Command):
             raise DistutilsOptionError('\'expath\' option is required')
         if self.rstpath is None:
             raise DistutilsOptionError('\'rstpath\' option is required')
-        self.expath = _path_rel2file(self.expath)
-        self.rstpath = _path_rel2file(self.rstpath)
         self.announce('Converting example scripts')
 
     def run(self):
@@ -222,7 +234,6 @@ class BuildConfigInfo(Command):
     def finalize_options(self):
         if self.rstpath is None:
             raise DistutilsOptionError('\'rstpath\' option is required')
-        self.rstpath = _path_rel2file(self.rstpath)
         self.announce('Generating configuration documentation')
 
     def run(self):

and here what I think it was before via `git diff 343b9a0..HEAD^`

diff --git a/setup.py b/setup.py
index b58d046..a4b92ed 100644
--- a/setup.py
+++ b/setup.py
@@ -8,45 +8,19 @@
 
 import datetime
 import os
-import platform
-import setuptools
-import sys
 
+from os.path import (
+    dirname,
+    join as opj,
+)
+from setuptools import Command, DistutilsOptionError
+from setuptools.config import read_configuration
 
-from distutils.core import Command
-from distutils.errors import DistutilsOptionError
-from distutils.version import LooseVersion
-from genericpath import exists
-from os import linesep, makedirs
-from os.path import dirname, join as opj, sep as pathsep, splitext
-from setuptools import findall, find_packages, setup
+import versioneer
 
 from . import formatters as fmt
 
 
-def _path_rel2file(*p):
-    # dirname instead of joining with pardir so it works if
-    # datalad_build_support/ is just symlinked into some extension
-    # while developing
-    return opj(dirname(dirname(__file__)), *p)
-
-
-def get_version(name):
-    """Load version from version.py without entailing any imports
-
-    Parameters
-    ----------
-    name: str
-      Name of the folder (package) where from to read version.py
-    """
-    # This might entail lots of imports which might not yet be available
-    # so let's do ad-hoc parsing of the version.py
-    with open(_path_rel2file(name, 'version.py')) as f:
-        version_lines = list(filter(lambda x: x.startswith('__version__'), f))
-    assert (len(version_lines) == 1)
-    return version_lines[0].split('=')[1].strip(" '\"\t\n")
-
-
 class BuildManPage(Command):
     # The BuildManPage code was originally distributed
     # under the same License of Python
@@ -55,17 +29,25 @@ class BuildManPage(Command):
     description = 'Generate man page from an ArgumentParser instance.'
 
     user_options = [
-        ('manpath=', None, 'output path for manpages'),
-        ('rstpath=', None, 'output path for RST files'),
+        ('manpath=', None,
+         'output path for manpages (relative paths are relative to the '
+         'datalad package)'),
+        ('rstpath=', None,
+         'output path for RST files (relative paths are relative to the '
+         'datalad package)'),
         ('parser=', None, 'module path to an ArgumentParser instance'
          '(e.g. mymod:func, where func is a method or function which return'
          'a dict with one or more arparse.ArgumentParser instances.'),
+        ('cmdsuite=', None, 'module path to an extension command suite '
+         '(e.g. mymod:command_suite) to limit the build to the contained '
+         'commands.'),
     ]
 
     def initialize_options(self):
         self.manpath = opj('build', 'man')
         self.rstpath = opj('docs', 'source', 'generated', 'man')
         self.parser = 'datalad.cmdline.main:setup_parser'
+        self.cmdsuite = None
 
     def finalize_options(self):
         if self.manpath is None:
@@ -74,8 +56,6 @@ class BuildManPage(Command):
             raise DistutilsOptionError('\'rstpath\' option is required')
         if self.parser is None:
             raise DistutilsOptionError('\'parser\' option is required')
-        self.manpath = _path_rel2file(self.manpath)
-        self.rstpath = _path_rel2file(self.rstpath)
         mod_name, func_name = self.parser.split(':')
         fromlist = mod_name.split('.')
         try:
@@ -84,10 +64,18 @@ class BuildManPage(Command):
                 ['datalad'],
                 formatter_class=fmt.ManPageFormatter,
                 return_subparsers=True,
-                help_ignore_extensions=True)
+                # ignore extensions only for the main package to avoid pollution
+                # with all extension commands that happen to be installed
+                help_ignore_extensions=self.distribution.get_name() == 'datalad')
 
         except ImportError as err:
             raise err
+        if self.cmdsuite:
+            mod_name, suite_name = self.cmdsuite.split(':')
+            mod = __import__(mod_name, fromlist=mod_name.split('.'))
+            suite = getattr(mod, suite_name)
+            self.cmdlist = [c[2] if len(c) > 2 else c[1].replace('_', '-').lower()
+                            for c in suite[1]]
 
         self.announce('Writing man page(s) to %s' % self.manpath)
         self._today = datetime.date.today()
@@ -137,9 +125,12 @@ class BuildManPage(Command):
         #appname = self._parser.prog
         appname = 'datalad'
 
+        cfg = read_configuration(
+            opj(dirname(dirname(__file__)), 'setup.cfg'))['metadata']
+
         sections = {
             'Authors': """{0} is developed by {1} <{2}>.""".format(
-                appname, dist.get_author(), dist.get_author_email()),
+                appname, cfg['author'], cfg['author_email']),
         }
 
         for cls, opath, ext in ((fmt.ManPageFormatter, self.manpath, '1'),
@@ -147,6 +138,8 @@ class BuildManPage(Command):
             if not os.path.exists(opath):
                 os.makedirs(opath)
             for cmdname in getattr(self, 'cmdline_names', list(self._parser)):
+                if hasattr(self, 'cmdlist') and cmdname not in self.cmdlist:
+                    continue
                 p = self._parser[cmdname]
                 cmdname = "{0}{1}".format(
                     'datalad ' if cmdname != 'datalad' else '',
@@ -154,7 +147,7 @@ class BuildManPage(Command):
                 format = cls(
                     cmdname,
                     ext_sections=sections,
-                    version=get_version(getattr(self, 'mod_name', appname)))
+                    version=versioneer.get_version())
                 formatted = format.format_man_page(p)
                 with open(opj(opath, '{0}.{1}'.format(
                         cmdname.replace(' ', '-'),
@@ -180,8 +173,6 @@ class BuildRSTExamplesFromScripts(Command):
             raise DistutilsOptionError('\'expath\' option is required')
         if self.rstpath is None:
             raise DistutilsOptionError('\'rstpath\' option is required')
-        self.expath = _path_rel2file(self.expath)
-        self.rstpath = _path_rel2file(self.rstpath)
         self.announce('Converting example scripts')
 
     def run(self):
@@ -212,7 +203,6 @@ class BuildConfigInfo(Command):
     def finalize_options(self):
         if self.rstpath is None:
             raise DistutilsOptionError('\'rstpath\' option is required')
-        self.rstpath = _path_rel2file(self.rstpath)
         self.announce('Generating configuration documentation')
 
     def run(self):
@@ -262,253 +252,3 @@ class BuildConfigInfo(Command):
                         desc_tmpl += 'undocumented\n'
                     v.update(docs)
                     rst.write(_indent(desc_tmpl.format(**v), '    '))
-
-
-class BuildSchema(Command):
-    description = 'Generate DataLad JSON-LD schema.'
-
-    user_options = [
-        ('path=', None, 'output path for schema file'),
-    ]
-
-    def initialize_options(self):
-        self.path = opj('docs', 'source', '_extras')
-
-    def finalize_options(self):
-        if self.path is None:
-            raise DistutilsOptionError('\'path\' option is required')
-        self.path = _path_rel2file(self.path)
-        self.announce('Generating JSON-LD schema file')
-
-    def run(self):
-        from datalad.metadata.definitions import common_defs
-        from datalad.metadata.definitions import version as schema_version
-        import json
-        import shutil
-
-        def _mk_fname(label, version):
-            return '{}{}{}.json'.format(
-                label,
-                '_v' if version else '',
-                version)
-
-        def _defs2context(defs, context_label, vocab_version, main_version=schema_version):
-            opath = opj(
-                self.path,
-                _mk_fname(context_label, vocab_version))
-            odir = dirname(opath)
-            if not os.path.exists(odir):
-                os.makedirs(odir)
-
-            # to become DataLad's own JSON-LD context
-            context = {}
-            schema = {"@context": context}
-            if context_label != 'schema':
-                schema['@vocab'] = 'http://docs.datalad.org/{}'.format(
-                    _mk_fname('schema', main_version))
-            for key, val in defs.items():
-                # git-annex doesn't allow ':', but in JSON-LD we need it for
-                # namespace separation -- let's make '.' in git-annex mean
-                # ':' in JSON-LD
-                key = key.replace('.', ':')
-                definition = val['def']
-                if definition.startswith('http://') or definition.startswith('https://'):
-                    # this is not a URL, hence an @id definitions that points
-                    # to another schema
-                    context[key] = definition
-                    continue
-                # the rest are compound definitions
-                props = {'@id': definition}
-                if 'unit' in val:
-                    props['unit'] = val['unit']
-                if 'descr' in val:
-                    props['description'] = val['descr']
-                context[key] = props
-
-            with open(opath, 'w') as fp:
-                json.dump(
-                    schema,
-                    fp,
-                    ensure_ascii=True,
-                    indent=1,
-                    separators=(', ', ': '),
-                    sort_keys=True)
-            print('schema written to {}'.format(opath))
-
-        # core vocabulary
-        _defs2context(common_defs, 'schema', schema_version)
-
-        # present the same/latest version also as the default
-        shutil.copy(
-            opj(self.path, _mk_fname('schema', schema_version)),
-            opj(self.path, 'schema.json'))
-
-
-def setup_entry_points(entry_points):
-    """Sneaky monkey patching could be fixed only via even sneakier monkey patching
-
-    It will never break, I promise!
-    """
-
-    def get_script_content(script_name, shebang="#!/usr/bin/env python"):
-        return linesep.join([
-            shebang,
-            "#",
-            "# Custom simplistic runner for DataLad. Assumes datalad module",
-            "# being available.  Generated by monkey patching monkey patched",
-            "# setuptools.",
-            "#",
-            "from %s import main" % entry_points[script_name],
-            "main()",
-            ""]).encode()
-
-    def patch_write_script(mod):
-        """Patches write_script of the module with our shim to provide
-        lightweight invocation script
-        """
-
-        orig_meth = getattr(mod, 'write_script')
-
-        def _provide_lean_script_contents(
-                self, script_name, contents, mode="t", *ignored):
-            # could be a script from another module -- let it be as is
-            if script_name in entry_points:
-                # keep shebang
-                contents = get_script_content(
-                    script_name,
-                    contents.splitlines()[0].decode())
-            return orig_meth(self, script_name, contents, mode=mode)
-
-        setattr(mod, 'write_script', _provide_lean_script_contents)
-
-    # We still need this one so that setuptools known about the scripts
-    # So we generate some bogus ones, and provide a list of them ;)
-    # pre-generate paths so we could give them to setuptools
-    scripts_build_dir = opj('build', 'scripts_generated')
-    scripts = [opj(scripts_build_dir, x) for x in entry_points]
-
-    if 'clean' not in sys.argv:
-        if not exists(scripts_build_dir):
-            makedirs(scripts_build_dir)
-        for s, mod in entry_points.items():
-            with open(opj(scripts_build_dir, s), 'wb') as f:
-                f.write(get_script_content(s))
-
-    platform_system = platform.system().lower()
-    setup_kwargs = {}
-
-    if platform_system == 'windows':
-        # TODO: investigate https://github.com/matthew-brett/myscripter,
-        # nibabel/nixext approach to support similar setup on Windows
-        setup_kwargs['entry_points'] = {
-            'console_scripts': ['%s=%s:main' % i for i in entry_points.items()]
-        }
-    else:
-        # Damn you sharktopus!
-        from setuptools.command.install_scripts import \
-            install_scripts as stinstall_scripts
-        from setuptools.command.easy_install import easy_install
-
-        patch_write_script(stinstall_scripts)
-        patch_write_script(easy_install)
-
-        setup_kwargs['scripts'] = scripts
-
-    return setup_kwargs
-
-
-def get_long_description_from_README():
-    """Read README.md, convert to .rst using pypandoc
-
-    If pypandoc is not available or fails - just output original .md.
-
-    Returns
-    -------
-    dict
-      with keys long_description and possibly long_description_content_type
-      for newer setuptools which support uploading of markdown as is.
-    """
-    # PyPI used to not render markdown. Workaround for a sane appearance
-    # https://github.com/pypa/pypi-legacy/issues/148#issuecomment-227757822
-    # is still in place for older setuptools
-
-    README = opj(_path_rel2file('README.md'))
-
-    ret = {}
-    if LooseVersion(setuptools.__version__) >= '38.6.0':
-        # check than this
-        ret['long_description'] = open(README).read()
-        ret['long_description_content_type'] = 'text/markdown'
-        return ret
-
-    # Convert or fall-back
-    try:
-        import pypandoc
-        return {'long_description': pypandoc.convert(README, 'rst')}
-    except (ImportError, OSError) as exc:
-        # attempting to install pandoc via brew on OSX currently hangs and
-        # pypandoc imports but throws OSError demanding pandoc
-        print(
-                "WARNING: pypandoc failed to import or thrown an error while "
-                "converting"
-                " README.md to RST: %r   .md version will be used as is" % exc
-        )
-        return {'long_description': open(README).read()}
-
-
-def findsome(subdir, extensions):
-    """Find files under subdir having specified extensions
-
-    Leading directory (datalad) gets stripped
-    """
-    return [
-        f.split(pathsep, 1)[1] for f in findall(opj('datalad', subdir))
-        if splitext(f)[-1].lstrip('.') in extensions
-    ]
-
-
-def datalad_setup(name, **kwargs):
-    """A helper for a typical invocation of setuptools.setup.
-
-    If not provided in kwargs, following fields will be autoset to the defaults
-    or obtained from the present on the file system files:
-
-    - author
-    - author_email
-    - packages -- all found packages which start with `name`
-    - long_description -- converted to .rst using pypandoc README.md
-    - version -- parsed `__version__` within `name/version.py`
-
-    Parameters
-    ----------
-    name: str
-        Name of the Python package
-    **kwargs:
-        The rest of the keyword arguments passed to setuptools.setup as is
-    """
-    # Simple defaults
-    for k, v in {
-        'author': "The DataLad Team and Contributors",
-        'author_email': "team@datalad.org"
-    }.items():
-        if kwargs.get(k) is None:
-            kwargs[k] = v
-
-    # More complex, requiring some function call
-
-    # Only recentish versions of find_packages support include
-    # packages = find_packages('.', include=['datalad*'])
-    # so we will filter manually for maximal compatibility
-    if kwargs.get('packages') is None:
-        kwargs['packages'] = [pkg for pkg in find_packages('.') if pkg.startswith(name)]
-    if kwargs.get('long_description') is None:
-        kwargs.update(get_long_description_from_README())
-    if kwargs.get('version') is None:
-        kwargs['version'] = get_version(name)
-
-    cmdclass = kwargs.get('cmdclass', {})
-    # Check if command needs some module specific handling
-    for v in cmdclass.values():
-        if hasattr(v, 'handle_module'):
-            getattr(v, 'handle_module')(name, **kwargs)
-    return setup(name=name, **kwargs)
\ No newline at end of file

difference was large because of fd99b4e which made this repo specificly applicable only for extensions, but then "why bother" if not to reuse with datalad core itself... so ideally we should indeed minimize the diff and make this repo used across core and extensions

[adswa]

isn't this a duplicate import? (see lines 20 and following)

[mslw]

I see one usage, in line 306; if Version(setuptools.__version__) >= Version('38.6.0'):

(that being said, setuptools 38.6.0 is March 2018, so positively ancient)

[adswa]

bearing the official metadata tag and ci upgrade, Python 3.7 is EOL and we should switch to require Py>=3.8

[adswa]

I completely support this, it would be nice to have something unified. A different reason is a pending deprecation in setup_tools which is still being used in the extension template (and all extensions). A part of this code relies on setuptool's read_configuration command:

https://github.com/datalad/datalad-extension-template/blob/d95eb9663fddcfeb22d1edd78dceb0c60a026bce/_datalad_buildsupport/setup.py#L17

With setuptools > 61.0.0, setuptools.config.read_configuration was deprecated. Building docs throws the following message:

(next) adina@muninn in ~/repos/datalad-next on git:main
❱ make -C docs html 
make: Entering directory '/home/adina/repos/datalad-next/docs'
python -m sphinx -b html -d build/doctrees  -W source build/html
Running Sphinx v6.2.1
running build_manpage
/home/adina/repos/datalad-next/_datalad_buildsupport/setup.py:128: SetuptoolsDeprecationWarning: Deprecated API usage.
!!

        ********************************************************************************
        As setuptools moves its configuration towards `pyproject.toml`,
        `setuptools.config.read_configuration` became deprecated.

        For the time being, you can use the `setuptools.config.setupcfg` module
        to access a backward compatible API, but this module is provisional
        and might be removed in the future.

        To read project metadata, consider using
        ``build.util.project_wheel_metadata`` (https://pypi.org/project/build/).
        For simple scenarios, you can also try parsing the file directly
        with the help of ``configparser``.
        ********************************************************************************

!!
  cfg = read_configuration(

The transitional backwards compatible API from setuptools.config.setupcfg import read_configuration works for now (though not without problems in some cases, e.g., pypa/setuptools#3231). The reason behind the deprecations are plans to eventually deprecate setup.cfg in favour of pyproject.toml: pypa/setuptools#3214. However, the code in datalad-core can live entirely without it, and that seems to be a nicer way forward.