From 0473c13e67d407193e012a1d7e26bb1b958d906b Mon Sep 17 00:00:00 2001 From: Fox Yu Date: Sat, 30 Sep 2023 00:27:21 +0800 Subject: [PATCH 1/3] support repo fetch of bundled libzmq via ZMQ_PREFIX --- buildutils/bundle.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ buildutils/config.py | 16 +++++++++++-- pyproject.toml | 1 + setup.py | 23 ++++++++++++++----- 4 files changed, 86 insertions(+), 8 deletions(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 6ee4d9694..316a82823 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -22,6 +22,11 @@ from .msg import fatal, info, warn +from git import Repo +from git.exc import InvalidGitRepositoryError, NoSuchPathError +from gitdb.exc import BadName +import re + pjoin = os.path.join # ----------------------------------------------------------------------------- @@ -160,6 +165,55 @@ def fetch_libzmq(savedir): savedir, 'zeromq', url=libzmq_url, fname=libzmq, checksum=libzmq_checksum ) +def fetch_libzmq_repo(savedir, url, ref): + """fetch libzmq from repo""" + dest = pjoin(savedir, 'zeromq') + #print("fetch_libzmq_repo: ", dest, url, ref) + try: + repo = Repo(dest) + except (InvalidGitRepositoryError, NoSuchPathError): + info("invalid local repo, clone from %s" % url) + if os.path.exists(dest): + shutil.rmtree(dest) + repo = Repo.clone_from(url, dest) + + if ref: + try: + commit = repo.commit(ref) + except BadName: + warn("invalid ref %s" % ref) + else: + if repo.head.commit != commit: + info("checking out %s" % ref) + repo.head.reference = commit + repo.head.reset(index=True, working_tree=True) + else: + info("repo head is already %s" % ref) + + # get repo version + zmq_hdr = pjoin(dest, 'include', 'zmq.h') + ver_maj = None + ver_min = None + ver_pat = None + repo_version = None + if os.path.exists(zmq_hdr): + for line in open(zmq_hdr, 'r'): + if re.search('^#define +ZMQ_VERSION_MAJOR +[0-9]+$', line): + ver_maj=line.split()[2] + elif re.search('^#define +ZMQ_VERSION_MINOR +[0-9]+$', line): + ver_min=line.split()[2] + elif re.search('^#define +ZMQ_VERSION_PATCH +[0-9]+$', line): + ver_pat=line.split()[2] + + if ver_maj and ver_min and ver_pat: + repo_version = (int(ver_maj), int(ver_min), int(ver_pat)) + else: + warn('unable to determine bundle_version, build may fail') + else: + warn('zmq header not found, build may fail') + + return repo_version + def stage_platform_hpp(zmqroot): """stage platform.hpp into libzmq sources diff --git a/buildutils/config.py b/buildutils/config.py index 6718bddff..67f0c6c47 100644 --- a/buildutils/config.py +++ b/buildutils/config.py @@ -110,14 +110,24 @@ def get_cfg_args(): def config_from_prefix(prefix): """Get config from zmq prefix""" settings = {} - if prefix.lower() in ('default', 'auto', ''): + prefix_lower = prefix.lower() + if prefix_lower in ('default', 'auto', ''): settings['zmq_prefix'] = '' settings['libzmq_extension'] = False settings['no_libzmq_extension'] = False - elif prefix.lower() in ('bundled', 'extension'): + elif prefix_lower in ('bundled', 'extension'): settings['zmq_prefix'] = '' settings['libzmq_extension'] = True settings['no_libzmq_extension'] = False + elif prefix_lower.startswith('git://') or prefix_lower.startswith('https://'): + settings['zmq_prefix'] = '' + settings['libzmq_extension'] = True + settings['no_libzmq_extension'] = False + + prefix_split = prefix.split('@', 1) + settings['zmq_repo_url'] = prefix_split[0] + if len(prefix_split) > 1: + settings['zmq_repo_ref'] = prefix_split[1] else: settings['zmq_prefix'] = os.path.abspath(prefix) settings['libzmq_extension'] = False @@ -157,6 +167,8 @@ def discover_settings(conf_base=None): 'build_ext': {}, 'bdist_egg': {}, 'win_ver': None, + 'zmq_repo_url': None, + 'zmq_repo_ref': None } if sys.platform.startswith('win'): settings['have_sys_un_h'] = False diff --git a/pyproject.toml b/pyproject.toml index a5f176e09..4fb1d7383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ requires = [ "setuptools_scm[toml]", "wheel", "packaging", + "GitPython", "cffi; implementation_name == 'pypy'", "cython>=0.29; implementation_name == 'cpython'", "cython>=0.29.35; implementation_name == 'cpython' and python_version >= '3.12'", diff --git a/setup.py b/setup.py index 090c1a545..c202d3499 100755 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ discover_settings, fatal, fetch_libzmq, + fetch_libzmq_repo, fetch_libzmq_dll, info, line, @@ -523,6 +524,7 @@ def check_zmq_version(self): ) def bundle_libzmq_extension(self): + global bundled_version bundledir = "bundled" ext_modules = self.distribution.ext_modules if ext_modules and any(m.name == 'zmq.libzmq' for m in ext_modules): @@ -536,7 +538,13 @@ def bundle_libzmq_extension(self): if not os.path.exists(bundledir): os.makedirs(bundledir) - fetch_libzmq(bundledir) + if self.config['zmq_repo_url']: + repo_version = fetch_libzmq_repo(bundledir, self.config['zmq_repo_url'], self.config['zmq_repo_ref']) + if repo_version and repo_version != bundled_version: + bundled_version = repo_version + info(f"bundled_version update with repo {bundled_version}") + else: + fetch_libzmq(bundledir) stage_platform_hpp(pjoin(bundledir, 'zeromq')) @@ -567,8 +575,8 @@ def bundle_libzmq_extension(self): sources += tweetnacl_sources includes.append(pjoin(tweetnacl, 'src')) includes.append(randombytes) - else: - # >= 4.2 + elif bundled_version <= (4, 3, 4): + # >= 4.2 and <= 4.3.4 sources += glob(pjoin(bundledir, 'zeromq', 'src', 'tweetnacl.c')) # construct the Extensions: @@ -583,9 +591,12 @@ def bundle_libzmq_extension(self): # before finalize_options in build_ext self.distribution.ext_modules.insert(0, libzmq) - # use tweetnacl to provide CURVE support - libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1)) - libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1)) + if bundled_version <= (4, 3, 4): + # use tweetnacl to provide CURVE support + libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1)) + libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1)) + else: + libzmq.define_macros.append(('ZMQ_HAVE_STRUCT_SOCKADDR_UN', 1)) # set draft flag if self.config["zmq_draft_api"]: From 74027fd78eacc77668a495b5515df1a59eb93c25 Mon Sep 17 00:00:00 2001 From: Fox Yu Date: Wed, 4 Oct 2023 00:45:21 +0800 Subject: [PATCH 2/3] Support the exact URL scheme of pip; support fetching zip archive --- buildutils/bundle.py | 95 ++++++++++++++++++++++++++++++-------------- buildutils/config.py | 22 +++++++--- setup.py | 9 +++-- 3 files changed, 87 insertions(+), 39 deletions(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 316a82823..0a3848167 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -9,23 +9,26 @@ # ----------------------------------------------------------------------------- +import errno import hashlib import os import platform +import re import shutil +import stat import sys import zipfile from subprocess import PIPE, Popen from tempfile import TemporaryDirectory from unittest import mock +from urllib.parse import urlparse from urllib.request import urlopen -from .msg import fatal, info, warn - from git import Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError from gitdb.exc import BadName -import re + +from .msg import fatal, info, warn pjoin = os.path.join @@ -165,31 +168,63 @@ def fetch_libzmq(savedir): savedir, 'zeromq', url=libzmq_url, fname=libzmq, checksum=libzmq_checksum ) -def fetch_libzmq_repo(savedir, url, ref): - """fetch libzmq from repo""" + +# On Windows, deleting a local git repo directory with shutil.rmtree +# fails with permission problem on some read-only files in .git/. +# Add a hook trying to fix the permission. +def handle_remove_readonly(func, path, exc): + excvalue = exc[1] + if func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES: + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 + func(path) + else: + raise + + +def fetch_libzmq_repo_zip(savedir, url): + fetch_path = urlparse(url) + fetch_name = os.path.basename(fetch_path.path) dest = pjoin(savedir, 'zeromq') - #print("fetch_libzmq_repo: ", dest, url, ref) - try: - repo = Repo(dest) - except (InvalidGitRepositoryError, NoSuchPathError): - info("invalid local repo, clone from %s" % url) + fname_file = pjoin(dest, fetch_name) + # checks for a file with the name of the zip archive + if os.path.exists(fname_file): + info("already have extracted sources from repo archive %s" % fetch_name) + return + else: if os.path.exists(dest): - shutil.rmtree(dest) - repo = Repo.clone_from(url, dest) + shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly) + fetch_and_extract(savedir, 'zeromq', url=url, fname=fetch_name, checksum=None) + open(fname_file, 'a') # touch the file with the name of the zip archive + + +def fetch_libzmq_repo(savedir, url, ref): + """fetch libzmq from repo""" + dest = pjoin(savedir, 'zeromq') - if ref: + if url.endswith('.zip'): + fetch_libzmq_repo_zip(savedir, url) + else: try: - commit = repo.commit(ref) - except BadName: - warn("invalid ref %s" % ref) - else: - if repo.head.commit != commit: - info("checking out %s" % ref) - repo.head.reference = commit - repo.head.reset(index=True, working_tree=True) + repo = Repo(dest) + except (InvalidGitRepositoryError, NoSuchPathError): + info("invalid local repo, clone from %s" % url) + if os.path.exists(dest): + shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly) + repo = Repo.clone_from(url, dest) + + if ref: + try: + commit = repo.commit(ref) + except BadName: + warn("invalid ref %s" % ref) else: - info("repo head is already %s" % ref) - + if repo.head.commit != commit: + info("checking out %s" % ref) + repo.head.reference = commit + repo.head.reset(index=True, working_tree=True) + else: + info("repo head is already %s" % ref) + # get repo version zmq_hdr = pjoin(dest, 'include', 'zmq.h') ver_maj = None @@ -197,21 +232,21 @@ def fetch_libzmq_repo(savedir, url, ref): ver_pat = None repo_version = None if os.path.exists(zmq_hdr): - for line in open(zmq_hdr, 'r'): + for line in open(zmq_hdr): if re.search('^#define +ZMQ_VERSION_MAJOR +[0-9]+$', line): - ver_maj=line.split()[2] + ver_maj = line.split()[2] elif re.search('^#define +ZMQ_VERSION_MINOR +[0-9]+$', line): - ver_min=line.split()[2] + ver_min = line.split()[2] elif re.search('^#define +ZMQ_VERSION_PATCH +[0-9]+$', line): - ver_pat=line.split()[2] - + ver_pat = line.split()[2] + if ver_maj and ver_min and ver_pat: repo_version = (int(ver_maj), int(ver_min), int(ver_pat)) else: - warn('unable to determine bundle_version, build may fail') + warn('unable to determine bundle_version, build may fail') else: warn('zmq header not found, build may fail') - + return repo_version diff --git a/buildutils/config.py b/buildutils/config.py index 67f0c6c47..bbf63ad1c 100644 --- a/buildutils/config.py +++ b/buildutils/config.py @@ -119,15 +119,25 @@ def config_from_prefix(prefix): settings['zmq_prefix'] = '' settings['libzmq_extension'] = True settings['no_libzmq_extension'] = False - elif prefix_lower.startswith('git://') or prefix_lower.startswith('https://'): + settings['zmq_repo_url'] = None + settings['zmq_repo_ref'] = None + elif prefix_lower.startswith('git@'): settings['zmq_prefix'] = '' settings['libzmq_extension'] = True settings['no_libzmq_extension'] = False - prefix_split = prefix.split('@', 1) - settings['zmq_repo_url'] = prefix_split[0] - if len(prefix_split) > 1: - settings['zmq_repo_ref'] = prefix_split[1] + prefix_split = prefix.split('@', 2) + settings['zmq_repo_url'] = prefix_split[1] + if len(prefix_split) > 2: + settings['zmq_repo_ref'] = prefix_split[2] + else: + settings['zmq_repo_ref'] = None + elif prefix_lower.startswith('https://') and prefix_lower.endswith('.zip'): + settings['zmq_prefix'] = '' + settings['libzmq_extension'] = True + settings['no_libzmq_extension'] = False + settings['zmq_repo_url'] = prefix_lower + settings['zmq_repo_ref'] = None else: settings['zmq_prefix'] = os.path.abspath(prefix) settings['libzmq_extension'] = False @@ -168,7 +178,7 @@ def discover_settings(conf_base=None): 'bdist_egg': {}, 'win_ver': None, 'zmq_repo_url': None, - 'zmq_repo_ref': None + 'zmq_repo_ref': None, } if sys.platform.startswith('win'): settings['have_sys_un_h'] = False diff --git a/setup.py b/setup.py index c202d3499..d5253488a 100755 --- a/setup.py +++ b/setup.py @@ -51,8 +51,8 @@ discover_settings, fatal, fetch_libzmq, - fetch_libzmq_repo, fetch_libzmq_dll, + fetch_libzmq_repo, info, line, localpath, @@ -539,7 +539,9 @@ def bundle_libzmq_extension(self): os.makedirs(bundledir) if self.config['zmq_repo_url']: - repo_version = fetch_libzmq_repo(bundledir, self.config['zmq_repo_url'], self.config['zmq_repo_ref']) + repo_version = fetch_libzmq_repo( + bundledir, self.config['zmq_repo_url'], self.config['zmq_repo_ref'] + ) if repo_version and repo_version != bundled_version: bundled_version = repo_version info(f"bundled_version update with repo {bundled_version}") @@ -596,7 +598,8 @@ def bundle_libzmq_extension(self): libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1)) libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1)) else: - libzmq.define_macros.append(('ZMQ_HAVE_STRUCT_SOCKADDR_UN', 1)) + if sys.platform.startswith('win'): + libzmq.define_macros.append(('ZMQ_HAVE_STRUCT_SOCKADDR_UN', 1)) # set draft flag if self.config["zmq_draft_api"]: From 3b922f5406dd0974c5db7093b0b6f680fcfb4858 Mon Sep 17 00:00:00 2001 From: Fox Yu Date: Mon, 16 Oct 2023 00:12:48 +0800 Subject: [PATCH 3/3] drop git support, since archive fetching is good enough --- buildutils/bundle.py | 42 ++++++------------------------------------ buildutils/config.py | 20 +++----------------- pyproject.toml | 1 - setup.py | 8 ++++---- 4 files changed, 13 insertions(+), 58 deletions(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 0a3848167..3dfd5ed06 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -24,10 +24,6 @@ from urllib.parse import urlparse from urllib.request import urlopen -from git import Repo -from git.exc import InvalidGitRepositoryError, NoSuchPathError -from gitdb.exc import BadName - from .msg import fatal, info, warn pjoin = os.path.join @@ -181,7 +177,10 @@ def handle_remove_readonly(func, path, exc): raise -def fetch_libzmq_repo_zip(savedir, url): +def fetch_libzmq_archive(savedir, url): + """fetch libzmq from archive zip""" + dest = pjoin(savedir, 'zeromq') + fetch_path = urlparse(url) fetch_name = os.path.basename(fetch_path.path) dest = pjoin(savedir, 'zeromq') @@ -189,41 +188,12 @@ def fetch_libzmq_repo_zip(savedir, url): # checks for a file with the name of the zip archive if os.path.exists(fname_file): info("already have extracted sources from repo archive %s" % fetch_name) - return else: if os.path.exists(dest): shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly) fetch_and_extract(savedir, 'zeromq', url=url, fname=fetch_name, checksum=None) - open(fname_file, 'a') # touch the file with the name of the zip archive - - -def fetch_libzmq_repo(savedir, url, ref): - """fetch libzmq from repo""" - dest = pjoin(savedir, 'zeromq') - - if url.endswith('.zip'): - fetch_libzmq_repo_zip(savedir, url) - else: - try: - repo = Repo(dest) - except (InvalidGitRepositoryError, NoSuchPathError): - info("invalid local repo, clone from %s" % url) - if os.path.exists(dest): - shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly) - repo = Repo.clone_from(url, dest) - - if ref: - try: - commit = repo.commit(ref) - except BadName: - warn("invalid ref %s" % ref) - else: - if repo.head.commit != commit: - info("checking out %s" % ref) - repo.head.reference = commit - repo.head.reset(index=True, working_tree=True) - else: - info("repo head is already %s" % ref) + with open(fname_file, 'a'): # touch the file with the name of the zip archive + pass # get repo version zmq_hdr = pjoin(dest, 'include', 'zmq.h') diff --git a/buildutils/config.py b/buildutils/config.py index bbf63ad1c..a38eefe36 100644 --- a/buildutils/config.py +++ b/buildutils/config.py @@ -119,25 +119,12 @@ def config_from_prefix(prefix): settings['zmq_prefix'] = '' settings['libzmq_extension'] = True settings['no_libzmq_extension'] = False - settings['zmq_repo_url'] = None - settings['zmq_repo_ref'] = None - elif prefix_lower.startswith('git@'): - settings['zmq_prefix'] = '' - settings['libzmq_extension'] = True - settings['no_libzmq_extension'] = False - - prefix_split = prefix.split('@', 2) - settings['zmq_repo_url'] = prefix_split[1] - if len(prefix_split) > 2: - settings['zmq_repo_ref'] = prefix_split[2] - else: - settings['zmq_repo_ref'] = None + settings['zmq_archive_url'] = None elif prefix_lower.startswith('https://') and prefix_lower.endswith('.zip'): settings['zmq_prefix'] = '' settings['libzmq_extension'] = True settings['no_libzmq_extension'] = False - settings['zmq_repo_url'] = prefix_lower - settings['zmq_repo_ref'] = None + settings['zmq_archive_url'] = prefix_lower else: settings['zmq_prefix'] = os.path.abspath(prefix) settings['libzmq_extension'] = False @@ -177,8 +164,7 @@ def discover_settings(conf_base=None): 'build_ext': {}, 'bdist_egg': {}, 'win_ver': None, - 'zmq_repo_url': None, - 'zmq_repo_ref': None, + 'zmq_archive_url': None, } if sys.platform.startswith('win'): settings['have_sys_un_h'] = False diff --git a/pyproject.toml b/pyproject.toml index 4fb1d7383..a5f176e09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ requires = [ "setuptools_scm[toml]", "wheel", "packaging", - "GitPython", "cffi; implementation_name == 'pypy'", "cython>=0.29; implementation_name == 'cpython'", "cython>=0.29.35; implementation_name == 'cpython' and python_version >= '3.12'", diff --git a/setup.py b/setup.py index d5253488a..86ea3054d 100755 --- a/setup.py +++ b/setup.py @@ -51,8 +51,8 @@ discover_settings, fatal, fetch_libzmq, + fetch_libzmq_archive, fetch_libzmq_dll, - fetch_libzmq_repo, info, line, localpath, @@ -538,9 +538,9 @@ def bundle_libzmq_extension(self): if not os.path.exists(bundledir): os.makedirs(bundledir) - if self.config['zmq_repo_url']: - repo_version = fetch_libzmq_repo( - bundledir, self.config['zmq_repo_url'], self.config['zmq_repo_ref'] + if self.config['zmq_archive_url']: + repo_version = fetch_libzmq_archive( + bundledir, self.config['zmq_archive_url'] ) if repo_version and repo_version != bundled_version: bundled_version = repo_version