Merge pull request #1902 from fox4u/repo_zmq_support

Support of git clone and zip archive fetching of bundled libzmq source in ZMQ_PREFIX

buildutils/bundle.py

@@ -9,15 +9,19 @@
 # -----------------------------------------------------------------------------
 
 
+import errno
 import hashlib
 import os
 import platform
+import re
 import shutil
+import stat
 import sys
 import zipfile
 from subprocess import PIPE, Popen
 from tempfile import TemporaryDirectory
 from unittest import mock
+from urllib.parse import urlparse
 from urllib.request import urlopen
 
 from .msg import fatal, info, warn
@@ -161,6 +165,61 @@ def fetch_libzmq(savedir):
     )
 
 
+# On Windows, deleting a local git repo directory with shutil.rmtree
+# fails with permission problem on some read-only files in .git/.
+# Add a hook trying to fix the permission.
+def handle_remove_readonly(func, path, exc):
+    excvalue = exc[1]
+    if func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES:
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # 0777
+        func(path)
+    else:
+        raise
+
+
+def fetch_libzmq_archive(savedir, url):
+    """fetch libzmq from archive zip"""
+    dest = pjoin(savedir, 'zeromq')
+
+    fetch_path = urlparse(url)
+    fetch_name = os.path.basename(fetch_path.path)
+    dest = pjoin(savedir, 'zeromq')
+    fname_file = pjoin(dest, fetch_name)
+    # checks for a file with the name of the zip archive
+    if os.path.exists(fname_file):
+        info("already have extracted sources from repo archive %s" % fetch_name)
+    else:
+        if os.path.exists(dest):
+            shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly)
+        fetch_and_extract(savedir, 'zeromq', url=url, fname=fetch_name, checksum=None)
+        with open(fname_file, 'a'):  # touch the file with the name of the zip archive
+            pass
+
+    # get repo version
+    zmq_hdr = pjoin(dest, 'include', 'zmq.h')
+    ver_maj = None
+    ver_min = None
+    ver_pat = None
+    repo_version = None
+    if os.path.exists(zmq_hdr):
+        for line in open(zmq_hdr):
+            if re.search('^#define +ZMQ_VERSION_MAJOR +[0-9]+$', line):
+                ver_maj = line.split()[2]
+            elif re.search('^#define +ZMQ_VERSION_MINOR +[0-9]+$', line):
+                ver_min = line.split()[2]
+            elif re.search('^#define +ZMQ_VERSION_PATCH +[0-9]+$', line):
+                ver_pat = line.split()[2]
+
+        if ver_maj and ver_min and ver_pat:
+            repo_version = (int(ver_maj), int(ver_min), int(ver_pat))
+        else:
+            warn('unable to determine bundle_version, build may fail')
+    else:
+        warn('zmq header not found, build may fail')
+
+    return repo_version
+
+
 def stage_platform_hpp(zmqroot):
     """stage platform.hpp into libzmq sources
 

buildutils/config.py

@@ -110,14 +110,21 @@ def get_cfg_args():
 def config_from_prefix(prefix):
     """Get config from zmq prefix"""
     settings = {}
-    if prefix.lower() in ('default', 'auto', ''):
+    prefix_lower = prefix.lower()
+    if prefix_lower in ('default', 'auto', ''):
         settings['zmq_prefix'] = ''
         settings['libzmq_extension'] = False
         settings['no_libzmq_extension'] = False
-    elif prefix.lower() in ('bundled', 'extension'):
+    elif prefix_lower in ('bundled', 'extension'):
         settings['zmq_prefix'] = ''
         settings['libzmq_extension'] = True
         settings['no_libzmq_extension'] = False
+        settings['zmq_archive_url'] = None
+    elif prefix_lower.startswith('https://') and prefix_lower.endswith('.zip'):
+        settings['zmq_prefix'] = ''
+        settings['libzmq_extension'] = True
+        settings['no_libzmq_extension'] = False
+        settings['zmq_archive_url'] = prefix_lower
     else:
         settings['zmq_prefix'] = os.path.abspath(prefix)
         settings['libzmq_extension'] = False
@@ -157,6 +164,7 @@ def discover_settings(conf_base=None):
         'build_ext': {},
         'bdist_egg': {},
         'win_ver': None,
+        'zmq_archive_url': None,
     }
     if sys.platform.startswith('win'):
         settings['have_sys_un_h'] = False

setup.py

@@ -52,6 +52,7 @@
     discover_settings,
     fatal,
     fetch_libzmq,
+    fetch_libzmq_archive,
     fetch_libzmq_dll,
     info,
     line,
@@ -524,6 +525,7 @@ def check_zmq_version(self):
                 )
 
     def bundle_libzmq_extension(self):
+        global bundled_version
         bundledir = "bundled"
         ext_modules = self.distribution.ext_modules
         if ext_modules and any(m.name == 'zmq.libzmq' for m in ext_modules):
@@ -537,7 +539,15 @@ def bundle_libzmq_extension(self):
         if not os.path.exists(bundledir):
             os.makedirs(bundledir)
 
-        fetch_libzmq(bundledir)
+        if self.config['zmq_archive_url']:
+            repo_version = fetch_libzmq_archive(
+                bundledir, self.config['zmq_archive_url']
+            )
+            if repo_version and repo_version != bundled_version:
+                bundled_version = repo_version
+                info(f"bundled_version update with repo {bundled_version}")
+        else:
+            fetch_libzmq(bundledir)
 
         stage_platform_hpp(pjoin(bundledir, 'zeromq'))
 
@@ -568,8 +578,8 @@ def bundle_libzmq_extension(self):
             sources += tweetnacl_sources
             includes.append(pjoin(tweetnacl, 'src'))
             includes.append(randombytes)
-        else:
-            # >= 4.2
+        elif bundled_version <= (4, 3, 4):
+            # >= 4.2 and <= 4.3.4
             sources += glob(pjoin(bundledir, 'zeromq', 'src', 'tweetnacl.c'))
 
         # construct the Extensions:
@@ -584,9 +594,13 @@ def bundle_libzmq_extension(self):
         # before finalize_options in build_ext
         self.distribution.ext_modules.insert(0, libzmq)
 
-        # use tweetnacl to provide CURVE support
-        libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1))
-        libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1))
+        if bundled_version <= (4, 3, 4):
+            # use tweetnacl to provide CURVE support
+            libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1))
+            libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1))
+        else:
+            if sys.platform.startswith('win'):
+                libzmq.define_macros.append(('ZMQ_HAVE_STRUCT_SOCKADDR_UN', 1))
 
         # set draft flag
         if self.config["zmq_draft_api"]: