Support the exact URL scheme of pip; support fetching zip archive

buildutils/bundle.py

@@ -9,23 +9,26 @@
 # -----------------------------------------------------------------------------
 
 
+import errno
 import hashlib
 import os
 import platform
+import re
 import shutil
+import stat
 import sys
 import zipfile
 from subprocess import PIPE, Popen
 from tempfile import TemporaryDirectory
 from unittest import mock
+from urllib.parse import urlparse
 from urllib.request import urlopen
 
-from .msg import fatal, info, warn
-
 from git import Repo
 from git.exc import InvalidGitRepositoryError, NoSuchPathError
 from gitdb.exc import BadName
-import re
+
+from .msg import fatal, info, warn
 
 pjoin = os.path.join
 
@@ -165,53 +168,85 @@ def fetch_libzmq(savedir):
         savedir, 'zeromq', url=libzmq_url, fname=libzmq, checksum=libzmq_checksum
     )
 
-def fetch_libzmq_repo(savedir, url, ref):
-    """fetch libzmq from repo"""    
+
+# On Windows, deleting a local git repo directory with shutil.rmtree
+# fails with permission problem on some read-only files in .git/.
+# Add a hook trying to fix the permission.
+def handle_remove_readonly(func, path, exc):
+    excvalue = exc[1]
+    if func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES:
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # 0777
+        func(path)
+    else:
+        raise
+
+
+def fetch_libzmq_repo_zip(savedir, url):
+    fetch_path = urlparse(url)
+    fetch_name = os.path.basename(fetch_path.path)
     dest = pjoin(savedir, 'zeromq')
-    #print("fetch_libzmq_repo: ", dest, url, ref)
-    try:
-        repo = Repo(dest)
-    except (InvalidGitRepositoryError, NoSuchPathError):
-        info("invalid local repo, clone from %s" % url)
+    fname_file = pjoin(dest, fetch_name)
+    # checks for a file with the name of the zip archive
+    if os.path.exists(fname_file):
+        info("already have extracted sources from repo archive %s" % fetch_name)
+        return
+    else:
         if os.path.exists(dest):
-            shutil.rmtree(dest)
-        repo = Repo.clone_from(url, dest)
+            shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly)
+        fetch_and_extract(savedir, 'zeromq', url=url, fname=fetch_name, checksum=None)
+        open(fname_file, 'a')  # touch the file with the name of the zip archive
+
+
+def fetch_libzmq_repo(savedir, url, ref):
+    """fetch libzmq from repo"""
+    dest = pjoin(savedir, 'zeromq')
 
-    if ref:
+    if url.endswith('.zip'):
+        fetch_libzmq_repo_zip(savedir, url)
+    else:
         try:
-            commit = repo.commit(ref)
-        except BadName:
-            warn("invalid ref %s" % ref)
-        else:
-            if repo.head.commit != commit:
-                info("checking out %s" % ref)
-                repo.head.reference = commit
-                repo.head.reset(index=True, working_tree=True)
+            repo = Repo(dest)
+        except (InvalidGitRepositoryError, NoSuchPathError):
+            info("invalid local repo, clone from %s" % url)
+            if os.path.exists(dest):
+                shutil.rmtree(dest, ignore_errors=False, onerror=handle_remove_readonly)
+            repo = Repo.clone_from(url, dest)
+
+        if ref:
+            try:
+                commit = repo.commit(ref)
+            except BadName:
+                warn("invalid ref %s" % ref)
             else:
-                info("repo head is already %s" % ref)
-
+                if repo.head.commit != commit:
+                    info("checking out %s" % ref)
+                    repo.head.reference = commit
+                    repo.head.reset(index=True, working_tree=True)
+                else:
+                    info("repo head is already %s" % ref)
+
     # get repo version
     zmq_hdr = pjoin(dest, 'include', 'zmq.h')
     ver_maj = None
     ver_min = None
     ver_pat = None
     repo_version = None
     if os.path.exists(zmq_hdr):
-        for line in open(zmq_hdr, 'r'):
+        for line in open(zmq_hdr):
             if re.search('^#define +ZMQ_VERSION_MAJOR +[0-9]+$', line):
-                ver_maj=line.split()[2]
+                ver_maj = line.split()[2]
             elif re.search('^#define +ZMQ_VERSION_MINOR +[0-9]+$', line):
-                ver_min=line.split()[2]
+                ver_min = line.split()[2]
             elif re.search('^#define +ZMQ_VERSION_PATCH +[0-9]+$', line):
-                ver_pat=line.split()[2]
-    
+                ver_pat = line.split()[2]
+
         if ver_maj and ver_min and ver_pat:
             repo_version = (int(ver_maj), int(ver_min), int(ver_pat))
         else:
-            warn('unable to determine bundle_version, build may fail') 
+            warn('unable to determine bundle_version, build may fail')
     else:
         warn('zmq header not found, build may fail')
-    
+
     return repo_version
 
 

buildutils/config.py

@@ -119,15 +119,25 @@ def config_from_prefix(prefix):
         settings['zmq_prefix'] = ''
         settings['libzmq_extension'] = True
         settings['no_libzmq_extension'] = False
-    elif prefix_lower.startswith('git://') or prefix_lower.startswith('https://'):
+        settings['zmq_repo_url'] = None
+        settings['zmq_repo_ref'] = None
+    elif prefix_lower.startswith('git@'):
         settings['zmq_prefix'] = ''
         settings['libzmq_extension'] = True
         settings['no_libzmq_extension'] = False
 
-        prefix_split = prefix.split('@', 1)
-        settings['zmq_repo_url'] = prefix_split[0]
-        if len(prefix_split) > 1:
-            settings['zmq_repo_ref'] = prefix_split[1]
+        prefix_split = prefix.split('@', 2)
+        settings['zmq_repo_url'] = prefix_split[1]
+        if len(prefix_split) > 2:
+            settings['zmq_repo_ref'] = prefix_split[2]
+        else:
+            settings['zmq_repo_ref'] = None
+    elif prefix_lower.startswith('https://') and prefix_lower.endswith('.zip'):
+        settings['zmq_prefix'] = ''
+        settings['libzmq_extension'] = True
+        settings['no_libzmq_extension'] = False
+        settings['zmq_repo_url'] = prefix_lower
+        settings['zmq_repo_ref'] = None
     else:
         settings['zmq_prefix'] = os.path.abspath(prefix)
         settings['libzmq_extension'] = False
@@ -168,7 +178,7 @@ def discover_settings(conf_base=None):
         'bdist_egg': {},
         'win_ver': None,
         'zmq_repo_url': None,
-        'zmq_repo_ref': None
+        'zmq_repo_ref': None,
     }
     if sys.platform.startswith('win'):
         settings['have_sys_un_h'] = False

setup.py

@@ -51,8 +51,8 @@
     discover_settings,
     fatal,
     fetch_libzmq,
-    fetch_libzmq_repo,
     fetch_libzmq_dll,
+    fetch_libzmq_repo,
     info,
     line,
     localpath,
@@ -539,7 +539,9 @@ def bundle_libzmq_extension(self):
             os.makedirs(bundledir)
 
         if self.config['zmq_repo_url']:
-            repo_version = fetch_libzmq_repo(bundledir, self.config['zmq_repo_url'], self.config['zmq_repo_ref'])
+            repo_version = fetch_libzmq_repo(
+                bundledir, self.config['zmq_repo_url'], self.config['zmq_repo_ref']
+            )
             if repo_version and repo_version != bundled_version:
                 bundled_version = repo_version
                 info(f"bundled_version update with repo {bundled_version}")
@@ -596,7 +598,8 @@ def bundle_libzmq_extension(self):
             libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1))
             libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1))
         else:
-            libzmq.define_macros.append(('ZMQ_HAVE_STRUCT_SOCKADDR_UN', 1))
+            if sys.platform.startswith('win'):
+                libzmq.define_macros.append(('ZMQ_HAVE_STRUCT_SOCKADDR_UN', 1))
 
         # set draft flag
         if self.config["zmq_draft_api"]: