refactor to use requests instead of own connection-foo

CHANGES.rst

@@ -6,7 +6,11 @@ Changelog
 
 Breaking changes:
 
-- *add item here*
+- Use `requests <http://docs.python-requests.org/>`_ library instead of handcrafting connection and requests on our own.
+  This avoids strange problems in real-world customers environments.
+  We do not need to reinvent the wheel here.
+  Requests always uses HTTP 1.1 and drops support for HTTP 1.0 only caches.
+  [jensens]
 
 New features:
 

bootstrap.py

@@ -27,7 +27,7 @@
 
 tmpeggs = tempfile.mkdtemp()
 
-usage = '''\
+usage = """\
 [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
 
 Bootstraps a buildout-based project.
@@ -37,30 +37,47 @@
 
 Note that by using --find-links to point to local resources, you can keep
 this script from going over the network.
-'''
+"""
 
 parser = OptionParser(usage=usage)
 parser.add_option("-v", "--version", help="use a specific zc.buildout version")
 
-parser.add_option("-t", "--accept-buildout-test-releases",
-                  dest='accept_buildout_test_releases',
-                  action="store_true", default=False,
-                  help=("Normally, if you do not specify a --version, the "
-                        "bootstrap script and buildout gets the newest "
-                        "*final* versions of zc.buildout and its recipes and "
-                        "extensions for you.  If you use this flag, "
-                        "bootstrap and buildout will get the newest releases "
-                        "even if they are alphas or betas."))
-parser.add_option("-c", "--config-file",
-                  help=("Specify the path to the buildout configuration "
-                        "file to be used."))
-parser.add_option("-f", "--find-links",
-                  help=("Specify a URL to search for buildout releases"))
-parser.add_option("--allow-site-packages",
-                  action="store_true", default=False,
-                  help=("Let bootstrap.py use existing site packages"))
-parser.add_option("--setuptools-version",
-                  help="use a specific setuptools version")
+parser.add_option(
+    "-t",
+    "--accept-buildout-test-releases",
+    dest="accept_buildout_test_releases",
+    action="store_true",
+    default=False,
+    help=(
+        "Normally, if you do not specify a --version, the "
+        "bootstrap script and buildout gets the newest "
+        "*final* versions of zc.buildout and its recipes and "
+        "extensions for you.  If you use this flag, "
+        "bootstrap and buildout will get the newest releases "
+        "even if they are alphas or betas."
+    ),
+)
+parser.add_option(
+    "-c",
+    "--config-file",
+    help=(
+        "Specify the path to the buildout configuration " "file to be used."
+    ),
+)
+parser.add_option(
+    "-f",
+    "--find-links",
+    help=("Specify a URL to search for buildout releases"),
+)
+parser.add_option(
+    "--allow-site-packages",
+    action="store_true",
+    default=False,
+    help=("Let bootstrap.py use existing site packages"),
+)
+parser.add_option(
+    "--setuptools-version", help="use a specific setuptools version"
+)
 
 
 options, args = parser.parse_args()
@@ -77,25 +94,26 @@
     from urllib2 import urlopen
 
 ez = {}
-exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
+exec(urlopen("https://bootstrap.pypa.io/ez_setup.py").read(), ez)
 
 if not options.allow_site_packages:
     # ez_setup imports site, which adds site packages
     # this will remove them from the path to ensure that incompatible versions
     # of setuptools are not in the path
     import site
+
     # inside a virtualenv, there is no 'getsitepackages'.
     # We can't remove these reliably
-    if hasattr(site, 'getsitepackages'):
+    if hasattr(site, "getsitepackages"):
         for sitepackage_path in site.getsitepackages():
             sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
 
 setup_args = dict(to_dir=tmpeggs, download_delay=0)
 
 if options.setuptools_version is not None:
-    setup_args['version'] = options.setuptools_version
+    setup_args["version"] = options.setuptools_version
 
-ez['use_setuptools'](**setup_args)
+ez["use_setuptools"](**setup_args)
 import setuptools
 import pkg_resources
 
@@ -110,41 +128,51 @@
 
 ws = pkg_resources.working_set
 
-cmd = [sys.executable, '-c',
-       'from setuptools.command.easy_install import main; main()',
-       '-mZqNxd', tmpeggs]
+cmd = [
+    sys.executable,
+    "-c",
+    "from setuptools.command.easy_install import main; main()",
+    "-mZqNxd",
+    tmpeggs,
+]
 
 find_links = os.environ.get(
-    'bootstrap-testing-find-links',
-    options.find_links or
-    ('http://downloads.buildout.org/'
-     if options.accept_buildout_test_releases else None)
-    )
+    "bootstrap-testing-find-links",
+    options.find_links
+    or (
+        "http://downloads.buildout.org/"
+        if options.accept_buildout_test_releases
+        else None
+    ),
+)
 if find_links:
-    cmd.extend(['-f', find_links])
+    cmd.extend(["-f", find_links])
 
 setuptools_path = ws.find(
-    pkg_resources.Requirement.parse('setuptools')).location
+    pkg_resources.Requirement.parse("setuptools")
+).location
 
-requirement = 'zc.buildout'
+requirement = "zc.buildout"
 version = options.version
 if version is None and not options.accept_buildout_test_releases:
     # Figure out the most recent final version of zc.buildout.
     import setuptools.package_index
-    _final_parts = '*final-', '*final'
+
+    _final_parts = "*final-", "*final"
 
     def _final_version(parsed_version):
         try:
             return not parsed_version.is_prerelease
         except AttributeError:
             # Older setuptools
             for part in parsed_version:
-                if (part[:1] == '*') and (part not in _final_parts):
+                if (part[:1] == "*") and (part not in _final_parts):
                     return False
             return True
 
     index = setuptools.package_index.PackageIndex(
-        search_path=[setuptools_path])
+        search_path=[setuptools_path]
+    )
     if find_links:
         index.add_find_links((find_links,))
     req = pkg_resources.Requirement.parse(requirement)
@@ -163,13 +191,13 @@ def _final_version(parsed_version):
             best.sort()
             version = best[-1].version
 if version:
-    requirement = '=='.join((requirement, version))
+    requirement = "==".join((requirement, version))
 cmd.append(requirement)
 
 import subprocess
+
 if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
-    raise Exception(
-        "Failed to execute command:\n%s" % repr(cmd)[1:-1])
+    raise Exception("Failed to execute command:\n%s" % repr(cmd)[1:-1])
 
 ######################################################################
 # Import and run buildout
@@ -178,12 +206,12 @@ def _final_version(parsed_version):
 ws.require(requirement)
 import zc.buildout.buildout
 
-if not [a for a in args if '=' not in a]:
-    args.append('bootstrap')
+if not [a for a in args if "=" not in a]:
+    args.append("bootstrap")
 
 # if -c was provided, we push it back into args for buildout' main function
 if options.config_file is not None:
-    args[0:0] = ['-c', options.config_file]
+    args[0:0] = ["-c", options.config_file]
 
 zc.buildout.buildout.main(args)
 shutil.rmtree(tmpeggs)

plone/__init__.py

@@ -1,2 +1,2 @@
 # -*- coding: utf-8 -*-
-__import__('pkg_resources').declare_namespace(__name__)
+__import__("pkg_resources").declare_namespace(__name__)

plone/cachepurging/browser.py

@@ -26,10 +26,10 @@ def __init__(self, context, request):
     def __call__(self):
 
         if not isCachePurgingEnabled():
-            return 'Caching not enabled'
+            return "Caching not enabled"
 
         notify(Purge(self.context))
-        return 'Queued'
+        return "Queued"
 
 
 class PurgeImmediately(object):
@@ -42,7 +42,7 @@ def __init__(self, context, request):
 
     def __call__(self):
         if not isCachePurgingEnabled():
-            return 'Caching not enabled'
+            return "Caching not enabled"
 
         registry = getUtility(IRegistry)
         settings = registry.forInterface(ICachePurgingSettings)
@@ -54,10 +54,7 @@ def __call__(self):
                 status, xcache, xerror = purger.purgeSync(url)
                 out.write(
                     RESULT_TPL.format(
-                        url=url,
-                        status=status,
-                        xcache=xcache,
-                        xerror=xerror,
+                        url=url, status=status, xcache=xcache, xerror=xerror
                     )
                 )
         return out.getvalue()

plone/cachepurging/interfaces.py

@@ -4,7 +4,7 @@
 from zope.interface import Interface
 
 
-_ = MessageFactory('plone')
+_ = MessageFactory("plone")
 
 
 class ICachePurgingSettings(Interface):
@@ -21,45 +21,51 @@ class ICachePurgingSettings(Interface):
 
     cachingProxies = schema.Tuple(
         title=_(u"Caching proxies"),
-        description=_(u"Provide the URLs of each proxy to which PURGE "
-                      u"requests should be sent."),
+        description=_(
+            u"Provide the URLs of each proxy to which PURGE "
+            u"requests should be sent."
+        ),
         value_type=schema.URI(),
     )
 
     virtualHosting = schema.Bool(
         title=_(u"Send PURGE requests with virtual hosting paths"),
-        description=_(u"This option is only relevant if you are using "
-                      u"virtual hosting with Zope's VirtualHostMonster. "
-                      u"This relies on special tokens (VirtualHostBase "
-                      u"and VirtualHostRoot) in the URL to instruct "
-                      u"Zope about the types of URLs that the user sees. "
-                      u"If virtual host URLs are in use and this option "
-                      u"is set, PURGE requests will be sent to the "
-                      u"caching proxy with the virtual hosting tokens "
-                      u"in place. This makes sense if there is a web "
-                      u"server in front of your caching proxy performing "
-                      u"the rewrites necessary to translate a user-"
-                      u"facing URL into a virtual hosting URL, so that "
-                      u"the requests the caching proxy sees have the "
-                      u"rewrite information in them. Conversely, if the "
-                      u"rewrite is done in or behind the caching proxy, "
-                      u"you want to disable this option, so that the "
-                      u"PURGE requests use URLs that match those seen "
-                      u"by the caching proxy as they come from the "
-                      u"client."),
+        description=_(
+            u"This option is only relevant if you are using "
+            u"virtual hosting with Zope's VirtualHostMonster. "
+            u"This relies on special tokens (VirtualHostBase "
+            u"and VirtualHostRoot) in the URL to instruct "
+            u"Zope about the types of URLs that the user sees. "
+            u"If virtual host URLs are in use and this option "
+            u"is set, PURGE requests will be sent to the "
+            u"caching proxy with the virtual hosting tokens "
+            u"in place. This makes sense if there is a web "
+            u"server in front of your caching proxy performing "
+            u"the rewrites necessary to translate a user-"
+            u"facing URL into a virtual hosting URL, so that "
+            u"the requests the caching proxy sees have the "
+            u"rewrite information in them. Conversely, if the "
+            u"rewrite is done in or behind the caching proxy, "
+            u"you want to disable this option, so that the "
+            u"PURGE requests use URLs that match those seen "
+            u"by the caching proxy as they come from the "
+            u"client."
+        ),
         required=True,
         default=False,
     )
 
     domains = schema.Tuple(
         title=_(u"Domains"),
-        description=_(u"This option is only relevant if you are using "
-                      u"virtual hosting and you have enabled the option "
-                      u"to send PURGE requests with virtual hosting URLs "
-                      u"above. If you your site is served on multiple "
-                      u"domains e.g. http://example.org and "
-                      u"http://www.example.org you may wish to purge "
-                      u"both. If so, list all your domains here"),
+        description=_(
+            u"This option is only relevant if you are using "
+            u"virtual hosting and you have enabled the option "
+            u"to send PURGE requests with virtual hosting URLs "
+            u"above. If you your site is served on multiple "
+            u"domains e.g. http://example.org and "
+            u"http://www.example.org you may wish to purge "
+            u"both. If so, list all your domains here"
+        ),
         required=False,
         default=(),
         missing_value=(),
@@ -86,12 +92,12 @@ class IPurger(Interface):
     """A utility used to manage the purging process.
     """
 
-    def purgeAsync(url, httpVerb='PURGE'):
+    def purgeAsync(url, httpVerb="PURGE"):
         """Send a PURGE request to a particular URL asynchronously in a
         worker thread.
         """
 
-    def purgeSync(url, httpVerb='PURGE'):
+    def purgeSync(url, httpVerb="PURGE"):
         """Send a PURGE request to a particular URL synchronosly.
 
         Returns a triple ``(status, xcache, xerror)`` where ``status`` is
@@ -112,10 +118,9 @@ def stopThreads(wait=False):
     errorHeaders = schema.Tuple(
         title=u"Error header names",
         value_type=schema.ASCIILine(),
-        default=('x-squid-error',)
+        default=("x-squid-error",),
     )
 
     http_1_1 = schema.Bool(
-        title=u"Use HTTP 1.1 for PURGE request",
-        default=True,
+        title=u"Use HTTP 1.1 for PURGE request", default=True
     )

plone/cachepurging/paths.py

@@ -15,7 +15,7 @@ def __init__(self, context):
         self.context = context
 
     def getRelativePaths(self):
-        return ['/' + self.context.virtual_url_path()]
+        return ["/" + self.context.virtual_url_path()]
 
     def getAbsolutePaths(self):
         return []