diff --git a/CHANGES.rst b/CHANGES.rst index b38b26f..694c5d6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,11 @@ Changelog Breaking changes: -- *add item here* +- Use `requests `_ library instead of handcrafting connection and requests on our own. + This avoids strange problems in real-world customers environments. + We do not need to reinvent the wheel here. + Requests always uses HTTP 1.1 and drops support for HTTP 1.0 only caches. + [jensens] New features: diff --git a/bootstrap.py b/bootstrap.py index a629566..2294e35 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -27,7 +27,7 @@ tmpeggs = tempfile.mkdtemp() -usage = '''\ +usage = """\ [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] Bootstraps a buildout-based project. @@ -37,30 +37,47 @@ Note that by using --find-links to point to local resources, you can keep this script from going over the network. -''' +""" parser = OptionParser(usage=usage) parser.add_option("-v", "--version", help="use a specific zc.buildout version") -parser.add_option("-t", "--accept-buildout-test-releases", - dest='accept_buildout_test_releases', - action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas.")) -parser.add_option("-c", "--config-file", - help=("Specify the path to the buildout configuration " - "file to be used.")) -parser.add_option("-f", "--find-links", - help=("Specify a URL to search for buildout releases")) -parser.add_option("--allow-site-packages", - action="store_true", default=False, - help=("Let bootstrap.py use existing site packages")) -parser.add_option("--setuptools-version", - help="use a specific setuptools version") +parser.add_option( + "-t", + "--accept-buildout-test-releases", + dest="accept_buildout_test_releases", + action="store_true", + default=False, + help=( + "Normally, if you do not specify a --version, the " + "bootstrap script and buildout gets the newest " + "*final* versions of zc.buildout and its recipes and " + "extensions for you. If you use this flag, " + "bootstrap and buildout will get the newest releases " + "even if they are alphas or betas." + ), +) +parser.add_option( + "-c", + "--config-file", + help=( + "Specify the path to the buildout configuration " "file to be used." + ), +) +parser.add_option( + "-f", + "--find-links", + help=("Specify a URL to search for buildout releases"), +) +parser.add_option( + "--allow-site-packages", + action="store_true", + default=False, + help=("Let bootstrap.py use existing site packages"), +) +parser.add_option( + "--setuptools-version", help="use a specific setuptools version" +) options, args = parser.parse_args() @@ -77,25 +94,26 @@ from urllib2 import urlopen ez = {} -exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez) +exec(urlopen("https://bootstrap.pypa.io/ez_setup.py").read(), ez) if not options.allow_site_packages: # ez_setup imports site, which adds site packages # this will remove them from the path to ensure that incompatible versions # of setuptools are not in the path import site + # inside a virtualenv, there is no 'getsitepackages'. # We can't remove these reliably - if hasattr(site, 'getsitepackages'): + if hasattr(site, "getsitepackages"): for sitepackage_path in site.getsitepackages(): sys.path[:] = [x for x in sys.path if sitepackage_path not in x] setup_args = dict(to_dir=tmpeggs, download_delay=0) if options.setuptools_version is not None: - setup_args['version'] = options.setuptools_version + setup_args["version"] = options.setuptools_version -ez['use_setuptools'](**setup_args) +ez["use_setuptools"](**setup_args) import setuptools import pkg_resources @@ -110,28 +128,37 @@ ws = pkg_resources.working_set -cmd = [sys.executable, '-c', - 'from setuptools.command.easy_install import main; main()', - '-mZqNxd', tmpeggs] +cmd = [ + sys.executable, + "-c", + "from setuptools.command.easy_install import main; main()", + "-mZqNxd", + tmpeggs, +] find_links = os.environ.get( - 'bootstrap-testing-find-links', - options.find_links or - ('http://downloads.buildout.org/' - if options.accept_buildout_test_releases else None) - ) + "bootstrap-testing-find-links", + options.find_links + or ( + "http://downloads.buildout.org/" + if options.accept_buildout_test_releases + else None + ), +) if find_links: - cmd.extend(['-f', find_links]) + cmd.extend(["-f", find_links]) setuptools_path = ws.find( - pkg_resources.Requirement.parse('setuptools')).location + pkg_resources.Requirement.parse("setuptools") +).location -requirement = 'zc.buildout' +requirement = "zc.buildout" version = options.version if version is None and not options.accept_buildout_test_releases: # Figure out the most recent final version of zc.buildout. import setuptools.package_index - _final_parts = '*final-', '*final' + + _final_parts = "*final-", "*final" def _final_version(parsed_version): try: @@ -139,12 +166,13 @@ def _final_version(parsed_version): except AttributeError: # Older setuptools for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): + if (part[:1] == "*") and (part not in _final_parts): return False return True index = setuptools.package_index.PackageIndex( - search_path=[setuptools_path]) + search_path=[setuptools_path] + ) if find_links: index.add_find_links((find_links,)) req = pkg_resources.Requirement.parse(requirement) @@ -163,13 +191,13 @@ def _final_version(parsed_version): best.sort() version = best[-1].version if version: - requirement = '=='.join((requirement, version)) + requirement = "==".join((requirement, version)) cmd.append(requirement) import subprocess + if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0: - raise Exception( - "Failed to execute command:\n%s" % repr(cmd)[1:-1]) + raise Exception("Failed to execute command:\n%s" % repr(cmd)[1:-1]) ###################################################################### # Import and run buildout @@ -178,12 +206,12 @@ def _final_version(parsed_version): ws.require(requirement) import zc.buildout.buildout -if not [a for a in args if '=' not in a]: - args.append('bootstrap') +if not [a for a in args if "=" not in a]: + args.append("bootstrap") # if -c was provided, we push it back into args for buildout' main function if options.config_file is not None: - args[0:0] = ['-c', options.config_file] + args[0:0] = ["-c", options.config_file] zc.buildout.buildout.main(args) shutil.rmtree(tmpeggs) diff --git a/plone/__init__.py b/plone/__init__.py index 68c04af..03d08ff 100644 --- a/plone/__init__.py +++ b/plone/__init__.py @@ -1,2 +1,2 @@ # -*- coding: utf-8 -*- -__import__('pkg_resources').declare_namespace(__name__) +__import__("pkg_resources").declare_namespace(__name__) diff --git a/plone/cachepurging/browser.py b/plone/cachepurging/browser.py index 8a0809d..4c20c42 100644 --- a/plone/cachepurging/browser.py +++ b/plone/cachepurging/browser.py @@ -26,10 +26,10 @@ def __init__(self, context, request): def __call__(self): if not isCachePurgingEnabled(): - return 'Caching not enabled' + return "Caching not enabled" notify(Purge(self.context)) - return 'Queued' + return "Queued" class PurgeImmediately(object): @@ -42,7 +42,7 @@ def __init__(self, context, request): def __call__(self): if not isCachePurgingEnabled(): - return 'Caching not enabled' + return "Caching not enabled" registry = getUtility(IRegistry) settings = registry.forInterface(ICachePurgingSettings) @@ -54,10 +54,7 @@ def __call__(self): status, xcache, xerror = purger.purgeSync(url) out.write( RESULT_TPL.format( - url=url, - status=status, - xcache=xcache, - xerror=xerror, + url=url, status=status, xcache=xcache, xerror=xerror ) ) return out.getvalue() diff --git a/plone/cachepurging/interfaces.py b/plone/cachepurging/interfaces.py index b65bab1..363eb71 100644 --- a/plone/cachepurging/interfaces.py +++ b/plone/cachepurging/interfaces.py @@ -4,7 +4,7 @@ from zope.interface import Interface -_ = MessageFactory('plone') +_ = MessageFactory("plone") class ICachePurgingSettings(Interface): @@ -21,45 +21,51 @@ class ICachePurgingSettings(Interface): cachingProxies = schema.Tuple( title=_(u"Caching proxies"), - description=_(u"Provide the URLs of each proxy to which PURGE " - u"requests should be sent."), + description=_( + u"Provide the URLs of each proxy to which PURGE " + u"requests should be sent." + ), value_type=schema.URI(), ) virtualHosting = schema.Bool( title=_(u"Send PURGE requests with virtual hosting paths"), - description=_(u"This option is only relevant if you are using " - u"virtual hosting with Zope's VirtualHostMonster. " - u"This relies on special tokens (VirtualHostBase " - u"and VirtualHostRoot) in the URL to instruct " - u"Zope about the types of URLs that the user sees. " - u"If virtual host URLs are in use and this option " - u"is set, PURGE requests will be sent to the " - u"caching proxy with the virtual hosting tokens " - u"in place. This makes sense if there is a web " - u"server in front of your caching proxy performing " - u"the rewrites necessary to translate a user-" - u"facing URL into a virtual hosting URL, so that " - u"the requests the caching proxy sees have the " - u"rewrite information in them. Conversely, if the " - u"rewrite is done in or behind the caching proxy, " - u"you want to disable this option, so that the " - u"PURGE requests use URLs that match those seen " - u"by the caching proxy as they come from the " - u"client."), + description=_( + u"This option is only relevant if you are using " + u"virtual hosting with Zope's VirtualHostMonster. " + u"This relies on special tokens (VirtualHostBase " + u"and VirtualHostRoot) in the URL to instruct " + u"Zope about the types of URLs that the user sees. " + u"If virtual host URLs are in use and this option " + u"is set, PURGE requests will be sent to the " + u"caching proxy with the virtual hosting tokens " + u"in place. This makes sense if there is a web " + u"server in front of your caching proxy performing " + u"the rewrites necessary to translate a user-" + u"facing URL into a virtual hosting URL, so that " + u"the requests the caching proxy sees have the " + u"rewrite information in them. Conversely, if the " + u"rewrite is done in or behind the caching proxy, " + u"you want to disable this option, so that the " + u"PURGE requests use URLs that match those seen " + u"by the caching proxy as they come from the " + u"client." + ), required=True, default=False, ) domains = schema.Tuple( title=_(u"Domains"), - description=_(u"This option is only relevant if you are using " - u"virtual hosting and you have enabled the option " - u"to send PURGE requests with virtual hosting URLs " - u"above. If you your site is served on multiple " - u"domains e.g. http://example.org and " - u"http://www.example.org you may wish to purge " - u"both. If so, list all your domains here"), + description=_( + u"This option is only relevant if you are using " + u"virtual hosting and you have enabled the option " + u"to send PURGE requests with virtual hosting URLs " + u"above. If you your site is served on multiple " + u"domains e.g. http://example.org and " + u"http://www.example.org you may wish to purge " + u"both. If so, list all your domains here" + ), required=False, default=(), missing_value=(), @@ -86,12 +92,12 @@ class IPurger(Interface): """A utility used to manage the purging process. """ - def purgeAsync(url, httpVerb='PURGE'): + def purgeAsync(url, httpVerb="PURGE"): """Send a PURGE request to a particular URL asynchronously in a worker thread. """ - def purgeSync(url, httpVerb='PURGE'): + def purgeSync(url, httpVerb="PURGE"): """Send a PURGE request to a particular URL synchronosly. Returns a triple ``(status, xcache, xerror)`` where ``status`` is @@ -112,10 +118,9 @@ def stopThreads(wait=False): errorHeaders = schema.Tuple( title=u"Error header names", value_type=schema.ASCIILine(), - default=('x-squid-error',) + default=("x-squid-error",), ) http_1_1 = schema.Bool( - title=u"Use HTTP 1.1 for PURGE request", - default=True, + title=u"Use HTTP 1.1 for PURGE request", default=True ) diff --git a/plone/cachepurging/paths.py b/plone/cachepurging/paths.py index b67c583..1f4b0cf 100644 --- a/plone/cachepurging/paths.py +++ b/plone/cachepurging/paths.py @@ -15,7 +15,7 @@ def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/' + self.context.virtual_url_path()] + return ["/" + self.context.virtual_url_path()] def getAbsolutePaths(self): return [] diff --git a/plone/cachepurging/purger.py b/plone/cachepurging/purger.py index d80a316..fb11a37 100644 --- a/plone/cachepurging/purger.py +++ b/plone/cachepurging/purger.py @@ -14,113 +14,104 @@ from App.config import getConfiguration from plone.cachepurging.interfaces import IPurger -from six.moves import http_client from six.moves import queue from six.moves import range -from six.moves import urllib +from six.moves.urllib.parse import urlparse +from traceback import format_exception from zope.interface import implementer from zope.testing.cleanup import addCleanUp import atexit import logging +import requests import six -import socket import sys import threading -import time -logger = logging.getLogger('plone.cachepurging') - - -class Connection(http_client.HTTPConnection): - """A connection that can handle either HTTP or HTTPS - """ - - def __init__(self, host, port=None, scheme="http", timeout=5): - self.scheme = scheme - if scheme == "http": - self.default_port = http_client.HTTP_PORT - elif scheme == "https": - self.default_port = http_client.HTTPS_PORT - else: - raise ValueError("Invalid scheme '%s'" % scheme) - http_client.HTTPConnection.__init__(self, host, port, timeout=timeout) - self.timeout = timeout - - def connect(self): - if self.scheme == "http": - http_client.HTTPConnection.connect(self) - elif self.scheme == "https": - import ssl # import here in case python has no ssl support - # Clone of http_client.HTTPSConnection.connect - sock = socket.create_connection((self.host, self.port), - timeout=self.timeout) - key_file = cert_file = None - self.sock = ssl.wrap_socket(sock, key_file, cert_file) - else: - raise ValueError("Invalid scheme '%s'" % self.scheme) +logger = logging.getLogger(__name__) @implementer(IPurger) class DefaultPurger(object): - - def __init__(self, factory=Connection, timeout=30, backlog=0, - errorHeaders=('x-squid-error', ), http_1_1=True): - self.factory = factory + def __init__( + self, timeout=(3, 27), backlog=0, errorHeaders=("x-squid-error",) + ): self.timeout = timeout self.queues = {} self.workers = {} self.backlog = backlog self.queueLock = threading.Lock() self.errorHeaders = errorHeaders - self.http_1_1 = http_1_1 - def purgeAsync(self, url, httpVerb='PURGE'): - (scheme, host, path, params, query, fragment) = urllib.parse.urlparse(url) # noqa: E501 - __traceback_info__ = (url, httpVerb, scheme, host, - path, params, query, fragment) + def purge(self, session, url, httpVerb="PURGE"): + """Perform the single purge request. - q, w = self.getQueueAndWorker(url) - try: - q.put((url, httpVerb), block=False) - logger.debug('Queued %s' % url) - except queue.Full: - # Make a loud noise. Ideally the queue size would be - # user-configurable - but the more likely case is that the purge - # host is down. - if not getConfiguration().debug_mode: - logger.warning("The purge queue for the URL %s is full - the " - "request will be discarded. Please check the " - "server is reachable, or disable this purge " - "host", url) + Returns a triple ``(resp, xcache, xerror)`` where ``resp`` is the + response object for the connection, ``xcache`` is the contents of the + X-Cache header, and ``xerror`` is the contents of the first header + found of the header list in ``self.errorHeaders``. + """ + __traceback_info__ = url + logger.debug("making %s request to %s", httpVerb, url) + resp = session.request(httpVerb, url, timeout=self.timeout) + xcache = resp.headers.get("x-cache", "") + xerror = "" + for header in self.errorHeaders: + xerror = resp.headers.get(header, "") + if xerror: + # Break on first found. + break + logger.debug( + "%s of %s: %s %s", httpVerb, url, resp.status_code, resp.reason + ) + return resp, xcache, xerror + + def purgeSync(self, url, httpVerb="PURGE"): + """Purge synchronous. - def purgeSync(self, url, httpVerb='PURGE'): + Fails if requests to cache fails. + """ try: - conn = self.getConnection(url) - try: - resp, xcache, xerror = self._purgeSync(conn, url, httpVerb) - status = resp.status - finally: - conn.close() - except: + with requests.Session() as session: + resp, xcache, xerror = self.purge(session, url, httpVerb) + status = str(resp.status_code) + except Exception: status = "ERROR" err, msg, tb = sys.exc_info() - from traceback import format_exception - xerror = '\n'.join(format_exception(err, msg, tb)) + xerror = "\n".join(format_exception(err, msg, tb)) # Avoid leaking a ref to traceback. del err, msg, tb - xcache = '' - logger.debug('Finished %s for %s: %s %s' - % (httpVerb, url, status, xcache)) + xcache = "" + logger.debug( + "Finished %s for %s: %s %s" % (httpVerb, url, status, xcache) + ) if xerror: - logger.debug('Error while purging %s:\n%s' % (url, xerror)) + logger.debug("Error while purging %s:\n%s" % (url, xerror)) logger.debug("Completed synchronous purge of %s", url) return status, xcache, xerror + def purgeAsync(self, url, httpVerb="PURGE"): + current_queue, worker = self.getQueueAndWorker(url) + try: + current_queue.put((url, httpVerb), block=False) + logger.debug("Queued %s" % url) + except queue.Full: + # Make a loud noise. Ideally the queue size would be + # user-configurable - but the more likely case is that the purge + # host is down. + if not getConfiguration().debug_mode: + logger.warning( + "The purge queue for the URL %s is full - the " + "request will be discarded. Please check the " + "server is reachable, or disable this purge " + "host", + url, + ) + def stopThreads(self, wait=False): - for w in six.itervalues(self.workers): - w.stopping = True + for worker in six.itervalues(self.workers): + worker.stop() # in case the queue is empty, wake it up so the .stopping flag is seen for q in self.queues.values(): try: @@ -137,84 +128,36 @@ def stopThreads(self, wait=False): ok = False return ok - def getConnection(self, url): - """Creates a new connection - returns a connection object that is - already connected. Exceptions raised by that connection are not - trapped. - """ - - (scheme, host, path, params, query, fragment) = urllib.parse.urlparse(url) - # - # process. - conn = self.factory(host, scheme=scheme, timeout=self.timeout) - conn.connect() - logger.debug("established connection to %s", host) - return conn - def getQueueAndWorker(self, url): """Create or retrieve a queue and a worker thread instance for the given URL. """ - (scheme, host, path, params, query, fragment) = urllib.parse.urlparse(url) + (scheme, host, path, params, query, fragment) = urlparse(url) key = (host, scheme) if key not in self.queues: self.queueLock.acquire() try: if key not in self.queues: - logger.debug("Creating worker thread for %s://%s", - scheme, host) - assert key not in self.workers + logger.debug( + "Creating worker thread for %s://%s", scheme, host + ) + if key in self.workers: + raise ValueError( + "Queue Key must not already exist in workers" + ) self.queues[key] = queue_ = queue.Queue(self.backlog) self.workers[key] = worker = Worker( - queue_, host, scheme, self) + queue_, host, scheme, self + ) worker.start() finally: self.queueLock.release() return self.queues[key], self.workers[key] - def _purgeSync(self, conn, url, httpVerb): - """Perform the purge request. Returns a triple - ``(resp, xcache, xerror)`` where ``resp`` is the response object for - the connection, ``xcache`` is the contents of the X-Cache header, - and ``xerror`` is the contents of the first header found of the - header list in ``self.errorHeaders``. - """ - - (scheme, host, path, params, query, fragment) = urllib.parse.urlparse(url) # noqa: E501 - __traceback_info__ = (url, httpVerb, scheme, host, - path, params, query, fragment) - - if self.http_1_1: - conn._http_vsn = 11 - conn._http_vsn_str = 'HTTP/1.1' - else: - conn._http_vsn = 10 - conn._http_vsn_str = 'HTTP/1.0' - # When using HTTP 1.0, to make up for the lack of a 'Host' header - # we use the full url as the purge path, to allow for virtual - # hosting in squid - path = url - - purge_path = urllib.parse.urlunparse( - ('', '', path, params, query, fragment)) - logger.debug('making %s request to %s for %s.', - httpVerb, host, purge_path) - conn.putrequest(httpVerb, purge_path, skip_accept_encoding=True) - conn.endheaders() - resp = conn.getresponse() - - xcache = resp.getheader('x-cache', '') - xerror = '' - for header in self.errorHeaders: - xerror = resp.getheader(header, '') - if xerror: - # Break on first found. - break - resp.read() - logger.debug("%s of %s: %s %s", - httpVerb, url, resp.status, resp.reason) - return resp, xcache, xerror + @property + def http_1_1(self): + return True class Worker(threading.Thread): @@ -228,7 +171,8 @@ def __init__(self, queue, host, scheme, producer): self.queue = queue self.stopping = False super(Worker, self).__init__( - name="PurgeThread for %s://%s" % (scheme, host)) + name="PurgeThread for %s://%s" % (scheme, host) + ) def stop(self): self.stopping = True @@ -236,100 +180,53 @@ def stop(self): def run(self): logger.debug("%s starting", self) # queue should always exist! - q = self.producer.queues[(self.host, self.scheme)] - connection = None + current_queue = self.producer.queues[(self.host, self.scheme)] atexit.register(self.stop) try: - while not self.stopping: - item = q.get() - if self.stopping or item is None: # Shut down thread signal - logger.debug('Stopping worker thread for ' - '(%s, %s).' % (self.host, self.scheme)) - break - url, httpVerb = item - - # Loop handling errors (other than connection errors) doing - # the actual purge. - for i in range(5): - if self.stopping: + with requests.Session() as session: + while not self.stopping: + item = current_queue.get() + if self.stopping or item is None: + # Shut down thread signal + logger.debug( + "Stopping worker thread for " + "(%s, %s)." % (self.host, self.scheme) + ) break - # Get a connection. - if connection is None: - connection = self.getConnection(url) - if connection is None: # stopping + url, httpVerb = item + + # Loop handling errors (other than connection errors) doing + # the actual purge. + for i in range(5): + if self.stopping: + break + # Got an item, purge it! + try: + resp, msg, err = self.producer.purge( + session, url, httpVerb + ) + # TODO: Check resp for errors, + # if all fine: + if resp.status_code == requests.codes.ok: + break # all done with this item! + except Exception: + # All other exceptions are evil - we just disard + # the item. This prevents other logic failures etc + # being retried. + logger.exception("Failed to purge %s", url) break - # Got an item, purge it! - try: - resp, msg, err = self.producer._purgeSync( - connection, - url, - httpVerb + logger.debug( + "Transient failure on %s for %s, " + "retrying: %s" % (httpVerb, url) ) - # worked! See if we can leave the connection open for - # the next item we need to process - # NOTE: If we make a HTTP 1.0 request to IIS, it - # returns a HTTP 1.1 request and closes the - # connection. It is not clear if IIS is evil for - # not returning a "connection: close" header in this - # case (ie, assuming HTTP 1.0 close semantics), or - # if http_client.py is evil for not detecting this - # situation and flagging will_close. - if not self.producer.http_1_1 or resp.will_close: - connection.close() - connection = None - break # all done with this item! - except (http_client.HTTPException, socket.error) as e: - # All errors 'connection' related errors are treated - # the same - simply drop the connection and retry. - # the process for establishing the connection handles - # other bad things that go wrong. - logger.debug('Transient failure on %s for %s, ' - 're-establishing connection and ' - 'retrying: %s' % (httpVerb, url, e)) - connection.close() - connection = None - except Exception: - # All other exceptions are evil - we just disard the - # item. This prevents other logic failures etc being - # retried. - connection.close() - connection = None - logger.exception('Failed to purge %s', url) - break - except: - logger.exception('Exception in worker thread ' - 'for (%s, %s)' % (self.host, self.scheme)) + except Exception: + logger.exception( + "Exception in worker thread " + "for (%s, %s)" % (self.host, self.scheme) + ) logger.debug("%s terminating", self) - def getConnection(self, url): - """Get a connection to the given URL. - - Blocks until either a connection is established, or we are asked to - shut-down. Includes a simple strategy for slowing down the retry rate, - retrying from 5 seconds to 20 seconds until the connection appears or - we waited a full minute. - """ - wait_time = 2.5 - while not self.stopping: - try: - return self.producer.getConnection(url) - except socket.error as e: - wait_time = int(min(wait_time * 2, 21)) - if wait_time > 20: - # we waited a full minute, we assume a permanent failure - logger.debug("Error %s connecting to %s - reconnect " - "failed.", e, url) - self.stopping = True - break - logger.debug("Error %s connecting to %s - will " - "retry in %d second(s)", e, url, wait_time) - for i in range(wait_time): - if self.stopping: - break - time.sleep(1) - return None # must be stopping! - DEFAULT_PURGER = DefaultPurger() diff --git a/plone/cachepurging/rewrite.py b/plone/cachepurging/rewrite.py index 0be3121..a3424f8 100644 --- a/plone/cachepurging/rewrite.py +++ b/plone/cachepurging/rewrite.py @@ -22,7 +22,7 @@ def __call__(self, path): request = self.request # No rewriting necessary - virtualURL = request.get('VIRTUAL_URL', None) + virtualURL = request.get("VIRTUAL_URL", None) if virtualURL is None: return [path] @@ -39,17 +39,17 @@ def __call__(self, path): return [path] # We need to reconstruct VHM URLs for each of the domains - virtualUrlParts = request.get('VIRTUAL_URL_PARTS') - virtualRootPhysicalPath = request.get('VirtualRootPhysicalPath') + virtualUrlParts = request.get("VIRTUAL_URL_PARTS") + virtualRootPhysicalPath = request.get("VirtualRootPhysicalPath") # Make sure request is compliant if ( - not virtualUrlParts or - not virtualRootPhysicalPath or - not isinstance(virtualUrlParts, (list, tuple,)) or - not isinstance(virtualRootPhysicalPath, (list, tuple,)) or - len(virtualUrlParts) < 2 or - len(virtualUrlParts) > 3 + not virtualUrlParts + or not virtualRootPhysicalPath + or not isinstance(virtualUrlParts, (list, tuple)) + or not isinstance(virtualRootPhysicalPath, (list, tuple)) + or len(virtualUrlParts) < 2 + or len(virtualUrlParts) > 3 ): return [path] @@ -58,31 +58,33 @@ def __call__(self, path): domains = [virtualUrlParts[0]] # Virtual root, e.g. /Plone. Clear if we don't have any virtual root - virtualRoot = '/'.join(virtualRootPhysicalPath) - if virtualRoot == '/': - virtualRoot = '' + virtualRoot = "/".join(virtualRootPhysicalPath) + if virtualRoot == "/": + virtualRoot = "" # Prefix, e.g. /_vh_foo/_vh_bar. Clear if we don't have any. - pathPrefix = len(virtualUrlParts) == 3 and virtualUrlParts[1] or '' + pathPrefix = len(virtualUrlParts) == 3 and virtualUrlParts[1] or "" if pathPrefix: - pathPrefix = '/' + \ - '/'.join(['_vh_%s' % p for p in pathPrefix.split('/')]) + pathPrefix = "/" + "/".join( + ["_vh_%s" % p for p in pathPrefix.split("/")] + ) # Path, e.g. /front-page - if len(path) > 0 and not path.startswith('/'): - path = '/' + path + if len(path) > 0 and not path.startswith("/"): + path = "/" + path paths = [] for domain in domains: scheme, host = urllib.parse.urlparse(domain)[:2] paths.append( - '/VirtualHostBase/%(scheme)s/%(host)s%(root)s/' - 'VirtualHostRoot%(prefix)s%(path)s' % { - 'scheme': scheme, - 'host': host, - 'root': virtualRoot, - 'prefix': pathPrefix, - 'path': path + "/VirtualHostBase/%(scheme)s/%(host)s%(root)s/" + "VirtualHostRoot%(prefix)s%(path)s" + % { + "scheme": scheme, + "host": host, + "root": virtualRoot, + "prefix": pathPrefix, + "path": path, } ) return paths diff --git a/plone/cachepurging/tests/test_hooks.py b/plone/cachepurging/tests/test_hooks.py index 13fdc8f..8e7fb94 100644 --- a/plone/cachepurging/tests/test_hooks.py +++ b/plone/cachepurging/tests/test_hooks.py @@ -34,7 +34,6 @@ class FauxRequest(dict): class TestQueueHandler(unittest.TestCase): - def setUp(self): provideAdapter(AttributeAnnotations) provideAdapter(persistentFieldAdapter) @@ -53,17 +52,16 @@ def test_no_request(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): return [] @@ -87,17 +85,16 @@ def test_request_not_annotatable(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): return [] @@ -119,12 +116,11 @@ def test_no_registry(self): @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): return [] @@ -148,17 +144,16 @@ def test_caching_disabled(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = False - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): return [] @@ -182,12 +177,13 @@ def test_enabled_no_paths(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) notify(Purge(context)) - self.assertEqual({'plone.cachepurging.urls': set()}, - dict(IAnnotations(request))) + self.assertEqual( + {"plone.cachepurging.urls": set()}, dict(IAnnotations(request)) + ) def test_enabled(self): context = FauxContext() @@ -202,17 +198,16 @@ def test_enabled(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): return [] @@ -221,12 +216,13 @@ def getAbsolutePaths(self): notify(Purge(context)) - self.assertEqual({'plone.cachepurging.urls': set(['/foo', '/bar'])}, - dict(IAnnotations(request))) + self.assertEqual( + {"plone.cachepurging.urls": set(["/foo", "/bar"])}, + dict(IAnnotations(request)), + ) class TestPurgeHandler(unittest.TestCase): - def setUp(self): provideAdapter(AttributeAnnotations) provideAdapter(persistentFieldAdapter) @@ -244,15 +240,14 @@ def test_request_not_annotatable(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurger) class FauxPurger(object): - def __init__(self): self.purged = [] - def purgeAsync(self, url, httpVerb='PURGE'): + def purgeAsync(self, url, httpVerb="PURGE"): self.purged.append(url) purger = FauxPurger() @@ -272,15 +267,14 @@ def test_no_path_key(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurger) class FauxPurger(object): - def __init__(self): self.purged = [] - def purgeAsync(self, url, httpVerb='PURGE'): + def purgeAsync(self, url, httpVerb="PURGE"): self.purged.append(url) purger = FauxPurger() @@ -294,7 +288,7 @@ def test_no_paths(self): request = FauxRequest() alsoProvides(request, IAttributeAnnotatable) - IAnnotations(request)['plone.cachepurging.urls'] = set() + IAnnotations(request)["plone.cachepurging.urls"] = set() registry = Registry() registry.registerInterface(ICachePurgingSettings) @@ -302,15 +296,14 @@ def test_no_paths(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurger) class FauxPurger(object): - def __init__(self): self.purged = [] - def purgeAsync(self, url, httpVerb='PURGE'): + def purgeAsync(self, url, httpVerb="PURGE"): self.purged.append(url) purger = FauxPurger() @@ -324,16 +317,16 @@ def test_no_registry(self): request = FauxRequest() alsoProvides(request, IAttributeAnnotatable) - IAnnotations(request)['plone.cachepurging.urls'] = set( - ['/foo', '/bar']) + IAnnotations(request)["plone.cachepurging.urls"] = set( + ["/foo", "/bar"] + ) @implementer(IPurger) class FauxPurger(object): - def __init__(self): self.purged = [] - def purgeAsync(self, url, httpVerb='PURGE'): + def purgeAsync(self, url, httpVerb="PURGE"): self.purged.append(url) purger = FauxPurger() @@ -347,8 +340,9 @@ def test_caching_disabled(self): request = FauxRequest() alsoProvides(request, IAttributeAnnotatable) - IAnnotations(request)['plone.cachepurging.urls'] = set( - ['/foo', '/bar']) + IAnnotations(request)["plone.cachepurging.urls"] = set( + ["/foo", "/bar"] + ) registry = Registry() registry.registerInterface(ICachePurgingSettings) @@ -356,15 +350,14 @@ def test_caching_disabled(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = False - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurger) class FauxPurger(object): - def __init__(self): self.purged = [] - def purgeAsync(self, url, httpVerb='PURGE'): + def purgeAsync(self, url, httpVerb="PURGE"): self.purged.append(url) purger = FauxPurger() @@ -378,8 +371,9 @@ def test_no_purger(self): request = FauxRequest() alsoProvides(request, IAttributeAnnotatable) - IAnnotations(request)['plone.cachepurging.urls'] = set( - ['/foo', '/bar']) + IAnnotations(request)["plone.cachepurging.urls"] = set( + ["/foo", "/bar"] + ) registry = Registry() registry.registerInterface(ICachePurgingSettings) @@ -387,7 +381,7 @@ def test_no_purger(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) try: notify(PubSuccess(request)) @@ -398,8 +392,9 @@ def test_purge(self): request = FauxRequest() alsoProvides(request, IAttributeAnnotatable) - IAnnotations(request)['plone.cachepurging.urls'] = set( - ['/foo', '/bar']) + IAnnotations(request)["plone.cachepurging.urls"] = set( + ["/foo", "/bar"] + ) registry = Registry() registry.registerInterface(ICachePurgingSettings) @@ -407,15 +402,14 @@ def test_purge(self): settings = registry.forInterface(ICachePurgingSettings) settings.enabled = True - settings.cachingProxies = ('http://localhost:1234',) + settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurger) class FauxPurger(object): - def __init__(self): self.purged = [] - def purgeAsync(self, url, httpVerb='PURGE'): + def purgeAsync(self, url, httpVerb="PURGE"): self.purged.append(url) purger = FauxPurger() @@ -423,7 +417,7 @@ def purgeAsync(self, url, httpVerb='PURGE'): notify(PubSuccess(request)) self.assertSetEqual( - {'http://localhost:1234/foo', 'http://localhost:1234/bar'}, + {"http://localhost:1234/foo", "http://localhost:1234/bar"}, set(purger.purged), ) diff --git a/plone/cachepurging/tests/test_purger.py b/plone/cachepurging/tests/test_purger.py index aec27d2..0be4ea4 100644 --- a/plone/cachepurging/tests/test_purger.py +++ b/plone/cachepurging/tests/test_purger.py @@ -16,14 +16,10 @@ # Define a test HTTP server that returns canned responses -SERVER_PORT = int(os.environ.get('ZSERVER_PORT', 8765)) +SERVER_PORT = int(os.environ.get("ZSERVER_PORT", 8765)) class TestHandler(BaseHTTPRequestHandler): - - def log_message(self, format, *args): - pass - def do_PURGE(self): # Get the pre-defined response from the server's queue. try: @@ -33,39 +29,38 @@ def do_PURGE(self): print(self.command, self.path, self.protocol_version) for h, v in self.headers.items(): print("%s: %s" % (h, v)) - raise RuntimeError('Unexpected connection') + raise RuntimeError("Unexpected connection") # We may have a function to call to check things. - validator = nr.get('validator') + validator = nr.get("validator") if validator: validator(self) # We may have to wake up some other code now the test connection # has been made, but before the response is sent. - waiter = nr.get('waiter') + waiter = nr.get("waiter") if waiter: waiter.acquire() waiter.release() # for now, response=None means simulate an unexpected error. - if nr['response'] is None: + if nr["response"] is None: self.rfile.close() return # Send the response. - self.send_response(nr['response']) - headers = nr.get('headers', None) + self.send_response(nr["response"]) + headers = nr.get("headers", None) if headers: for h, v in headers.items(): self.send_header(h, v) - data = nr.get('data', b'') + data = nr.get("data", b"") self.send_header("Content-Length", len(data)) self.end_headers() self.wfile.write(data) class TestHTTPServer(HTTPServer): - def __init__(self, address, handler): HTTPServer.__init__(self, address, handler) self.response_queue = queue.Queue() @@ -73,11 +68,11 @@ def __init__(self, address, handler): def queue_response(self, **kw): self.response_queue.put(kw) + # Finally the test suites. class TestCase(unittest.TestCase): - def setUp(self): self.purger = DefaultPurger() self.httpd, self.httpt = self.startServer() @@ -91,8 +86,10 @@ def tearDown(self): if self.httpd.response_queue.empty(): break time.sleep(0.1) - self.assertTrue(self.httpd.response_queue.empty(), - "response queue not consumed") + self.assertTrue( + self.httpd.response_queue.empty(), + "response queue not consumed", + ) if not self.purger.stopThreads(wait=True): self.fail("The purge threads did not stop") finally: @@ -112,7 +109,7 @@ def startServer(self, port=SERVER_PORT, start=True): """Start a TestHTTPServer in a separate thread, returning a tuple (server, thread). If start is true, the thread is started. """ - server_address = ('localhost', port) + server_address = ("localhost", port) httpd = TestHTTPServer(server_address, TestHandler) t = threading.Thread(target=httpd.serve_forever) if start: @@ -121,7 +118,6 @@ def startServer(self, port=SERVER_PORT, start=True): class TestSync(TestCase): - def setUp(self): super(TestSync, self).setUp() self.purger.http_1_1 = True @@ -137,33 +133,29 @@ def dispatchURL(self, path, method="PURGE", port=SERVER_PORT): def testSimpleSync(self): self.httpd.queue_response(response=200) resp = self.dispatchURL("/foo") - self.assertEqual((200, '', ''), resp) + self.assertEqual((200, "", ""), resp) def testHeaders(self): - headers = {'X-Squid-Error': 'error text', - 'X-Cache': 'a message', - } + headers = {"X-Squid-Error": "error text", "X-Cache": "a message"} self.httpd.queue_response(response=200, headers=headers) status, msg, err = self.dispatchURL("/foo") - self.assertEqual(msg, 'a message') - self.assertEqual(err, 'error text') + self.assertEqual(msg, "a message") + self.assertEqual(err, "error text") self.assertEqual(status, 200) def testError(self): self.httpd.queue_response(response=None) status, msg, err = self.dispatchURL("/foo") - self.assertEqual(status, 'ERROR') + self.assertEqual(status, "ERROR") class TestSyncHTTP10(TestSync): - def setUp(self): super(TestSync, self).setUp() self.purger.http_1_1 = False class TestAsync(TestCase): - def dispatchURL(self, path, method="PURGE", port=SERVER_PORT): url = "http://localhost:%s%s" % (port, path) self.purger.purgeAsync(url, method) @@ -195,7 +187,6 @@ def testAsyncError(self): class TestAsyncConnectionFailure(TestCase): - def setUp(self): # Override setup to not start the server immediately self.purger = DefaultPurger() diff --git a/plone/cachepurging/tests/test_rewrite.py b/plone/cachepurging/tests/test_rewrite.py index 238bc26..49e5193 100644 --- a/plone/cachepurging/tests/test_rewrite.py +++ b/plone/cachepurging/tests/test_rewrite.py @@ -16,7 +16,6 @@ class FauxRequest(dict): class TestRewrite(unittest.TestCase): - def setUp(self): self.request = FauxRequest() self.rewriter = DefaultRewriter(self.request) @@ -28,43 +27,51 @@ def tearDown(self): def _prepareVHMRequest( self, path, - domain='example.com', - root='/plone', - prefix='', - protocol='http' + domain="example.com", + root="/plone", + prefix="", + protocol="http", ): - translatedPrefix = '/'.join(['_vh_%s' % p for p in prefix.split('/')]) - - self.request['URL'] = '%s://%s%s%s' % (protocol, domain, prefix, path,) - self.request[ - 'ACTUAL_URL'] = '%s://%s%s%s' % (protocol, domain, prefix, path,) - self.request['SERVER_URL'] = '%s://%s' % (protocol, domain,) - self.request['PATH_INFO'] = ( - '/VirtualHostBase/%s/%s:80%s/' - 'VirtualHostRoot%s%s' % ( - protocol, domain, root, translatedPrefix, path, - ) + translatedPrefix = "/".join(["_vh_%s" % p for p in prefix.split("/")]) + + self.request["URL"] = "%s://%s%s%s" % (protocol, domain, prefix, path) + self.request["ACTUAL_URL"] = "%s://%s%s%s" % ( + protocol, + domain, + prefix, + path, + ) + self.request["SERVER_URL"] = "%s://%s" % (protocol, domain) + self.request["PATH_INFO"] = ( + "/VirtualHostBase/%s/%s:80%s/" + "VirtualHostRoot%s%s" + % (protocol, domain, root, translatedPrefix, path) ) - self.request['VIRTUAL_URL'] = '%s://%s%s' % (protocol, domain, path) + self.request["VIRTUAL_URL"] = "%s://%s%s" % (protocol, domain, path) if prefix: - self.request['VIRTUAL_URL_PARTS'] = ( - '%s://%s' % (protocol, domain,), prefix[1:], path[1:]) + self.request["VIRTUAL_URL_PARTS"] = ( + "%s://%s" % (protocol, domain), + prefix[1:], + path[1:], + ) else: - self.request['VIRTUAL_URL_PARTS'] = ( - '%s://%s' % (protocol, domain,), path[1:]) + self.request["VIRTUAL_URL_PARTS"] = ( + "%s://%s" % (protocol, domain), + path[1:], + ) - self.request['VirtualRootPhysicalPath'] = tuple(root.split('/')) + self.request["VirtualRootPhysicalPath"] = tuple(root.split("/")) def test_no_registry(self): - self._prepareVHMRequest('/foo') - self.assertEqual(['/foo'], self.rewriter('/foo')) + self._prepareVHMRequest("/foo") + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_no_settings(self): registry = Registry() provideUtility(registry, IRegistry) - self._prepareVHMRequest('/foo') - self.assertEqual(['/foo'], self.rewriter('/foo')) + self._prepareVHMRequest("/foo") + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_virtual_hosting_disabled(self): registry = Registry() @@ -73,8 +80,8 @@ def test_virtual_hosting_disabled(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = False - self._prepareVHMRequest('/foo') - self.assertEqual(['/foo'], self.rewriter('/foo')) + self._prepareVHMRequest("/foo") + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_empty_request(self): registry = Registry() @@ -84,7 +91,7 @@ def test_empty_request(self): settings.virtualHosting = True self.request.clear() - self.assertEqual(['/foo'], self.rewriter('/foo')) + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_no_virtual_url(self): registry = Registry() @@ -93,9 +100,9 @@ def test_no_virtual_url(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo') - del self.request['VIRTUAL_URL'] - self.assertEqual(['/foo'], self.rewriter('/foo')) + self._prepareVHMRequest("/foo") + del self.request["VIRTUAL_URL"] + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_no_virtual_url_parts(self): registry = Registry() @@ -104,9 +111,9 @@ def test_no_virtual_url_parts(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo') - del self.request['VIRTUAL_URL_PARTS'] - self.assertEqual(['/foo'], self.rewriter('/foo')) + self._prepareVHMRequest("/foo") + del self.request["VIRTUAL_URL_PARTS"] + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_no_virtual_root_physical_path(self): registry = Registry() @@ -115,9 +122,9 @@ def test_no_virtual_root_physical_path(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo') - del self.request['VirtualRootPhysicalPath'] - self.assertEqual(['/foo'], self.rewriter('/foo')) + self._prepareVHMRequest("/foo") + del self.request["VirtualRootPhysicalPath"] + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_malformed_virtual_url_parts(self): registry = Registry() @@ -126,20 +133,24 @@ def test_malformed_virtual_url_parts(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo') + self._prepareVHMRequest("/foo") - self.request['VIRTUAL_URL_PARTS'] = ('foo',) - self.assertEqual(['/foo'], self.rewriter('/foo')) + self.request["VIRTUAL_URL_PARTS"] = ("foo",) + self.assertEqual(["/foo"], self.rewriter("/foo")) - self.request['VIRTUAL_URL_PARTS'] = () - self.assertEqual(['/foo'], self.rewriter('/foo')) + self.request["VIRTUAL_URL_PARTS"] = () + self.assertEqual(["/foo"], self.rewriter("/foo")) - self.request['VIRTUAL_URL_PARTS'] = ( - 'http://example.com', '', '/foo', 'x') - self.assertEqual(['/foo'], self.rewriter('/foo')) + self.request["VIRTUAL_URL_PARTS"] = ( + "http://example.com", + "", + "/foo", + "x", + ) + self.assertEqual(["/foo"], self.rewriter("/foo")) - self.request['VIRTUAL_URL_PARTS'] = 'foo' - self.assertEqual(['/foo'], self.rewriter('/foo')) + self.request["VIRTUAL_URL_PARTS"] = "foo" + self.assertEqual(["/foo"], self.rewriter("/foo")) def test_standard_vhm(self): registry = Registry() @@ -148,10 +159,10 @@ def test_standard_vhm(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo') + self._prepareVHMRequest("/foo") self.assertEqual( - ['/VirtualHostBase/http/example.com/plone/VirtualHostRoot/foo'], - self.rewriter('/foo') + ["/VirtualHostBase/http/example.com/plone/VirtualHostRoot/foo"], + self.rewriter("/foo"), ) def test_virtual_root_is_app_root(self): @@ -161,11 +172,11 @@ def test_virtual_root_is_app_root(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo', root='/') + self._prepareVHMRequest("/foo", root="/") self.assertEqual( - ['/VirtualHostBase/http/example.com/VirtualHostRoot/foo'], - self.rewriter('/foo') + ["/VirtualHostBase/http/example.com/VirtualHostRoot/foo"], + self.rewriter("/foo"), ) def test_virtual_root_is_deep(self): @@ -175,12 +186,14 @@ def test_virtual_root_is_deep(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo', root='/bar/plone') + self._prepareVHMRequest("/foo", root="/bar/plone") self.assertEqual( - ['/VirtualHostBase/http/example.com/bar/plone/' - 'VirtualHostRoot/foo'], - self.rewriter('/foo') + [ + "/VirtualHostBase/http/example.com/bar/plone/" + "VirtualHostRoot/foo" + ], + self.rewriter("/foo"), ) def test_inside_out_hosting(self): @@ -190,12 +203,15 @@ def test_inside_out_hosting(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo', root='/bar/plone', prefix='/foo/bar') + self._prepareVHMRequest("/foo", root="/bar/plone", prefix="/foo/bar") self.assertEqual( - ['/VirtualHostBase/http/example.com/bar/plone/' - 'VirtualHostRoot/_vh_foo/_vh_bar/foo'], - self.rewriter('/foo')) + [ + "/VirtualHostBase/http/example.com/bar/plone/" + "VirtualHostRoot/_vh_foo/_vh_bar/foo" + ], + self.rewriter("/foo"), + ) def test_inside_out_hosting_root_empty_path(self): registry = Registry() @@ -204,12 +220,15 @@ def test_inside_out_hosting_root_empty_path(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/', root='/plone', prefix='/plone') + self._prepareVHMRequest("/", root="/plone", prefix="/plone") self.assertEqual( - ['/VirtualHostBase/http/example.com/plone/' - 'VirtualHostRoot/_vh_plone'], - self.rewriter('')) + [ + "/VirtualHostBase/http/example.com/plone/" + "VirtualHostRoot/_vh_plone" + ], + self.rewriter(""), + ) def test_virtual_path_is_root(self): registry = Registry() @@ -218,11 +237,11 @@ def test_virtual_path_is_root(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/', root='/plone') + self._prepareVHMRequest("/", root="/plone") - self.assertEqual([ - '/VirtualHostBase/http/example.com/plone/VirtualHostRoot/'], - self.rewriter('/') + self.assertEqual( + ["/VirtualHostBase/http/example.com/plone/VirtualHostRoot/"], + self.rewriter("/"), ) def test_virtual_path_is_empty(self): @@ -232,11 +251,11 @@ def test_virtual_path_is_empty(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('', root='/plone') + self._prepareVHMRequest("", root="/plone") self.assertEqual( - ['/VirtualHostBase/http/example.com/plone/VirtualHostRoot'], - self.rewriter('') + ["/VirtualHostBase/http/example.com/plone/VirtualHostRoot"], + self.rewriter(""), ) def test_virtual_path_is_deep(self): @@ -246,12 +265,14 @@ def test_virtual_path_is_deep(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo/bar', root='/plone') + self._prepareVHMRequest("/foo/bar", root="/plone") self.assertEqual( - ['/VirtualHostBase/http/example.com/plone/' - 'VirtualHostRoot/foo/bar'], - self.rewriter('/foo/bar') + [ + "/VirtualHostBase/http/example.com/plone/" + "VirtualHostRoot/foo/bar" + ], + self.rewriter("/foo/bar"), ) def test_nonstandard_port(self): @@ -261,10 +282,10 @@ def test_nonstandard_port(self): settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - self._prepareVHMRequest('/foo', domain='example.com:81') + self._prepareVHMRequest("/foo", domain="example.com:81") self.assertEqual( - ['/VirtualHostBase/http/example.com:81/plone/VirtualHostRoot/foo'], - self.rewriter('/foo') + ["/VirtualHostBase/http/example.com:81/plone/VirtualHostRoot/foo"], + self.rewriter("/foo"), ) def test_https(self): @@ -275,11 +296,14 @@ def test_https(self): settings.virtualHosting = True self._prepareVHMRequest( - '/foo', domain='example.com:81', protocol='https') + "/foo", domain="example.com:81", protocol="https" + ) self.assertEqual( - ['/VirtualHostBase/https/example.com:81/plone/' - 'VirtualHostRoot/foo'], - self.rewriter('/foo') + [ + "/VirtualHostBase/https/example.com:81/plone/" + "VirtualHostRoot/foo" + ], + self.rewriter("/foo"), ) def test_domains(self): @@ -288,18 +312,19 @@ def test_domains(self): registry.registerInterface(ICachePurgingSettings) settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - settings.domains = ('http://example.org:81', 'https://example.com:82') + settings.domains = ("http://example.org:81", "https://example.com:82") self._prepareVHMRequest( - '/foo', domain='example.com:81', protocol='https') + "/foo", domain="example.com:81", protocol="https" + ) self.assertEqual( [ - '/VirtualHostBase/http/example.org:81/plone/VirtualHostRoot/' - 'foo', - '/VirtualHostBase/https/example.com:82/plone/VirtualHostRoot/' - 'foo' + "/VirtualHostBase/http/example.org:81/plone/VirtualHostRoot/" + "foo", + "/VirtualHostBase/https/example.com:82/plone/VirtualHostRoot/" + "foo", ], - self.rewriter('/foo') + self.rewriter("/foo"), ) def test_domains_w_different_path_in_request(self): @@ -308,18 +333,19 @@ def test_domains_w_different_path_in_request(self): registry.registerInterface(ICachePurgingSettings) settings = registry.forInterface(ICachePurgingSettings) settings.virtualHosting = True - settings.domains = ('http://example.org:81', 'https://example.com:82') + settings.domains = ("http://example.org:81", "https://example.com:82") self._prepareVHMRequest( - '/bar', domain='example.com:81', protocol='https') + "/bar", domain="example.com:81", protocol="https" + ) self.assertEqual( [ - '/VirtualHostBase/http/example.org:81/plone/VirtualHostRoot/' - 'foo', - '/VirtualHostBase/https/example.com:82/plone/VirtualHostRoot/' - 'foo' + "/VirtualHostBase/http/example.org:81/plone/VirtualHostRoot/" + "foo", + "/VirtualHostBase/https/example.com:82/plone/VirtualHostRoot/" + "foo", ], - self.rewriter('/foo') + self.rewriter("/foo"), ) diff --git a/plone/cachepurging/tests/test_traversable_paths.py b/plone/cachepurging/tests/test_traversable_paths.py index cb5fcec..06fd19a 100644 --- a/plone/cachepurging/tests/test_traversable_paths.py +++ b/plone/cachepurging/tests/test_traversable_paths.py @@ -8,19 +8,17 @@ @implementer(ITraversable) class FauxTraversable(object): - def virtual_url_path(self): - return 'foo' + return "foo" class TestTraversablePaths(unittest.TestCase): - def test_traversable_paths(self): context = FauxTraversable() paths = TraversablePurgePaths(context) - self.assertEqual(['/foo'], paths.getRelativePaths()) + self.assertEqual(["/foo"], paths.getRelativePaths()) self.assertEqual([], paths.getAbsolutePaths()) diff --git a/plone/cachepurging/tests/test_utils.py b/plone/cachepurging/tests/test_utils.py index 89a3ce3..73b3dfb 100644 --- a/plone/cachepurging/tests/test_utils.py +++ b/plone/cachepurging/tests/test_utils.py @@ -24,7 +24,6 @@ class FauxRequest(dict): class TestIsCachingEnabled(unittest.TestCase): - def setUp(self): provideAdapter(persistentFieldAdapter) @@ -70,7 +69,6 @@ def test_passed_registry(self): class TestGetPathsToPurge(unittest.TestCase): - def setUp(self): self.context = FauxContext() self.request = FauxRequest() @@ -80,14 +78,13 @@ def tearDown(self): def test_no_purge_paths(self): self.assertEqual( - [], list(utils.getPathsToPurge(self.context, self.request))) + [], list(utils.getPathsToPurge(self.context, self.request)) + ) def test_empty_relative_paths(self): - @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context @@ -100,153 +97,146 @@ def getAbsolutePaths(self): provideAdapter(FauxPurgePaths, name="test1") self.assertEqual( - [], list(utils.getPathsToPurge(self.context, self.request))) + [], list(utils.getPathsToPurge(self.context, self.request)) + ) def test_no_rewriter(self): @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): - return ['/baz'] + return ["/baz"] provideAdapter(FauxPurgePaths, name="test1") - self.assertEqual(['/foo', '/bar', '/baz'], - list(utils.getPathsToPurge(self.context, self.request))) + self.assertEqual( + ["/foo", "/bar", "/baz"], + list(utils.getPathsToPurge(self.context, self.request)), + ) def test_test_rewriter(self): @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): - return ['/baz'] + return ["/baz"] provideAdapter(FauxPurgePaths, name="test1") @implementer(IPurgePathRewriter) @adapter(FauxRequest) class DefaultRewriter(object): - def __init__(self, request): self.request = request def __call__(self, path): - return ['/vhm1' + path, '/vhm2' + path] + return ["/vhm1" + path, "/vhm2" + path] provideAdapter(DefaultRewriter) - self.assertEqual(['/vhm1/foo', '/vhm2/foo', - '/vhm1/bar', '/vhm2/bar', - '/baz'], - list(utils.getPathsToPurge(self.context, self.request))) + self.assertEqual( + ["/vhm1/foo", "/vhm2/foo", "/vhm1/bar", "/vhm2/bar", "/baz"], + list(utils.getPathsToPurge(self.context, self.request)), + ) def test_multiple_purge_paths(self): @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths1(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): - return ['/baz'] + return ["/baz"] provideAdapter(FauxPurgePaths1, name="test1") @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths2(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo/view'] + return ["/foo/view"] def getAbsolutePaths(self): - return ['/quux'] + return ["/quux"] provideAdapter(FauxPurgePaths2, name="test2") @implementer(IPurgePathRewriter) @adapter(FauxRequest) class DefaultRewriter(object): - def __init__(self, request): self.request = request def __call__(self, path): - return ['/vhm1' + path, '/vhm2' + path] + return ["/vhm1" + path, "/vhm2" + path] provideAdapter(DefaultRewriter) self.assertEqual( [ - '/vhm1/foo', - '/vhm2/foo', - '/vhm1/bar', - '/vhm2/bar', - '/baz', - '/vhm1/foo/view', - '/vhm2/foo/view', - '/quux' + "/vhm1/foo", + "/vhm2/foo", + "/vhm1/bar", + "/vhm2/bar", + "/baz", + "/vhm1/foo/view", + "/vhm2/foo/view", + "/quux", ], - list(utils.getPathsToPurge(self.context, self.request)) + list(utils.getPathsToPurge(self.context, self.request)), ) def test_rewriter_abort(self): - @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths1(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): - return ['/baz'] + return ["/baz"] provideAdapter(FauxPurgePaths1, name="test1") @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths2(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo/view'] + return ["/foo/view"] def getAbsolutePaths(self): - return ['/quux'] + return ["/quux"] provideAdapter(FauxPurgePaths2, name="test2") @implementer(IPurgePathRewriter) @adapter(FauxRequest) class DefaultRewriter(object): - def __init__(self, request): self.request = request @@ -256,36 +246,35 @@ def __call__(self, path): provideAdapter(DefaultRewriter) self.assertEqual( - ['/baz', '/quux'], - list(utils.getPathsToPurge(self.context, self.request)) + ["/baz", "/quux"], + list(utils.getPathsToPurge(self.context, self.request)), ) class TestGetURLsToPurge(unittest.TestCase): - def test_no_proxies(self): - self.assertEqual([], list(utils.getURLsToPurge('/foo', []))) + self.assertEqual([], list(utils.getURLsToPurge("/foo", []))) def test_absolute_path(self): self.assertEqual( - ['http://localhost:1234/foo/bar', 'http://localhost:2345/foo/bar'], + ["http://localhost:1234/foo/bar", "http://localhost:2345/foo/bar"], list( utils.getURLsToPurge( - '/foo/bar', - ['http://localhost:1234', 'http://localhost:2345/'] + "/foo/bar", + ["http://localhost:1234", "http://localhost:2345/"], ) - ) + ), ) def test_relative_path(self): self.assertEqual( - ['http://localhost:1234/foo/bar', 'http://localhost:2345/foo/bar'], + ["http://localhost:1234/foo/bar", "http://localhost:2345/foo/bar"], list( utils.getURLsToPurge( - 'foo/bar', - ['http://localhost:1234', 'http://localhost:2345/'] + "foo/bar", + ["http://localhost:1234", "http://localhost:2345/"], ) - ) + ), ) diff --git a/plone/cachepurging/tests/test_views.py b/plone/cachepurging/tests/test_views.py index 53baf3b..894276c 100644 --- a/plone/cachepurging/tests/test_views.py +++ b/plone/cachepurging/tests/test_views.py @@ -27,7 +27,6 @@ class FauxRequest(dict): class Handler(object): - def __init__(self): self.invocations = [] @@ -37,7 +36,6 @@ def handler(self, event): class TestQueuePurge(unittest.TestCase): - def setUp(self): provideAdapter(persistentFieldAdapter) self.registry = Registry() @@ -46,7 +44,7 @@ def setUp(self): self.settings = self.registry.forInterface(ICachePurgingSettings) self.settings.enabled = True - self.settings.cachingProxies = ('http://localhost:1234',) + self.settings.cachingProxies = ("http://localhost:1234",) self.handler = Handler() provideHandler(self.handler.handler) @@ -58,7 +56,7 @@ def test_disabled(self): self.settings.enabled = False view = QueuePurge(FauxContext(), FauxRequest()) - self.assertEqual('Caching not enabled', view()) + self.assertEqual("Caching not enabled", view()) self.assertEqual([], self.handler.invocations) def test_enabled(self): @@ -66,13 +64,12 @@ def test_enabled(self): context = FauxContext() view = QueuePurge(context, FauxRequest) - self.assertEqual('Queued', view()) + self.assertEqual("Queued", view()) self.assertEqual(1, len(self.handler.invocations)) self.assertTrue(self.handler.invocations[0].object is context) class TestPurgeImmediately(unittest.TestCase): - def setUp(self): provideAdapter(persistentFieldAdapter) self.registry = Registry() @@ -81,17 +78,16 @@ def setUp(self): self.settings = self.registry.forInterface(ICachePurgingSettings) self.settings.enabled = True - self.settings.cachingProxies = ('http://localhost:1234',) + self.settings.cachingProxies = ("http://localhost:1234",) @implementer(IPurgePaths) @adapter(FauxContext) class FauxPurgePaths(object): - def __init__(self, context): self.context = context def getRelativePaths(self): - return ['/foo', '/bar'] + return ["/foo", "/bar"] def getAbsolutePaths(self): return [] @@ -100,8 +96,7 @@ def getAbsolutePaths(self): @implementer(IPurger) class FauxPurger(object): - - def purgeSync(self, url, httpVerb='PURGE'): + def purgeSync(self, url, httpVerb="PURGE"): return "200 OK", "cached", None provideUtility(FauxPurger()) @@ -112,20 +107,20 @@ def tearDown(self): def test_disabled(self): self.settings.enabled = False view = PurgeImmediately(FauxContext(), FauxRequest()) - self.assertEqual('Caching not enabled', view()) + self.assertEqual("Caching not enabled", view()) def test_purge(self): view = PurgeImmediately(FauxContext(), FauxRequest()) self.assertEqual( - 'Purged: http://localhost:1234/foo, ' - 'Status: 200 OK, ' - 'X-Cache: cached, ' - 'Error: None\n' - 'Purged: http://localhost:1234/bar, ' - 'Status: 200 OK, ' - 'X-Cache: cached, ' - 'Error: None\n', - view() + "Purged: http://localhost:1234/foo, " + "Status: 200 OK, " + "X-Cache: cached, " + "Error: None\n" + "Purged: http://localhost:1234/bar, " + "Status: 200 OK, " + "X-Cache: cached, " + "Error: None\n", + view(), ) diff --git a/plone/cachepurging/utils.py b/plone/cachepurging/utils.py index 7f2a5c5..73d67ec 100644 --- a/plone/cachepurging/utils.py +++ b/plone/cachepurging/utils.py @@ -52,10 +52,10 @@ def getURLsToPurge(path, proxies): listed in the registry into account. """ - if not path.startswith('/'): - path = '/' + path + if not path.startswith("/"): + path = "/" + path for proxy in proxies: - if proxy.endswith('/'): + if proxy.endswith("/"): proxy = proxy[:-1] yield proxy + path diff --git a/setup.py b/setup.py index f49b2fb..8a3700a 100644 --- a/setup.py +++ b/setup.py @@ -1,49 +1,47 @@ from setuptools import setup, find_packages -version = '1.0.16.dev0' +version = "2.0.dev0" setup( - name='plone.cachepurging', + name="plone.cachepurging", version=version, description="Cache purging support for Zope 2 applications", - long_description=(open('README.rst').read() + '\n' + - open('CHANGES.rst').read()), + long_description=( + open("README.rst").read() + "\n" + open("CHANGES.rst").read() + ), classifiers=[ - 'Framework :: Plone', - 'Framework :: Plone :: 4.3', - 'Framework :: Plone :: 5.0', - 'Framework :: Plone :: 5.1', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', + "Framework :: Plone", + "Framework :: Plone :: 5.1", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.6", ], - keywords='plone cache purge', - author='Plone Foundation', - author_email='plone-developers@lists.sourceforge.net', - url='https://pypi.python.org/pypi/plone.cachepurging', - license='GPL version 2', - packages=find_packages(exclude=['ez_setup']), - namespace_packages=['plone'], + keywords="plone cache purge", + author="Plone Foundation", + author_email="plone-developers@lists.sourceforge.net", + url="https://pypi.python.org/pypi/plone.cachepurging", + license="GPL version 2", + packages=find_packages(exclude=["ez_setup"]), + namespace_packages=["plone"], include_package_data=True, zip_safe=False, install_requires=[ - 'setuptools', - 'five.globalrequest', - 'plone.registry', - 'six', - 'z3c.caching', - 'zope.annotation', - 'zope.component', - 'zope.event', - 'zope.i18nmessageid', - 'zope.interface', - 'zope.lifecycleevent', - 'zope.schema', - 'zope.testing', - 'Zope2' + "setuptools", + "five.globalrequest", + "plone.registry", + "requests", + "six", + "z3c.caching", + "zope.annotation", + "zope.component", + "zope.event", + "zope.i18nmessageid", + "zope.interface", + "zope.lifecycleevent", + "zope.schema", + "zope.testing", + "Zope2", ], - extras_require={ - 'test': ['plone.app.testing'], - }, + extras_require={"test": ["plone.app.testing"]}, )