From ce74f9e9081d267ed57d736e24324ae97fdfe9f4 Mon Sep 17 00:00:00 2001 From: Megan Ketelaar Date: Wed, 17 Oct 2018 18:16:22 -0400 Subject: [PATCH 1/3] Issue a purge request when we get a stale serial. --- src/bandersnatch/master.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/bandersnatch/master.py b/src/bandersnatch/master.py index 06dab2eba..15ee163e0 100644 --- a/src/bandersnatch/master.py +++ b/src/bandersnatch/master.py @@ -60,6 +60,23 @@ def get(self, path, required_serial, **kw): required_serial, path, got_serial ) ) + # HACK: The following attempts to purge the cache of the page we + # just tried to fetch. This works around PyPI's caches sometimes + # returning a stale serial for a package. Ideally, this should + # be fixed on the PyPI side, at which point the following code + # should be removed. + logger.debug( + "Issuing a PURGE for {} to clear the cache".format( + path + ) + ) + try: + self.session.request("PURGE", path, timeout=self.timeout) + except requests.exceptions.HTTPError: + logger.warning( + "Got an error when attempting to clear the cache", exc_info=True + ) + raise StalePage( "Expected PyPI serial {} for request {} but got {}".format( required_serial, path, got_serial From a892bda09abbcedccb9ce090d281ab693463b78c Mon Sep 17 00:00:00 2001 From: Megan Ketelaar Date: Wed, 17 Oct 2018 18:47:13 -0400 Subject: [PATCH 2/3] Fix formatting --- src/bandersnatch/master.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/bandersnatch/master.py b/src/bandersnatch/master.py index 15ee163e0..c723a0b34 100644 --- a/src/bandersnatch/master.py +++ b/src/bandersnatch/master.py @@ -65,11 +65,7 @@ def get(self, path, required_serial, **kw): # returning a stale serial for a package. Ideally, this should # be fixed on the PyPI side, at which point the following code # should be removed. - logger.debug( - "Issuing a PURGE for {} to clear the cache".format( - path - ) - ) + logger.debug(f"Issuing a PURGE for {path} to clear the cache") try: self.session.request("PURGE", path, timeout=self.timeout) except requests.exceptions.HTTPError: From 4031009dff7bc7d8ac50471536d7838ca3e77ffd Mon Sep 17 00:00:00 2001 From: Megan Ketelaar Date: Thu, 18 Oct 2018 13:31:31 -0400 Subject: [PATCH 3/3] Add purge to exception method --- src/bandersnatch/master.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bandersnatch/master.py b/src/bandersnatch/master.py index c723a0b34..0433d0b61 100644 --- a/src/bandersnatch/master.py +++ b/src/bandersnatch/master.py @@ -74,7 +74,8 @@ def get(self, path, required_serial, **kw): ) raise StalePage( - "Expected PyPI serial {} for request {} but got {}".format( + "Expected PyPI serial {} for request {} but got {}. " + + "HTTP PURGE has been issued to the request url".format( required_serial, path, got_serial ) )