Refinements

rotating_proxies/expire.py

@@ -41,17 +41,28 @@ def __init__(self, proxy_list, backoff=None):
         self.unchecked = set(self.proxies.keys())
         self.good = set()
         self.dead = set()
+        self.used = set()
 
         if backoff is None:
             backoff = exp_backoff_full_jitter
         self.backoff = backoff
 
     def get_random(self):
         """ Return a random available proxy (either good or unchecked) """
-        available = list(self.unchecked | self.good)
+
+        available = self.unchecked | self.good
         if not available:
             return None
-        return random.choice(available)
+
+        unused = available - self.used
+        if not unused:
+            self.used.clear()
+            unused = available
+
+        proxy = random.choice(list(unused))
+        self.used.add(proxy)
+
+        return proxy
 
     def get_proxy(self, proxy_address):
         """

rotating_proxies/middlewares.py

@@ -165,17 +165,27 @@ def _handle_result(self, request, spider):
         proxy = self.proxies.get_proxy(request.meta.get('proxy', None))
         if not (proxy and request.meta.get('_rotating_proxy')):
             return
-        self.stats.set_value('proxies/unchecked', len(self.proxies.unchecked) - len(self.proxies.reanimated))
-        self.stats.set_value('proxies/reanimated', len(self.proxies.reanimated))
-        self.stats.set_value('proxies/mean_backoff', self.proxies.mean_backoff_time)
+
         ban = request.meta.get('_ban', None)
+        retry = False
+
         if ban is True:
             self.proxies.mark_dead(proxy)
-            self.stats.set_value('proxies/dead', len(self.proxies.dead))
-            return self._retry(request, spider)
+            retry = True
         elif ban is False:
             self.proxies.mark_good(proxy)
-            self.stats.set_value('proxies/good', len(self.proxies.good))
+
+        self._update_proxy_stats()
+
+        if retry:
+            return self._retry(request, spider)
+
+    def _update_proxy_stats(self):
+        self.stats.set_value('proxies/unchecked', len(self.proxies.unchecked) - len(self.proxies.reanimated))
+        self.stats.set_value('proxies/reanimated', len(self.proxies.reanimated))
+        self.stats.set_value('proxies/mean_backoff', self.proxies.mean_backoff_time)
+        self.stats.set_value('proxies/dead', len(self.proxies.dead))
+        self.stats.set_value('proxies/good', len(self.proxies.good))
 
     def _retry(self, request, spider):
         retries = request.meta.get('proxy_retry_times', 0) + 1