Use API call to check if archive.org overloaded (#253)

netkan/netkan/mirrorer.py

@@ -9,7 +9,6 @@
 from importlib.resources import read_text
 from typing import Optional, List, Union, Iterable, BinaryIO, Dict, Any
 import boto3
-import requests
 import github
 import internetarchive
 from jinja2 import Template
@@ -311,7 +310,7 @@ def process_queue(self, queue_name: str, timeout: int) -> None:
                     to_delete = []
                     for msg in messages:
                         # Check if archive.org is overloaded before each upload
-                        if self.ia_overloaded():
+                        if self.ia_session.s3_is_overloaded(access_key=self.ia_access):
                             logging.info('The Internet Archive is overloaded, try again later')
                             break
                         path = Path(self.ckm_repo.git_repo.working_dir, msg.body)
@@ -362,11 +361,6 @@ def _default_branch(self, ckan: Ckan) -> str:
                 return self._gh.get_repo(full_name).default_branch
         return 'master'
 
-    def ia_overloaded(self) -> bool:
-        response = requests.get(
-            f'https://s3.us.archive.org/?check_limit=1&accesskey={self.ia_access}')
-        return response.status_code == 503 or response.json().get('over_limit')
-
     def purge_epochs(self, dry_run: bool) -> None:
         if dry_run:
             logging.info('Dry run mode enabled, no changes will be made')