Merge pull request #26 from itrujnara/bug_fixes

Added better request error handling
nf-core · May 14, 2024 · 22f1f7d · 22f1f7d
2 parents 8ba2862 + 1bcc0a9
commit 22f1f7d
Show file tree

Hide file tree

Showing 11 changed files with 67 additions and 28 deletions.
diff --git a/bin/ensembl2uniprot.py b/bin/ensembl2uniprot.py
@@ -5,8 +5,7 @@
 
 import sys
 
-import requests
-from utils import check_id_mapping_results_ready
+from utils import check_id_mapping_results_ready, safe_get, safe_post
 
 
 def ensembl2uniprot(ensembl_ids: list[str]) -> list[str]:
@@ -22,7 +21,8 @@ def ensembl2uniprot(ensembl_ids: list[str]) -> list[str]:
         "to": "UniProtKB"
     }
 
-    res = requests.post("https://rest.uniprot.org/idmapping/run", data=payload)
+    res = safe_post("https://rest.uniprot.org/idmapping/run", data=payload)
+
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")
 
@@ -31,7 +31,7 @@ def ensembl2uniprot(ensembl_ids: list[str]) -> list[str]:
     # wait for the job to finish
     check_id_mapping_results_ready(job_id)
 
-    res = requests.get(f"https://rest.uniprot.org/idmapping/results/{job_id}")
+    res = safe_get(f"https://rest.uniprot.org/idmapping/results/{job_id}")
 
     json = res.json()
 

diff --git a/bin/fetch_afdb_structures.py b/bin/fetch_afdb_structures.py
@@ -5,7 +5,7 @@
 
 import sys
 
-import requests
+from utils import safe_get
 
 
 def fetch_structures(path: str, prefix: str) -> None:
@@ -21,14 +21,15 @@ def fetch_structures(path: str, prefix: str) -> None:
 
     for id in ids:
         url = f"https://alphafold.ebi.ac.uk/api/prediction/{id}"
-        res = requests.get(url)
+        res = safe_get(url)
+
         if res.ok:
             pdb_url = res.json()[0]["pdbUrl"]
             version = res.json()[0]["latestVersion"]
 
             print(f"{id}: {version}", file=sys.stderr)
 
-            res = requests.get(pdb_url)
+            res = safe_get(pdb_url)
 
             if res.ok:
                 print(res.text, file=open(f"{id}.pdb", 'w'))

diff --git a/bin/fetch_inspector_group.py b/bin/fetch_inspector_group.py
@@ -5,15 +5,15 @@
 
 import sys
 
-import requests
+from utils import safe_get
 
 
 def fetch_inspector_by_id(uniprot_id: str, db_id: str = "Eukaryota2019") -> None:
     """
     Fetch orthologs for a given UniProt ID from the OrthoInspector database.
     """
     url = f"https://lbgi.fr/api/orthoinspector/{db_id}/protein/{uniprot_id}/orthologs"
-    res = requests.get(url)
+    res = safe_get(url)
 
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")

diff --git a/bin/fetch_oma_group.py b/bin/fetch_oma_group.py
@@ -5,7 +5,7 @@
 
 import sys
 
-import requests
+from utils import safe_get
 
 
 def main() -> None:
@@ -17,7 +17,7 @@ def main() -> None:
 
     id = sys.argv[1]
 
-    res = requests.get(f"https://omabrowser.org/api/group/{id}")
+    res = safe_get(f"https://omabrowser.org/api/group/{id}")
 
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")

diff --git a/bin/fetch_panther_group.py b/bin/fetch_panther_group.py
@@ -5,7 +5,7 @@
 
 import sys
 
-import requests
+from utils import safe_get
 
 
 def main() -> None:
@@ -15,7 +15,7 @@ def main() -> None:
     if len(sys.argv) < 3:
         raise ValueError("Too few arguments. Usage: fetch_panther_group.py <id> <organism>")
 
-    res = requests.get(f"https://www.pantherdb.org/services/oai/pantherdb/ortholog/matchortho?geneInputList={sys.argv[1]}&organism={sys.argv[2]}&orthologType=all")
+    res = safe_get(f"https://www.pantherdb.org/services/oai/pantherdb/ortholog/matchortho?geneInputList={sys.argv[1]}&organism={sys.argv[2]}&orthologType=all")
 
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")

diff --git a/bin/fetch_sequences.py b/bin/fetch_sequences.py
@@ -5,7 +5,7 @@
 
 import sys
 
-import requests
+from utils import safe_get, safe_post
 
 
 def fetch_seqs_oma(path: str, prefix: str) -> list[str]:
@@ -18,7 +18,9 @@ def fetch_seqs_oma(path: str, prefix: str) -> list[str]:
 
     payload = {"ids": ids}
 
-    res = requests.post("https://omabrowser.org/api/protein/bulk_retrieve/", json=payload)
+    print(payload, file=sys.stderr)
+
+    res = safe_post("https://omabrowser.org/api/protein/bulk_retrieve/", json=payload)
 
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")
@@ -50,7 +52,7 @@ def fetch_seqs_uniprot(oma_misses: list, prefix: str) -> None:
     misses = []
 
     for id in oma_misses:
-        res = requests.get(f"https://rest.uniprot.org/uniprotkb/{id}.fasta")
+        res = safe_get(f"https://rest.uniprot.org/uniprotkb/{id}.fasta")
         if res.ok:
             hits.append((id, res.text.split("\n", 1)[1].replace("\n", "")))
         else:

diff --git a/bin/get_oma_version.py b/bin/get_oma_version.py
@@ -3,14 +3,14 @@
 # Written by Igor Trujnara, released under the MIT license
 # See https://opensource.org/license/mit for details
 
-import requests
+from utils import safe_get
 
 
 def main() -> None:
     """
     Get the version of the OMA database and API.
     """
-    res = requests.get("https://omabrowser.org/api/version")
+    res = safe_get("https://omabrowser.org/api/version")
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")
     json = res.json()

diff --git a/bin/refseq2uniprot.py b/bin/refseq2uniprot.py
@@ -5,8 +5,7 @@
 
 import sys
 
-import requests
-from utils import check_id_mapping_results_ready
+from utils import check_id_mapping_results_ready, safe_get, safe_post
 
 
 def refseq2uniprot(refseq_ids: list[str]) -> list[str]:
@@ -22,15 +21,15 @@ def refseq2uniprot(refseq_ids: list[str]) -> list[str]:
         "to": "UniProtKB"
     }
 
-    res = requests.post("https://rest.uniprot.org/idmapping/run", data=payload)
+    res = safe_post("https://rest.uniprot.org/idmapping/run", data=payload)
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")
 
     job_id = res.json()["jobId"]
 
     check_id_mapping_results_ready(job_id)
 
-    res = requests.get(f"https://rest.uniprot.org/idmapping/results/{job_id}")
+    res = safe_get(f"https://rest.uniprot.org/idmapping/results/{job_id}")
 
     json = res.json()
 

diff --git a/bin/uniprot2uniprot.py b/bin/uniprot2uniprot.py
@@ -5,8 +5,7 @@
 
 import sys
 
-import requests
-from utils import check_id_mapping_results_ready
+from utils import check_id_mapping_results_ready, safe_post, safe_get
 
 
 def uniprot2uniprot(uniprot_names: list[str]) -> list[str]:
@@ -22,15 +21,15 @@ def uniprot2uniprot(uniprot_names: list[str]) -> list[str]:
         "to": "UniProtKB"
     }
 
-    res = requests.post("https://rest.uniprot.org/idmapping/run", data=payload)
+    res = safe_post("https://rest.uniprot.org/idmapping/run", data=payload)
     if not res.ok:
         raise ValueError(f"HTTP error: {res.status_code}")
 
     job_id = res.json()["jobId"]
 
     check_id_mapping_results_ready(job_id)
 
-    res = requests.get(f"https://rest.uniprot.org/idmapping/results/{job_id}")
+    res = safe_get(f"https://rest.uniprot.org/idmapping/results/{job_id}")
 
     json = res.json()
 

diff --git a/bin/utils.py b/bin/utils.py
@@ -2,19 +2,43 @@
 # See https://opensource.org/license/mit for details
 # Includes code written by UniProt contributors published under CC-BY 4.0 license
 
+import sys
 import time
 from typing import Any
 
 import requests
+from requests.exceptions import RequestException
 
 POLLING_INTERVAL = 0.5
 
+def safe_get(url: str):
+    """
+    Get a URL and return the response.
+    """
+    try:
+        return requests.get(url)
+    except requests.exceptions.RequestException as e:
+        print(f"A network issue occurred. Retrying request. Details:\n{e}", file=sys.stderr)
+        sys.exit(10)
+
+
+def safe_post(url: str, data: dict = dict(), json: dict = dict()):
+    """
+    Post data to a URL and return the response.
+    """
+    try:
+        return requests.post(url, data=data, json=json)
+    except requests.exceptions.RequestException as e:
+        print(f"A network issue occurred. Retrying request. Details:\n{e}", file=sys.stderr)
+        sys.exit(10)
+
+
 def check_id_mapping_results_ready(job_id):
     """
     Wait until the ID mapping job is finished.
     """
     while True:
-        request = requests.get(f"https://rest.uniprot.org/idmapping/status/{job_id}")
+        request = safe_get(f"https://rest.uniprot.org/idmapping/status/{job_id}")
         j = request.json()
         if "jobStatus" in j:
             if j["jobStatus"] == "RUNNING":
@@ -29,7 +53,7 @@ def fetch_seq(url: str) -> tuple[bool, dict]:
     """
     Get JSON from a URL.
     """
-    res = requests.get(url)
+    res = safe_get(url)
     if not res.ok:
         print(f"HTTP error. Code: {res.status_code}")
         return (False, dict())

diff --git a/conf/modules.config b/conf/modules.config
@@ -28,6 +28,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'FETCH_OMA_GROUP_LOCAL|FETCH_OMA_GROUP_ONLINE' {
@@ -36,6 +38,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'FETCH_PANTHER_GROUP_LOCAL|FETCH_PANTHER_GROUP_ONLINE' {
@@ -44,6 +48,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'FETCH_INSPECTOR_GROUP_ONLINE' {
@@ -52,6 +58,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'FETCH_EGGNOG_GROUP_LOCAL|FETCH_EGGNOG_GROUP_ONLINE' {
@@ -60,6 +68,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'MERGE_CSV' {
@@ -149,6 +159,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'FETCH_AFDB_STRUCTURES' {
@@ -157,6 +169,8 @@ process {
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
+        errorStrategy = {task.exitStatus == 10 ? 'retry' : 'finish'}
+        maxRetries = 3
     }
 
     withName: 'FILTER_FASTA' {