Skip to content

Commit

Permalink
V1.7.8 (#183)
Browse files Browse the repository at this point in the history 
* improves: display fixes

* improves: sample return one if k=1

* improves: sample return one if k=1

* improves: sample return one if k=1
  • Loading branch information
@jlsneto
jlsneto authored Sep 25, 2022
1 parent 433eecc commit 6d2c4ae
Show file tree
Hide file tree
Showing 39 changed files with 2,578 additions and 1,126 deletions.
11 changes: 7 additions & 4 deletions cereja/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,7 @@
from . import experimental
from ._requests import request


VERSION = "1.7.7.final.0"

VERSION = "1.7.8.final.0"

__version__ = get_version_pep440_compliant(VERSION)

Expand All @@ -59,5 +57,10 @@
unicode_ = f"\033[31m\U0001F352\033[0;0m"
print(f"{unicode_} Using Cereja v.{get_version_pep440_compliant()}\r")
NON_BMP_SUPPORTED = True
except (UnicodeEncodeError, UnicodeDecodeError, UnicodeError, UnicodeTranslateError):
except (
UnicodeEncodeError,
UnicodeDecodeError,
UnicodeError,
UnicodeTranslateError,
):
NON_BMP_SUPPORTED = False
12 changes: 7 additions & 5 deletions cereja/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,17 @@
if __name__ == "__main__":
sys.stdout.write("\U0001F352 Cereja Tools\n")
sys.stdout.flush()
parser = argparse.ArgumentParser(description='Cereja Tools.')
parser.add_argument('--version', action='version', version=get_version_pep440_compliant())
parser.add_argument('--startmodule', type=str)
parser = argparse.ArgumentParser(description="Cereja Tools.")
parser.add_argument(
"--version", action="version", version=get_version_pep440_compliant()
)
parser.add_argument("--startmodule", type=str)
args = parser.parse_args()
if args.startmodule:
base_dir = Path(BASE_DIR)
license_ = b''.join(FileIO.load(base_dir.parent.join('LICENSE')).data).decode()
license_ = b"".join(FileIO.load(base_dir.parent.join("LICENSE")).data).decode()
license_ = '"""\n' + license_ + '"""'
new_module_path = base_dir.join(*args.startmodule.split('/'))
new_module_path = base_dir.join(*args.startmodule.split("/"))
if new_module_path.parent.exists and new_module_path.parent.is_dir:
FileIO.create(new_module_path, license_).save()
else:
Expand Down
102 changes: 60 additions & 42 deletions cereja/_requests/_http.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import io
from ..config import PROXIES_URL

__all__ = ['HttpRequest', 'HttpResponse']
__all__ = ["HttpRequest", "HttpResponse"]

from urllib.error import HTTPError, URLError
from urllib.parse import urlparse
Expand All @@ -35,10 +35,11 @@


class _Http:

def __init__(self, url, data=None, headers=None, port=None):
self.headers = headers or {}
self._protocol, self._port, self._domains, self._endpoint = self.parse_url(url=url, port=port)
self._protocol, self._port, self._domains, self._endpoint = self.parse_url(
url=url, port=port
)
self._data = data or None

@staticmethod
Expand Down Expand Up @@ -71,44 +72,44 @@ def data(self):

@property
def url(self):
port = f':{self._port}' if self._port else self._port
port = f":{self._port}" if self._port else self._port
endpoint = f"/{self._endpoint}" if self._endpoint else self._endpoint
return f'{self._protocol}://{self._domains}{port}{endpoint}'
return f"{self._protocol}://{self._domains}{port}{endpoint}"

@property
def content_type(self):
return self.headers.get('Content-type')
return self.headers.get("Content-type")

@property
def content_length(self):
return self.headers.get('Content-length')
return self.headers.get("Content-length")

@classmethod
def parse_url(cls, url: str, port=None):

url = url.replace('://', '.')
url = url.split('/', maxsplit=1)
url = url.replace("://", ".")
url = url.split("/", maxsplit=1)

domains = url.pop(0).split('.')
if ':' in domains[-1]:
domain, port = domains[-1].split(':')
domains = url.pop(0).split(".")
if ":" in domains[-1]:
domain, port = domains[-1].split(":")
domains[-1] = domain

protocol = domains.pop(0) if domains[0].startswith('http') else 'https'
protocol = domains.pop(0) if domains[0].startswith("http") else "https"

if not port:
port = ''
port = ""

domains = '.'.join(domains)
endpoint = '/'.join(url) if url else ''
domains = ".".join(domains)
endpoint = "/".join(url) if url else ""

return protocol, port, domains, endpoint


class HttpResponse:
CHUNK_SIZE = 1024 * 1024 * 1 # 1MB

def __init__(self, request: 'HttpRequest', save_on_path=None, timeout=None):
def __init__(self, request: "HttpRequest", save_on_path=None, timeout=None):
self._finished = False
self._timeout = timeout
self._code = None
Expand All @@ -117,7 +118,7 @@ def __init__(self, request: 'HttpRequest', save_on_path=None, timeout=None):
self._headers = {}
self._request = request
self._save_on_path = save_on_path
self._total_completed = '0.0%'
self._total_completed = "0.0%"
self._th_request = threading.Thread(target=self.__request)
self._th_request.start()
while not self.headers and not self._finished:
Expand All @@ -126,18 +127,27 @@ def __init__(self, request: 'HttpRequest', save_on_path=None, timeout=None):
def __request(self):
# TODO: send to HttpRequest ?
try:
with urllib_req.urlopen(self._request.urllib_req, timeout=self._timeout) as req_file:
with urllib_req.urlopen(
self._request.urllib_req, timeout=self._timeout
) as req_file:
self._code = req_file.status
self._status = req_file.reason
if hasattr(req_file, 'headers'):
if hasattr(req_file, "headers"):
self._headers = dict(req_file.headers.items())
if self.CHUNK_SIZE * 3 > (req_file.length or 0):
self._data = req_file.read()
else:
with Progress(name=f'Fetching data', max_value=int(req_file.getheader('Content-Length')),
states=('download', 'time')) as prog:

with (open(self._save_on_path, 'wb') if self._save_on_path else io.BytesIO()) as f:
with Progress(
name=f"Fetching data",
max_value=int(req_file.getheader("Content-Length")),
states=("download", "time"),
) as prog:

with (
open(self._save_on_path, "wb")
if self._save_on_path
else io.BytesIO()
) as f:
total_downloaded = 0
while True:
chunk = req_file.read(self.CHUNK_SIZE)
Expand All @@ -153,7 +163,7 @@ def __request(self):
self._data = err.read()
self._code = err.code
self._status = err.reason
if hasattr(err, 'headers'):
if hasattr(err, "headers"):
self._headers = dict(err.headers.items())
except URLError as err:
msg = f"{err.reason}: {self._request.url}"
Expand All @@ -164,13 +174,13 @@ def __request(self):
def __repr__(self):
if not self._finished:
return f"<HttpResponse: Fetching data {self._total_completed}>"
return f'<HttpResponse: code={self.code}, status={self._status}>'
return f"<HttpResponse: code={self.code}, status={self._status}>"

@property
def content_type(self):
if not self._finished:
self._th_request.join()
return self._headers.get('Content-Type')
return self._headers.get("Content-Type")

@property
def headers(self):
Expand Down Expand Up @@ -210,7 +220,7 @@ def json(self):
@property
def data(self):
self._th_request.join() # await for request
if self.content_type == 'application/json':
if self.content_type == "application/json":
if not self._data:
return {}
return self.json()
Expand All @@ -223,40 +233,46 @@ class HttpRequest(_Http):
def __init__(self, method, url, *args, **kwargs):
super().__init__(url=url, *args, **kwargs)
if isinstance(self.data, dict):
self.headers.update({'Content-type': 'application/json'})
self.headers.update({"Content-type": "application/json"})
self._count = 0
self._method = method
self._req = None

def __repr__(self):
return f'Request(url={self.url}, method={self._method})'
return f"Request(url={self.url}, method={self._method})"

@classmethod
def get_proxies_list(cls):
if cls.PROXIES:
print('já tem proxies')
print("já tem proxies")
return cls.PROXIES
try:
cls.PROXIES = json.loads(cls('GET', PROXIES_URL).send_request().data)
cls.PROXIES = json.loads(cls("GET", PROXIES_URL).send_request().data)
except:
pass
return cls.PROXIES

def send_request(self, save_on=None, timeout=None, **kwargs):
if 'data' in kwargs:
self._data = self.parser(kwargs.pop('data'))

if 'headers' in kwargs:
headers = kwargs.pop('headers')
assert isinstance(headers, dict), TypeError("Headers type is invalid. send a dict")
if "data" in kwargs:
self._data = self.parser(kwargs.pop("data"))

if "headers" in kwargs:
headers = kwargs.pop("headers")
assert isinstance(headers, dict), TypeError(
"Headers type is invalid. send a dict"
)
self.headers.update(headers)
self._count += 1
return HttpResponse(request=self, save_on_path=save_on, timeout=timeout)

@property
def urllib_req(self):
return urllib_req.Request(url=self.url, data=self.parser(self.data), headers=self.headers,
method=self._method)
return urllib_req.Request(
url=self.url,
data=self.parser(self.data),
headers=self.headers,
method=self._method,
)

@property
def total_request(self):
Expand All @@ -271,8 +287,10 @@ def parser(cls, data) -> bytes:
if isinstance(data, dict):
return json.dumps(data).encode()

return str(data).encode() if data else b''
return str(data).encode() if data else b""

@classmethod
def build_and_send(cls, method, url, data=None, port=None, headers=None, **kwargs):
return cls(method=method, url=url, data=data, port=port, headers=headers).send_request(**kwargs)
return cls(
method=method, url=url, data=data, port=port, headers=headers
).send_request(**kwargs)
42 changes: 30 additions & 12 deletions cereja/_requests/request.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,29 @@
"""
from ._http import HttpRequest, HttpResponse

__all__ = ['is_url', 'download', 'get_proxies_list', 'post', 'get', 'put', 'head', 'delete', 'connect', 'options',
'trace', 'patch']
__all__ = [
"is_url",
"download",
"get_proxies_list",
"post",
"get",
"put",
"head",
"delete",
"connect",
"options",
"trace",
"patch",
]


def download(url, save_on=None, timeout=None, **kwargs) -> HttpResponse:
"""
The GET method requests a representation of the specified resource. Requests using GET should only retrieve data.
"""
return HttpRequest.build_and_send(method='GET', url=url, save_on=save_on, timeout=timeout, **kwargs)
return HttpRequest.build_and_send(
method="GET", url=url, save_on=save_on, timeout=timeout, **kwargs
)


def is_url(url):
Expand All @@ -52,60 +66,64 @@ def post(url, data=None, headers=None, **kwargs) -> HttpResponse:
effects on the server.
"""
return HttpRequest.build_and_send(method='POST', url=url, data=data, headers=headers, **kwargs)
return HttpRequest.build_and_send(
method="POST", url=url, data=data, headers=headers, **kwargs
)


def get(url, data=None, headers=None, **kwargs) -> HttpResponse:
"""
The GET method requests a representation of the specified resource. Requests using GET should only retrieve data.
"""
return HttpRequest.build_and_send(method='GET', url=url, data=data, headers=headers, **kwargs)
return HttpRequest.build_and_send(
method="GET", url=url, data=data, headers=headers, **kwargs
)


def put(url, **kwargs) -> HttpResponse:
"""
The PUT method replaces all current representations of the target resource with the request payload.
"""
return HttpRequest.build_and_send(method='PUT', url=url, **kwargs)
return HttpRequest.build_and_send(method="PUT", url=url, **kwargs)


def head(url, **kwargs) -> HttpResponse:
"""
The HEAD method asks for a response identical to that of a GET request, but without the response body.
"""
return HttpRequest.build_and_send(method='HEAD', url=url, **kwargs)
return HttpRequest.build_and_send(method="HEAD", url=url, **kwargs)


def delete(url, **kwargs) -> HttpResponse:
"""
The DELETE method deletes the specified resource.
"""
return HttpRequest.build_and_send(method='DELETE', url=url, **kwargs)
return HttpRequest.build_and_send(method="DELETE", url=url, **kwargs)


def connect(url, **kwargs) -> HttpResponse:
"""
The CONNECT method establishes a tunnel to the server identified by the target resource.
"""
return HttpRequest.build_and_send(method='CONNECT', url=url, **kwargs)
return HttpRequest.build_and_send(method="CONNECT", url=url, **kwargs)


def options(url, **kwargs) -> HttpResponse:
"""
The OPTIONS method is used to describe the communication options for the target resource.
"""
return HttpRequest.build_and_send(method='OPTIONS', url=url, **kwargs)
return HttpRequest.build_and_send(method="OPTIONS", url=url, **kwargs)


def trace(url, **kwargs) -> HttpResponse:
"""
The TRACE method performs a message loop-back test along the path to the target resource.
"""
return HttpRequest.build_and_send(method='TRACE', url=url, **kwargs)
return HttpRequest.build_and_send(method="TRACE", url=url, **kwargs)


def patch(url, **kwargs) -> HttpResponse:
"""
The PATCH method is used to apply partial modifications to a resource.
"""
return HttpRequest.build_and_send(method='PATCH', url=url, **kwargs)
return HttpRequest.build_and_send(method="PATCH", url=url, **kwargs)
Loading

0 comments on commit 6d2c4ae

Please sign in to comment.