From 426244b172a3b5d5ba1905fee47e55a531c7a98c Mon Sep 17 00:00:00 2001 From: "rex.zheng" Date: Wed, 14 Apr 2021 16:57:05 +0800 Subject: [PATCH 1/5] refactor(client): delete protecting code in case response is not json PR Closed: https://github.com/Graviti-AI/tensorbay-python-sdk/pull/384 --- tensorbay/client/log.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorbay/client/log.py b/tensorbay/client/log.py index 0184749cb..b485c94ae 100644 --- a/tensorbay/client/log.py +++ b/tensorbay/client/log.py @@ -165,10 +165,7 @@ def _dump_response(response: Response) -> str: if "Content-Type" in response.headers: content_type = response.headers["Content-Type"] if content_type.startswith("application/json"): - try: - content = json.dumps(response.json(), indent=2) - except json.decoder.JSONDecodeError: - content = "" + content = json.dumps(response.json(), indent=2) elif content_type.startswith("text"): content = response.text elif len(response.content) > 512: From d3b1b24f5600eb309d508630b37999b0c72b1241 Mon Sep 17 00:00:00 2001 From: "yexuan.li" Date: Wed, 14 Apr 2021 17:19:31 +0800 Subject: [PATCH 2/5] refactor(opendataset): build the exception system for opendataset module PR Closed: https://github.com/Graviti-AI/tensorbay-python-sdk/pull/385 --- tensorbay/exception.py | 47 ++++++++++++++++++- .../opendataset/LISATrafficLight/loader.py | 5 +- tensorbay/opendataset/_utility/exceptions.py | 22 +++------ tensorbay/opendataset/_utility/glob.py | 6 +-- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/tensorbay/exception.py b/tensorbay/exception.py index 4d40f5cb7..4d39a4dad 100644 --- a/tensorbay/exception.py +++ b/tensorbay/exception.py @@ -3,8 +3,53 @@ # Copyright 2021 Graviti. Licensed under MIT License. # -"""TensorBayException.""" +"""TensorBay cutoms exceptions. + +The class hierarchy for TensorBay custom exceptions is:: + + +-- :class:`TensorBayException` + +-- :class:`TensorBayOpenDatasetError` + +-- :class:`NoFileError` + +-- :class:`FileStructureError` + +""" class TensorBayException(Exception): """This is the base class for TensorBay custom exceptions.""" + + +class TensorBayOpendatasetException(TensorBayException): + """This is the base class for custom exceptions in TensorBay opendataset module.""" + + +class NoFileError(TensorBayOpendatasetException): + """This class defines the exception for no matching file found in the opendataset directory. + + Arguments: + pattern: Glob pattern. + + """ + + def __init__(self, pattern: str) -> None: + super().__init__() + self._pattern = pattern + + def __str__(self) -> str: + return f'No file follows the giving pattern "{self._pattern}"' + + +class FileStructureError(TensorBayOpendatasetException): + """This class defines the exception for incorrect file structure in the opendataset directory. + + Arguments: + message: The error message. + + """ + + def __init__(self, message: str) -> None: + super().__init__() + self._message = message + + def __str__(self) -> str: + return self._message diff --git a/tensorbay/opendataset/LISATrafficLight/loader.py b/tensorbay/opendataset/LISATrafficLight/loader.py index 63006b0fb..b3e922667 100644 --- a/tensorbay/opendataset/LISATrafficLight/loader.py +++ b/tensorbay/opendataset/LISATrafficLight/loader.py @@ -11,6 +11,7 @@ import re from ...dataset import Data, Dataset, Segment +from ...exception import FileStructureError from ...label import Classification, LabeledBox2D from .._utility import glob @@ -64,7 +65,7 @@ def LISATrafficLight(path: str) -> Dataset: Loaded `Dataset` object. Raises: - TypeError: When frame number is discontinuous. + FileStructureError: When frame number is discontinuous. """ root_path = os.path.abspath(os.path.expanduser(path)) @@ -87,7 +88,7 @@ def LISATrafficLight(path: str) -> Dataset: # Check the frame_number from filename: "daySequence1--00345.jpg" if _get_frame_number(image_paths[-1]) + 1 != len(image_paths): - raise TypeError(f"Discontinuous frame number in '{filedir}'") + raise FileStructureError(f"Discontinuous frame number in '{filedir}'") for image_path in image_paths: data = Data(image_path) diff --git a/tensorbay/opendataset/_utility/exceptions.py b/tensorbay/opendataset/_utility/exceptions.py index cbce3bdf8..54c96ef2e 100644 --- a/tensorbay/opendataset/_utility/exceptions.py +++ b/tensorbay/opendataset/_utility/exceptions.py @@ -3,24 +3,14 @@ # Copyright 2021 Graviti. Licensed under MIT License. # -"""Exceptions about open data loader.""" +"""Exceptions about open data loader. +The content in this module is deprecated since v1.3.0, and will be removed in v1.5.0. -class OpenDatasetException(Exception): - """This is the parent class to all open dataset exceptions.""" +Please use :class:`~tensorbay.exception.NoFileError` instead of :class:`OpenDatasetNoFileError`. +""" -class OpenDatasetNoFileError(OpenDatasetException): - """Exception for no file found in the opendataset directory. +from ...exception import NoFileError - Arguments: - pattern: Glob pattern. - - """ - - def __init__(self, pattern: str) -> None: - super().__init__() - self._pattern = pattern - - def __str__(self) -> str: - return f'No file follows the giving pattern "{self._pattern}"' +OpenDatasetNoFileError = NoFileError diff --git a/tensorbay/opendataset/_utility/glob.py b/tensorbay/opendataset/_utility/glob.py index cf254adfb..e9d2292dc 100644 --- a/tensorbay/opendataset/_utility/glob.py +++ b/tensorbay/opendataset/_utility/glob.py @@ -8,7 +8,7 @@ from glob import glob as buildin_glob from typing import List -from .exceptions import OpenDatasetNoFileError +from ...exception import NoFileError def glob(pathname: str, *, recursive: bool = False) -> List[str]: @@ -27,12 +27,12 @@ def glob(pathname: str, *, recursive: bool = False) -> List[str]: A sorted list of paths matching a pathname pattern. Raises: - OpenDatasetNoFileError: When there is no file matching the given pathname pattern. + NoFileError: When there is no file matching the given pathname pattern. """ paths = buildin_glob(pathname, recursive=recursive) if not paths: - raise OpenDatasetNoFileError(pathname) + raise NoFileError(pathname) paths.sort() return paths From 2e2a97e4e66a0120cf29fdf07f160cb768351816 Mon Sep 17 00:00:00 2001 From: "linji.xue" Date: Wed, 14 Apr 2021 21:31:37 +0800 Subject: [PATCH 3/5] feat(client): add thread lock to PagingList to avoid resending requests PR Closed: https://github.com/Graviti-AI/tensorbay-python-sdk/pull/386 --- tensorbay/client/requests.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/tensorbay/client/requests.py b/tensorbay/client/requests.py index 383691a33..0adfd1992 100644 --- a/tensorbay/client/requests.py +++ b/tensorbay/client/requests.py @@ -12,11 +12,14 @@ """ import logging +from collections import defaultdict from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait from itertools import count, repeat, zip_longest +from threading import Lock from typing import ( Any, Callable, + DefaultDict, Dict, Generator, Iterable, @@ -302,6 +305,7 @@ class PagingList(Sequence[_T], ReprMixin): # pylint: disable=too-many-ancestors """ _S = TypeVar("_S", bound="PagingList[_T]") + _K = TypeVar("_K") _repr_type = ReprType.SEQUENCE @@ -313,6 +317,7 @@ def __init__( ) -> None: self._data: Dict[int, _T] = {} self._attr: Dict[str, int] = {} + self._locks: DefaultDict[Union[int, str], Lock] = defaultdict(Lock) self._func = func self._limit = limit @@ -343,12 +348,12 @@ def __getitem__(self: _S, index: Union[int, slice]) -> Union[_T, _S]: paging_index = self._slice.start + index if paging_index not in self._data: - self._extend(paging_index) + offset = self._get_offset(paging_index) + self._call_with_lock(offset, self._extend, offset) return self._data[paging_index] - def _extend(self, index: int) -> int: - offset = index // self._limit * self._limit + def _extend(self, offset: int) -> int: generator = self._func(offset, self._limit) try: for i in count(offset): @@ -359,10 +364,14 @@ def _extend(self, index: int) -> int: raise TypeError("Impossible to be here, add this to make pylint and mypy happy") + def _get_total_count(self, index: int) -> None: + offset = self._get_offset(index) + self._attr["totalCount"] = self._extend(offset) + def _get_len(self, index: int = 0) -> int: if self._len is None: if "totalCount" not in self._attr: - self._attr["totalCount"] = self._extend(index) + self._call_with_lock("totalCount", self._get_total_count, index) total_count = self._attr["totalCount"] stop = total_count if self._slice.stop is None else min(total_count, self._slice.stop) @@ -374,6 +383,21 @@ def _get_len(self, index: int = 0) -> int: def _make_index_positive(self, index: int) -> int: return index if index >= 0 else self._get_len() + index + def _get_offset(self, index: int) -> int: + return index // self._limit * self._limit + + def _call_with_lock(self, key: Union[int, str], func: Callable[[_K], Any], arg: _K) -> None: + lock = self._locks[key] + acquire = lock.acquire(blocking=False) + try: + if acquire: + func(arg) + del self._locks[key] + else: + lock.acquire() + finally: + lock.release() + def _get_slice(self: _S, input_slice: slice) -> _S: start = self._slice.start if input_slice.start is not None: @@ -385,9 +409,10 @@ def _get_slice(self: _S, input_slice: slice) -> _S: paging_list = self.__class__(self._func, self._limit, slice(start, stop)) - # The sliced PagingList shares the "_data" and "_attr" with the root PagingList + # The sliced PagingList shares the "_data", "_attr" and "_locks" with the root PagingList paging_list._data = self._data # pylint: disable=protected-access paging_list._attr = self._attr # pylint: disable=protected-access + paging_list._locks = self._locks # pylint: disable=protected-access return paging_list From 735d687e033d1758ac2e1ef9d2d8d796cd695205 Mon Sep 17 00:00:00 2001 From: "luyao.wang" Date: Mon, 12 Apr 2021 19:37:58 +0800 Subject: [PATCH 4/5] docs: hide all rst files that are not included in toctrees PR Closed: https://github.com/Graviti-AI/tensorbay-python-sdk/pull/378 --- docs/source/api/exception.rst | 3 +++ docs/source/api/healthcheck/healthcheck_module.rst | 3 +++ docs/source/community/contribution.rst | 3 +++ docs/source/community/roadmap.rst | 3 +++ docs/source/reference/release_note.rst | 3 +++ 5 files changed, 15 insertions(+) diff --git a/docs/source/api/exception.rst b/docs/source/api/exception.rst index f4267721a..8af027278 100644 --- a/docs/source/api/exception.rst +++ b/docs/source/api/exception.rst @@ -1,3 +1,6 @@ +:orphan: +:nosearch: + ############ Exceptions ############ diff --git a/docs/source/api/healthcheck/healthcheck_module.rst b/docs/source/api/healthcheck/healthcheck_module.rst index f5a47a733..7f697a335 100644 --- a/docs/source/api/healthcheck/healthcheck_module.rst +++ b/docs/source/api/healthcheck/healthcheck_module.rst @@ -1,3 +1,6 @@ +:orphan: +:nosearch: + tensorbay.healthcheck ===================== diff --git a/docs/source/community/contribution.rst b/docs/source/community/contribution.rst index f3215cc9a..68f299446 100644 --- a/docs/source/community/contribution.rst +++ b/docs/source/community/contribution.rst @@ -1,3 +1,6 @@ +:orphan: +:nosearch: + ############## Contribution ############## diff --git a/docs/source/community/roadmap.rst b/docs/source/community/roadmap.rst index 869570ea8..d5810ea0b 100644 --- a/docs/source/community/roadmap.rst +++ b/docs/source/community/roadmap.rst @@ -1,3 +1,6 @@ +:orphan: +:nosearch: + ######### Roadmap ######### diff --git a/docs/source/reference/release_note.rst b/docs/source/reference/release_note.rst index 07dcf8160..2500d194f 100644 --- a/docs/source/reference/release_note.rst +++ b/docs/source/reference/release_note.rst @@ -1,3 +1,6 @@ +:orphan: +:nosearch: + ############## Release Note ############## From 76fe65fbd2004cb493133b3dad566524005a6a35 Mon Sep 17 00:00:00 2001 From: "luyao.wang" Date: Mon, 12 Apr 2021 15:22:26 +0800 Subject: [PATCH 5/5] docs: disable properties, add them into the attributes list PR Closed: https://github.com/Graviti-AI/tensorbay-python-sdk/pull/373 --- docs/source/api/client/dataset.rst | 2 +- docs/source/api/client/requests.rst | 1 + docs/source/api/client/segment.rst | 2 +- docs/source/api/dataset/data.rst | 1 + docs/source/api/dataset/dataset.rst | 2 +- docs/source/api/label/supports.rst | 1 + docs/source/api/sensor/intrinsics.rst | 1 + docs/source/api/utility/name.rst | 1 + docs/source/api/utility/tbrn.rst | 1 + tensorbay/client/dataset.py | 5 +++++ tensorbay/client/requests.py | 7 ++++++- tensorbay/client/segment.py | 4 ++++ tensorbay/dataset/data.py | 1 + tensorbay/dataset/dataset.py | 4 ++++ tensorbay/label/supports.py | 1 + tensorbay/sensor/intrinsics.py | 6 +++--- tensorbay/utility/name.py | 3 +++ tensorbay/utility/tbrn.py | 8 ++++++++ 18 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/source/api/client/dataset.rst b/docs/source/api/client/dataset.rst index d200cf93e..e8fb363b6 100644 --- a/docs/source/api/client/dataset.rst +++ b/docs/source/api/client/dataset.rst @@ -4,4 +4,4 @@ tensorbay.client.dataset .. automodule:: tensorbay.client.dataset :members: :show-inheritance: - :exclude-members: + :exclude-members: name, dataset_id, status diff --git a/docs/source/api/client/requests.rst b/docs/source/api/client/requests.rst index de524ec3b..32bdc5924 100644 --- a/docs/source/api/client/requests.rst +++ b/docs/source/api/client/requests.rst @@ -4,3 +4,4 @@ tensorbay.client.requests .. automodule:: tensorbay.client.requests :members: :show-inheritance: + :exclude-members: is_intern diff --git a/docs/source/api/client/segment.rst b/docs/source/api/client/segment.rst index aaef0c55a..436195d42 100644 --- a/docs/source/api/client/segment.rst +++ b/docs/source/api/client/segment.rst @@ -4,4 +4,4 @@ tensorbay.client.segment .. automodule:: tensorbay.client.segment :members: :show-inheritance: - :exclude-members: + :exclude-members: name, status diff --git a/docs/source/api/dataset/data.rst b/docs/source/api/dataset/data.rst index 5922df1ee..d35ce88e0 100644 --- a/docs/source/api/dataset/data.rst +++ b/docs/source/api/dataset/data.rst @@ -4,3 +4,4 @@ tensorbay.dataset.data .. automodule:: tensorbay.dataset.data :members: :show-inheritance: + :exclude-members: target_remote_path diff --git a/docs/source/api/dataset/dataset.rst b/docs/source/api/dataset/dataset.rst index 392a22084..09166f47d 100644 --- a/docs/source/api/dataset/dataset.rst +++ b/docs/source/api/dataset/dataset.rst @@ -4,5 +4,5 @@ tensorbay.dataset.dataset .. automodule:: tensorbay.dataset.dataset :members: :show-inheritance: - :exclude-members: + :exclude-members: catalog, notes \ No newline at end of file diff --git a/docs/source/api/label/supports.rst b/docs/source/api/label/supports.rst index c7eb3a934..aa02209de 100644 --- a/docs/source/api/label/supports.rst +++ b/docs/source/api/label/supports.rst @@ -4,3 +4,4 @@ tensorbay.label.supports .. automodule:: tensorbay.label.supports :members: :show-inheritance: + :exclude-members: number diff --git a/docs/source/api/sensor/intrinsics.rst b/docs/source/api/sensor/intrinsics.rst index 7f0dedfcb..e799a0366 100644 --- a/docs/source/api/sensor/intrinsics.rst +++ b/docs/source/api/sensor/intrinsics.rst @@ -4,3 +4,4 @@ tensorbay.sensor.intrinsics .. automodule:: tensorbay.sensor.intrinsics :members: :show-inheritance: + :exclude-members: camera_matrix, distortion_coefficients diff --git a/docs/source/api/utility/name.rst b/docs/source/api/utility/name.rst index c4b16c5ac..96945aaaa 100644 --- a/docs/source/api/utility/name.rst +++ b/docs/source/api/utility/name.rst @@ -4,3 +4,4 @@ tensorbay.utility.name .. automodule:: tensorbay.utility.name :members: :show-inheritance: + :exclude-members: name diff --git a/docs/source/api/utility/tbrn.rst b/docs/source/api/utility/tbrn.rst index 3edfdbc14..a792677bd 100644 --- a/docs/source/api/utility/tbrn.rst +++ b/docs/source/api/utility/tbrn.rst @@ -4,3 +4,4 @@ tensorbay.utility.tbrn .. automodule:: tensorbay.utility.tbrn :members: :show-inheritance: + :exclude-members: dataset_name, segment_name, frame_index, sensor_name, remote_path, type diff --git a/tensorbay/client/dataset.py b/tensorbay/client/dataset.py index 1aa0a04f7..44d8b08ce 100644 --- a/tensorbay/client/dataset.py +++ b/tensorbay/client/dataset.py @@ -51,6 +51,11 @@ class DatasetClientBase: # pylint: disable=too-many-public-methods dataset_id: Dataset ID. gas_client: The initial client to interact between local and TensorBay. + Attributes: + name: Dataset name. + dataset_id: Dataset ID. + status: The status of the dataset client. + """ _client: Client diff --git a/tensorbay/client/requests.py b/tensorbay/client/requests.py index 0adfd1992..09e1d4cf9 100644 --- a/tensorbay/client/requests.py +++ b/tensorbay/client/requests.py @@ -48,7 +48,12 @@ class Config: # pylint: disable=too-few-public-methods - """This is a base class defining the concept of Request Config.""" + """This is a base class defining the concept of Request Config. + + Attributes: + is_intern: Whether the request is from intern. + + """ def __init__(self) -> None: diff --git a/tensorbay/client/segment.py b/tensorbay/client/segment.py index 65c0af6f2..cf13e9aaf 100644 --- a/tensorbay/client/segment.py +++ b/tensorbay/client/segment.py @@ -61,6 +61,10 @@ class SegmentClientBase: # pylint: disable=too-many-instance-attributes name: Segment name. dataset_client: The dataset client. + Attributes: + name: Segment name. + status: The status of the dataset client. + """ _EXPIRED_IN_SECOND = 240 diff --git a/tensorbay/dataset/data.py b/tensorbay/dataset/data.py index 4970d463e..8ffe26226 100644 --- a/tensorbay/dataset/data.py +++ b/tensorbay/dataset/data.py @@ -117,6 +117,7 @@ class Data(DataBase): path: The file local path. timestamp: The timestamp for the file. labels: The :class:`Labels` that contains all the label information of the file. + target_remote_path: The target remote path of the data. """ diff --git a/tensorbay/dataset/dataset.py b/tensorbay/dataset/dataset.py index 68ae558b3..f02ddc7c7 100644 --- a/tensorbay/dataset/dataset.py +++ b/tensorbay/dataset/dataset.py @@ -93,6 +93,10 @@ class DatasetBase(NameMixin, Sequence[_T]): # pylint: disable=too-many-ancestor Arguments: name: The name of the dataset. + Attributes: + catalog: The :class:`~tensorbay.label.catalog.Catalog` of the dataset. + notes: The :class:`Notes` of the dataset. + """ _repr_type = ReprType.SEQUENCE diff --git a/tensorbay/label/supports.py b/tensorbay/label/supports.py index 98d482262..81e9b7d6d 100644 --- a/tensorbay/label/supports.py +++ b/tensorbay/label/supports.py @@ -105,6 +105,7 @@ class KeypointsInfo(ReprMixin, EqMixin): description: The description of the keypoints. Attributes: + number: The number of the set of keypoints. names: All the names of the keypoints. skeleton: The skeleton of the keypoints indicating which keypoint should connect with another. diff --git a/tensorbay/sensor/intrinsics.py b/tensorbay/sensor/intrinsics.py index ba94e2abb..7b141b1d5 100644 --- a/tensorbay/sensor/intrinsics.py +++ b/tensorbay/sensor/intrinsics.py @@ -493,9 +493,9 @@ class CameraIntrinsics(ReprMixin): **kwargs: Float values to initialize :class:`DistortionCoefficients`. Attributes: - _camera_matrix: A 3x3 Sequence of the camera matrix. - _distortion_coefficients: It is the deviation from rectilinear projection. It includes - radial distortion and tangential distortion. + camera_matrix: A 3x3 Sequence of the camera matrix. + distortion_coefficients: It is the deviation from rectilinear projection. It includes + radial distortion and tangential distortion. Examples: >>> matrix = [[1, 3, 3], diff --git a/tensorbay/utility/name.py b/tensorbay/utility/name.py index 1a737a4df..67da0a304 100644 --- a/tensorbay/utility/name.py +++ b/tensorbay/utility/name.py @@ -35,6 +35,9 @@ class NameMixin(ReprMixin, EqMixin): name: Name of the class. description: Description of the class. + Attributes: + name: Name of the class. + """ _P = TypeVar("_P", bound="NameMixin") diff --git a/tensorbay/utility/tbrn.py b/tensorbay/utility/tbrn.py index 29e2afba2..004813970 100644 --- a/tensorbay/utility/tbrn.py +++ b/tensorbay/utility/tbrn.py @@ -142,6 +142,14 @@ class TBRN: remote_path: Object path of the file. tbrn: Full TBRN string. + Attributes: + dataset_name: Name of the dataset. + segment_name: Name of the segment. + frame_index: Index of the frame. + sensor_name: Name of the sensor. + remote_path: Object path of the file. + type: The type of this TBRN. + Raises: TypeError: The TBRN is invalid.