Merge branch 'main' into T14418_multiprocess_safe

docs/source/api/client/dataset.rst

@@ -4,4 +4,4 @@ tensorbay.client.dataset
 .. automodule:: tensorbay.client.dataset
    :members:
    :show-inheritance:
-   :exclude-members:
+   :exclude-members: name, dataset_id, status

docs/source/api/client/requests.rst

@@ -4,3 +4,4 @@ tensorbay.client.requests
 .. automodule:: tensorbay.client.requests
    :members:
    :show-inheritance:
+   :exclude-members: is_intern

docs/source/api/client/segment.rst

@@ -4,4 +4,4 @@ tensorbay.client.segment
 .. automodule:: tensorbay.client.segment
    :members:
    :show-inheritance:
-   :exclude-members:
+   :exclude-members: name, status

docs/source/api/dataset/data.rst

@@ -4,3 +4,4 @@ tensorbay.dataset.data
 .. automodule:: tensorbay.dataset.data
    :members:
    :show-inheritance:
+   :exclude-members: target_remote_path

docs/source/api/dataset/dataset.rst

@@ -4,5 +4,5 @@ tensorbay.dataset.dataset
 .. automodule:: tensorbay.dataset.dataset
    :members:
    :show-inheritance:
-   :exclude-members:
+   :exclude-members: catalog, notes
 

docs/source/api/exception.rst

@@ -1,3 +1,6 @@
+:orphan:
+:nosearch:
+
 ############
  Exceptions
 ############

docs/source/api/healthcheck/healthcheck_module.rst

@@ -1,3 +1,6 @@
+:orphan:
+:nosearch:
+
 tensorbay.healthcheck
 =====================
 

docs/source/api/label/supports.rst

@@ -4,3 +4,4 @@ tensorbay.label.supports
 .. automodule:: tensorbay.label.supports
    :members:
    :show-inheritance:
+   :exclude-members: number

docs/source/api/sensor/intrinsics.rst

@@ -4,3 +4,4 @@ tensorbay.sensor.intrinsics
 .. automodule:: tensorbay.sensor.intrinsics
    :members:
    :show-inheritance:
+   :exclude-members: camera_matrix, distortion_coefficients

docs/source/api/utility/name.rst

@@ -4,3 +4,4 @@ tensorbay.utility.name
 .. automodule:: tensorbay.utility.name
    :members:
    :show-inheritance:
+   :exclude-members: name

docs/source/api/utility/tbrn.rst

@@ -4,3 +4,4 @@ tensorbay.utility.tbrn
 .. automodule:: tensorbay.utility.tbrn
    :members:
    :show-inheritance:
+   :exclude-members: dataset_name, segment_name, frame_index, sensor_name, remote_path, type

docs/source/community/contribution.rst

@@ -1,3 +1,6 @@
+:orphan:
+:nosearch:
+
 ##############
  Contribution
 ##############

docs/source/community/roadmap.rst

@@ -1,3 +1,6 @@
+:orphan:
+:nosearch:
+
 #########
  Roadmap
 #########

docs/source/reference/release_note.rst

@@ -1,3 +1,6 @@
+:orphan:
+:nosearch:
+
 ##############
  Release Note
 ##############

tensorbay/client/dataset.py

@@ -51,6 +51,11 @@ class DatasetClientBase:  # pylint: disable=too-many-public-methods
         dataset_id: Dataset ID.
         gas_client: The initial client to interact between local and TensorBay.
 
+    Attributes:
+        name: Dataset name.
+        dataset_id: Dataset ID.
+        status: The status of the dataset client.
+
     """
 
     _client: Client

tensorbay/client/log.py

@@ -165,10 +165,7 @@ def _dump_response(response: Response) -> str:
     if "Content-Type" in response.headers:
         content_type = response.headers["Content-Type"]
         if content_type.startswith("application/json"):
-            try:
-                content = json.dumps(response.json(), indent=2)
-            except json.decoder.JSONDecodeError:
-                content = ""
+            content = json.dumps(response.json(), indent=2)
         elif content_type.startswith("text"):
             content = response.text
         elif len(response.content) > 512:

tensorbay/client/requests.py

@@ -16,6 +16,7 @@
 from collections import defaultdict
 from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait
 from itertools import count, repeat, zip_longest
+from threading import Lock
 from typing import (
     Any,
     Callable,
@@ -48,7 +49,12 @@
 
 
 class Config:  # pylint: disable=too-few-public-methods
-    """This is a base class defining the concept of Request Config."""
+    """This is a base class defining the concept of Request Config.
+
+    Attributes:
+         is_intern: Whether the request is from intern.
+
+    """
 
     def __init__(self) -> None:
 
@@ -314,6 +320,7 @@ class PagingList(Sequence[_T], ReprMixin):  # pylint: disable=too-many-ancestors
     """
 
     _S = TypeVar("_S", bound="PagingList[_T]")
+    _K = TypeVar("_K")
 
     _repr_type = ReprType.SEQUENCE
 
@@ -325,6 +332,7 @@ def __init__(
     ) -> None:
         self._data: Dict[int, _T] = {}
         self._attr: Dict[str, int] = {}
+        self._locks: DefaultDict[Union[int, str], Lock] = defaultdict(Lock)
         self._func = func
         self._limit = limit
 
@@ -355,12 +363,12 @@ def __getitem__(self: _S, index: Union[int, slice]) -> Union[_T, _S]:
 
         paging_index = self._slice.start + index
         if paging_index not in self._data:
-            self._extend(paging_index)
+            offset = self._get_offset(paging_index)
+            self._call_with_lock(offset, self._extend, offset)
 
         return self._data[paging_index]
 
-    def _extend(self, index: int) -> int:
-        offset = index // self._limit * self._limit
+    def _extend(self, offset: int) -> int:
         generator = self._func(offset, self._limit)
         try:
             for i in count(offset):
@@ -371,10 +379,14 @@ def _extend(self, index: int) -> int:
 
         raise TypeError("Impossible to be here, add this to make pylint and mypy happy")
 
+    def _get_total_count(self, index: int) -> None:
+        offset = self._get_offset(index)
+        self._attr["totalCount"] = self._extend(offset)
+
     def _get_len(self, index: int = 0) -> int:
         if self._len is None:
             if "totalCount" not in self._attr:
-                self._attr["totalCount"] = self._extend(index)
+                self._call_with_lock("totalCount", self._get_total_count, index)
 
             total_count = self._attr["totalCount"]
             stop = total_count if self._slice.stop is None else min(total_count, self._slice.stop)
@@ -386,6 +398,21 @@ def _get_len(self, index: int = 0) -> int:
     def _make_index_positive(self, index: int) -> int:
         return index if index >= 0 else self._get_len() + index
 
+    def _get_offset(self, index: int) -> int:
+        return index // self._limit * self._limit
+
+    def _call_with_lock(self, key: Union[int, str], func: Callable[[_K], Any], arg: _K) -> None:
+        lock = self._locks[key]
+        acquire = lock.acquire(blocking=False)
+        try:
+            if acquire:
+                func(arg)
+                del self._locks[key]
+            else:
+                lock.acquire()
+        finally:
+            lock.release()
+
     def _get_slice(self: _S, input_slice: slice) -> _S:
         start = self._slice.start
         if input_slice.start is not None:
@@ -397,9 +424,10 @@ def _get_slice(self: _S, input_slice: slice) -> _S:
 
         paging_list = self.__class__(self._func, self._limit, slice(start, stop))
 
-        # The sliced PagingList shares the "_data" and "_attr" with the root PagingList
+        # The sliced PagingList shares the "_data", "_attr" and "_locks" with the root PagingList
         paging_list._data = self._data  # pylint: disable=protected-access
         paging_list._attr = self._attr  # pylint: disable=protected-access
+        paging_list._locks = self._locks  # pylint: disable=protected-access
 
         return paging_list
 

tensorbay/client/segment.py

@@ -61,6 +61,10 @@ class SegmentClientBase:  # pylint: disable=too-many-instance-attributes
         name: Segment name.
         dataset_client: The dataset client.
 
+    Attributes:
+        name: Segment name.
+        status: The status of the dataset client.
+
     """
 
     _EXPIRED_IN_SECOND = 240

tensorbay/dataset/data.py

@@ -117,6 +117,7 @@ class Data(DataBase):
         path: The file local path.
         timestamp: The timestamp for the file.
         labels: The :class:`Labels` that contains all the label information of the file.
+        target_remote_path: The target remote path of the data.
 
     """
 

tensorbay/dataset/dataset.py

@@ -93,6 +93,10 @@ class DatasetBase(NameMixin, Sequence[_T]):  # pylint: disable=too-many-ancestor
     Arguments:
         name: The name of the dataset.
 
+    Attributes:
+        catalog: The :class:`~tensorbay.label.catalog.Catalog` of the dataset.
+        notes: The :class:`Notes` of the dataset.
+
     """
 
     _repr_type = ReprType.SEQUENCE

tensorbay/exception.py

@@ -3,8 +3,53 @@
 # Copyright 2021 Graviti. Licensed under MIT License.
 #
 
-"""TensorBayException."""
+"""TensorBay cutoms exceptions.
+
+The class hierarchy for TensorBay custom exceptions is::
+
+     +-- :class:`TensorBayException`
+         +-- :class:`TensorBayOpenDatasetError`
+             +-- :class:`NoFileError`
+             +-- :class:`FileStructureError`
+
+"""
 
 
 class TensorBayException(Exception):
     """This is the base class for TensorBay custom exceptions."""
+
+
+class TensorBayOpendatasetException(TensorBayException):
+    """This is the base class for custom exceptions in TensorBay opendataset module."""
+
+
+class NoFileError(TensorBayOpendatasetException):
+    """This class defines the exception for no matching file found in the opendataset directory.
+
+    Arguments:
+        pattern: Glob pattern.
+
+    """
+
+    def __init__(self, pattern: str) -> None:
+        super().__init__()
+        self._pattern = pattern
+
+    def __str__(self) -> str:
+        return f'No file follows the giving pattern "{self._pattern}"'
+
+
+class FileStructureError(TensorBayOpendatasetException):
+    """This class defines the exception for incorrect file structure in the opendataset directory.
+
+    Arguments:
+        message: The error message.
+
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__()
+        self._message = message
+
+    def __str__(self) -> str:
+        return self._message

tensorbay/label/supports.py

@@ -105,6 +105,7 @@ class KeypointsInfo(ReprMixin, EqMixin):
         description: The description of the keypoints.
 
     Attributes:
+        number: The number of the set of keypoints.
         names: All the names of the keypoints.
         skeleton: The skeleton of the keypoints
             indicating which keypoint should connect with another.

tensorbay/opendataset/LISATrafficLight/loader.py

@@ -11,6 +11,7 @@
 import re
 
 from ...dataset import Data, Dataset, Segment
+from ...exception import FileStructureError
 from ...label import Classification, LabeledBox2D
 from .._utility import glob
 
@@ -64,7 +65,7 @@ def LISATrafficLight(path: str) -> Dataset:
         Loaded `Dataset` object.
 
     Raises:
-        TypeError: When frame number is discontinuous.
+        FileStructureError: When frame number is discontinuous.
 
     """
     root_path = os.path.abspath(os.path.expanduser(path))
@@ -87,7 +88,7 @@ def LISATrafficLight(path: str) -> Dataset:
 
         # Check the frame_number from filename: "daySequence1--00345.jpg"
         if _get_frame_number(image_paths[-1]) + 1 != len(image_paths):
-            raise TypeError(f"Discontinuous frame number in '{filedir}'")
+            raise FileStructureError(f"Discontinuous frame number in '{filedir}'")
 
         for image_path in image_paths:
             data = Data(image_path)

tensorbay/opendataset/_utility/exceptions.py

@@ -3,24 +3,14 @@
 # Copyright 2021 Graviti. Licensed under MIT License.
 #
 
-"""Exceptions about open data loader."""
+"""Exceptions about open data loader.
 
+The content in this module is deprecated since v1.3.0, and will be removed in v1.5.0.
 
-class OpenDatasetException(Exception):
-    """This is the parent class to all open dataset exceptions."""
+Please use :class:`~tensorbay.exception.NoFileError` instead of :class:`OpenDatasetNoFileError`.
 
+"""
 
-class OpenDatasetNoFileError(OpenDatasetException):
-    """Exception for no file found in the opendataset directory.
+from ...exception import NoFileError
 
-    Arguments:
-        pattern: Glob pattern.
-
-    """
-
-    def __init__(self, pattern: str) -> None:
-        super().__init__()
-        self._pattern = pattern
-
-    def __str__(self) -> str:
-        return f'No file follows the giving pattern "{self._pattern}"'
+OpenDatasetNoFileError = NoFileError

tensorbay/opendataset/_utility/glob.py

@@ -8,7 +8,7 @@
 from glob import glob as buildin_glob
 from typing import List
 
-from .exceptions import OpenDatasetNoFileError
+from ...exception import NoFileError
 
 
 def glob(pathname: str, *, recursive: bool = False) -> List[str]:
@@ -27,12 +27,12 @@ def glob(pathname: str, *, recursive: bool = False) -> List[str]:
         A sorted list of paths matching a pathname pattern.
 
     Raises:
-        OpenDatasetNoFileError: When there is no file matching the given pathname pattern.
+        NoFileError: When there is no file matching the given pathname pattern.
 
     """
     paths = buildin_glob(pathname, recursive=recursive)
     if not paths:
-        raise OpenDatasetNoFileError(pathname)
+        raise NoFileError(pathname)
     paths.sort()
 
     return paths

tensorbay/sensor/intrinsics.py

@@ -493,9 +493,9 @@ class CameraIntrinsics(ReprMixin):
         **kwargs: Float values to initialize :class:`DistortionCoefficients`.
 
     Attributes:
-        _camera_matrix: A 3x3 Sequence of the camera matrix.
-        _distortion_coefficients: It is the deviation from rectilinear projection. It includes
-            radial distortion and tangential distortion.
+        camera_matrix: A 3x3 Sequence of the camera matrix.
+        distortion_coefficients: It is the deviation from rectilinear projection. It includes
+        radial distortion and tangential distortion.
 
     Examples:
         >>> matrix = [[1, 3, 3],