Skip to content

Commit

Permalink
Convert dummy datasets to toy datasets (#1988)
Browse files Browse the repository at this point in the history
* Update cls, det datsets

* Remove useless files

* Change action datasets

* Edit action dataset

* change dir

* Add xml files

* Remove useless

* Edite tets

* Fix tests

* Fix tests

* Remove ptc

* Remove

* Fix precommit

* Update dataset, fix cls bug

* Remove useless dataset

* Edit drop_last

* Fix missed part

* Change threshold values to unifying

* bugfix: squeezing to 1 dimenetion

* Change threshold for deployment

* Fix multi-gpu issue, e2e tests

* Decrease num_workers for tiling test and tiling processes

* Revert num_workers for tests

* Fix datsets

---------

Co-authored-by: eunwoosh <eunwoo.shin@intel.com>
  • Loading branch information
sungmanc and eunwoosh authored Apr 18, 2023
1 parent 861157a commit ebb2d77
Show file tree
Hide file tree
Showing 217 changed files with 289 additions and 269 deletions.
2 changes: 1 addition & 1 deletion otx/algorithms/classification/adapters/mmcls/configurer.py
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,7 @@ def configure_samples_per_gpu(
# drop the last batch if the last batch size is 1
# batch size of 1 is a runtime error for training batch normalization layer
if subset in ("train", "unlabeled") and dataset_len % samples_per_gpu == 1:
dataloader_cfg.drop_last = True
dataloader_cfg["drop_last"] = True

cfg.data[f"{subset}_dataloader"] = dataloader_cfg

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def forward_train(self, imgs, **kwargs):
raise ValueError("'gt_label' does not exist in the labeled image")
if "extra_0" not in kwargs:
raise ValueError("'extra_0' does not exist in the dataset")
target = kwargs["gt_label"].squeeze()
target = kwargs["gt_label"].squeeze(dim=1)
unlabeled_data = kwargs["extra_0"]
x = {}
x["labeled"] = self.extract_feat(imgs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def update_pipeline(cfg):

# For stable hierarchical information indexing
if subset == "train" and kwargs["type"] == "OTXHierarchicalClsDataset":
config.data[f"{subset}_dataloader"].drop_last = True
config.data[f"{subset}_dataloader"]["drop_last"] = True

cfgs = get_dataset_configs(config, subset)
for cfg in cfgs:
Expand Down
2 changes: 1 addition & 1 deletion otx/algorithms/detection/adapters/mmdet/configurer.py
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,7 @@ def configure_samples_per_gpu(
# drop the last batch if the last batch size is 1
# batch size of 1 is a runtime error for training batch normalization layer
if subset in ("train", "unlabeled") and dataset_len % samples_per_gpu == 1:
dataloader_cfg.drop_last = True
dataloader_cfg["drop_last"] = True

cfg.data[f"{subset}_dataloader"] = dataloader_cfg

Expand Down
2 changes: 1 addition & 1 deletion otx/algorithms/detection/adapters/mmdet/datasets/tiling.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def __init__(
iou_threshold: float = 0.45,
max_per_img: int = 1500,
filter_empty_gt: bool = True,
nproc: int = 4,
nproc: int = 2,
):
self.min_area_ratio = min_area_ratio
self.filter_empty_gt = filter_empty_gt
Expand Down
2 changes: 1 addition & 1 deletion otx/algorithms/segmentation/adapters/mmseg/configurer.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@ def configure_samples_per_gpu(self, cfg: Config, subset: str) -> None:
# drop the last batch if the last batch size is 1
# batch size of 1 is a runtime error for training batch normalization layer
if subset in ("train", "unlabeled") and dataset_len % samples_per_gpu == 1:
dataloader_cfg.drop_last = True
dataloader_cfg["drop_last"] = True

cfg.data[f"{subset}_dataloader"] = dataloader_cfg

Expand Down
2 changes: 1 addition & 1 deletion otx/cli/utils/multi_gpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ def setup_multi_gpu_train(
def finalize(self):
"""Join all child processes."""
for p in self._processes:
if p.join(10) is None and p.exitcode is None:
if p.join(30) is None and p.exitcode is None:
p.kill()

@staticmethod
Expand Down
Binary file added tests/assets/classification_dataset/0/11.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/14.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/16.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/17.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/18.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/26.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/3.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/30.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/4.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/5.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/6.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/0/8.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/0.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/1.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/10.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/12.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/15.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/19.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/2.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/21.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/27.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/28.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added tests/assets/classification_dataset/1/29.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -23,39 +23,39 @@
</labels>
<segments></segments>
<original_size>
<width>256</width>
<height>192</height>
<width>24</width>
<height>24</height>
</original_size>
</task>
</meta>
<image id="0" name="00020.jpg" width="256" height="192">
<image id="0" name="00020.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="1" name="00021.jpg" width="256" height="192">
<image id="1" name="00021.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="2" name="00022.jpg" width="256" height="192">
<image id="2" name="00022.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="3" name="00023.jpg" width="256" height="192">
<image id="3" name="00023.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="4" name="00024.jpg" width="256" height="192">
<image id="4" name="00024.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="5" name="00025.jpg" width="256" height="192">
<image id="5" name="00025.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="6" name="00026.jpg" width="256" height="192">
<image id="6" name="00026.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="7" name="00027.jpg" width="256" height="192">
<image id="7" name="00027.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="8" name="00028.jpg" width="256" height="192">
<image id="8" name="00028.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
<image id="9" name="00029.jpg" width="256" height="192">
<image id="9" name="00029.jpg" width="24" height="24">
<tag label="0" source="manual"></tag>
</image>
</annotations>
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
<version>1.1</version>
<meta>
<task>
<id>2</id>
<name>v2</name>
<id>1</id>
<name>v1</name>
<size>10</size>
<mode>interpolation</mode>
<overlap>2</overlap>
Expand All @@ -23,39 +23,39 @@
</labels>
<segments></segments>
<original_size>
<width>256</width>
<height>192</height>
<width>24</width>
<height>24</height>
</original_size>
</task>
</meta>
<image id="0" name="00020.jpg" width="256" height="192">
<image id="0" name="00020.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="1" name="00021.jpg" width="256" height="192">
<image id="1" name="00021.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="2" name="00022.jpg" width="256" height="192">
<image id="2" name="00022.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="3" name="00023.jpg" width="256" height="192">
<image id="3" name="00023.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="4" name="00024.jpg" width="256" height="192">
<image id="4" name="00024.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="5" name="00025.jpg" width="256" height="192">
<image id="5" name="00025.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="6" name="00026.jpg" width="256" height="192">
<image id="6" name="00026.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="7" name="00027.jpg" width="256" height="192">
<image id="7" name="00027.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="8" name="00028.jpg" width="256" height="192">
<image id="8" name="00028.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
<image id="9" name="00029.jpg" width="256" height="192">
<image id="9" name="00029.jpg" width="24" height="24">
<tag label="1" source="manual"></tag>
</image>
</annotations>

This file was deleted.

Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Binary file modified tests/assets/cvat_dataset/action_detection/train.pkl
100755 → 100644
Binary file not shown.
46 changes: 23 additions & 23 deletions tests/assets/cvat_dataset/action_detection/train/0/annotations.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,45 +21,45 @@
<attributes></attributes>
</label>
<label>
<name>1</name>
<name>Right</name>
<attributes></attributes>
</label>
</labels>
<segments></segments>
<original_size>
<width>256</width>
<height>192</height>
<width>24</width>
<height>24</height>
</original_size>
</task>
</meta>
<track id="0" label="EmptyFrame">
<box frame="20" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="00" outside="0" occluded="0" xtl="0.08" ytl="0.25" xbr="0.50" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="21" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="01" outside="0" occluded="0" xtl="0.12" ytl="0.25" xbr="0.54" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="22" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="02" outside="0" occluded="0" xtl="0.16" ytl="0.25" xbr="0.58" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="23" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="03" outside="0" occluded="0" xtl="0.20" ytl="0.25" xbr="0.62" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="1">
<box frame="24" outside="0" occluded="0" xtl="0.27298716" ytl="0.0073581696" xbr="0.67030156" ybr="0.9858064" z_order="0" label="1"></box>
<track id="0" label="Right">
<box frame="04" outside="0" occluded="0" xtl="0.25" ytl="0.25" xbr="0.67" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="25" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="05" outside="0" occluded="0" xtl="0.29" ytl="0.25" xbr="0.71" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="26" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="06" outside="0" occluded="0" xtl="0.33" ytl="0.25" xbr="0.75" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="27" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="07" outside="0" occluded="0" xtl="0.37" ytl="0.25" xbr="0.79" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="28" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="08" outside="0" occluded="0" xtl="0.41" ytl="0.25" xbr="0.83" ybr="0.75" z_order="0" label="Right"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="29" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Right">
<box frame="09" outside="0" occluded="0" xtl="0.45" ytl="0.25" xbr="0.87" ybr="0.75" z_order="0" label="Right"></box>
</track>
</annotations>
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
46 changes: 23 additions & 23 deletions tests/assets/cvat_dataset/action_detection/train/1/annotations.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,45 +21,45 @@
<attributes></attributes>
</label>
<label>
<name>2</name>
<name>Left</name>
<attributes></attributes>
</label>
</labels>
<segments></segments>
<original_size>
<width>256</width>
<height>192</height>
<width>24</width>
<height>24</height>
</original_size>
</task>
</meta>
<track id="0" label="EmptyFrame">
<box frame="20" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="00" outside="0" occluded="0" xtl="0.45" ytl="0.25" xbr="0.87" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="21" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="01" outside="0" occluded="0" xtl="0.41" ytl="0.25" xbr="0.83" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="22" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="02" outside="0" occluded="0" xtl="0.37" ytl="0.25" xbr="0.79" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="2">
<box frame="23" outside="0" occluded="0" xtl="0.22982445" ytl="0.0969286" xbr="0.72086316" ybr="0.9888343" z_order="0" label="2"></box>
<track id="0" label="Left">
<box frame="03" outside="0" occluded="0" xtl="0.33" ytl="0.25" xbr="0.75" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="24" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="04" outside="0" occluded="0" xtl="0.29" ytl="0.25" xbr="0.71" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="25" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="05" outside="0" occluded="0" xtl="0.25" ytl="0.25" xbr="0.67" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="26" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="06" outside="0" occluded="0" xtl="0.20" ytl="0.25" xbr="0.62" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="27" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="07" outside="0" occluded="0" xtl="0.16" ytl="0.25" xbr="0.58" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="28" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="08" outside="0" occluded="0" xtl="0.12" ytl="0.25" xbr="0.54" ybr="0.75" z_order="0" label="Left"></box>
</track>
<track id="0" label="EmptyFrame">
<box frame="29" outside="0" occluded="0" xtl="0" ytl="0" xbr="0" ybr="0" z_order="0" label="EmptyFrame"></box>
<track id="0" label="Left">
<box frame="09" outside="0" occluded="0" xtl="0.08" ytl="0.25" xbr="0.50" ybr="0.75" z_order="0" label="Left"></box>
</track>
</annotations>
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Diff not rendered.
Loading

0 comments on commit ebb2d77

Please sign in to comment.