Skip to content

Commit

Permalink
benchmark debug
Browse files Browse the repository at this point in the history
  • Loading branch information
@kprokofi
kprokofi committed Mar 15, 2024
1 parent bac7e72 commit 1292d1c
Showing 10 changed files with 111 additions and 42 deletions.
10 changes: 6 additions & 4 deletions src/otx/engine/engine.py
View file
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@

import inspect
from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterable, Literal
from typing import TYPE_CHECKING, Any, Iterable, Literal, Union
from warnings import warn

import torch
@@ -138,14 +138,16 @@ def __init__(
label_info=self._datamodule.label_info if self._datamodule is not None else None,
)
)
if self.task == OTXTaskType.DETECTION:
if self.task in [OTXTaskType.DETECTION, OTXTaskType.INSTANCE_SEGMENTATION] and self.device.accelerator == "xpu":
from mmcv.ops.nms import NMSop
from mmcv.ops.roi_align import RoIAlign
from otx.algo.detection.utils import monkey_patched_nms, monkey_patched_roi_align
from mmengine.structures import instance_data
import numpy as np

instance_data.LongTypeTensor = [torch.LongTensor, torch.xpu.LongTensor]
instance_data.BoolTypeTensor = [torch.BoolTensor, torch.xpu.BoolTensor]
LongTypeTensor = Union[torch.LongTensor, torch.xpu.LongTensor]
BoolTypeTensor = Union[torch.BoolTensor, torch.xpu.BoolTensor]
instance_data.IndexType = Union[str, slice, int, list, LongTypeTensor, BoolTypeTensor, np.ndarray]
NMSop.forward = monkey_patched_nms
RoIAlign.forward = monkey_patched_roi_align

2 changes: 1 addition & 1 deletion src/otx/recipe/instance_segmentation/maskrcnn_efficientnetb2b_tile.yaml
View file
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ callback_monitor: val/map_50
data: ../_base_/data/mmdet_base.yaml
overrides:
max_epochs: 100
gradient_clip_val: 35.0
# gradient_clip_val: 35.0
data:
task: INSTANCE_SEGMENTATION
config:
2 changes: 1 addition & 1 deletion src/otx/recipe/instance_segmentation/maskrcnn_r50.yaml
View file
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ callback_monitor: val/map_50
data: ../_base_/data/mmdet_base.yaml
overrides:
max_epochs: 100
gradient_clip_val: 35.0
# gradient_clip_val: 35.0
data:
task: INSTANCE_SEGMENTATION
config:
2 changes: 1 addition & 1 deletion src/otx/recipe/instance_segmentation/maskrcnn_r50_tile.yaml
View file
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ callback_monitor: val/map_50
data: ../_base_/data/mmdet_base.yaml
overrides:
max_epochs: 100
gradient_clip_val: 35.0
# gradient_clip_val: 35.0
data:
task: INSTANCE_SEGMENTATION
config:
2 changes: 1 addition & 1 deletion src/otx/recipe/instance_segmentation/rtmdet_inst_tiny.yaml
View file
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ data: ../_base_/data/mmdet_base.yaml
overrides:
precision: 32 # 16/"16-true" does not work
max_epochs: 100
gradient_clip_val: 35.0
# gradient_clip_val: 35.0
data:
task: INSTANCE_SEGMENTATION
config:
1 change: 1 addition & 0 deletions tests/perf/benchmark.py
View file
Original file line number Diff line number Diff line change
@@ -125,6 +125,7 @@ def run(
num_repeat = self.num_repeat # Override by global setting

for seed in range(num_repeat):
seed = 42
sub_work_dir = work_dir / str(seed)
tags["seed"] = str(seed)
extra_metrics = {}
62 changes: 46 additions & 16 deletions tests/perf/test_classification.py
View file
Original file line number Diff line number Diff line change
@@ -27,31 +27,58 @@ class TestPerfSingleLabelClassification(PerfTestBase):
DATASET_TEST_CASES = [
Benchmark.Dataset(
name=f"multiclass_CUB_small_{idx}",
path=Path("multiclass_classification/multiclass_CUB_small") / f"{idx}",
path=Path("classification/single_label/multiclass_CUB_small") / f"{idx}",
size="small",
data_format="imagenet_with_subset_dirs",
num_classes=2,
num_repeat=5,
num_repeat=1,
extra_overrides={},
)
for idx in (1, 2, 3)
] + [
Benchmark.Dataset(
name="multiclass_CUB_medium",
path=Path("multiclass_classification/multiclass_CUB_medium"),
path=Path("classification/single_label/multiclass_CUB_medium"),
size="medium",
data_format="imagenet_with_subset_dirs",
num_classes=67,
num_repeat=5,
num_repeat=1,
extra_overrides={},
),
Benchmark.Dataset(
name="multiclass_food101_large",
path=Path("multiclass_classification/multiclass_food101_large"),
path=Path("classification/single_label/multiclass_food101_large"),
size="large",
data_format="imagenet_with_subset_dirs",
num_classes=20,
num_repeat=5,
num_repeat=1,
extra_overrides={},
),
Benchmark.Dataset(
name="cars",
path=Path("car_data/car_data"),
size="large",
data_format="imagenet_with_subset_dirs",
num_classes=196,
num_repeat=1,
extra_overrides={},
),
Benchmark.Dataset(
name="flowers",
path=Path("flowers"),
size="large",
data_format="imagenet_with_subset_dirs",
num_classes=102,
num_repeat=1,
extra_overrides={},
),
Benchmark.Dataset(
name="skin",
path=Path("skin"),
size="large",
data_format="imagenet_with_subset_dirs",
num_classes=14,
num_repeat=1,
extra_overrides={},
),
]
@@ -87,6 +114,7 @@ def test_perf(
fxt_dataset: Benchmark.Dataset,
fxt_benchmark: Benchmark,
):
fxt_benchmark.accelerator = "gpu"
self._test_perf(
model=fxt_model,
dataset=fxt_dataset,
@@ -108,31 +136,31 @@ class TestPerfMultiLabelClassification(PerfTestBase):
DATASET_TEST_CASES = [
Benchmark.Dataset(
name=f"multilabel_CUB_small_{idx}",
path=Path("multilabel_classification/multilabel_CUB_small") / f"{idx}",
path=Path("classification/multi_label/multilabel_CUB_small") / f"{idx}",
size="small",
data_format="datumaro",
num_classes=3,
num_repeat=5,
num_repeat=1,
extra_overrides={},
)
for idx in (1, 2, 3)
] + [
Benchmark.Dataset(
name="multilabel_CUB_medium",
path=Path("multilabel_classification/multilabel_CUB_medium"),
path=Path("classification/multi_label/multilabel_CUB_medium"),
size="medium",
data_format="datumaro",
num_classes=68,
num_repeat=5,
num_repeat=1,
extra_overrides={},
),
Benchmark.Dataset(
name="multilabel_food101_large",
path=Path("multilabel_classification/multilabel_food101_large"),
path=Path("classification/multi_label/multilabel_food101_large"),
size="large",
data_format="datumaro",
num_classes=21,
num_repeat=5,
num_repeat=1,
extra_overrides={},
),
]
@@ -168,6 +196,7 @@ def test_perf(
fxt_dataset: Benchmark.Dataset,
fxt_benchmark: Benchmark,
):
fxt_benchmark.accelerator = "gpu"
self._test_perf(
model=fxt_model,
dataset=fxt_dataset,
@@ -189,11 +218,11 @@ class TestPerfHierarchicalLabelClassification(PerfTestBase):
DATASET_TEST_CASES = [
Benchmark.Dataset(
name=f"hlabel_CUB_small_{idx}",
path=Path("hlabel_classification/hlabel_CUB_small") / f"{idx}",
path=Path("classification/h_label/hlabel_CUB_small") / f"{idx}",
size="small",
data_format="datumaro",
num_classes=6,
num_repeat=5,
num_repeat=1,
extra_overrides={
"model.num_multiclass_heads": "3",
"model.num_multilabel_classes": "0",
@@ -203,11 +232,11 @@ class TestPerfHierarchicalLabelClassification(PerfTestBase):
] + [
Benchmark.Dataset(
name="hlabel_CUB_medium",
path=Path("hlabel_classification/hlabel_CUB_medium"),
path=Path("classification/h_label/hlabel_CUB_medium"),
size="medium",
data_format="datumaro",
num_classes=102,
num_repeat=5,
num_repeat=1,
extra_overrides={
"model.num_multiclass_heads": "23",
"model.num_multilabel_classes": "0",
@@ -247,6 +276,7 @@ def test_perf(
fxt_dataset: Benchmark.Dataset,
fxt_benchmark: Benchmark,
):
fxt_benchmark.accelerator = "gpu"
self._test_perf(
model=fxt_model,
dataset=fxt_dataset,
27 changes: 21 additions & 6 deletions tests/perf/test_detection.py
View file
Original file line number Diff line number Diff line change
@@ -33,9 +33,9 @@ class TestPerfObjectDetection(PerfTestBase):
size="small",
data_format="coco",
num_classes=1,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
# "deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
"callback_monitor": "val/f1-score",
"scheduler.monitor": "val/f1-score",
@@ -49,9 +49,9 @@ class TestPerfObjectDetection(PerfTestBase):
size="medium",
data_format="coco",
num_classes=1,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
# "deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
"callback_monitor": "val/f1-score",
"scheduler.monitor": "val/f1-score",
@@ -63,9 +63,23 @@ class TestPerfObjectDetection(PerfTestBase):
size="large",
data_format="coco",
num_classes=1,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
# "deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
"callback_monitor": "val/f1-score",
"scheduler.monitor": "val/f1-score",
},
),
Benchmark.Dataset(
name="pascal_tiny",
path=Path("pascal_tiny/coco_otx"),
size="large",
data_format="coco",
num_classes=20,
num_repeat=1,
extra_overrides={
# "deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
"callback_monitor": "val/f1-score",
"scheduler.monitor": "val/f1-score",
@@ -104,6 +118,7 @@ def test_perf(
fxt_dataset: Benchmark.Dataset,
fxt_benchmark: Benchmark,
):
fxt_benchmark.accelerator = "gpu"
self._test_perf(
model=fxt_model,
dataset=fxt_dataset,
12 changes: 7 additions & 5 deletions tests/perf/test_instance_segmentation.py
View file
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ class TestPerfInstanceSegmentation(PerfTestBase):
size="small",
data_format="coco",
num_classes=5,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
@@ -45,7 +45,7 @@ class TestPerfInstanceSegmentation(PerfTestBase):
size="medium",
data_format="coco",
num_classes=2,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
@@ -59,7 +59,7 @@ class TestPerfInstanceSegmentation(PerfTestBase):
size="large",
data_format="coco",
num_classes=1,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
@@ -100,6 +100,7 @@ def test_perf(
fxt_dataset: Benchmark.Dataset,
fxt_benchmark: Benchmark,
):
fxt_benchmark.accelerator = "gpu"
self._test_perf(
model=fxt_model,
dataset=fxt_dataset,
@@ -124,7 +125,7 @@ class TestPerfTilingInstanceSegmentation(PerfTestBase):
size="small",
data_format="coco",
num_classes=1,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
@@ -140,7 +141,7 @@ class TestPerfTilingInstanceSegmentation(PerfTestBase):
size="medium",
data_format="coco",
num_classes=1,
num_repeat=5,
num_repeat=1,
extra_overrides={
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasure",
@@ -182,6 +183,7 @@ def test_perf(
fxt_dataset: Benchmark.Dataset,
fxt_benchmark: Benchmark,
):
fxt_benchmark.accelerator = "gpu"
self._test_perf(
model=fxt_model,
dataset=fxt_dataset,
Loading

0 comments on commit 1292d1c

Please sign in to comment.