Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update benchmark tool for otx2.1 #3652

Merged
merged 5 commits into from
Jun 21, 2024
Merged

Update benchmark tool for otx2.1 #3652

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b8b7d75
Update benchmark tool for otx2.1
chuneuny-emily Jun 20, 2024
c00fce0
Fix pre-commit issues
chuneuny-emily Jun 20, 2024
77f543d
Fix pre-commit issues
chuneuny-emily Jun 20, 2024
be0d677
Add maskrcnn_r50_tv for instance seg
chuneuny-emily Jun 21, 2024
fe1dcca
Update code feedbacks
chuneuny-emily Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@ def pytest_addoption(parser: pytest.Parser):
)
parser.addoption(
"--deterministic",
action="store_true",
default=False,
help="Turn on deterministic training.",
action="store",
eunwoosh marked this conversation as resolved.
Show resolved Hide resolved
default=None,
help="Turn on deterministic training (true/false).",
)
parser.addoption(
"--user-name",
Expand Down
5 changes: 1 addition & 4 deletions tests/perf/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,10 +175,7 @@ def fxt_dry_run(request: pytest.FixtureRequest) -> str:
@pytest.fixture(scope="session")
def fxt_deterministic(request: pytest.FixtureRequest) -> str:
"""Option to turn on deterministic training."""
deterministic = request.config.getoption("--deterministic")
msg = f"{deterministic = }"
log.info(msg)
return deterministic
return request.config.getoption("--deterministic")


@pytest.fixture(scope="session")
Expand Down
16 changes: 13 additions & 3 deletions tests/perf/test_action.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,26 @@

from __future__ import annotations

import logging
from pathlib import Path

import pytest

from .benchmark import Benchmark
from .conftest import PerfTestBase

log = logging.getLogger(__name__)


@pytest.fixture(scope="session")
def fxt_deterministic(request: pytest.FixtureRequest) -> bool:
"""Override the deterministic setting for action classification task."""
deterministic = request.config.getoption("--deterministic")
deterministic = True if deterministic is None else deterministic.lower() == "true"
msg = f"deterministic={deterministic}"
log.info(msg)
eunwoosh marked this conversation as resolved.
Show resolved Hide resolved
return deterministic


class TestPerfActionClassification(PerfTestBase):
"""Benchmark action classification."""
Expand All @@ -30,7 +43,6 @@ class TestPerfActionClassification(PerfTestBase):
extra_overrides={
"train": {
"max_epochs": "10",
"deterministic": "True",
},
},
),
Expand All @@ -42,7 +54,6 @@ class TestPerfActionClassification(PerfTestBase):
extra_overrides={
"train": {
"max_epochs": "10",
"deterministic": "True",
},
},
),
Expand All @@ -54,7 +65,6 @@ class TestPerfActionClassification(PerfTestBase):
extra_overrides={
"train": {
"max_epochs": "3",
"deterministic": "True",
},
},
),
Expand Down
62 changes: 29 additions & 33 deletions tests/perf/test_anomaly.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,40 @@

from __future__ import annotations

import logging
from pathlib import Path
from typing import ClassVar

import pytest

from .benchmark import Benchmark
from .conftest import PerfTestBase

log = logging.getLogger(__name__)


@pytest.fixture(scope="session")
eunwoosh marked this conversation as resolved.
Show resolved Hide resolved
def fxt_deterministic(request: pytest.FixtureRequest) -> bool:
"""Override the deterministic setting for anomaly tasks."""
deterministic = request.config.getoption("--deterministic")
deterministic = False if deterministic is None else deterministic.lower() == "true"
msg = f"deterministic={deterministic}"
log.info(msg)
return deterministic


class TestPerfAnomalyClassification(PerfTestBase):
"""Benchmark anomaly classification."""

MODEL_TEST_CASES = [ # noqa: RUF012
MODEL_TEST_CASES: ClassVar[list[Benchmark.Model]] = [
Benchmark.Model(task="anomaly_classification", name="padim", category="speed"),
Benchmark.Model(task="anomaly_classification", name="stfpm", category="accuracy"),
]

DATASET_TEST_CASES = [
Benchmark.Dataset(
name=f"mvtec_bottle_small_{idx}",
path=Path("anomaly/mvtec/bottle_small") / f"{idx}",
group="small",
num_repeat=5,
extra_overrides={},
)
for idx in (1, 2, 3)
] + [
DATASET_TEST_CASES: ClassVar[list[Benchmark.Dataset]] = [
# TODO(Emily): Need to replace small datasets with appropriate ones.
# Currently excluding small datasets from benchmark testing until replacements are ready.
# Small datasets to be replaced include: mvtec_bottle_small_1, mvtec_bottle_small_2, mvtec_bottle_small_3.
Benchmark.Dataset(
name="mvtec_wood_medium",
path=Path("anomaly/mvtec/wood_medium"),
Expand Down Expand Up @@ -93,21 +101,15 @@ def test_perf(
class TestPerfAnomalyDetection(PerfTestBase):
"""Benchmark anomaly detection."""

MODEL_TEST_CASES = [ # noqa: RUF012
MODEL_TEST_CASES: ClassVar[list[Benchmark.Model]] = [
Benchmark.Model(task="anomaly_detection", name="padim", category="speed"),
Benchmark.Model(task="anomaly_detection", name="stfpm", category="accuracy"),
]

DATASET_TEST_CASES = [
Benchmark.Dataset(
name=f"mvtec_bottle_small_{idx}",
path=Path("anomaly/mvtec/bottle_small") / f"{idx}",
group="small",
num_repeat=5,
extra_overrides={},
)
for idx in (1, 2, 3)
] + [
DATASET_TEST_CASES: ClassVar[list[Benchmark.Dataset]] = [
# TODO(Emily): Need to replace small datasets with appropriate ones.
# Currently excluding small datasets from benchmark testing until replacements are ready.
# Small datasets to be replaced include: mvtec_bottle_small_1, mvtec_bottle_small_2, mvtec_bottle_small_3.
Benchmark.Dataset(
name="mvtec_wood_medium",
path=Path("anomaly/mvtec/wood_medium"),
Expand Down Expand Up @@ -170,21 +172,15 @@ def test_perf(
class TestPerfAnomalySegmentation(PerfTestBase):
"""Benchmark anomaly segmentation."""

MODEL_TEST_CASES = [ # noqa: RUF012
MODEL_TEST_CASES: ClassVar[list[Benchmark.Model]] = [
Benchmark.Model(task="anomaly_segmentation", name="padim", category="speed"),
Benchmark.Model(task="anomaly_segmentation", name="stfpm", category="accuracy"),
]

DATASET_TEST_CASES = [
Benchmark.Dataset(
name=f"mvtec_bottle_small_{idx}",
path=Path("anomaly/mvtec/bottle_small") / f"{idx}",
group="small",
num_repeat=5,
extra_overrides={},
)
for idx in (1, 2, 3)
] + [
DATASET_TEST_CASES: ClassVar[list[Benchmark.Dataset]] = [
# TODO(Emily): Need to replace small datasets with appropriate ones.
# Currently excluding small datasets from benchmark testing until replacements are ready.
# Small datasets to be replaced include: mvtec_bottle_small_1, mvtec_bottle_small_2, mvtec_bottle_small_3.
Benchmark.Dataset(
name="mvtec_wood_medium",
path=Path("anomaly/mvtec/wood_medium"),
Expand Down
18 changes: 14 additions & 4 deletions tests/perf/test_classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,26 @@

from __future__ import annotations

import logging
from pathlib import Path

import pytest

from .benchmark import Benchmark
from .conftest import PerfTestBase

log = logging.getLogger(__name__)


@pytest.fixture(scope="session")
def fxt_deterministic(request: pytest.FixtureRequest) -> bool:
"""Override the deterministic setting for classification tasks."""
deterministic = request.config.getoption("--deterministic")
deterministic = True if deterministic is None else deterministic.lower() == "true"
msg = f"deterministic={deterministic}"
log.info(msg)
return deterministic


class TestPerfSingleLabelClassification(PerfTestBase):
"""Benchmark single-label classification."""
Expand Down Expand Up @@ -267,7 +280,7 @@ class TestPerfSemiSLMultiClass(PerfTestBase):
"""Benchmark single-label classification for Semi-SL task."""

MODEL_TEST_CASES = [ # noqa: RUF012
Benchmark.Model(task="classification/multi_class_cls", name="efficientnet_b0_semisl", category="speed"),
Benchmark.Model(task="classification/multi_class_cls", name="efficientnet_b0_semisl", category="balance"),
Benchmark.Model(task="classification/multi_class_cls", name="mobilenet_v3_large_semisl", category="speed"),
Benchmark.Model(task="classification/multi_class_cls", name="efficientnet_v2_semisl", category="accuracy"),
Benchmark.Model(task="classification/multi_class_cls", name="deit_tiny_semisl", category="other"),
Expand All @@ -290,7 +303,6 @@ class TestPerfSemiSLMultiClass(PerfTestBase):
"data.config.train_subset.subset_name": "train_data",
"data.config.val_subset.subset_name": "val_data",
"data.config.test_subset.subset_name": "val_data",
"deterministic": "True",
},
},
)
Expand All @@ -309,7 +321,6 @@ class TestPerfSemiSLMultiClass(PerfTestBase):
"data.config.train_subset.subset_name": "train_data",
"data.config.val_subset.subset_name": "val_data",
"data.config.test_subset.subset_name": "val_data",
"deterministic": "True",
},
},
)
Expand All @@ -328,7 +339,6 @@ class TestPerfSemiSLMultiClass(PerfTestBase):
"data.config.train_subset.subset_name": "train_data",
"data.config.val_subset.subset_name": "val_data",
"data.config.test_subset.subset_name": "val_data",
"deterministic": "True",
},
},
)
Expand Down
23 changes: 14 additions & 9 deletions tests/perf/test_detection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,26 @@

from __future__ import annotations

import logging
from pathlib import Path

import pytest

from .benchmark import Benchmark
from .conftest import PerfTestBase

log = logging.getLogger(__name__)


@pytest.fixture(scope="session")
def fxt_deterministic(request: pytest.FixtureRequest) -> bool:
"""Override the deterministic setting for detection task."""
deterministic = request.config.getoption("--deterministic")
deterministic = True if deterministic is None else deterministic.lower() == "true"
msg = f"deterministic={deterministic}"
log.info(msg)
return deterministic


class TestPerfObjectDetection(PerfTestBase):
"""Benchmark object detection."""
Expand All @@ -24,6 +37,7 @@ class TestPerfObjectDetection(PerfTestBase):
Benchmark.Model(task="detection", name="yolox_s", category="other"),
Benchmark.Model(task="detection", name="yolox_l", category="other"),
Benchmark.Model(task="detection", name="yolox_x", category="other"),
Benchmark.Model(task="detection", name="rtmdet_tiny", category="other"),
]

DATASET_TEST_CASES = [
Expand All @@ -33,9 +47,6 @@ class TestPerfObjectDetection(PerfTestBase):
group="small",
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
},
"test": {
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
},
Expand All @@ -49,9 +60,6 @@ class TestPerfObjectDetection(PerfTestBase):
group="medium",
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
},
"test": {
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
},
Expand All @@ -63,9 +71,6 @@ class TestPerfObjectDetection(PerfTestBase):
group="large",
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
},
"test": {
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
},
Expand Down
21 changes: 16 additions & 5 deletions tests/perf/test_instance_segmentation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,26 @@

from __future__ import annotations

import logging
from pathlib import Path

import pytest

from .benchmark import Benchmark
from .conftest import PerfTestBase

log = logging.getLogger(__name__)


@pytest.fixture(scope="session")
def fxt_deterministic(request: pytest.FixtureRequest) -> bool:
"""Override the deterministic setting for instance segmentation task."""
deterministic = request.config.getoption("--deterministic")
deterministic = False if deterministic is None else deterministic.lower() == "true"
msg = f"deterministic={deterministic}"
log.info(msg)
return deterministic


class TestPerfInstanceSegmentation(PerfTestBase):
"""Benchmark instance segmentation."""
Expand All @@ -20,6 +33,8 @@ class TestPerfInstanceSegmentation(PerfTestBase):
Benchmark.Model(task="instance_segmentation", name="maskrcnn_efficientnetb2b", category="speed"),
Benchmark.Model(task="instance_segmentation", name="maskrcnn_r50", category="accuracy"),
Benchmark.Model(task="instance_segmentation", name="maskrcnn_swint", category="other"),
Benchmark.Model(task="instance_segmentation", name="rtmdet_inst_tiny", category="other"),
Benchmark.Model(task="instance_segmentation", name="maskrcnn_r50_tv", category="other"),
]

DATASET_TEST_CASES = [
Expand All @@ -30,7 +45,6 @@ class TestPerfInstanceSegmentation(PerfTestBase):
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
"callback_monitor": "val/f1-score",
"model.scheduler.monitor": "val/f1-score",
Expand All @@ -49,7 +63,6 @@ class TestPerfInstanceSegmentation(PerfTestBase):
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
"callback_monitor": "val/f1-score",
"model.scheduler.monitor": "val/f1-score",
Expand All @@ -66,7 +79,6 @@ class TestPerfInstanceSegmentation(PerfTestBase):
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
"callback_monitor": "val/f1-score",
"model.scheduler.monitor": "val/f1-score",
Expand Down Expand Up @@ -133,6 +145,7 @@ class TestPerfTilingInstanceSegmentation(PerfTestBase):
Benchmark.Model(task="instance_segmentation", name="maskrcnn_efficientnetb2b_tile", category="speed"),
Benchmark.Model(task="instance_segmentation", name="maskrcnn_r50_tile", category="accuracy"),
Benchmark.Model(task="instance_segmentation", name="maskrcnn_swint_tile", category="other"),
Benchmark.Model(task="instance_segmentation", name="rtmdet_inst_tiny_tile", category="other"),
]

DATASET_TEST_CASES = [
Expand All @@ -143,7 +156,6 @@ class TestPerfTilingInstanceSegmentation(PerfTestBase):
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
"callback_monitor": "val/f1-score",
"model.scheduler.monitor": "val/f1-score",
Expand All @@ -162,7 +174,6 @@ class TestPerfTilingInstanceSegmentation(PerfTestBase):
num_repeat=5,
extra_overrides={
"train": {
"deterministic": "True",
"metric": "otx.core.metrics.fmeasure.FMeasureCallable",
"callback_monitor": "val/f1-score",
"model.scheduler.monitor": "val/f1-score",
Expand Down
Loading
Loading