Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor configuration & CLI #2805

Merged
merged 37 commits into from
Jan 23, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
58c5d5a
Initial CLI commit
harimkang Jan 16, 2024
78c48cb
Align core module's arguments
harimkang Jan 16, 2024
8f17fbc
Fix some minor dict issues
harimkang Jan 16, 2024
b28e8d0
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 16, 2024
19448c1
Fix jsonargparse to recognize Trainer arguments
harimkang Jan 16, 2024
6c9189e
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 16, 2024
784027a
Add callback_monitor
harimkang Jan 17, 2024
98b5bfd
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 17, 2024
6e2a077
Remove hydra
harimkang Jan 17, 2024
5554ed7
Fix tests & Add cli tests
harimkang Jan 17, 2024
ec40904
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 17, 2024
6a6e374
Refactor configuration files
harimkang Jan 17, 2024
5d60742
Add fxt_accelerator in cli intg test
harimkang Jan 17, 2024
3679a70
Fix some docstring & add comments in cli
harimkang Jan 18, 2024
1a254d9
Reflect review comments
harimkang Jan 18, 2024
768951e
Replace monkey-patching to contextmanager
harimkang Jan 18, 2024
f4af2b0
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 18, 2024
c9cf77c
Fix indent in docstring
harimkang Jan 18, 2024
ebc0361
Add review comment fix
harimkang Jan 19, 2024
0f0c890
Rebase latest commit
harimkang Jan 19, 2024
9f9088d
Fix unittest
harimkang Jan 19, 2024
c7ddf78
Merge branch 'v2' into harimkan/cli-configurations
harimkang Jan 19, 2024
0450edc
Rebase latest commit
harimkang Jan 19, 2024
a15e980
Add yaml representer for saving
harimkang Jan 22, 2024
13b0785
Reverse model config type
harimkang Jan 22, 2024
1be81c5
Fix cli intg tests
harimkang Jan 22, 2024
fa6f1ef
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 22, 2024
75f4451
Remove wrong file
harimkang Jan 22, 2024
4bb17b9
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 22, 2024
e842876
Modify Regression Test
harimkang Jan 22, 2024
3214f22
Disable SAM intg test temporarily
harimkang Jan 22, 2024
55c36a2
Rebase latest configuration
harimkang Jan 22, 2024
07ccf1a
Fix Regression Test
harimkang Jan 22, 2024
1bad751
Rebase latest commit
harimkang Jan 22, 2024
2462b16
Merge branch 'v2' of https://github.com/openvinotoolkit/training_exte…
harimkang Jan 22, 2024
933d5be
Hot-fix config
harimkang Jan 22, 2024
7c3fe8c
Minor fix
harimkang Jan 23, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 3 additions & 11 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,14 @@ classifiers = [
dependencies = [
"datumaro",
"omegaconf",
"hydra-core",
"rich",
"jsonargparse==4.27.1",
"psutil", # Mem cache needs system checks
"ftfy",
"regex",
"importlib_resources"
"importlib_resources",
"docstring_parser", # CLI help-formatter
"rich_argparse", # CLI help-formatter
]

[project.optional-dependencies]
Expand Down Expand Up @@ -245,21 +246,12 @@ exclude = [
"tests/assets",

# it will be cleaned up later
"src/otx/core/engine/utils/*",
"src/otx/algo/classification/backbones/*",
"for_developers/helpers.py",

# Ruff complains it but don't know how to fix since it literally showed no useful logs.
# https://github.com/openvinotoolkit/training_extensions/actions/runs/7176557723/job/19541622452?pr=2718#step:5:170
"tests/regression/*.py",

# Temporarily disable it. This will be fixed in the next PR.
# [TODO] harimkang
"src/otx/core/utils/instantiators.py",
"src/otx/core/utils/logging_utils.py",
"src/otx/core/utils/pylogger.py",
"src/otx/core/utils/rich_utils.py",
"src/otx/core/utils/utils.py",
]

# Same as Black.
Expand Down
10 changes: 4 additions & 6 deletions src/otx/algo/classification/otx_dino_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@

from __future__ import annotations

from typing import TYPE_CHECKING, Any
from typing import Any

import torch
from omegaconf import DictConfig
from torch import nn

from otx.core.data.entity.base import OTXBatchLossEntity
Expand All @@ -17,9 +18,6 @@
)
from otx.core.model.entity.classification import OTXMulticlassClsModel

if TYPE_CHECKING:
from omegaconf import DictConfig


class DINOv2(nn.Module):
"""DINO-v2 Model."""
Expand Down Expand Up @@ -65,8 +63,8 @@ def forward(self, imgs: torch.Tensor, labels: torch.Tensor = None) -> torch.Tens
class DINOv2RegisterClassifier(OTXMulticlassClsModel):
"""DINO-v2 Classification Model with register."""

def __init__(self, num_classes: int, config: DictConfig) -> None:
self.config = config
def __init__(self, num_classes: int, config: DictConfig | dict) -> None:
vinnamkim marked this conversation as resolved.
Show resolved Hide resolved
self.config = DictConfig(config)
super().__init__(num_classes=num_classes) # create the model

def _create_model(self) -> nn.Module:
Expand Down
2 changes: 1 addition & 1 deletion src/otx/algo/detection/atss.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
class ATSS(MMDetCompatibleModel):
"""ATSS Model."""

def __init__(self, num_classes: int, variant: Literal["mobilenetv2", "f50_fpn", "resnext101"]) -> None:
def __init__(self, num_classes: int, variant: Literal["mobilenetv2", "r50_fpn", "resnext101"]) -> None:
model_name = f"atss_{variant}"
config = read_mmconfig(model_name=model_name)
super().__init__(num_classes=num_classes, config=config)
2 changes: 1 addition & 1 deletion src/otx/algo/segmentation/litehrnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
class LiteHRNet(MMSegCompatibleModel):
"""LiteHRNet Model."""

def __init__(self, num_classes: int, variant: Literal["18", "s", "x"]) -> None:
def __init__(self, num_classes: int, variant: Literal["18", 18, "s", "x"]) -> None:
jaegukhyun marked this conversation as resolved.
Show resolved Hide resolved
model_name = f"litehrnet_{variant}"
config = read_mmconfig(model_name=model_name)
super().__init__(num_classes=num_classes, config=config)
70 changes: 1 addition & 69 deletions src/otx/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,75 +3,7 @@

"""CLI entrypoints."""

from __future__ import annotations

from jsonargparse import ArgumentParser

from otx import __version__


class OTXCLI:
"""OTX CLI entrypoint."""

def __init__(self) -> None:
"""Initialize OTX CLI."""
self.parser = self.init_parser()
self.setup_subcommands()
self.config = self.parser.parse_args()

self.run()

def init_parser(self, **kwargs) -> ArgumentParser:
"""Initialize the argument parser for the OTX CLI.

Args:
**kwargs: Additional keyword arguments to pass to the ArgumentParser constructor.

Returns:
ArgumentParser: The initialized argument parser.
"""
parser = ArgumentParser(description="OpenVINO Training-Extension command line tool", env_prefix="otx")
parser.add_argument(
"-V",
"--version",
action="version",
version=f"%(prog)s {__version__}",
help="Display OTX version number.",
)
return parser

def setup_subcommands(self) -> None:
"""Setup subcommands for the OTX CLI."""
parser_subcommands = self.parser.add_subcommands()
from otx.cli.install import add_install_parser

add_install_parser(parser_subcommands)

# otx train parser
train_parser = ArgumentParser()
train_parser.add_argument("overrides", help="overrides values", default=[], nargs="+")
parser_subcommands.add_subcommand("train", train_parser, help="Training subcommand for OTX")

# otx test parser
test_parser = ArgumentParser()
test_parser.add_argument("overrides", help="overrides values", default=[], nargs="+")
parser_subcommands.add_subcommand("test", test_parser, help="Testing subcommand for OTX")

def run(self) -> None:
"""Run the OTX CLI."""
subcommand = self.config["subcommand"]
if subcommand == "install":
from otx.cli.install import otx_install

otx_install(**self.config["install"])
elif subcommand == "train":
from otx.cli.train import otx_train

otx_train(**self.config["train"])
elif subcommand == "test":
from otx.cli.test import otx_test

otx_test(**self.config["test"])
from .cli import OTXCLI


def main() -> None:
Expand Down
Loading