From b044dd228c66e13aa23d6829f7de49c7af76f77d Mon Sep 17 00:00:00 2001 From: Harim Kang Date: Tue, 2 Apr 2024 23:06:10 +0900 Subject: [PATCH] Add Configuration Converter in otx.tools (#3254) --- src/otx/engine/engine.py | 2 + src/otx/tools/converter.py | 434 +++++++++++++++ tests/assets/geti-configs/det.json | 839 +++++++++++++++++++++++++++++ tests/unit/tools/__init__.py | 2 + tests/unit/tools/test_converter.py | 48 ++ 5 files changed, 1325 insertions(+) create mode 100644 src/otx/tools/converter.py create mode 100644 tests/assets/geti-configs/det.json create mode 100644 tests/unit/tools/__init__.py create mode 100644 tests/unit/tools/test_converter.py diff --git a/src/otx/engine/engine.py b/src/otx/engine/engine.py index 97bd8feb635..c00a0fe2f4b 100644 --- a/src/otx/engine/engine.py +++ b/src/otx/engine/engine.py @@ -286,6 +286,8 @@ def train( raise TypeError(msg) best_checkpoint_symlink = Path(self.work_dir) / "best_checkpoint.ckpt" + if best_checkpoint_symlink.is_symlink(): + best_checkpoint_symlink.unlink() best_checkpoint_symlink.symlink_to(self.checkpoint) return self.trainer.callback_metrics diff --git a/src/otx/tools/converter.py b/src/otx/tools/converter.py new file mode 100644 index 00000000000..d982f6754b8 --- /dev/null +++ b/src/otx/tools/converter.py @@ -0,0 +1,434 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Converter for v1 config.""" + +from __future__ import annotations + +import argparse +import json +from copy import deepcopy +from pathlib import Path +from typing import Any +from warnings import warn + +from jsonargparse import ArgumentParser, Namespace + +from otx.core.config.data import DataModuleConfig, SamplerConfig, SubsetConfig, TileConfig +from otx.core.data.module import OTXDataModule +from otx.core.model.base import OTXModel +from otx.core.types import PathLike +from otx.core.types.task import OTXTaskType +from otx.engine import Engine +from otx.engine.utils.auto_configurator import AutoConfigurator + +TEMPLATE_ID_DICT = { + # MULTI_CLASS_CLS + "Custom_Image_Classification_DeiT-Tiny": { + "task": OTXTaskType.MULTI_CLASS_CLS, + "model_name": "otx_deit_tiny", + }, + "Custom_Image_Classification_EfficinetNet-B0": { + "task": OTXTaskType.MULTI_CLASS_CLS, + "model_name": "efficientnet_b0_light", + }, + "Custom_Image_Classification_EfficientNet-V2-S": { + "task": OTXTaskType.MULTI_CLASS_CLS, + "model_name": "efficientnet_v2_light", + }, + "Custom_Image_Classification_MobileNet-V3-large-1x": { + "task": OTXTaskType.MULTI_CLASS_CLS, + "model_name": "mobilenet_v3_large_light", + }, + # DETECTION + "Custom_Object_Detection_Gen3_ATSS": { + "task": OTXTaskType.DETECTION, + "model_name": "atss_mobilenetv2", + }, + "Object_Detection_ResNeXt101_ATSS": { + "task": OTXTaskType.DETECTION, + "model_name": "atss_resnext101", + }, + "Custom_Object_Detection_Gen3_SSD": { + "task": OTXTaskType.DETECTION, + "model_name": "ssd_mobilenetv2", + }, + "Object_Detection_YOLOX_X": { + "task": OTXTaskType.DETECTION, + "model_name": "yolox_x", + }, + "Object_Detection_YOLOX_L": { + "task": OTXTaskType.DETECTION, + "model_name": "yolox_l", + }, + "Object_Detection_YOLOX_S": { + "task": OTXTaskType.DETECTION, + "model_name": "yolox_s", + }, + "Custom_Object_Detection_YOLOX": { + "task": OTXTaskType.DETECTION, + "model_name": "yolox_tiny", + }, + # INSTANCE_SEGMENTATION + "Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50": { + "task": OTXTaskType.INSTANCE_SEGMENTATION, + "model_name": "maskrcnn_r50", + }, + "Custom_Counting_Instance_Segmentation_MaskRCNN_SwinT_FP16": { + "task": OTXTaskType.INSTANCE_SEGMENTATION, + "model_name": "maskrcnn_swint", + }, + "Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B": { + "task": OTXTaskType.INSTANCE_SEGMENTATION, + "model_name": "maskrcnn_efficientnetb2b", + }, + # ROTATED_DETECTION + "Custom_Rotated_Detection_via_Instance_Segmentation_MaskRCNN_ResNet50": { + "task": OTXTaskType.ROTATED_DETECTION, + "model_name": "maskrcnn_r50", + }, + "Custom_Rotated_Detection_via_Instance_Segmentation_MaskRCNN_EfficientNetB2B": { + "task": OTXTaskType.ROTATED_DETECTION, + "model_name": "maskrcnn_efficientnetb2b", + }, + # SEMANTIC_SEGMENTATION + "Custom_Semantic_Segmentation_Lite-HRNet-18-mod2_OCR": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "litehrnet_18", + }, + "Custom_Semantic_Segmentation_Lite-HRNet-18_OCR": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "litehrnet_18", + }, + "Custom_Semantic_Segmentation_Lite-HRNet-s-mod2_OCR": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "litehrnet_s", + }, + "Custom_Semantic_Segmentation_Lite-HRNet-x-mod3_OCR": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "litehrnet_x", + }, + "Custom_Semantic_Segmentation_SegNext_t": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "segnext_t", + }, + "Custom_Semantic_Segmentation_SegNext_s": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "segnext_s", + }, + "Custom_Semantic_Segmentation_SegNext_B": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "segnext_b", + }, + # ANOMALY_CLASSIFICATION + "ote_anomaly_classification_padim": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "padim", + }, + "ote_anomaly_classification_stfpm": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "stfpm", + }, + # ANOMALY_DETECTION + "ote_anomaly_detection_padim": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "padim", + }, + "ote_anomaly_detection_stfpm": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "stfpm", + }, + # ANOMALY_SEGMENTATION + "ote_anomaly_segmentation_padim": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "padim", + }, + "ote_anomaly_segmentation_stfpm": { + "task": OTXTaskType.SEMANTIC_SEGMENTATION, + "model_name": "stfpm", + }, +} + + +class ConfigConverter: + """Convert ModelTemplate for OTX v1 to OTX v2 recipe dictionary. + + This class is used to convert ModelTemplate for OTX v1 to OTX v2 recipe dictionary. + + Example: + The following examples show how to use the Converter class. + We expect a config file with ModelTemplate information in json form. + + Convert template.json to dictionary:: + + converter = ConfigConverter() + config = converter.convert("template.json") + + Instantiate an object from the configuration dictionary:: + + engine, train_kwargs = converter.instantiate( + config=config, + work_dir="otx-workspace", + data_root="tests/assets/car_tree_bug", + ) + + Train the model:: + + engine.train(**train_kwargs) + """ + + @staticmethod + def convert(config_path: str) -> dict: + """Convert a configuration file to a default configuration dictionary. + + Args: + config_path (str): The path to the configuration file. + + Returns: + dict: The default configuration dictionary. + + """ + with Path(config_path).open() as f: + template_config = json.load(f) + + hyperparameters = template_config["hyperparameters"] + param_dict = ConfigConverter._get_params(hyperparameters) + + task_info = TEMPLATE_ID_DICT[template_config["model_template_id"]] + if param_dict.get("enable_tiling", None) and not task_info["model_name"].endswith("_tile"): + task_info["model_name"] += "_tile" + default_config = ConfigConverter._get_default_config(task_info) + ConfigConverter._update_params(default_config, param_dict) + ConfigConverter._remove_unused_key(default_config) + return default_config + + @staticmethod + def _get_default_config(task_info: dict) -> dict: + """Return default otx conifg for template use.""" + return AutoConfigurator(**task_info).config # type: ignore[arg-type] + + @staticmethod + def _get_params(hyperparameters: dict) -> dict: + """Get configuraable parameters from ModelTemplate config hyperparameters field.""" + param_dict = {} + for param_name, param_info in hyperparameters.items(): + if isinstance(param_info, dict): + if "value" in param_info: + param_dict[param_name] = param_info["value"] + else: + param_dict = param_dict | ConfigConverter._get_params(param_info) + + return param_dict + + @staticmethod + def _update_params(config: dict, param_dict: dict) -> None: # noqa: C901 + """Update params of OTX recipe from Geit configurable params.""" + unused_params = deepcopy(param_dict) + + def update_mem_cache_size(param_value: int) -> None: + config["data"]["config"]["mem_cache_size"] = f"{int(param_value / 1000000)}MB" + + def update_batch_size(param_value: int) -> None: + config["data"]["config"]["train_subset"]["batch_size"] = param_value + + def update_inference_batch_size(param_value: int) -> None: + config["data"]["config"]["val_subset"]["batch_size"] = param_value + config["data"]["config"]["test_subset"]["batch_size"] = param_value + + def update_learning_rate(param_value: float) -> None: + config["model"]["init_args"]["optimizer"]["init_args"]["lr"] = param_value + + def update_learning_rate_warmup_iters(param_value: int) -> None: + scheduler = config["model"]["init_args"]["scheduler"] + if scheduler["class_path"] == "otx.core.schedulers.LinearWarmupSchedulerCallable": + scheduler["init_args"]["num_warmup_steps"] = param_value + + def update_num_iters(param_value: int) -> None: + config["max_epochs"] = param_value + + def update_num_workers(param_value: int) -> None: + config["data"]["config"]["train_subset"]["num_workers"] = param_value + config["data"]["config"]["val_subset"]["num_workers"] = param_value + config["data"]["config"]["test_subset"]["num_workers"] = param_value + + def update_enable_early_stopping(param_value: bool) -> None: + idx = ConfigConverter._get_callback_idx(config["callbacks"], "lightning.pytorch.callbacks.EarlyStopping") + if not param_value and idx > -1: + config["callbacks"].pop(idx) + + def update_early_stop_patience(param_value: int) -> None: + for callback in config["callbacks"]: + if callback["class_path"] == "lightning.pytorch.callbacks.EarlyStopping": + callback["init_args"]["patience"] = param_value + break + + def update_use_adaptive_interval(param_value: bool) -> None: + idx = ConfigConverter._get_callback_idx( + config["callbacks"], + "otx.algo.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling", + ) + if not param_value and idx > -1: + config["callbacks"].pop(idx) + + def update_auto_num_workers(param_value: bool) -> None: + config["data"]["config"]["auto_num_workers"] = param_value + + def update_enable_tiling(param_value: bool) -> None: + config["data"]["config"]["tile_config"]["enable_tiler"] = param_value + if param_value: + config["data"]["config"]["tile_config"]["enable_adaptive_tiling"] = param_dict["enable_adaptive_params"] + config["data"]["config"]["tile_config"]["tile_size"] = ( + param_dict["tile_size"], + param_dict["tile_size"], + ) + config["data"]["config"]["tile_config"]["overlap"] = param_dict["tile_overlap"] + config["data"]["config"]["tile_config"]["max_num_instances"] = param_dict["tile_max_number"] + config["data"]["config"]["tile_config"]["sampling_ratio"] = param_dict["tile_sampling_ratio"] + config["data"]["config"]["tile_config"]["object_tile_ratio"] = param_dict["object_tile_ratio"] + tile_params = [ + "enable_adaptive_params", + "tile_size", + "tile_overlap", + "tile_max_number", + "tile_sampling_ratio", + "object_tile_ratio", + ] + for tile_param in tile_params: + unused_params.pop(tile_param) + + param_update_funcs = { + "mem_cache_size": update_mem_cache_size, + "batch_size": update_batch_size, + "inference_batch_size": update_inference_batch_size, + "learning_rate": update_learning_rate, + "learning_rate_warmup_iters": update_learning_rate_warmup_iters, + "num_iters": update_num_iters, + "num_workers": update_num_workers, + "enable_early_stopping": update_enable_early_stopping, + "early_stop_patience": update_early_stop_patience, + "use_adaptive_interval": update_use_adaptive_interval, + "auto_num_workers": update_auto_num_workers, + "enable_tiling": update_enable_tiling, + } + for param_name, param_value in param_dict.items(): + update_func = param_update_funcs.get(param_name) + if update_func: + update_func(param_value) + unused_params.pop(param_name) + + warn("Warning: These parameters are not updated", stacklevel=1) + for param_name, param_value in unused_params.items(): + print(f"\t {param_name}: {param_value}") + + @staticmethod + def _get_callback_idx(callbacks: list, name: str) -> int: + """Return required callbacks index from callback list.""" + for idx, callback in enumerate(callbacks): + if callback["class_path"] == name: + return idx + return -1 + + @staticmethod + def _remove_unused_key(config: dict) -> None: + """Remove unused keys from the config dictionary. + + Args: + config (dict): The configuration dictionary. + """ + config.pop("config") # Remove config key that for CLI + config["data"].pop("__path__") # Remove __path__ key that for CLI overriding + + @staticmethod + def instantiate( + config: dict, + work_dir: PathLike | None = None, + data_root: PathLike | None = None, + **kwargs, + ) -> tuple[Engine, dict[str, Any]]: + """Instantiate an object from the configuration dictionary. + + Args: + config (dict): The configuration dictionary. + work_dir (PathLike): Path to the working directory. + data_root (PathLike): The root directory for data. + + Returns: + tuple: A tuple containing the engine and the train kwargs dictionary. + """ + config.update(kwargs) + + # Instantiate datamodule + data_config = config.pop("data") + if data_root is not None: + data_config["config"]["data_root"] = data_root + + train_config = data_config["config"].pop("train_subset") + val_config = data_config["config"].pop("val_subset") + test_config = data_config["config"].pop("test_subset") + datamodule = OTXDataModule( + task=data_config["task"], + config=DataModuleConfig( + train_subset=SubsetConfig(sampler=SamplerConfig(**train_config.pop("sampler", {})), **train_config), + val_subset=SubsetConfig(sampler=SamplerConfig(**val_config.pop("sampler", {})), **val_config), + test_subset=SubsetConfig(sampler=SamplerConfig(**test_config.pop("sampler", {})), **test_config), + tile_config=TileConfig(**data_config["config"].pop("tile_config", {})), + **data_config["config"], + ), + ) + + num_classes = datamodule.label_info.num_classes + + # Update num_classes & Instantiate Model + model_config = config.pop("model") + model_config["init_args"]["num_classes"] = num_classes + + model_parser = ArgumentParser() + model_parser.add_subclass_arguments(OTXModel, "model", required=False, fail_untyped=False) + model = model_parser.instantiate_classes(Namespace(model=model_config)).get("model") + + # Instantiate Engine + config_work_dir = config.pop("work_dir", config["engine"].pop("work_dir", None)) + config["engine"]["work_dir"] = work_dir if work_dir is not None else config_work_dir + engine = Engine( + model=model, + datamodule=datamodule, + **config.pop("engine"), + ) + + # Instantiate Engine.train Arguments + engine_parser = ArgumentParser() + train_arguments = engine_parser.add_method_arguments( + Engine, + "train", + skip={"accelerator", "devices"}, + fail_untyped=False, + ) + # Update callbacks & logger dir as engine.work_dir + for callback in config["callbacks"]: + if "dirpath" in callback["init_args"]: + callback["init_args"]["dirpath"] = engine.work_dir + for logger in config["logger"]: + if "save_dir" in logger["init_args"]: + logger["init_args"]["save_dir"] = engine.work_dir + if "log_dir" in logger["init_args"]: + logger["init_args"]["log_dir"] = engine.work_dir + instantiated_kwargs = engine_parser.instantiate_classes(Namespace(**config)) + + train_kwargs = {k: v for k, v in instantiated_kwargs.items() if k in train_arguments} + + return engine, train_kwargs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config", help="Input ModelTemplate config") + parser.add_argument("-i", "--data_root", help="Input dataset root path") + parser.add_argument("-o", "--work_dir", help="Input work directory path") + args = parser.parse_args() + otx_config = ConfigConverter.convert(config_path=args.config) + engine, train_kwargs = ConfigConverter.instantiate( + config=otx_config, + data_root=args.data_root, + work_dir=args.work_dir, + ) + engine.train(**train_kwargs) diff --git a/tests/assets/geti-configs/det.json b/tests/assets/geti-configs/det.json new file mode 100644 index 00000000000..4a80baeb55e --- /dev/null +++ b/tests/assets/geti-configs/det.json @@ -0,0 +1,839 @@ +{ + "job_type": "train", + "model_template_id": "Custom_Object_Detection_Gen3_ATSS", + "hyperparameters": { + "header": "Configuration for an object detection task", + "description": "Configuration for an object detection task", + "visible_in_ui": true, + "id": "65eec304dd9da90646288d30", + "type": "CONFIGURABLE_PARAMETERS", + "algo_backend": { + "header": "Algo backend parameters", + "description": "parameters for algo backend", + "visible_in_ui": false, + "type": "PARAMETER_GROUP", + "train_type": { + "value": "Incremental", + "default_value": "Incremental", + "description": "Training scheme option that determines how to train the model", + "header": "Train type", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "SELECTABLE", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "enum_name": "TrainType", + "options": { + "Incremental": "Incremental", + "Semisupervised": "Semisupervised" + } + }, + "mem_cache_size": { + "value": 100000000, + "default_value": 100000000, + "description": "Size of memory pool for caching decoded data to load data faster (bytes).", + "header": "Size of memory pool", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 10000000000 + }, + "storage_cache_scheme": { + "value": "NONE", + "default_value": "NONE", + "description": "Scheme for storage cache", + "header": "Scheme for storage cache", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "SELECTABLE", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "enum_name": "StorageCacheScheme", + "options": { + "NONE": "NONE", + "AS_IS": "AS-IS", + "JPEG_75": "JPEG/75", + "JPEG_95": "JPEG/95", + "PNG": "PNG", + "TIFF": "TIFF" + } + }, + "enable_noisy_label_detection": { + "value": false, + "default_value": false, + "description": "Set to True to enable loss dynamics tracking for each sample to detect noisy labeled samples.", + "header": "Enable loss dynamics tracking for noisy label detection", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + } + }, + "learning_parameters": { + "header": "Learning Parameters", + "description": "Learning Parameters", + "visible_in_ui": true, + "type": "PARAMETER_GROUP", + "batch_size": { + "value": 16, + "default_value": 8, + "description": "The number of training samples seen in each iteration of training. Increasing this value improves training time and may make the training more stable. A larger batch size has higher memory requirements.", + "header": "Batch size", + "warning": "Increasing this value may cause the system to use more memory than available, potentially causing out of memory errors, please update with caution.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "possible", + "auto_hpo_value": null, + "min_value": 1, + "max_value": 512 + }, + "inference_batch_size": { + "value": 8, + "default_value": 8, + "description": "The number of samples seen in each iteration of inference. Increasing this value improves inference time and may make the inference more stable. A larger batch size has higher memory requirements.", + "header": "Inference batch size", + "warning": "Increasing this value may cause the system to use more memory than available, potentially causing out of memory errors, please update with caution.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 1, + "max_value": 512 + }, + "learning_rate": { + "value": 0.01, + "default_value": 0.004, + "description": "Increasing this value will speed up training convergence but might make it unstable.", + "header": "Learning rate", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "FLOAT", + "auto_hpo_state": "possible", + "auto_hpo_value": null, + "min_value": 1e-7, + "max_value": 0.1, + "step_size": null + }, + "learning_rate_warmup_iters": { + "value": 6, + "default_value": 3, + "description": "In this periods of initial training iterations, the model will be trained in low learning rate, which will be increased incrementally up to the expected learning rate setting. This warm-up phase is known to be helpful to stabilize training, thus result in better performance.", + "header": "Number of iterations for learning rate warmup", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 10000 + }, + "num_iters": { + "value": 50, + "default_value": 200, + "description": "Increasing this value causes the results to be more robust but training time will be longer.", + "header": "Number of training iterations", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 1, + "max_value": 100000 + }, + "num_workers": { + "value": 8, + "default_value": 2, + "description": "Increasing this value might improve training speed however it might cause out of memory errors. If the number of workers is set to zero, data loading will happen in the main training thread.", + "header": "Number of cpu threads to use during batch generation", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "NONE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 8 + }, + "enable_early_stopping": { + "value": true, + "default_value": true, + "description": "Early exit from training when validation accuracy isn't changed or decreased for several epochs.", + "header": "Enable early stopping of the training", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "early_stop_start": { + "value": 3, + "default_value": 3, + "description": "Default integer description", + "header": "Start epoch for early stopping", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 1000 + }, + "early_stop_patience": { + "value": 4, + "default_value": 10, + "description": "Training will stop if the model does not improve within the number of epochs of patience.", + "header": "Patience for early stopping", + "warning": "This is applied exclusively when early stopping is enabled.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 50 + }, + "early_stop_iteration_patience": { + "value": 0, + "default_value": 0, + "description": "Training will stop if the model does not improve within the number of iterations of patience. This ensures the model is trained enough with the number of iterations of patience before early stopping.", + "header": "Iteration patience for early stopping", + "warning": "This is applied exclusively when early stopping is enabled.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 1000 + }, + "use_adaptive_interval": { + "value": false, + "default_value": true, + "description": "Depending on the size of iteration per epoch, adaptively update the validation interval and related values.", + "header": "Use adaptive validation interval", + "warning": "This will automatically control the patience and interval when early stopping is enabled.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "auto_adapt_batch_size": { + "value": "None", + "default_value": "None", + "description": "Safe => Prevent GPU out of memory. Full => Find a batch size using most of GPU memory.", + "header": "Decrease batch size if current batch size isn't fit to CUDA memory.", + "warning": "Enabling this could change the actual batch size depending on the current GPU status. The learning rate also could be adjusted according to the adapted batch size. This process might change a model performance and take some extra computation time to try a few batch size candidates.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "SELECTABLE", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "enum_name": "BatchSizeAdaptType", + "options": { + "NONE": "None", + "SAFE": "Safe", + "FULL": "Full" + } + }, + "auto_num_workers": { + "value": false, + "default_value": false, + "description": "Adapt num_workers according to current hardware status automatically.", + "header": "Enable auto adaptive num_workers", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "input_size": { + "value": "Default", + "default_value": "Default", + "description": "The input size of the given model could be configured to one of the predefined resolutions. Reduced training and inference time could be expected by using smaller input size. In Auto mode, the input size is automatically determined based on dataset statistics. Defaults to per-model default resolution.", + "header": "Configure model input size.", + "warning": "Modifying input size may decrease model performance.", + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "SELECTABLE", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "enum_name": "InputSizePreset", + "options": { + "DEFAULT": "Default", + "AUTO": "Auto", + "_256x256": "256x256", + "_384x384": "384x384", + "_512x512": "512x512", + "_768x768": "768x768", + "_1024x1024": "1024x1024" + } + } + }, + "nncf_optimization": { + "header": "Optimization by NNCF", + "description": "Optimization by NNCF", + "visible_in_ui": true, + "type": "PARAMETER_GROUP", + "enable_quantization": { + "value": true, + "default_value": true, + "description": "Enable quantization algorithm", + "header": "Enable quantization algorithm", + "warning": null, + "editable": false, + "visible_in_ui": false, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "enable_pruning": { + "value": false, + "default_value": false, + "description": "Enable filter pruning algorithm", + "header": "Enable filter pruning algorithm", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "pruning_supported": { + "value": true, + "default_value": true, + "description": "Whether filter pruning is supported", + "header": "Whether filter pruning is supported", + "warning": null, + "editable": false, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "maximal_accuracy_degradation": { + "value": 1.0, + "default_value": 1.0, + "description": "The maximal allowed accuracy metric drop in absolute values", + "header": "Maximum accuracy degradation", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "NONE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "FLOAT", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0.0, + "max_value": 100.0, + "step_size": null + } + }, + "postprocessing": { + "header": "Postprocessing", + "description": "Postprocessing", + "visible_in_ui": true, + "type": "PARAMETER_GROUP", + "confidence_threshold": { + "value": 0.35, + "default_value": 0.35, + "description": "This threshold only takes effect if the threshold is not set based on the result.", + "header": "Confidence threshold", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "FLOAT", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 1, + "step_size": null + }, + "max_num_detections": { + "value": 0, + "default_value": 0, + "description": "Extra detection outputs will be discared in non-maximum suppression process. Defaults to 0, which means per-model default values.", + "header": "Maximum number of detections per image", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 10000 + }, + "use_ellipse_shapes": { + "value": false, + "default_value": false, + "description": "Use direct ellipse shape in inference instead of polygon from mask", + "header": "Use ellipse shapes", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "result_based_confidence_threshold": { + "value": true, + "default_value": true, + "description": "Confidence threshold is derived from the results", + "header": "Result based confidence threshold", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "INFERENCE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + } + }, + "pot_parameters": { + "header": "POT Parameters", + "description": "POT Parameters", + "visible_in_ui": true, + "type": "PARAMETER_GROUP", + "preset": { + "value": "Performance", + "default_value": "Performance", + "description": "Quantization preset that defines quantization scheme", + "header": "Preset", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "NONE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "SELECTABLE", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "enum_name": "POTQuantizationPreset", + "options": { + "MIXED": "Mixed", + "PERFORMANCE": "Performance" + } + }, + "stat_subset_size": { + "value": 300, + "default_value": 300, + "description": "Number of data samples used for post-training optimization", + "header": "Number of data samples", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "NONE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 1, + "max_value": 100000 + }, + "stat_requests_number": { + "value": 0, + "default_value": 0, + "description": "Number of requests during statistics collection", + "header": "Number of requests", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "NONE", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0, + "max_value": 100000 + } + }, + "tiling_parameters": { + "header": "Tiling", + "description": "Crop dataset to tiles", + "visible_in_ui": true, + "type": "PARAMETER_GROUP", + "enable_tiling": { + "value": true, + "default_value": false, + "description": "Set to True to allow tiny objects to be better detected.", + "header": "Enable tiling", + "warning": "Tiling trades off speed for accuracy as it increases the number of images to be processed. In turn, it's memory efficient as smaller resolution patches are handled at onces so that the possibility of OOM issues could be reduced. Important: In the current version, depending on the dataset size and the available hardware resources, a model may not train successfully when tiling is enabled.", + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "enable_adaptive_params": { + "value": true, + "default_value": true, + "description": "Config tile size and tile overlap adaptively based on annotated dataset statistic. Manual settings well be ignored if it's turned on. Please turn off this option in order to tune tiling parameters manually.", + "header": "Enable adaptive tiling parameters", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "BOOLEAN", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null + }, + "tile_size": { + "value": 400, + "default_value": 400, + "description": "Tile image size. (tile_size x tile_size) sub images will be the unit of computation.", + "header": "Tile Image Size", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 100, + "max_value": 4096 + }, + "tile_overlap": { + "value": 0.5, + "default_value": 0.2, + "description": "Overlap ratio between each two neighboring tiles. Recommend to set as large_object_size / tile_size.", + "header": "Tile Overlap", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "FLOAT", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0.0, + "max_value": 0.9, + "step_size": null + }, + "tile_max_number": { + "value": 1500, + "default_value": 1500, + "description": "Maximum number of objects per tile", + "header": "Max object per tile", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "INTEGER", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 1, + "max_value": 5000 + }, + "tile_sampling_ratio": { + "value": 1.0, + "default_value": 1.0, + "description": "Since tiling train and validation to all tile from large image, usually it takes lots of time than normal training. The tile_sampling_ratio is ratio for sampling entire tile dataset. Sampling tile dataset would save lots of time for training and validation time. Note that sampling will be applied to training and validation dataset, not test dataset.", + "header": "Sampling Ratio for entire tiling", + "warning": null, + "editable": true, + "visible_in_ui": true, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "FLOAT", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 1e-6, + "max_value": 1.0, + "step_size": null + }, + "object_tile_ratio": { + "value": 0.03, + "default_value": 0.03, + "description": "The desired ratio of min object size and tile size.", + "header": "Object tile ratio", + "warning": null, + "editable": true, + "visible_in_ui": false, + "affects_outcome_of": "TRAINING", + "ui_rules": { + "operator": "AND", + "action": "DISABLE_EDITING", + "type": "UI_RULES", + "rules": [] + }, + "type": "FLOAT", + "auto_hpo_state": "not_possible", + "auto_hpo_value": null, + "min_value": 0.0, + "max_value": 1.0, + "step_size": null + } + } + }, + "export_parameters": [ + { + "type": "openvino", + "output_model_id": "65f283d5cf4d9b8e1ba2e5c8", + "precision": "FP32", + "with_xai": true + }, + { + "type": "openvino", + "output_model_id": "65f283d5cf4d9b8e1ba2e5c9", + "precision": "FP32", + "with_xai": false + }, + { + "type": "openvino", + "output_model_id": "65f283d5cf4d9b8e1ba2e5ca", + "precision": "FP16", + "with_xai": false + }, + { + "type": "onnx", + "output_model_id": "65f283d5cf4d9b8e1ba2e5cb", + "precision": "FP32", + "with_xai": false + } + ], + "optimization_type": "NONE" +} diff --git a/tests/unit/tools/__init__.py b/tests/unit/tools/__init__.py new file mode 100644 index 00000000000..916f3a44b27 --- /dev/null +++ b/tests/unit/tools/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/unit/tools/test_converter.py b/tests/unit/tools/test_converter.py new file mode 100644 index 00000000000..395183bf32a --- /dev/null +++ b/tests/unit/tools/test_converter.py @@ -0,0 +1,48 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from otx.tools.converter import ConfigConverter + + +class TestConfigConverter: + def test_convert(self): + config = ConfigConverter.convert("tests/assets/geti-configs/det.json") + + assert config["data"]["config"]["mem_cache_size"] == "100MB" + assert config["data"]["config"]["train_subset"]["batch_size"] == 16 + assert config["data"]["config"]["val_subset"]["batch_size"] == 8 + assert config["data"]["config"]["test_subset"]["batch_size"] == 8 + assert config["model"]["init_args"]["optimizer"]["init_args"]["lr"] == 0.01 + assert config["model"]["init_args"]["scheduler"]["init_args"]["num_warmup_steps"] == 6 + assert config["max_epochs"] == 50 + assert config["data"]["config"]["train_subset"]["num_workers"] == 8 + assert config["data"]["config"]["val_subset"]["num_workers"] == 8 + assert config["data"]["config"]["test_subset"]["num_workers"] == 8 + assert config["callbacks"][0]["init_args"]["patience"] == 4 + assert config["data"]["config"]["tile_config"]["enable_tiler"] is True + assert config["data"]["config"]["tile_config"]["overlap"] == 0.5 + + def test_instantiate(self, tmp_path): + data_root = "tests/assets/car_tree_bug" + config = ConfigConverter.convert(config_path="tests/assets/geti-configs/det.json") + engine, train_kwargs = ConfigConverter.instantiate( + config=config, + work_dir=tmp_path, + data_root=data_root, + ) + assert engine.work_dir == tmp_path + + assert engine.datamodule.config.data_root == data_root + assert engine.datamodule.config.mem_cache_size == "100MB" + assert engine.datamodule.config.train_subset.batch_size == 16 + assert engine.datamodule.config.val_subset.batch_size == 8 + assert engine.datamodule.config.test_subset.batch_size == 8 + assert engine.datamodule.config.train_subset.num_workers == 8 + assert engine.datamodule.config.val_subset.num_workers == 8 + assert engine.datamodule.config.test_subset.num_workers == 8 + assert engine.datamodule.config.tile_config.enable_tiler + + assert len(train_kwargs["callbacks"]) == len(config["callbacks"]) + assert train_kwargs["callbacks"][0].patience == 4 + assert len(train_kwargs["logger"]) == len(config["logger"]) + assert train_kwargs["max_epochs"] == 50