diff --git a/external/mmsegmentation/tests/test_ote_training.py b/external/mmsegmentation/tests/test_ote_training.py index 4355c715be9..20c0a70fcf3 100644 --- a/external/mmsegmentation/tests/test_ote_training.py +++ b/external/mmsegmentation/tests/test_ote_training.py @@ -276,5 +276,8 @@ def test(self, test_parameters, test_case_fx, data_collector_fx, cur_test_expected_metrics_callback_fx): + if "18_OCR" in test_parameters["model_name"] \ + or "x-mod3_OCR" in test_parameters["model_name"]: + pytest.skip("Known issue CVS-83781") test_case_fx.run_stage(test_parameters['test_stage'], data_collector_fx, cur_test_expected_metrics_callback_fx) diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py index 43ced561e11..3569b1ba3ff 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py @@ -15,6 +15,7 @@ from ote_sdk.entities.datasets import DatasetEntity from ote_sdk.entities.inference_parameters import InferenceParameters, default_progress_callback +from ote_sdk.entities.train_parameters import default_progress_callback as train_default_progress_callback from ote_sdk.entities.model import ModelEntity, ModelPrecision # ModelStatus from ote_sdk.entities.resultset import ResultSetEntity from mmcv.utils import ConfigDict @@ -217,6 +218,7 @@ def cancel_training(self): The stopping mechanism allows stopping after each iteration, but validation will still be carried out. Stopping will therefore take some time. """ + self._should_stop = True logger.info("Cancel training requested.") if self.cancel_interface is not None: self.cancel_interface.cancel() @@ -229,8 +231,16 @@ def train(self, output_model: ModelEntity, train_parameters: Optional[TrainParameters] = None): logger.info('train()') + # Check for stop signal between pre-eval and training. + # If training is cancelled at this point, + if self._should_stop: + logger.info('Training cancelled.') + self._should_stop = False + self._is_training = False + return + # Set OTE LoggerHook & Time Monitor - update_progress_callback = default_progress_callback + update_progress_callback = train_default_progress_callback if train_parameters is not None: update_progress_callback = train_parameters.update_progress self._time_monitor = TrainingProgressCallback(update_progress_callback) @@ -238,8 +248,17 @@ def train(self, stage_module = 'ClsTrainer' self._data_cfg = self._init_train_data_cfg(dataset) + self._is_training = True results = self._run_task(stage_module, mode='train', dataset=dataset, parameters=train_parameters) + # Check for stop signal between pre-eval and training. + # If training is cancelled at this point, + if self._should_stop: + logger.info('Training cancelled.') + self._should_stop = False + self._is_training = False + return + # get output model model_ckpt = results.get('final_ckpt') if model_ckpt is None: @@ -257,6 +276,7 @@ def train(self, dashboard_metrics=training_metrics) logger.info(f'Final model performance: {str(performance)}') output_model.performance = performance + self._is_training = False logger.info('train done.') def _init_train_data_cfg(self, dataset: DatasetEntity): diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py index a176834f2f6..81f8827ab47 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py @@ -11,7 +11,7 @@ import torch from mmcv.utils import ConfigDict from detection_tasks.apis.detection.config_utils import remove_from_config -from detection_tasks.apis.detection.ote_utils import TrainingProgressCallback +from detection_tasks.apis.detection.ote_utils import TrainingProgressCallback, InferenceProgressCallback from detection_tasks.extension.utils.hooks import OTELoggerHook from mpa_tasks.apis import BaseTask, TrainType from mpa_tasks.apis.detection import DetectionConfig @@ -67,6 +67,11 @@ def infer(self, ) -> DatasetEntity: logger.info('infer()') + update_progress_callback = default_progress_callback + if inference_parameters is not None: + update_progress_callback = inference_parameters.update_progress + + self._time_monitor = InferenceProgressCallback(len(dataset), update_progress_callback) # If confidence threshold is adaptive then up-to-date value should be stored in the model # and should not be changed during inference. Otherwise user-specified value should be taken. if not self._hyperparams.postprocessing.result_based_confidence_threshold: @@ -75,7 +80,7 @@ def infer(self, stage_module = 'DetectionInferrer' self._data_cfg = self._init_test_data_cfg(dataset) - results = self._run_task(stage_module, mode='train', dataset=dataset) + results = self._run_task(stage_module, mode='train', dataset=dataset, parameters=inference_parameters) # TODO: InferenceProgressCallback register logger.debug(f'result of run_task {stage_module} module = {results}') output = results['outputs'] @@ -310,7 +315,7 @@ def cancel_training(self): will therefore take some time. """ logger.info("Cancel training requested.") - # self._should_stop = True + self._should_stop = True # stop_training_filepath = os.path.join(self._training_work_dir, '.stop_training') # open(stop_training_filepath, 'a').close() if self.cancel_interface is not None: @@ -324,6 +329,14 @@ def train(self, output_model: ModelEntity, train_parameters: Optional[TrainParameters] = None): logger.info('train()') + # Check for stop signal when training has stopped. + # If should_stop is true, training was cancelled and no new + if self._should_stop: + logger.info('Training cancelled.') + self._should_stop = False + self._is_training = False + return + # Set OTE LoggerHook & Time Monitor update_progress_callback = default_progress_callback if train_parameters is not None: @@ -333,8 +346,15 @@ def train(self, stage_module = 'DetectionTrainer' self._data_cfg = self._init_train_data_cfg(dataset) + self._is_training = True results = self._run_task(stage_module, mode='train', dataset=dataset, parameters=train_parameters) - # logger.info(f'result of run_task {stage_module} module = {results}') + + # Check for stop signal when training has stopped. If should_stop is true, training was cancelled and no new + if self._should_stop: + logger.info('Training cancelled.') + self._should_stop = False + self._is_training = False + return # get output model model_ckpt = results.get('final_ckpt') @@ -389,6 +409,7 @@ def train(self, self.save_model(output_model) output_model.performance = performance # output_model.model_status = ModelStatus.SUCCESS + self._is_training = False logger.info('train done.') def _init_train_data_cfg(self, dataset: DatasetEntity): diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py index dda489328e0..7a523507dd9 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py @@ -11,7 +11,7 @@ import torch from mmcv.utils import ConfigDict from segmentation_tasks.apis.segmentation.config_utils import remove_from_config -from segmentation_tasks.apis.segmentation.ote_utils import TrainingProgressCallback +from segmentation_tasks.apis.segmentation.ote_utils import TrainingProgressCallback, InferenceProgressCallback from segmentation_tasks.extension.utils.hooks import OTELoggerHook from mpa import MPAConstants from mpa_tasks.apis import BaseTask, TrainType @@ -22,6 +22,7 @@ from ote_sdk.configuration.helper.utils import ids_to_strings from ote_sdk.entities.datasets import DatasetEntity from ote_sdk.entities.inference_parameters import InferenceParameters +from ote_sdk.entities.inference_parameters import default_progress_callback as default_infer_progress_callback from ote_sdk.entities.label import Domain from ote_sdk.entities.metrics import (CurveMetric, InfoMetric, LineChartInfo, MetricsGroup, Performance, ScoreMetric, @@ -48,8 +49,6 @@ create_annotation_from_segmentation_map, create_hard_prediction_from_soft_prediction) -# from mmdet.apis import export_model - logger = get_logger() @@ -70,12 +69,14 @@ def infer(self, logger.info('infer()') if inference_parameters is not None: - # update_progress_callback = inference_parameters.update_progress + update_progress_callback = inference_parameters.update_progress is_evaluation = inference_parameters.is_evaluation else: - # update_progress_callback = default_infer_progress_callback + update_progress_callback = default_infer_progress_callback is_evaluation = False + self._time_monitor = InferenceProgressCallback(len(dataset), update_progress_callback) + stage_module = 'SegInferrer' self._data_cfg = self._init_test_data_cfg(dataset) self._label_dictionary = dict(enumerate(self._labels, 1)) @@ -187,8 +188,10 @@ def _init_test_data_cfg(self, dataset: DatasetEntity): data_cfg = ConfigDict( data=ConfigDict( train=ConfigDict( - ote_dataset=None, - labels=self._labels, + dataset=ConfigDict( + ote_dataset=None, + labels=self._labels, + ) ), test=ConfigDict( ote_dataset=dataset, @@ -311,7 +314,7 @@ def cancel_training(self): will therefore take some time. """ logger.info("Cancel training requested.") - # self._should_stop = True + self._should_stop = True # stop_training_filepath = os.path.join(self._training_work_dir, '.stop_training') # open(stop_training_filepath, 'a').close() if self.cancel_interface is not None: @@ -325,6 +328,14 @@ def train(self, output_model: ModelEntity, train_parameters: Optional[TrainParameters] = None): logger.info('train()') + # Check for stop signal between pre-eval and training. + # If training is cancelled at this point, + if self._should_stop: + logger.info('Training cancelled.') + self._should_stop = False + self._is_training = False + return + # Set OTE LoggerHook & Time Monitor if train_parameters is not None: update_progress_callback = train_parameters.update_progress @@ -336,8 +347,17 @@ def train(self, # learning_curves = defaultdict(OTELoggerHook.Curve) stage_module = 'SegTrainer' self._data_cfg = self._init_train_data_cfg(dataset) + self._is_training = True results = self._run_task(stage_module, mode='train', dataset=dataset, parameters=train_parameters) + # Check for stop signal when training has stopped. + # If should_stop is true, training was cancelled and no new + if self._should_stop: + logger.info('Training cancelled.') + self._should_stop = False + self._is_training = False + return + # get output model model_ckpt = results.get('final_ckpt') if model_ckpt is None: @@ -358,6 +378,7 @@ def train(self, self.save_model(output_model) output_model.performance = performance # output_model.model_status = ModelStatus.SUCCESS + self._is_training = False logger.info('train done.') def _init_train_data_cfg(self, dataset: DatasetEntity): diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/task.py index f96562185a3..1c24be77b94 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/task.py @@ -29,27 +29,6 @@ logger = get_logger() -class _MPAUpdateProgressCallbackWrapper(UpdateProgressCallback): - """ UpdateProgressCallback wrapper - just wrapping the callback instance and provides error free representation as 'pretty_text' - """ - - def __init__(self, callback, **kwargs): - if not callable(callback): - raise RuntimeError(f'cannot accept a not callable object!! {callback}') - self._callback = callback - super().__init__(**kwargs) - - def __repr__(self): - return f"'{__name__}._MPAUpdateProgressCallbackWrapper'" - - def __reduce__(self): - return (self.__class__, (id(self),)) - - def __call__(self, progress: float, score: Optional[float] = None): - self._callback(progress, score) - - class BaseTask: def __init__(self, task_config, task_environment: TaskEnvironment): self._task_config = task_config @@ -83,6 +62,8 @@ def __init__(self, task_config, task_environment: TaskEnvironment): self._mode = None self._time_monitor = None self._learning_curves = None + self._is_training = False + self._should_stop = False self.cancel_interface = None self.reserved_cancel = False self.on_hook_initialized = self.OnHookInitialized(self) @@ -104,30 +85,9 @@ def _run_task(self, stage_module, mode=None, dataset=None, parameters=None, **kw raise RuntimeError( "'recipe_cfg' is not initialized yet." "call prepare() method before calling this method") - # self._stage_module = stage_module + if mode is not None: self._mode = mode - if parameters is not None: - if isinstance(parameters, TrainParameters): - hook_name = 'TrainProgressUpdateHook' - progress_callback = _MPAUpdateProgressCallbackWrapper(parameters.update_progress) - # TODO: update recipe to do RESUME - if parameters.resume: - pass - elif isinstance(parameters, InferenceParameters): - hook_name = 'InferenceProgressUpdateHook' - progress_callback = _MPAUpdateProgressCallbackWrapper(parameters.update_progress) - else: - hook_name = 'ProgressUpdateHook' - progress_callback = None - logger.info(f'progress callback = {progress_callback}, hook name = {hook_name}') - if progress_callback is not None: - progress_update_hook_cfg = ConfigDict( - type='ProgressUpdateHook', - name=hook_name, - callback=progress_callback - ) - update_or_add_custom_hook(self._recipe_cfg, progress_update_hook_cfg) common_cfg = ConfigDict(dict(output_path=self._output_path)) @@ -152,6 +112,14 @@ def finalize(self): if os.path.exists(self._output_path): shutil.rmtree(self._output_path, ignore_errors=False) + def _delete_scratch_space(self): + """ + Remove model checkpoints and mpa logs + """ + + if os.path.exists(self._output_path): + shutil.rmtree(self._output_path, ignore_errors=False) + def __del__(self): self.finalize() diff --git a/external/model-preparation-algorithm/ote_tests_pytest.ini b/external/model-preparation-algorithm/ote_tests_pytest.ini index 708cf1020ff..440f43f8b26 100644 --- a/external/model-preparation-algorithm/ote_tests_pytest.ini +++ b/external/model-preparation-algorithm/ote_tests_pytest.ini @@ -1,2 +1,2 @@ [pytest] -python_files = test_*_cls_il.py +python_files = test_ote_*.py diff --git a/external/model-preparation-algorithm/submodule b/external/model-preparation-algorithm/submodule index e97cd17d1e1..8d36bf59448 160000 --- a/external/model-preparation-algorithm/submodule +++ b/external/model-preparation-algorithm/submodule @@ -1 +1 @@ -Subproject commit e97cd17d1e19e00b74d16140ac77ea846d791f67 +Subproject commit 8d36bf5944837b7a3d22fc2c3a4cb93423619fc2 diff --git a/external/model-preparation-algorithm/tests/config.py b/external/model-preparation-algorithm/tests/config.py new file mode 100644 index 00000000000..1c65369aebe --- /dev/null +++ b/external/model-preparation-algorithm/tests/config.py @@ -0,0 +1,11 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +try: + import os + from e2e import config as config_e2e + + config_e2e.repository_name = os.environ.get('TT_REPOSITORY_NAME', 'ote/training_extensions/external/model-preparation-algorithm') +except ImportError: + pass diff --git a/external/model-preparation-algorithm/tests/conftest.py b/external/model-preparation-algorithm/tests/conftest.py new file mode 100644 index 00000000000..718e41b1051 --- /dev/null +++ b/external/model-preparation-algorithm/tests/conftest.py @@ -0,0 +1,62 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +try: + import e2e.fixtures + + from e2e.conftest_utils import * # noqa + from e2e.conftest_utils import pytest_addoption as _e2e_pytest_addoption # noqa + from e2e import config # noqa + from e2e.utils import get_plugins_from_packages + pytest_plugins = get_plugins_from_packages([e2e]) +except ImportError: + _e2e_pytest_addoption = None + pass +import config +import pytest +from ote_sdk.test_suite.pytest_insertions import * +from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT + +pytest_plugins = get_pytest_plugins_from_ote() + +ote_conftest_insertion(default_repository_name='ote/training_extensions/external/model-preparation-algorithm') + +@pytest.fixture +def ote_test_domain_fx(): + return 'model-preparation-algorithm' + +@pytest.fixture +def ote_test_scenario_fx(current_test_parameters_fx): + assert isinstance(current_test_parameters_fx, dict) + if current_test_parameters_fx.get('usecase') == REALLIFE_USECASE_CONSTANT: + return 'performance' + else: + return 'integration' + +@pytest.fixture(scope='session') +def ote_templates_root_dir_fx(): + import os.path as osp + import logging + logger = logging.getLogger(__name__) + root = osp.dirname(osp.dirname(osp.realpath(__file__))) + root = f'{root}/configs/' + logger.debug(f'overloaded ote_templates_root_dir_fx: return {root}') + return root + +@pytest.fixture(scope='session') +def ote_reference_root_dir_fx(): + import os.path as osp + import logging + logger = logging.getLogger(__name__) + root = osp.dirname(osp.dirname(osp.realpath(__file__))) + root = f'{root}/tests/reference/' + logger.debug(f'overloaded ote_reference_root_dir_fx: return {root}') + return root + +# pytest magic +def pytest_generate_tests(metafunc): + ote_pytest_generate_tests_insertion(metafunc) + +def pytest_addoption(parser): + ote_pytest_addoption_insertion(parser) diff --git a/external/model-preparation-algorithm/tests/expected_metrics/metrics_test_ote_training.yml b/external/model-preparation-algorithm/tests/expected_metrics/metrics_test_ote_training.yml new file mode 100644 index 00000000000..5a6d96a567b --- /dev/null +++ b/external/model-preparation-algorithm/tests/expected_metrics/metrics_test_ote_training.yml @@ -0,0 +1,104 @@ +? "ACTION-training_evaluation,model-ClassIncremental_Image_Classification_EfficinetNet-B0,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "target_value": 0.96 + "max_diff_if_less_threshold": 0.015 + "max_diff_if_greater_threshold": 0.02 +? "ACTION-export_evaluation,model-ClassIncremental_Image_Classification_EfficinetNet-B0,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "training_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Image_Classification_EfficinetNet-B0,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "export_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.05 +? "ACTION-training_evaluation,model-ClassIncremental_Image_Classification_EfficinetNet-V2-S,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "target_value": 0.96 + "max_diff_if_less_threshold": 0.015 + "max_diff_if_greater_threshold": 0.02 +? "ACTION-export_evaluation,model-ClassIncremental_Image_Classification_EfficinetNet-V2-S,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "training_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Image_Classification_EfficinetNet-V2-S,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "export_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.05 +? "ACTION-training_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-large-1x,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "target_value": 0.94 + "max_diff_if_less_threshold": 0.015 + "max_diff_if_greater_threshold": 0.02 +? "ACTION-export_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-large-1x,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "training_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-large-1x,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "export_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.05 +? "ACTION-training_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-large-0.75x,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "target_value": 0.94 + "max_diff_if_less_threshold": 0.015 + "max_diff_if_greater_threshold": 0.02 +? "ACTION-export_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-large-0.75x,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "training_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-large-0.75x,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "export_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.05 +? "ACTION-training_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-small,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "target_value": 0.94 + "max_diff_if_less_threshold": 0.015 + "max_diff_if_greater_threshold": 0.02 +? "ACTION-export_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-small,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "training_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Image_Classification_MobileNet-V3-small,dataset-cifar10_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Accuracy": + "base": "export_evaluation.metrics.accuracy.Accuracy" + "max_diff": 0.05 +? "ACTION-training_evaluation,model-ClassIncremental_Object_Detection_Gen3_ATSS,dataset-coco_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.f-measure": + "target_value": 0.66 + "max_diff_if_less_threshold": 0.01 + "max_diff_if_greater_threshold": 0.04 +? "ACTION-export_evaluation,model-ClassIncremental_Object_Detection_Gen3_ATSS,dataset-coco_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.f-measure": + "base": "training_evaluation.metrics.accuracy.f-measure" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Object_Detection_Gen3_ATSS,dataset-coco_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.f-measure": + "base": "export_evaluation.metrics.accuracy.f-measure" + "max_diff": 0.04 +? "ACTION-training_evaluation,model-ClassIncremental_Object_Detection_Gen3_VFNet,dataset-coco_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.f-measure": + "target_value": 0.65 + "max_diff_if_less_threshold": 0.01 + "max_diff_if_greater_threshold": 0.04 +? "ACTION-export_evaluation,model-ClassIncremental_Object_Detection_Gen3_VFNet,dataset-coco_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.f-measure": + "base": "training_evaluation.metrics.accuracy.f-measure" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Object_Detection_Gen3_VFNet,dataset-coco_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.f-measure": + "base": "export_evaluation.metrics.accuracy.f-measure" + "max_diff": 0.04 +? "ACTION-training_evaluation,model-ClassIncremental_Semantic_Segmentation_Lite-HRNet-18_OCR,dataset-voc_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Dice Average": + "target_value": 0.48 + "max_diff_if_less_threshold": 0.01 + "max_diff_if_greater_threshold": 0.05 +? "ACTION-export_evaluation,model-ClassIncremental_Semantic_Segmentation_Lite-HRNet-18_OCR,dataset-voc_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Dice Average": + "base": "training_evaluation.metrics.accuracy.Dice Average" + "max_diff": 0.01 +? "ACTION-pot_evaluation,model-ClassIncremental_Semantic_Segmentation_Lite-HRNet-18_OCR,dataset-voc_cls_incr,num_iters-CONFIG,batch-CONFIG,usecase-reallife" +: "metrics.accuracy.Dice Average": + "base": "export_evaluation.metrics.accuracy.Dice Average" + "max_diff": 0.04 diff --git a/external/model-preparation-algorithm/tests/ote_cli/test_cls_cls_il.py b/external/model-preparation-algorithm/tests/ote_cli/test_ote_cls_cls_il.py similarity index 100% rename from external/model-preparation-algorithm/tests/ote_cli/test_cls_cls_il.py rename to external/model-preparation-algorithm/tests/ote_cli/test_ote_cls_cls_il.py diff --git a/external/model-preparation-algorithm/tests/ote_cli/test_det_cls_il.py b/external/model-preparation-algorithm/tests/ote_cli/test_ote_det_cls_il.py similarity index 100% rename from external/model-preparation-algorithm/tests/ote_cli/test_det_cls_il.py rename to external/model-preparation-algorithm/tests/ote_cli/test_ote_det_cls_il.py diff --git a/external/model-preparation-algorithm/tests/ote_cli/test_seg_cls_il.py b/external/model-preparation-algorithm/tests/ote_cli/test_ote_seg_cls_il.py similarity index 100% rename from external/model-preparation-algorithm/tests/ote_cli/test_seg_cls_il.py rename to external/model-preparation-algorithm/tests/ote_cli/test_ote_seg_cls_il.py diff --git a/external/model-preparation-algorithm/tests/test_ote_api.py b/external/model-preparation-algorithm/tests/test_ote_api.py new file mode 100644 index 00000000000..50dd09e38cc --- /dev/null +++ b/external/model-preparation-algorithm/tests/test_ote_api.py @@ -0,0 +1,684 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import glob +import os.path as osp +import random +import time +import unittest +import warnings +from concurrent.futures import ThreadPoolExecutor +from typing import Optional + +import cv2 as cv +import numpy as np +from bson import ObjectId +from detection_tasks.apis.detection.ote_utils import generate_label_schema +from mpa_tasks.apis import BaseTask +from mpa_tasks.apis.classification import ( + ClassificationInferenceTask, + ClassificationTrainTask, +) +from mpa_tasks.apis.detection import DetectionInferenceTask, DetectionTrainTask +from mpa_tasks.apis.segmentation import SegmentationInferenceTask, SegmentationTrainTask +from ote_sdk.configuration.helper import create +from ote_sdk.entities.annotation import ( + Annotation, + AnnotationSceneEntity, + AnnotationSceneKind, +) +from ote_sdk.entities.color import Color +from ote_sdk.entities.dataset_item import DatasetItemEntity +from ote_sdk.entities.datasets import DatasetEntity +from ote_sdk.entities.id import ID +from ote_sdk.entities.image import Image +from ote_sdk.entities.inference_parameters import InferenceParameters +from ote_sdk.entities.label import Domain, LabelEntity +from ote_sdk.entities.label_schema import LabelGroup, LabelGroupType, LabelSchemaEntity +from ote_sdk.entities.metrics import Performance +from ote_sdk.entities.model import ModelEntity +from ote_sdk.entities.model_template import ( + TaskType, + parse_model_template, + task_type_to_label_domain, +) +from ote_sdk.entities.resultset import ResultSetEntity +from ote_sdk.entities.scored_label import ScoredLabel +from ote_sdk.entities.shapes.ellipse import Ellipse +from ote_sdk.entities.shapes.polygon import Point, Polygon +from ote_sdk.entities.shapes.rectangle import Rectangle +from ote_sdk.entities.subset import Subset +from ote_sdk.entities.task_environment import TaskEnvironment +from ote_sdk.entities.train_parameters import TrainParameters +from ote_sdk.test_suite.e2e_test_system import e2e_pytest_api +from ote_sdk.tests.test_helpers import generate_random_annotated_image +from ote_sdk.usecases.tasks.interfaces.export_interface import ExportType +from ote_sdk.utils.shape_factory import ShapeFactory + +DEFAULT_CLS_TEMPLATE_DIR = osp.join('configs', 'classification', 'efficientnet_b0_cls_incr') +DEFAULT_DET_TEMPLATE_DIR = osp.join('configs', 'detection', 'mobilenetv2_atss_cls_incr') +DEFAULT_SEG_TEMPLATE_DIR = osp.join('configs', 'segmentation', 'ocr-lite-hrnet-18-cls-incr') + + +def eval(task: BaseTask, model: ModelEntity, dataset: DatasetEntity) -> Performance: + start_time = time.time() + result_dataset = task.infer(dataset.with_empty_annotations()) + end_time = time.time() + print(f'{len(dataset)} analysed in {end_time - start_time} seconds') + result_set = ResultSetEntity( + model=model, + ground_truth_dataset=dataset, + prediction_dataset=result_dataset + ) + task.evaluate(result_set) + assert result_set.performance is not None + return result_set.performance + + +class MPAClsAPI(unittest.TestCase): + @e2e_pytest_api + def test_reading_classification_cls_incr_model_template(self): + classification_template = ['efficientnet_b0_cls_incr', 'efficientnet_v2_s_cls_incr', + 'mobilenet_v3_large_1_cls_incr', 'mobilenet_v3_large_075_cls_incr', + 'mobilenet_v3_small_cls_incr'] + for model_template in classification_template: + parse_model_template(osp.join('configs', 'classification', model_template, 'template.yaml')) + + @staticmethod + def generate_label_schema(not_empty_labels, multilabel=False): + assert len(not_empty_labels) > 1 + + label_schema = LabelSchemaEntity() + if multilabel: + emptylabel = LabelEntity(name="Empty label", is_empty=True, domain=Domain.CLASSIFICATION) + empty_group = LabelGroup(name="empty", labels=[emptylabel], group_type=LabelGroupType.EMPTY_LABEL) + for label in not_empty_labels: + label_schema.add_group(LabelGroup(name=label.name, labels=[label], group_type=LabelGroupType.EXCLUSIVE)) + label_schema.add_group(empty_group) + else: + main_group = LabelGroup(name="labels", labels=not_empty_labels, group_type=LabelGroupType.EXCLUSIVE) + label_schema.add_group(main_group) + return label_schema + + @staticmethod + def setup_configurable_parameters(template_dir, num_iters=10): + model_template = parse_model_template(osp.join(template_dir, 'template.yaml')) + hyper_parameters = create(model_template.hyper_parameters.data) + hyper_parameters.learning_parameters.num_iters = num_iters + return hyper_parameters, model_template + + def init_environment(self, params, model_template, number_of_images=10): + resolution = (224, 224) + colors = [(0,255,0), (0,0,255)] + cls_names = ['b', 'g'] + texts = ['Blue', 'Green'] + env_labels = [LabelEntity(name=name, domain=Domain.CLASSIFICATION, is_empty=False, id=ID(i)) for i, name in + enumerate(cls_names)] + + items = [] + + for _ in range(0, number_of_images): + for j, lbl in enumerate(env_labels): + class_img = np.zeros((*resolution, 3), dtype=np.uint8) + class_img[:] = colors[j] + class_img = cv.putText(class_img, texts[j], (50, 50), cv.FONT_HERSHEY_SIMPLEX, + .8 + j*.2, colors[j - 1], 2, cv.LINE_AA) + + image = Image(data=class_img) + labels = [ScoredLabel(label=lbl, probability=1.0)] + shapes = [Annotation(Rectangle.generate_full_box(), labels)] + annotation_scene = AnnotationSceneEntity(kind=AnnotationSceneKind.ANNOTATION, + annotations=shapes) + items.append(DatasetItemEntity(media=image, annotation_scene=annotation_scene)) + + rng = random.Random() + rng.seed(100) + rng.shuffle(items) + for i, _ in enumerate(items): + subset_region = i / number_of_images + if subset_region >= 0.9: + subset = Subset.TESTING + elif subset_region >= 0.6: + subset = Subset.VALIDATION + else: + subset = Subset.TRAINING + items[i].subset = subset + + dataset = DatasetEntity(items) + labels_schema = self.generate_label_schema(dataset.get_labels(), multilabel=False) + environment = TaskEnvironment(model=None, hyper_parameters=params, label_schema=labels_schema, + model_template=model_template) + return environment, dataset + + @e2e_pytest_api + def test_training_progress_tracking(self): + print('Task initialized, model training starts.') + training_progress_curve = [] + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_CLS_TEMPLATE_DIR, num_iters=5) + task_environment, dataset = self.init_environment(hyper_parameters, model_template, 20) + task = ClassificationTrainTask(task_environment=task_environment) + task._delete_scratch_space() + + def progress_callback(progress: float, score: Optional[float] = None): + training_progress_curve.append(progress) + + train_parameters = TrainParameters + train_parameters.update_progress = progress_callback + output_model = ModelEntity( + dataset, + task_environment.get_model_configuration(), + ) + task.train(dataset, output_model, train_parameters) + + assert len(training_progress_curve) > 0 + training_progress_curve = np.asarray(training_progress_curve) + print(training_progress_curve) + assert np.all(training_progress_curve[1:] >= training_progress_curve[:-1]) + + @e2e_pytest_api + def test_inference_progress_tracking(self): + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_CLS_TEMPLATE_DIR, num_iters=5) + task_environment, dataset = self.init_environment(hyper_parameters, model_template, 20) + task = ClassificationInferenceTask(task_environment=task_environment) + task._delete_scratch_space() + + print('Task initialized, model inference starts.') + inference_progress_curve = [] + + def progress_callback(progress: int): + inference_progress_curve.append(progress) + + inference_parameters = InferenceParameters + inference_parameters.update_progress = progress_callback + + task.infer(dataset.with_empty_annotations(), inference_parameters) + + assert len(inference_progress_curve) > 0 + inference_progress_curve = np.asarray(inference_progress_curve) + assert np.all(inference_progress_curve[1:] >= inference_progress_curve[:-1]) + + @e2e_pytest_api + def test_inference_task(self): + # Prepare pretrained weights + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_CLS_TEMPLATE_DIR, num_iters=2) + classification_environment, dataset = self.init_environment(hyper_parameters, model_template, 50) + val_dataset = dataset.get_subset(Subset.VALIDATION) + + train_task = ClassificationTrainTask(task_environment=classification_environment) + self.addCleanup(train_task._delete_scratch_space) + + trained_model = ModelEntity( + dataset, + classification_environment.get_model_configuration(), + ) + train_task.train(dataset, trained_model, TrainParameters) + performance_after_train = eval(train_task, trained_model, val_dataset) + + # Create InferenceTask + classification_environment.model = trained_model + inference_task = ClassificationInferenceTask(task_environment=classification_environment) + self.addCleanup(inference_task._delete_scratch_space) + + performance_after_load = eval(inference_task, trained_model, val_dataset) + + assert performance_after_train == performance_after_load + + # Export + exported_model = ModelEntity( + dataset, + classification_environment.get_model_configuration(), + _id=ObjectId()) + inference_task.export(ExportType.OPENVINO, exported_model) + + +class MPADetAPI(unittest.TestCase): + """ + Collection of tests for OTE API and OTE Model Templates + """ + @e2e_pytest_api + def test_reading_detection_cls_incr_model_template(self): + detection_template = ['mobilenetv2_atss_cls_incr', 'resnet50_vfnet_cls_incr'] + for model_template in detection_template: + parse_model_template(osp.join('configs', 'detection', model_template, 'template.yaml')) + + def init_environment( + self, + params, + model_template, + number_of_images=500, + task_type=TaskType.DETECTION): + + labels_names = ('rectangle', 'ellipse', 'triangle') + labels_schema = generate_label_schema(labels_names, task_type_to_label_domain(task_type)) + labels_list = labels_schema.get_labels(False) + environment = TaskEnvironment(model=None, hyper_parameters=params, label_schema=labels_schema, + model_template=model_template) + + warnings.filterwarnings('ignore', message='.* coordinates .* are out of bounds.*') + items = [] + for i in range(0, number_of_images): + image_numpy, annos = generate_random_annotated_image( + image_width=640, + image_height=480, + labels=labels_list, + max_shapes=20, + min_size=50, + max_size=100, + random_seed=None) + # Convert shapes according to task + for anno in annos: + if task_type == TaskType.INSTANCE_SEGMENTATION: + anno.shape = ShapeFactory.shape_as_polygon(anno.shape) + else: + anno.shape = ShapeFactory.shape_as_rectangle(anno.shape) + + image = Image(data=image_numpy) + annotation_scene = AnnotationSceneEntity( + kind=AnnotationSceneKind.ANNOTATION, + annotations=annos) + items.append(DatasetItemEntity(media=image, annotation_scene=annotation_scene)) + warnings.resetwarnings() + + rng = random.Random() + rng.shuffle(items) + for i, _ in enumerate(items): + subset_region = i / number_of_images + if subset_region >= 0.8: + subset = Subset.TESTING + elif subset_region >= 0.6: + subset = Subset.VALIDATION + else: + subset = Subset.TRAINING + items[i].subset = subset + + dataset = DatasetEntity(items) + return environment, dataset + + @staticmethod + def setup_configurable_parameters(template_dir, num_iters=10): + glb = glob.glob(f'{template_dir}/template*.yaml') + template_path = glb[0] if glb else None + if not template_path: + raise RuntimeError(f"Template YAML not found: {template_dir}") + + model_template = parse_model_template(template_path) + hyper_parameters = create(model_template.hyper_parameters.data) + hyper_parameters.learning_parameters.num_iters = num_iters + hyper_parameters.postprocessing.result_based_confidence_threshold = False + hyper_parameters.postprocessing.confidence_threshold = 0.1 + return hyper_parameters, model_template + + @e2e_pytest_api + def test_cancel_training_detection(self): + """ + Tests starting and cancelling training. + + Flow of the test: + - Creates a randomly annotated project with a small dataset containing 3 classes: + ['rectangle', 'triangle', 'circle']. + - Start training and give cancel training signal after 10 seconds. Assert that training + stops within 35 seconds after that + - Start training and give cancel signal immediately. Assert that training stops within 25 seconds. + + This test should be finished in under one minute on a workstation. + """ + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_DET_TEMPLATE_DIR, num_iters=500) + detection_environment, dataset = self.init_environment(hyper_parameters, model_template, 64) + + detection_task = DetectionTrainTask(task_environment=detection_environment) + + executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='train_thread') + + output_model = ModelEntity( + dataset, + detection_environment.get_model_configuration(), + ) + + training_progress_curve = [] + def progress_callback(progress: float, score: Optional[float] = None): + training_progress_curve.append(progress) + + train_parameters = TrainParameters + train_parameters.update_progress = progress_callback + + # Test stopping after some time + start_time = time.time() + train_future = executor.submit(detection_task.train, dataset, output_model, train_parameters) + # give train_thread some time to initialize the model + while not detection_task._is_training: + time.sleep(10) + detection_task.cancel_training() + + # stopping process has to happen in less than 35 seconds + train_future.result() + self.assertEqual(training_progress_curve[-1], 100) + self.assertLess(time.time() - start_time, 100, 'Expected to stop within 100 seconds.') + + # Test stopping immediately + start_time = time.time() + train_future = executor.submit(detection_task.train, dataset, output_model) + detection_task.cancel_training() + + train_future.result() + self.assertLess(time.time() - start_time, 25) # stopping process has to happen in less than 25 seconds + + @e2e_pytest_api + def test_training_progress_tracking(self): + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_DET_TEMPLATE_DIR, num_iters=5) + detection_environment, dataset = self.init_environment(hyper_parameters, model_template, 50) + + task = DetectionTrainTask(task_environment=detection_environment) + self.addCleanup(task._delete_scratch_space) + + print('Task initialized, model training starts.') + training_progress_curve = [] + + def progress_callback(progress: float, score: Optional[float] = None): + training_progress_curve.append(progress) + + train_parameters = TrainParameters + train_parameters.update_progress = progress_callback + output_model = ModelEntity( + dataset, + detection_environment.get_model_configuration(), + ) + task.train(dataset, output_model, train_parameters) + + self.assertGreater(len(training_progress_curve), 0) + training_progress_curve = np.asarray(training_progress_curve) + self.assertTrue(np.all(training_progress_curve[1:] >= training_progress_curve[:-1])) + + @e2e_pytest_api + def test_inference_progress_tracking(self): + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_DET_TEMPLATE_DIR, num_iters=10) + detection_environment, dataset = self.init_environment(hyper_parameters, model_template, 50) + + task = DetectionInferenceTask(task_environment=detection_environment) + self.addCleanup(task._delete_scratch_space) + + print('Task initialized, model inference starts.') + inference_progress_curve = [] + + def progress_callback(progress: int): + assert isinstance(progress, int) + inference_progress_curve.append(progress) + + inference_parameters = InferenceParameters + inference_parameters.update_progress = progress_callback + + task.infer(dataset.with_empty_annotations(), inference_parameters) + + self.assertGreater(len(inference_progress_curve), 0) + inference_progress_curve = np.asarray(inference_progress_curve) + self.assertTrue(np.all(inference_progress_curve[1:] >= inference_progress_curve[:-1])) + + @e2e_pytest_api + def test_inference_task(self): + # Prepare pretrained weights + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_DET_TEMPLATE_DIR, num_iters=2) + detection_environment, dataset = self.init_environment(hyper_parameters, model_template, 50) + val_dataset = dataset.get_subset(Subset.VALIDATION) + + train_task = DetectionTrainTask(task_environment=detection_environment) + self.addCleanup(train_task._delete_scratch_space) + + trained_model = ModelEntity( + dataset, + detection_environment.get_model_configuration(), + ) + train_task.train(dataset, trained_model, TrainParameters) + performance_after_train = eval(train_task, trained_model, val_dataset) + + # Create InferenceTask + detection_environment.model = trained_model + inference_task = DetectionInferenceTask(task_environment=detection_environment) + self.addCleanup(inference_task._delete_scratch_space) + + performance_after_load = eval(inference_task, trained_model, val_dataset) + + assert performance_after_train == performance_after_load + + # Export + exported_model = ModelEntity( + dataset, + detection_environment.get_model_configuration(), + _id=ObjectId()) + inference_task.export(ExportType.OPENVINO, exported_model) + + +class MPASegAPI(unittest.TestCase): + """ + Collection of tests for OTE API and OTE Model Templates + """ + @e2e_pytest_api + def test_reading_segmentation_cls_incr_model_template(self): + segmentation_template = ['ocr-lite-hrnet-18-cls-incr'] + for model_template in segmentation_template: + parse_model_template(osp.join('configs', 'segmentation', model_template, 'template.yaml')) + + @staticmethod + def generate_label_schema(label_names): + label_domain = Domain.SEGMENTATION + rgb = [int(i) for i in np.random.randint(0, 256, 3)] + colors = [Color(*rgb) for _ in range(len(label_names))] + not_empty_labels = [LabelEntity(name=name, color=colors[i], domain=label_domain, id=i) for i, name in + enumerate(label_names)] + empty_label = LabelEntity(name=f"Empty label", color=Color(42, 43, 46), + is_empty=True, domain=label_domain, id=len(not_empty_labels)) + + label_schema = LabelSchemaEntity() + exclusive_group = LabelGroup(name="labels", labels=not_empty_labels, group_type=LabelGroupType.EXCLUSIVE) + empty_group = LabelGroup(name="empty", labels=[empty_label], group_type=LabelGroupType.EMPTY_LABEL) + label_schema.add_group(exclusive_group) + label_schema.add_group(empty_group) + return label_schema + + def init_environment(self, params, model_template, number_of_images=10): + labels_names = ('rectangle', 'ellipse', 'triangle') + labels_schema = self.generate_label_schema(labels_names) + labels_list = labels_schema.get_labels(False) + environment = TaskEnvironment(model=None, hyper_parameters=params, label_schema=labels_schema, + model_template=model_template) + + warnings.filterwarnings('ignore', message='.* coordinates .* are out of bounds.*') + items = [] + for i in range(0, number_of_images): + image_numpy, shapes = generate_random_annotated_image(image_width=640, + image_height=480, + labels=labels_list, + max_shapes=20, + min_size=50, + max_size=100, + random_seed=None) + # Convert all shapes to polygons + out_shapes = [] + for shape in shapes: + shape_labels = shape.get_labels(include_empty=True) + + in_shape = shape.shape + if isinstance(in_shape, Rectangle): + points = [ + Point(in_shape.x1, in_shape.y1), + Point(in_shape.x2, in_shape.y1), + Point(in_shape.x2, in_shape.y2), + Point(in_shape.x1, in_shape.y2), + ] + elif isinstance(in_shape, Ellipse): + points = [Point(x, y) for x, y in in_shape.get_evenly_distributed_ellipse_coordinates()] + elif isinstance(in_shape, Polygon): + points = in_shape.points + + out_shapes.append(Annotation(Polygon(points=points), labels=shape_labels)) + + image = Image(data=image_numpy) + annotation = AnnotationSceneEntity( + kind=AnnotationSceneKind.ANNOTATION, + annotations=out_shapes) + items.append(DatasetItemEntity(media=image, annotation_scene=annotation)) + warnings.resetwarnings() + + rng = random.Random() + rng.shuffle(items) + for i, _ in enumerate(items): + subset_region = i / number_of_images + if subset_region >= 0.8: + subset = Subset.TESTING + elif subset_region >= 0.6: + subset = Subset.VALIDATION + else: + subset = Subset.TRAINING + + items[i].subset = subset + + dataset = DatasetEntity(items) + + return environment, dataset + + @staticmethod + def setup_configurable_parameters(template_dir, num_iters=10): + model_template = parse_model_template(osp.join(template_dir, 'template.yaml')) + + hyper_parameters = create(model_template.hyper_parameters.data) + hyper_parameters.learning_parameters.learning_rate_fixed_iters = 0 + hyper_parameters.learning_parameters.learning_rate_warmup_iters = 1 + hyper_parameters.learning_parameters.num_iters = num_iters + hyper_parameters.learning_parameters.num_checkpoints = 1 + + return hyper_parameters, model_template + + @e2e_pytest_api + def test_cancel_training_segmentation(self): + """ + Tests starting and cancelling training. + + Flow of the test: + - Creates a randomly annotated project with a small dataset. + - Start training and give cancel training signal after 10 seconds. Assert that training + stops within 35 seconds after that + - Start training and give cancel signal immediately. Assert that training stops within 25 seconds. + + This test should be finished in under one minute on a workstation. + """ + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_SEG_TEMPLATE_DIR, num_iters=200) + segmentation_environment, dataset = self.init_environment(hyper_parameters, model_template, 64) + + segmentation_task = SegmentationTrainTask(task_environment=segmentation_environment) + + executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='train_thread') + + output_model = ModelEntity( + dataset, + segmentation_environment.get_model_configuration(), + ) + + training_progress_curve = [] + def progress_callback(progress: float, score: Optional[float] = None): + training_progress_curve.append(progress) + + train_parameters = TrainParameters + train_parameters.update_progress = progress_callback + + # Test stopping after some time + start_time = time.time() + train_future = executor.submit(segmentation_task.train, dataset, output_model, train_parameters) + # give train_thread some time to initialize the model + while not segmentation_task._is_training: + time.sleep(10) + segmentation_task.cancel_training() + + # stopping process has to happen in less than 35 seconds + train_future.result() + self.assertEqual(training_progress_curve[-1], 100) + self.assertLess(time.time() - start_time, 100, 'Expected to stop within 100 seconds.') + + # Test stopping immediately + start_time = time.time() + train_future = executor.submit(segmentation_task.train, dataset, output_model) + segmentation_task.cancel_training() + + train_future.result() + self.assertLess(time.time() - start_time, 25) # stopping process has to happen in less than 25 seconds + + @e2e_pytest_api + def test_training_progress_tracking(self): + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_SEG_TEMPLATE_DIR, num_iters=5) + segmentation_environment, dataset = self.init_environment(hyper_parameters, model_template, 12) + + task = SegmentationTrainTask(task_environment=segmentation_environment) + #self.addCleanup(task._delete_scratch_space) + + print('Task initialized, model training starts.') + training_progress_curve = [] + + def progress_callback(progress: float, score: Optional[float] = None): + training_progress_curve.append(progress) + + train_parameters = TrainParameters + train_parameters.update_progress = progress_callback + output_model = ModelEntity( + dataset, + segmentation_environment.get_model_configuration(), + ) + task.train(dataset, output_model, train_parameters) + + self.assertGreater(len(training_progress_curve), 0) + training_progress_curve = np.asarray(training_progress_curve) + self.assertTrue(np.all(training_progress_curve[1:] >= training_progress_curve[:-1])) + + @e2e_pytest_api + def test_inference_progress_tracking(self): + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_SEG_TEMPLATE_DIR, num_iters=10) + segmentation_environment, dataset = self.init_environment(hyper_parameters, model_template, 12) + + task = SegmentationInferenceTask(task_environment=segmentation_environment) + self.addCleanup(task._delete_scratch_space) + + print('Task initialized, model inference starts.') + inference_progress_curve = [] + + def progress_callback(progress: int): + assert isinstance(progress, int) + inference_progress_curve.append(progress) + + inference_parameters = InferenceParameters + inference_parameters.update_progress = progress_callback + + task.infer(dataset.with_empty_annotations(), inference_parameters) + + self.assertGreater(len(inference_progress_curve), 0) + inference_progress_curve = np.asarray(inference_progress_curve) + self.assertTrue(np.all(inference_progress_curve[1:] >= inference_progress_curve[:-1])) + + @e2e_pytest_api + def test_inference_task(self): + # Prepare pretrained weights + hyper_parameters, model_template = self.setup_configurable_parameters(DEFAULT_SEG_TEMPLATE_DIR, num_iters=2) + segmentation_environment, dataset = self.init_environment(hyper_parameters, model_template, 30) + val_dataset = dataset.get_subset(Subset.VALIDATION) + + train_task = SegmentationTrainTask(task_environment=segmentation_environment) + self.addCleanup(train_task._delete_scratch_space) + + trained_model = ModelEntity( + dataset, + segmentation_environment.get_model_configuration(), + ) + train_task.train(dataset, trained_model, TrainParameters) + performance_after_train = eval(train_task, trained_model, val_dataset) + + # Create InferenceTask + segmentation_environment.model = trained_model + inference_task = SegmentationInferenceTask(task_environment=segmentation_environment) + self.addCleanup(inference_task._delete_scratch_space) + + performance_after_load = eval(inference_task, trained_model, val_dataset) + + assert performance_after_train == performance_after_load + + # Export + exported_model = ModelEntity( + dataset, + segmentation_environment.get_model_configuration(), + _id=ObjectId()) + inference_task.export(ExportType.OPENVINO, exported_model) diff --git a/external/model-preparation-algorithm/tests/test_ote_training.py b/external/model-preparation-algorithm/tests/test_ote_training.py new file mode 100644 index 00000000000..5312700bef1 --- /dev/null +++ b/external/model-preparation-algorithm/tests/test_ote_training.py @@ -0,0 +1,574 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import logging +import os +import os.path as osp +from collections import namedtuple +from copy import deepcopy +from pprint import pformat +from typing import Any, Callable, Dict, List, Optional, Type + +import pytest +from ote_sdk.entities.datasets import DatasetEntity +from ote_sdk.entities.label import Domain +from ote_sdk.entities.label_schema import LabelSchemaEntity +from ote_sdk.entities.subset import Subset + +from torchreid_tasks.utils import ClassificationDatasetAdapter +from detection_tasks.extension.datasets.data_utils import load_dataset_items_coco_format +from segmentation_tasks.extension.datasets.mmdataset import load_dataset_items + +from ote_sdk.test_suite.e2e_test_system import DataCollector, e2e_pytest_performance +from ote_sdk.test_suite.training_test_case import (OTETestCaseInterface, + generate_ote_integration_test_case_class) +from ote_sdk.test_suite.training_tests_common import (make_path_be_abs, + make_paths_be_abs, + KEEP_CONFIG_FIELD_VALUE, + REALLIFE_USECASE_CONSTANT, + ROOT_PATH_KEY) +from ote_sdk.test_suite.training_tests_helper import (OTETestHelper, + DefaultOTETestCreationParametersInterface, + OTETrainingTestInterface) +from ote_sdk.test_suite.training_tests_actions import (OTETestTrainingAction, + BaseOTETestAction, + OTETestTrainingEvaluationAction, + OTETestExportAction, + OTETestExportEvaluationAction, + OTETestPotAction, + OTETestPotEvaluationAction) + + +logger = logging.getLogger(__name__) + +def DATASET_PARAMETERS_FIELDS() -> List[str]: + return deepcopy(['annotations_train', + 'images_train_dir', + 'annotations_val', + 'images_val_dir', + 'annotations_test', + 'images_test_dir', + 'pre_trained_model', + ]) + +DatasetParameters = namedtuple('DatasetParameters', DATASET_PARAMETERS_FIELDS()) + +def get_test_action_classes() -> List[Type[BaseOTETestAction]]: + return [ + OTETestTrainingAction, + OTETestTrainingEvaluationAction, + OTETestExportAction, + OTETestExportEvaluationAction, + OTETestPotAction, + OTETestPotEvaluationAction, + ] + + +def _get_dataset_params_from_dataset_definitions(dataset_definitions, dataset_name): + if dataset_name not in dataset_definitions: + raise ValueError(f'dataset {dataset_name} is absent in dataset_definitions, ' + f'dataset_definitions.keys={list(dataset_definitions.keys())}') + cur_dataset_definition = dataset_definitions[dataset_name] + training_parameters_fields = {k: v for k, v in cur_dataset_definition.items() + if k in DATASET_PARAMETERS_FIELDS()} + make_paths_be_abs(training_parameters_fields, dataset_definitions[ROOT_PATH_KEY]) + + assert set(DATASET_PARAMETERS_FIELDS()) == set(training_parameters_fields.keys()), \ + f'ERROR: dataset definitions for name={dataset_name} does not contain all required fields' + assert all(training_parameters_fields.values()), \ + f'ERROR: dataset definitions for name={dataset_name} contains empty values for some required fields' + + params = DatasetParameters(**training_parameters_fields) + return params + +def _create_classification_dataset_and_labels_schema(dataset_params, model_name): + logger.debug(f'Using for train annotation file {dataset_params.annotations_train}') + logger.debug(f'Using for val annotation file {dataset_params.annotations_val}') + + dataset = ClassificationDatasetAdapter( + train_data_root=osp.join(dataset_params.images_train_dir), + train_ann_file=osp.join(dataset_params.annotations_train), + val_data_root=osp.join(dataset_params.images_val_dir), + val_ann_file=osp.join(dataset_params.annotations_val), + test_data_root=osp.join(dataset_params.images_test_dir), + test_ann_file=osp.join(dataset_params.annotations_test)) + + labels_schema = LabelSchemaEntity.from_labels(dataset.get_labels()) + return dataset, labels_schema + +def _create_object_detection_dataset_and_labels_schema(dataset_params): + logger.debug(f'Using for train annotation file {dataset_params.annotations_train}') + logger.debug(f'Using for val annotation file {dataset_params.annotations_val}') + labels_list = [] + items = [] + items.extend(load_dataset_items_coco_format( + ann_file_path=dataset_params.annotations_train, + data_root_dir=dataset_params.images_train_dir, + domain=Domain.DETECTION, + subset=Subset.TRAINING, + labels_list=labels_list)) + items.extend(load_dataset_items_coco_format( + ann_file_path=dataset_params.annotations_val, + data_root_dir=dataset_params.images_val_dir, + domain=Domain.DETECTION, + subset=Subset.VALIDATION, + labels_list=labels_list)) + items.extend(load_dataset_items_coco_format( + ann_file_path=dataset_params.annotations_test, + data_root_dir=dataset_params.images_test_dir, + domain=Domain.DETECTION, + subset=Subset.TESTING, + labels_list=labels_list)) + dataset = DatasetEntity(items=items) + labels_schema = LabelSchemaEntity.from_labels(dataset.get_labels()) + return dataset, labels_schema + +def _create_segmentation_dataset_and_labels_schema(dataset_params): + logger.debug(f'Using for train annotation file {dataset_params.annotations_train}') + logger.debug(f'Using for val annotation file {dataset_params.annotations_val}') + labels_list = [] + items = load_dataset_items( + ann_file_path=dataset_params.annotations_train, + data_root_dir=dataset_params.images_train_dir, + subset=Subset.TRAINING, + labels_list=labels_list) + items.extend(load_dataset_items( + ann_file_path=dataset_params.annotations_val, + data_root_dir=dataset_params.images_val_dir, + subset=Subset.VALIDATION, + labels_list=labels_list)) + items.extend(load_dataset_items( + ann_file_path=dataset_params.annotations_test, + data_root_dir=dataset_params.images_test_dir, + subset=Subset.TESTING, + labels_list=labels_list)) + dataset = DatasetEntity(items=items) + labels_schema = LabelSchemaEntity.from_labels(labels_list) + return dataset, labels_schema + + +class ClassificationClsIncrTrainingTestParameters(DefaultOTETestCreationParametersInterface): + def test_case_class(self) -> Type[OTETestCaseInterface]: + return generate_ote_integration_test_case_class( + get_test_action_classes() + ) + + def test_bunches(self) -> List[Dict[str, Any]]: + test_bunches = [ + dict( + model_name=[ + 'ClassIncremental_Image_Classification_EfficinetNet-B0', + 'ClassIncremental_Image_Classification_EfficinetNet-V2-S', + 'ClassIncremental_Image_Classification_MobileNet-V3-large-1x', + 'ClassIncremental_Image_Classification_MobileNet-V3-large-0.75x', + 'ClassIncremental_Image_Classification_MobileNet-V3-small' + ], + dataset_name=['cifar10_cls_incr'], + usecase='precommit', + ), + dict( + model_name=[ + 'ClassIncremental_Image_Classification_EfficinetNet-B0', + 'ClassIncremental_Image_Classification_EfficinetNet-V2-S', + 'ClassIncremental_Image_Classification_MobileNet-V3-large-1x', + 'ClassIncremental_Image_Classification_MobileNet-V3-large-0.75x', + 'ClassIncremental_Image_Classification_MobileNet-V3-small' + ], + dataset_name=['cifar10_cls_incr'], + num_training_iters=KEEP_CONFIG_FIELD_VALUE, + batch_size=KEEP_CONFIG_FIELD_VALUE, + usecase=REALLIFE_USECASE_CONSTANT, + ), + ] + + return deepcopy(test_bunches) + + def default_test_parameters(self) -> Dict[str, Any]: + DEFAULT_TEST_PARAMETERS = { + "num_training_iters": 2, + "batch_size": 16, + } + return deepcopy(DEFAULT_TEST_PARAMETERS) + + +class DetectionClsIncrTrainingTestParameters(DefaultOTETestCreationParametersInterface): + def test_case_class(self) -> Type[OTETestCaseInterface]: + return generate_ote_integration_test_case_class( + get_test_action_classes() + ) + + def test_bunches(self) -> List[Dict[str, Any]]: + test_bunches = [ + dict( + model_name=[ + 'ClassIncremental_Object_Detection_Gen3_ATSS', + 'ClassIncremental_Object_Detection_Gen3_VFNet', + ], + dataset_name='coco_cls_incr', + usecase='precommit', + ), + dict( + model_name=[ + 'ClassIncremental_Object_Detection_Gen3_ATSS', + 'ClassIncremental_Object_Detection_Gen3_VFNet', + ], + dataset_name='coco_cls_incr', + num_training_iters=KEEP_CONFIG_FIELD_VALUE, + batch_size=KEEP_CONFIG_FIELD_VALUE, + usecase=REALLIFE_USECASE_CONSTANT, + ), + + ] + return deepcopy(test_bunches) + + +class SegmentationClsIncrTrainingTestParameters(DefaultOTETestCreationParametersInterface): + def test_case_class(self) -> Type[OTETestCaseInterface]: + return generate_ote_integration_test_case_class( + get_test_action_classes() + ) + + def test_bunches(self) -> List[Dict[str, Any]]: + test_bunches = [ + dict( + model_name=[ + 'ClassIncremental_Semantic_Segmentation_Lite-HRNet-18_OCR', + ], + dataset_name='voc_cls_incr', + usecase='precommit', + ), + dict( + model_name=[ + 'ClassIncremental_Semantic_Segmentation_Lite-HRNet-18_OCR', + ], + dataset_name='voc_cls_incr', + num_training_iters=KEEP_CONFIG_FIELD_VALUE, + batch_size=KEEP_CONFIG_FIELD_VALUE, + usecase=REALLIFE_USECASE_CONSTANT, + ), + ] + return deepcopy(test_bunches) + + +class TestOTEReallifeClassificationClsIncr(OTETrainingTestInterface): + """ + The main class of running test in this file. + """ + PERFORMANCE_RESULTS = None # it is required for e2e system + helper = OTETestHelper(ClassificationClsIncrTrainingTestParameters()) + + @classmethod + def get_list_of_tests(cls, usecase: Optional[str] = None): + """ + This method should be a classmethod. It is called before fixture initialization, during + tests discovering. + """ + return cls.helper.get_list_of_tests(usecase) + + @pytest.fixture + def params_factories_for_test_actions_fx(self, current_test_parameters_fx, + dataset_definitions_fx, template_paths_fx, + ote_current_reference_dir_fx) -> Dict[str,Callable[[], Dict]]: + logger.debug('params_factories_for_test_actions_fx: begin') + + test_parameters = deepcopy(current_test_parameters_fx) + dataset_definitions = deepcopy(dataset_definitions_fx) + template_paths = deepcopy(template_paths_fx) + def _training_params_factory() -> Dict: + if dataset_definitions is None: + pytest.skip('The parameter "--dataset-definitions" is not set') + + model_name = test_parameters['model_name'] + dataset_name = test_parameters['dataset_name'] + num_training_iters = test_parameters['num_training_iters'] + batch_size = test_parameters['batch_size'] + + dataset_params = _get_dataset_params_from_dataset_definitions(dataset_definitions, dataset_name) + + if model_name not in template_paths: + raise ValueError(f'Model {model_name} is absent in template_paths, ' + f'template_paths.keys={list(template_paths.keys())}') + template_path = make_path_be_abs(template_paths[model_name], template_paths[ROOT_PATH_KEY]) + + logger.debug('training params factory: Before creating dataset and labels_schema') + dataset, labels_schema = _create_classification_dataset_and_labels_schema(dataset_params, model_name) + ckpt_path = None + if hasattr(dataset_params, 'pre_trained_model'): + ckpt_path = osp.join(osp.join(dataset_params.pre_trained_model, model_name),"weights.pth") + logger.info(f"Pretrained path : {ckpt_path}") + logger.debug('training params factory: After creating dataset and labels_schema') + + return { + 'dataset': dataset, + 'labels_schema': labels_schema, + 'template_path': template_path, + 'num_training_iters': num_training_iters, + 'batch_size': batch_size, + 'checkpoint': ckpt_path + } + params_factories_for_test_actions = { + 'training': _training_params_factory, + } + logger.debug('params_factories_for_test_actions_fx: end') + return params_factories_for_test_actions + + @pytest.fixture + def test_case_fx(self, current_test_parameters_fx, params_factories_for_test_actions_fx): + """ + This fixture returns the test case class OTEIntegrationTestCase that should be used for the current test. + Note that the cache from the test helper allows to store the instance of the class + between the tests. + If the main parameters used for this test are the same as the main parameters used for the previous test, + the instance of the test case class will be kept and re-used. It is helpful for tests that can + re-use the result of operations (model training, model optimization, etc) made for the previous tests, + if these operations are time-consuming. + If the main parameters used for this test differs w.r.t. the previous test, a new instance of + test case class will be created. + """ + test_case = type(self).helper.get_test_case(current_test_parameters_fx, + params_factories_for_test_actions_fx) + return test_case + + @e2e_pytest_performance + def test(self, + test_parameters, + test_case_fx, data_collector_fx, + cur_test_expected_metrics_callback_fx): + test_case_fx.run_stage(test_parameters['test_stage'], data_collector_fx, + cur_test_expected_metrics_callback_fx) + + +class TestOTEReallifeObjectDetectionClsIncr(OTETrainingTestInterface): + """ + The main class of running test in this file. + """ + PERFORMANCE_RESULTS = None # it is required for e2e system + helper = OTETestHelper(DetectionClsIncrTrainingTestParameters()) + + @classmethod + def get_list_of_tests(cls, usecase: Optional[str] = None): + """ + This method should be a classmethod. It is called before fixture initialization, during + tests discovering. + """ + return cls.helper.get_list_of_tests(usecase) + + @pytest.fixture + def params_factories_for_test_actions_fx(self, current_test_parameters_fx, + dataset_definitions_fx, template_paths_fx, + ote_current_reference_dir_fx) -> Dict[str,Callable[[], Dict]]: + logger.debug('params_factories_for_test_actions_fx: begin') + + test_parameters = deepcopy(current_test_parameters_fx) + dataset_definitions = deepcopy(dataset_definitions_fx) + template_paths = deepcopy(template_paths_fx) + def _training_params_factory() -> Dict: + if dataset_definitions is None: + pytest.skip('The parameter "--dataset-definitions" is not set') + + model_name = test_parameters['model_name'] + dataset_name = test_parameters['dataset_name'] + num_training_iters = test_parameters['num_training_iters'] + batch_size = test_parameters['batch_size'] + + dataset_params = _get_dataset_params_from_dataset_definitions(dataset_definitions, dataset_name) + + if model_name not in template_paths: + raise ValueError(f'Model {model_name} is absent in template_paths, ' + f'template_paths.keys={list(template_paths.keys())}') + template_path = make_path_be_abs(template_paths[model_name], template_paths[ROOT_PATH_KEY]) + + logger.debug('training params factory: Before creating dataset and labels_schema') + dataset, labels_schema = _create_object_detection_dataset_and_labels_schema(dataset_params) + ckpt_path = None + if hasattr(dataset_params, 'pre_trained_model'): + ckpt_path = osp.join(osp.join(dataset_params.pre_trained_model, model_name),"weights.pth") + logger.debug('training params factory: After creating dataset and labels_schema') + + return { + 'dataset': dataset, + 'labels_schema': labels_schema, + 'template_path': template_path, + 'num_training_iters': num_training_iters, + 'batch_size': batch_size, + 'checkpoint': ckpt_path + } + + params_factories_for_test_actions = { + 'training': _training_params_factory, + } + logger.debug('params_factories_for_test_actions_fx: end') + return params_factories_for_test_actions + + @pytest.fixture + def test_case_fx(self, current_test_parameters_fx, params_factories_for_test_actions_fx): + """ + This fixture returns the test case class OTEIntegrationTestCase that should be used for the current test. + Note that the cache from the test helper allows to store the instance of the class + between the tests. + If the main parameters used for this test are the same as the main parameters used for the previous test, + the instance of the test case class will be kept and re-used. It is helpful for tests that can + re-use the result of operations (model training, model optimization, etc) made for the previous tests, + if these operations are time-consuming. + If the main parameters used for this test differs w.r.t. the previous test, a new instance of + test case class will be created. + """ + test_case = type(self).helper.get_test_case(current_test_parameters_fx, + params_factories_for_test_actions_fx) + return test_case + + # TODO(lbeynens): move to common fixtures + @pytest.fixture + def data_collector_fx(self, request) -> DataCollector: + setup = deepcopy(request.node.callspec.params) + setup['environment_name'] = os.environ.get('TT_ENVIRONMENT_NAME', 'no-env') + setup['test_type'] = os.environ.get('TT_TEST_TYPE', 'no-test-type') # TODO: get from e2e test type + setup['scenario'] = 'api' # TODO(lbeynens): get from a fixture! + setup['test'] = request.node.name + setup['subject'] = 'detection-cls-incr' + setup['project'] = 'ote' + if 'test_parameters' in setup: + assert isinstance(setup['test_parameters'], dict) + if 'dataset_name' not in setup: + setup['dataset_name'] = setup['test_parameters'].get('dataset_name') + if 'model_name' not in setup: + setup['model_name'] = setup['test_parameters'].get('model_name') + if 'test_stage' not in setup: + setup['test_stage'] = setup['test_parameters'].get('test_stage') + if 'usecase' not in setup: + setup['usecase'] = setup['test_parameters'].get('usecase') + logger.info(f'creating DataCollector: setup=\n{pformat(setup, width=140)}') + data_collector = DataCollector(name='TestOTEIntegration', + setup=setup) + with data_collector: + logger.info('data_collector is created') + yield data_collector + logger.info('data_collector is released') + + @e2e_pytest_performance + def test(self, + test_parameters, + test_case_fx, data_collector_fx, + cur_test_expected_metrics_callback_fx): + test_case_fx.run_stage(test_parameters['test_stage'], data_collector_fx, + cur_test_expected_metrics_callback_fx) + + +class TestOTEReallifeSegmentationClsIncr(OTETrainingTestInterface): + """ + The main class of running test in this file. + """ + PERFORMANCE_RESULTS = None # it is required for e2e system + helper = OTETestHelper(SegmentationClsIncrTrainingTestParameters()) + + @classmethod + def get_list_of_tests(cls, usecase: Optional[str] = None): + """ + This method should be a classmethod. It is called before fixture initialization, during + tests discovering. + """ + return cls.helper.get_list_of_tests(usecase) + + @pytest.fixture + def params_factories_for_test_actions_fx(self, current_test_parameters_fx, + dataset_definitions_fx, template_paths_fx, + ote_current_reference_dir_fx) -> Dict[str,Callable[[], Dict]]: + logger.debug('params_factories_for_test_actions_fx: begin') + + test_parameters = deepcopy(current_test_parameters_fx) + dataset_definitions = deepcopy(dataset_definitions_fx) + template_paths = deepcopy(template_paths_fx) + def _training_params_factory() -> Dict: + if dataset_definitions is None: + pytest.skip('The parameter "--dataset-definitions" is not set') + + model_name = test_parameters['model_name'] + dataset_name = test_parameters['dataset_name'] + num_training_iters = test_parameters['num_training_iters'] + batch_size = test_parameters['batch_size'] + + dataset_params = _get_dataset_params_from_dataset_definitions(dataset_definitions, dataset_name) + + if model_name not in template_paths: + raise ValueError(f'Model {model_name} is absent in template_paths, ' + f'template_paths.keys={list(template_paths.keys())}') + template_path = make_path_be_abs(template_paths[model_name], template_paths[ROOT_PATH_KEY]) + + logger.debug('training params factory: Before creating dataset and labels_schema') + dataset, labels_schema = _create_segmentation_dataset_and_labels_schema(dataset_params) + import os.path as osp + ckpt_path = None + if hasattr(dataset_params, 'pre_trained_model'): + ckpt_path = osp.join(osp.join(dataset_params.pre_trained_model, model_name), "weights.pth") + logger.debug('training params factory: After creating dataset and labels_schema') + + return { + 'dataset': dataset, + 'labels_schema': labels_schema, + 'template_path': template_path, + 'num_training_iters': num_training_iters, + 'batch_size': batch_size, + 'checkpoint' : ckpt_path + } + + params_factories_for_test_actions = { + 'training': _training_params_factory, + } + logger.debug('params_factories_for_test_actions_fx: end') + return params_factories_for_test_actions + + @pytest.fixture + def test_case_fx(self, current_test_parameters_fx, params_factories_for_test_actions_fx): + """ + This fixture returns the test case class OTEIntegrationTestCase that should be used for the current test. + Note that the cache from the test helper allows to store the instance of the class + between the tests. + If the main parameters used for this test are the same as the main parameters used for the previous test, + the instance of the test case class will be kept and re-used. It is helpful for tests that can + re-use the result of operations (model training, model optimization, etc) made for the previous tests, + if these operations are time-consuming. + If the main parameters used for this test differs w.r.t. the previous test, a new instance of + test case class will be created. + """ + test_case = type(self).helper.get_test_case(current_test_parameters_fx, + params_factories_for_test_actions_fx) + return test_case + + # TODO(lbeynens): move to common fixtures + @pytest.fixture + def data_collector_fx(self, request) -> DataCollector: + setup = deepcopy(request.node.callspec.params) + setup['environment_name'] = os.environ.get('TT_ENVIRONMENT_NAME', 'no-env') + setup['test_type'] = os.environ.get('TT_TEST_TYPE', 'no-test-type') # TODO: get from e2e test type + setup['scenario'] = 'api' # TODO(lbeynens): get from a fixture! + setup['test'] = request.node.name + setup['subject'] = 'segmentation-cls-incr' + setup['project'] = 'ote' + if 'test_parameters' in setup: + assert isinstance(setup['test_parameters'], dict) + if 'dataset_name' not in setup: + setup['dataset_name'] = setup['test_parameters'].get('dataset_name') + if 'model_name' not in setup: + setup['model_name'] = setup['test_parameters'].get('model_name') + if 'test_stage' not in setup: + setup['test_stage'] = setup['test_parameters'].get('test_stage') + if 'usecase' not in setup: + setup['usecase'] = setup['test_parameters'].get('usecase') + logger.info(f'creating DataCollector: setup=\n{pformat(setup, width=140)}') + data_collector = DataCollector(name='TestOTEIntegration', + setup=setup) + with data_collector: + logger.info('data_collector is created') + yield data_collector + logger.info('data_collector is released') + + @e2e_pytest_performance + def test(self, + test_parameters, + test_case_fx, data_collector_fx, + cur_test_expected_metrics_callback_fx): + if "pot_evaluation" in test_parameters["test_stage"]: + pytest.xfail("Known issue CVS-84576") + test_case_fx.run_stage(test_parameters['test_stage'], data_collector_fx, + cur_test_expected_metrics_callback_fx) diff --git a/ote_sdk/ote_sdk/test_suite/training_tests_actions.py b/ote_sdk/ote_sdk/test_suite/training_tests_actions.py index 1d51945b4ed..8e5fc0b380b 100644 --- a/ote_sdk/ote_sdk/test_suite/training_tests_actions.py +++ b/ote_sdk/ote_sdk/test_suite/training_tests_actions.py @@ -3,7 +3,9 @@ # import importlib +import json import os +import os.path as osp from abc import ABC, abstractmethod from collections import OrderedDict from copy import deepcopy @@ -19,6 +21,8 @@ from ote_sdk.entities.resultset import ResultSetEntity from ote_sdk.entities.subset import Subset from ote_sdk.entities.task_environment import TaskEnvironment +from ote_sdk.serialization.label_mapper import LabelSchemaMapper, label_schema_to_bytes +from ote_sdk.usecases.adapters.model_adapter import ModelAdapter from ote_sdk.usecases.tasks.interfaces.export_interface import ExportType from ote_sdk.usecases.tasks.interfaces.optimization_interface import OptimizationType from ote_sdk.utils.importing import get_impl_class @@ -74,13 +78,24 @@ def __call__(self, data_collector: DataCollector, results_prev_stages: OrderedDi raise NotImplementedError("The main action method is not implemented") -def create_environment_and_task(params, labels_schema, model_template): +def create_environment_and_task( + params, labels_schema, model_template, dataset, model_adapters=None +): + environment = TaskEnvironment( model=None, hyper_parameters=params, label_schema=labels_schema, model_template=model_template, ) + + if model_adapters is not None: + environment.model = ModelEntity( + train_dataset=dataset, + configuration=environment.get_model_configuration(), + model_adapters=model_adapters, + ) + logger.info("Create base Task") task_impl_path = model_template.entrypoints.base task_cls = get_impl_class(task_impl_path) @@ -92,13 +107,20 @@ class OTETestTrainingAction(BaseOTETestAction): _name = "training" def __init__( - self, dataset, labels_schema, template_path, num_training_iters, batch_size + self, + dataset, + labels_schema, + template_path, + num_training_iters, + batch_size, + checkpoint=None, ): self.dataset = dataset self.labels_schema = labels_schema self.template_path = template_path self.num_training_iters = num_training_iters self.batch_size = batch_size + self.checkpoint = checkpoint def _get_training_performance_as_score_name_value(self): training_performance = getattr(self.output_model, "performance", None) @@ -146,12 +168,36 @@ def _run_ote_training(self, data_collector): f"{params.learning_parameters.batch_size}" ) - logger.debug("Setup environment") + model_adapters = None + if self.checkpoint is not None: + logger.debug("Load pretrained model") + model_adapters = { + "weights.pth": ModelAdapter(open(self.checkpoint, "rb").read()), + } + label_schema_path = osp.join( + osp.dirname(self.checkpoint), "label_schema.json" + ) + if osp.exists(label_schema_path): + with open(label_schema_path, encoding="UTF-8") as read_file: + serialized_label_schema = LabelSchemaMapper.backward( + json.load(read_file) + ) + model_adapters.update( + { + "label_schema.json": ModelAdapter( + label_schema_to_bytes(serialized_label_schema) + ) + } + ) + self.environment, self.task = create_environment_and_task( - params, self.labels_schema, self.model_template + params, + self.labels_schema, + self.model_template, + self.dataset, + model_adapters, ) - logger.debug("Train model") self.output_model = ModelEntity( self.dataset, self.environment.get_model_configuration(), @@ -159,6 +205,8 @@ def _run_ote_training(self, data_collector): self.copy_hyperparams = deepcopy(self.task._hyperparams) + logger.debug("Train model") + try: self.task.train(self.dataset, self.output_model) except Exception as ex: