[Anomaly] Added real-life training tests

external/anomaly/ote_anomalib/inference_task.py

@@ -228,6 +228,7 @@ def _export_to_onnx(self, onnx_path: str):
         )
 
     def export(self, export_type: ExportType, output_model: ModelEntity) -> None:
+        output_model.optimization_type = ModelOptimizationType.MO
         """Export model to OpenVINO IR.
 
         Args:

external/anomaly/ote_anomalib/nncf_task.py

@@ -36,6 +36,7 @@
 from ote_sdk.entities.datasets import DatasetEntity
 from ote_sdk.entities.model import (
     ModelEntity,
+    ModelFormat,
     ModelOptimizationType,
     ModelPrecision,
     OptimizationMethod,
@@ -183,6 +184,8 @@ def optimize(
         self.trainer = Trainer(**self.config.trainer, logger=False, callbacks=callbacks)
         self.trainer.fit(model=self.model, datamodule=datamodule)
         self.compression_ctrl = nncf_callback.nncf_ctrl
+        output_model.model_format = ModelFormat.BASE_FRAMEWORK
+        output_model.optimization_type = ModelOptimizationType.NNCF
         self.save_model(output_model)
 
         logger.info("Training completed.")

external/anomaly/ote_tests_pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+python_files = test_ote_api.py test_ote_inference.py test_ote_training.py

external/anomaly/tests/conftest.py

@@ -0,0 +1,59 @@
+# Copyright (C) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from ote_sdk.test_suite.pytest_insertions import *
+from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT
+
+pytest_plugins = get_pytest_plugins_from_ote()
+
+ote_conftest_insertion(default_repository_name='ote/training_extensions/external/anomaly')
+
+@pytest.fixture
+def ote_test_domain_fx():
+    return 'custom-anomaly-classification'
+
+@pytest.fixture
+def ote_test_scenario_fx(current_test_parameters_fx):
+    assert isinstance(current_test_parameters_fx, dict)
+    if current_test_parameters_fx.get('usecase') == REALLIFE_USECASE_CONSTANT:
+        return 'performance'
+    else:
+        return 'integration'
+
+@pytest.fixture(scope='session')
+def ote_templates_root_dir_fx():
+    import os.path as osp
+    import logging
+    logger = logging.getLogger(__name__)
+    root = osp.dirname(osp.dirname(osp.realpath(__file__)))
+    root = f'{root}/anomaly_classification/'
+    logger.debug(f'overloaded ote_templates_root_dir_fx: return {root}')
+    return root
+
+@pytest.fixture(scope='session')
+def ote_reference_root_dir_fx():
+    import os.path as osp
+    import logging
+    logger = logging.getLogger(__name__)
+    root = osp.dirname(osp.dirname(osp.realpath(__file__)))
+    root = f'{root}/tests/reference/'
+    logger.debug(f'overloaded ote_reference_root_dir_fx: return {root}')
+    return root
+
+# pytest magic
+def pytest_generate_tests(metafunc):
+    ote_pytest_generate_tests_insertion(metafunc)
+
+def pytest_addoption(parser):
+    ote_pytest_addoption_insertion(parser)

external/anomaly/tests/expected_metrics/metrics_test_ote_training.yml

@@ -0,0 +1,43 @@
+'ACTION-training_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'target_value': 0.73
+                'max_diff_if_less_threshold': 0.01
+                'max_diff_if_greater_threshold': 0.04
+'ACTION-export_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'training_evaluation.metrics.accuracy.f-measure'
+                'max_diff': 0.01
+'ACTION-pot_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'export_evaluation.metrics.accuracy.f-measure'
+                'max_diff': 0.03
+'ACTION-nncf_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'training_evaluation.metrics.accuracy.f-measure'
+                'max_diff_if_less_threshold': 0.01
+'ACTION-nncf_export_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'nncf_evaluation.metrics.accuracy.f-measure'
+                'max_diff': 0.01
+
+'ACTION-training_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'target_value': 0.74
+                'max_diff_if_less_threshold': 0.01
+                'max_diff_if_greater_threshold': 0.04
+'ACTION-export_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'training_evaluation.metrics.accuracy.f-measure'
+                'max_diff': 0.01
+'ACTION-pot_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'export_evaluation.metrics.accuracy.f-measure'
+                'max_diff': 0.03
+'ACTION-nncf_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'training_evaluation.metrics.accuracy.f-measure'
+                'max_diff_if_less_threshold': 0.01
+'ACTION-nncf_export_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife':
+        'metrics.accuracy.f-measure':
+                'base': 'nncf_evaluation.metrics.accuracy.f-measure'
+                'max_diff': 0.01