Merge pull request #3762 from sovrasov/vs/micro_benchmark

Add otx micro benchmark

CHANGELOG.md

@@ -18,6 +18,8 @@ All notable changes to this project will be documented in this file.
   (https://github.com/openvinotoolkit/training_extensions/pull/3749)
 - Enable torch.compile to work with classification
   (https://github.com/openvinotoolkit/training_extensions/pull/3758)
+- Add `otx benchmark` subcommand
+  (https://github.com/openvinotoolkit/training_extensions/pull/3762)
 
 ### Enhancements
 

docs/source/guide/get_started/api_tutorial.rst

@@ -107,10 +107,10 @@ If you want to use other models offered by OpenVINO™ Training Extension beside
             model_lists = list_models(task="DETECTION", print_table=True)
 
             '''
-            ┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓            
-            ┃ Task      ┃ Model Name            ┃ Recipe Path                                                    ┃            
-            ┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩            
-            │ DETECTION │ yolox_tiny            │ src/otx/recipe/detection/yolox_tiny.yaml                       │            
+            ┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+            ┃ Task      ┃ Model Name            ┃ Recipe Path                                                    ┃
+            ┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+            │ DETECTION │ yolox_tiny            │ src/otx/recipe/detection/yolox_tiny.yaml                       │
             │ ...       │                       │                                                                │
             └───────────┴───────────────────────┴────────────────────────────────────────────────────────────────┘
             '''
@@ -328,7 +328,7 @@ The datamodule used by the Engine is of type ``otx.core.data.module.OTXDataModul
             engine.train(precision="16")
 
         .. note::
-            
+
             This uses lightning's precision value. You can use the values below:
             - "64", "32", "16", "bf16",
             - 64, 32, 16
@@ -518,4 +518,32 @@ You can validate the optimized model as the usual model. For example for the NNC
 
     engine.test(checkpoint="<path/to/optimized/ir/xml>")
 
-That's it. Now, we can use OpenVINO™ Training Extensions APIs to create, train, and deploy deep learning models using the OpenVINO™ Training Extension.
+************
+Benchmarking
+************
+
+``Engine`` allows to perform benchmarking of the trained model, and provide theoretical complexity information in case of torch model.
+The estimated by ``Engine.benchmark()`` performance may differ from the performance of the deployed model, since the measurements are conducted
+via OTX inference API, which can introduce additional burden.
+
+.. tab-set::
+
+    .. tab-item:: Benchmark Model
+
+        .. code-block:: python
+
+            engine.benchmark()
+
+    .. tab-item:: Benchmark OpenVINO™ IR model
+
+        .. code-block:: python
+
+            engine.benchmark(checkpoint="<path/to/exported_model.xml>")
+
+        .. note::
+
+            Specifying a checkpoint only makes sense for OpenVINO™ IR models.
+
+Conclusion
+"""""""""""
+That's it! Now, we can use OpenVINO™ Training Extensions APIs to create, train, and deploy deep learning models using the OpenVINO™ Training Extensions.

docs/source/guide/get_started/cli_commands.rst

@@ -50,6 +50,7 @@ Help
     │     export              Export the trained model to OpenVINO Intermediate Representation (IR) or ONNX formats.  │
     │     optimize            Applies NNCF.PTQ to the underlying models (now works only for OV models).               │
     │     explain             Run XAI using the specified model and data (test subset).                               │
+    |     benchmark           Executes model micro benchmarking on random data.                                       |
     │                                                                                                                 │
     ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 
@@ -232,43 +233,43 @@ Example to find ready-to-use recipes for the detection task:
 .. code-block:: shell
 
     (otx) ...$ otx find --task DETECTION
-    ┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓                                  
-    ┃ Task      ┃ Model Name            ┃ Recipe Path                                 ┃                                  
-    ┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩                                  
-    │ DETECTION │ yolox_tiny            │ recipe/detection/yolox_tiny.yaml            │                                  
-    │ DETECTION │ atss_mobilenetv2_tile │ recipe/detection/atss_mobilenetv2_tile.yaml │                                  
-    │ DETECTION │ openvino_model        │ recipe/detection/openvino_model.yaml        │                                  
-    │ DETECTION │ atss_mobilenetv2      │ recipe/detection/atss_mobilenetv2.yaml      │                                  
-    │ DETECTION │ atss_resnext101       │ recipe/detection/atss_resnext101.yaml       │                                  
-    │ DETECTION │ yolox_l_tile          │ recipe/detection/yolox_l_tile.yaml          │                                  
-    │ DETECTION │ ssd_mobilenetv2_tile  │ recipe/detection/ssd_mobilenetv2_tile.yaml  │                                  
-    │ DETECTION │ atss_r50_fpn          │ recipe/detection/atss_r50_fpn.yaml          │                                  
-    │ DETECTION │ yolox_tiny_tile       │ recipe/detection/yolox_tiny_tile.yaml       │                                  
-    │ DETECTION │ yolox_s               │ recipe/detection/yolox_s.yaml               │                                  
-    │ DETECTION │ yolox_s_tile          │ recipe/detection/yolox_s_tile.yaml          │                                  
-    │ DETECTION │ rtmdet_tiny           │ recipe/detection/rtmdet_tiny.yaml           │                                  
-    │ DETECTION │ yolox_x               │ recipe/detection/yolox_x.yaml               │                                  
-    │ DETECTION │ yolox_x_tile          │ recipe/detection/yolox_x_tile.yaml          │                                  
-    │ DETECTION │ ssd_mobilenetv2       │ recipe/detection/ssd_mobilenetv2.yaml       │                                  
-    │ DETECTION │ yolox_l               │ recipe/detection/yolox_l.yaml               │                                  
+    ┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+    ┃ Task      ┃ Model Name            ┃ Recipe Path                                 ┃
+    ┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+    │ DETECTION │ yolox_tiny            │ recipe/detection/yolox_tiny.yaml            │
+    │ DETECTION │ atss_mobilenetv2_tile │ recipe/detection/atss_mobilenetv2_tile.yaml │
+    │ DETECTION │ openvino_model        │ recipe/detection/openvino_model.yaml        │
+    │ DETECTION │ atss_mobilenetv2      │ recipe/detection/atss_mobilenetv2.yaml      │
+    │ DETECTION │ atss_resnext101       │ recipe/detection/atss_resnext101.yaml       │
+    │ DETECTION │ yolox_l_tile          │ recipe/detection/yolox_l_tile.yaml          │
+    │ DETECTION │ ssd_mobilenetv2_tile  │ recipe/detection/ssd_mobilenetv2_tile.yaml  │
+    │ DETECTION │ atss_r50_fpn          │ recipe/detection/atss_r50_fpn.yaml          │
+    │ DETECTION │ yolox_tiny_tile       │ recipe/detection/yolox_tiny_tile.yaml       │
+    │ DETECTION │ yolox_s               │ recipe/detection/yolox_s.yaml               │
+    │ DETECTION │ yolox_s_tile          │ recipe/detection/yolox_s_tile.yaml          │
+    │ DETECTION │ rtmdet_tiny           │ recipe/detection/rtmdet_tiny.yaml           │
+    │ DETECTION │ yolox_x               │ recipe/detection/yolox_x.yaml               │
+    │ DETECTION │ yolox_x_tile          │ recipe/detection/yolox_x_tile.yaml          │
+    │ DETECTION │ ssd_mobilenetv2       │ recipe/detection/ssd_mobilenetv2.yaml       │
+    │ DETECTION │ yolox_l               │ recipe/detection/yolox_l.yaml               │
     └───────────┴───────────────────────┴─────────────────────────────────────────────┘
 
 Example to find yolo named model for the detection task:
 
 .. code-block:: shell
 
     (otx) ...$ otx find --task DETECTION --pattern 'yolo*'
-    ┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓                                        
-    ┃ Task      ┃ Model Name      ┃ Recipe Path                           ┃                                        
-    ┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩                                        
-    │ DETECTION │ yolox_tiny      │ recipe/detection/yolox_tiny.yaml      │                                        
-    │ DETECTION │ yolox_x         │ recipe/detection/yolox_x.yaml         │                                        
-    │ DETECTION │ yolox_l_tile    │ recipe/detection/yolox_l_tile.yaml    │                                        
-    │ DETECTION │ yolox_s         │ recipe/detection/yolox_s.yaml         │                                        
-    │ DETECTION │ yolox_l         │ recipe/detection/yolox_l.yaml         │                                        
-    │ DETECTION │ yolox_x_tile    │ recipe/detection/yolox_x_tile.yaml    │                                        
-    │ DETECTION │ yolox_s_tile    │ recipe/detection/yolox_s_tile.yaml    │                                        
-    │ DETECTION │ yolox_tiny_tile │ recipe/detection/yolox_tiny_tile.yaml │                                        
+    ┏━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+    ┃ Task      ┃ Model Name      ┃ Recipe Path                           ┃
+    ┡━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+    │ DETECTION │ yolox_tiny      │ recipe/detection/yolox_tiny.yaml      │
+    │ DETECTION │ yolox_x         │ recipe/detection/yolox_x.yaml         │
+    │ DETECTION │ yolox_l_tile    │ recipe/detection/yolox_l_tile.yaml    │
+    │ DETECTION │ yolox_s         │ recipe/detection/yolox_s.yaml         │
+    │ DETECTION │ yolox_l         │ recipe/detection/yolox_l.yaml         │
+    │ DETECTION │ yolox_x_tile    │ recipe/detection/yolox_x_tile.yaml    │
+    │ DETECTION │ yolox_s_tile    │ recipe/detection/yolox_s_tile.yaml    │
+    │ DETECTION │ yolox_tiny_tile │ recipe/detection/yolox_tiny_tile.yaml │
     └───────────┴─────────────────┴───────────────────────────────────────┘
 
 
@@ -476,7 +477,7 @@ The command below will generate saliency maps (heatmaps with red colored areas o
 
 .. note::
 
-    It is possible to pass both PyTorch weights ``.ckpt`` or OpenVINO™ IR ``exported_model.xml`` to ``--load-weights`` option.
+    It is possible to pass both PyTorch weights ``.ckpt`` or OpenVINO™ IR ``exported_model.xml`` to ``--checkpoint`` option.
 
 By default, the model is exported to the OpenVINO™ IR format without extra feature information needed for the ``explain`` function. To use OpenVINO™ IR model in ``otx explain``, please first export it with ``--explain`` parameter:
 
@@ -486,6 +487,24 @@ By default, the model is exported to the OpenVINO™ IR format without extra fea
                               --explain True
     (otx) ...$ otx explain ... --checkpoint outputs/openvino/with_features \
 
+*******************
+Micro-benchmarking
+*******************
+
+``otx benchmark`` tool allows performing a fast in-place benchmarking on randomly generated data. The benchmark excludes data loading cost, but takes into account extra burden of OTX API and ModelAPI (in case of OpenVINO™ IR models).
+
+The command requires checkpoint path for OpenVINO™ IR models. In case of torch models, this parameter is optional. Also, for torch model number of trainable parameters and theoretical computational complexity are estimated when model's structure allows that.
+It worth noticing that the latency and throughput are depend on batch size. Varying the batch size parameter, one can quickly explore the trade-off for the considered model. For OpenVINO™ IR models batching is imitated by using async inference API.
+
+.. code-block:: shell
+
+    (otx) ...$ otx benchmark ... --data_root <path/to/test/root> \
+                                 --checkpoint <path/to/model_weights> \
+                                 --batch_size 1
+
+.. note::
+
+    It is possible to pass both PyTorch weights ``.ckpt`` or OpenVINO™ IR ``exported_model.xml`` to ``--checkpoint`` option.
 
 ***********
 Workspace

src/otx/algo/classification/dino_v2.py

@@ -280,7 +280,7 @@ def _customize_inputs(self, inputs: MulticlassClsBatchDataEntity) -> dict[str, A
                 "mode": mode,
             }
         return {
-            "images": inputs.images,
+            "images": inputs.stacked_images,
             "labels": torch.cat(inputs.labels, dim=0),
             "imgs_info": inputs.imgs_info,
             "mode": mode,

src/otx/algo/classification/efficientnet.py

@@ -145,7 +145,7 @@ def _exporter(self) -> OTXModelExporter:
         """Creates OTXModelExporter object that can export the model."""
         return OTXNativeModelExporter(
             task_level_export_parameters=self._export_parameters,
-            input_size=(1, 3, 224, 224),
+            input_size=self.image_size,
             mean=(123.675, 116.28, 103.53),
             std=(58.395, 57.12, 57.375),
             resize_mode="standard",
@@ -230,7 +230,7 @@ def _customize_inputs(self, inputs: MulticlassClsBatchDataEntity) -> dict[str, A
                 "mode": mode,
             }
         return {
-            "images": inputs.images,
+            "images": inputs.stacked_images,
             "labels": torch.cat(inputs.labels, dim=0),
             "imgs_info": inputs.imgs_info,
             "mode": mode,

src/otx/algo/classification/efficientnet_v2.py

@@ -140,7 +140,7 @@ def _exporter(self) -> OTXModelExporter:
         """Creates OTXModelExporter object that can export the model."""
         return OTXNativeModelExporter(
             task_level_export_parameters=self._export_parameters,
-            input_size=(1, 3, 224, 224),
+            input_size=self.image_size,
             mean=(123.675, 116.28, 103.53),
             std=(58.395, 57.12, 57.375),
             resize_mode="standard",
@@ -222,7 +222,7 @@ def _customize_inputs(self, inputs: MulticlassClsBatchDataEntity) -> dict[str, A
                 "mode": mode,
             }
         return {
-            "images": inputs.images,
+            "images": inputs.stacked_images,
             "labels": torch.cat(inputs.labels, dim=0),
             "imgs_info": inputs.imgs_info,
             "mode": mode,

src/otx/algo/classification/heads/vision_transformer_head.py

@@ -141,4 +141,4 @@ def predict(
 
     def _get_predictions(self, cls_score: torch.Tensor) -> torch.Tensor:
         """Get the score from the classification score."""
-        return functional.softmax(cls_score, dim=1)
+        return functional.softmax(cls_score, dim=-1)

src/otx/algo/classification/huggingface_model.py

@@ -110,7 +110,7 @@ def _exporter(self) -> OTXModelExporter:
         """Creates OTXModelExporter object that can export the model."""
         return OTXNativeModelExporter(
             task_level_export_parameters=self._export_parameters,
-            input_size=(1, 3, 224, 224),
+            input_size=self.image_size,
             mean=(123.675, 116.28, 103.53),
             std=(58.395, 57.12, 57.375),
             resize_mode="standard",

src/otx/algo/classification/mobilenet_v3.py

@@ -152,7 +152,7 @@ def _exporter(self) -> OTXModelExporter:
         """Creates OTXModelExporter object that can export the model."""
         return OTXNativeModelExporter(
             task_level_export_parameters=self._export_parameters,
-            input_size=(1, 3, 224, 224),
+            input_size=self.image_size,
             mean=(123.675, 116.28, 103.53),
             std=(58.395, 57.12, 57.375),
             resize_mode="standard",
@@ -237,7 +237,7 @@ def _customize_inputs(self, inputs: MulticlassClsBatchDataEntity) -> dict[str, A
                 "mode": mode,
             }
         return {
-            "images": inputs.images,
+            "images": inputs.stacked_images,
             "labels": torch.cat(inputs.labels, dim=0),
             "imgs_info": inputs.imgs_info,
             "mode": mode,