From a26f7c3957841c8387d4e9d93643744c1a18cfdc Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 10:22:11 +0900 Subject: [PATCH 01/17] OTX output alignment --- .../quick_start_guide/cli_commands.rst | 56 ++++---- .../guide/tutorials/advanced/self_sl.rst | 4 +- .../guide/tutorials/advanced/semi_sl.rst | 6 +- docs/source/guide/tutorials/base/deploy.rst | 2 +- .../how_to_train/action_classification.rst | 10 +- .../base/how_to_train/action_detection.rst | 4 +- .../base/how_to_train/anomaly_detection.rst | 10 +- .../base/how_to_train/classification.rst | 12 +- .../tutorials/base/how_to_train/detection.rst | 18 +-- .../how_to_train/instance_segmentation.rst | 14 +- .../how_to_train/semantic_segmentation.rst | 12 +- otx/algorithms/common/tasks/training_base.py | 5 + otx/cli/manager/config_manager.py | 16 +++ otx/cli/tools/build.py | 10 +- otx/cli/tools/deploy.py | 15 +-- otx/cli/tools/eval.py | 18 +-- otx/cli/tools/export.py | 28 ++-- otx/cli/tools/optimize.py | 45 ++++--- otx/cli/tools/train.py | 25 ++-- otx/cli/utils/hpo.py | 2 +- otx/cli/utils/multi_gpu.py | 2 +- tests/conftest.py | 2 +- .../cli/classification/test_classification.py | 20 ++- tests/e2e/cli/detection/test_detection.py | 6 +- .../detection/test_instance_segmentation.py | 6 +- .../cli/detection/test_tiling_detection.py | 4 +- .../e2e/cli/detection/test_tiling_instseg.py | 4 +- .../e2e/cli/segmentation/test_segmentation.py | 8 +- .../cli/classification/test_classification.py | 4 +- .../cli/detection/test_detection.py | 4 +- .../detection/test_instance_segmentation.py | 4 +- .../cli/segmentation/test_segmentation.py | 4 +- .../classification/test_classification.py | 10 +- tests/regression/detection/test_detection.py | 4 +- .../detection/test_instnace_segmentation.py | 4 +- .../segmentation/test_segmentation.py | 6 +- tests/test_suite/pytest_insertions.py | 2 +- tests/test_suite/run_test_command.py | 123 +++++++++--------- tests/unit/cli/tools/test_build.py | 6 +- tests/unit/cli/tools/test_deploy.py | 6 +- tests/unit/cli/tools/test_eval.py | 10 +- tests/unit/cli/tools/test_export.py | 6 +- tests/unit/cli/tools/test_optimize.py | 15 +-- tests/unit/cli/tools/test_train.py | 13 +- tests/unit/cli/utils/test_hpo.py | 2 +- tests/unit/cli/utils/test_multi_gpu.py | 4 +- 46 files changed, 331 insertions(+), 260 deletions(-) diff --git a/docs/source/guide/get_started/quick_start_guide/cli_commands.rst b/docs/source/guide/get_started/quick_start_guide/cli_commands.rst index 2d628d9c4a9..f5bf7cf3efc 100644 --- a/docs/source/guide/get_started/quick_start_guide/cli_commands.rst +++ b/docs/source/guide/get_started/quick_start_guide/cli_commands.rst @@ -71,7 +71,7 @@ Building workspace folder (otx) ...$ otx build --help usage: otx build [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--test-data-roots TEST_DATA_ROOTS] [--unlabeled-data-roots UNLABELED_DATA_ROOTS] - [--unlabeled-file-list UNLABELED_FILE_LIST] [--task TASK] [--train-type TRAIN_TYPE] [--work-dir WORK_DIR] [--model MODEL] [--backbone BACKBONE] + [--unlabeled-file-list UNLABELED_FILE_LIST] [--task TASK] [--train-type TRAIN_TYPE] [--workspace WORKSPACE] [--model MODEL] [--backbone BACKBONE] [template] positional arguments: @@ -93,7 +93,7 @@ Building workspace folder --task TASK The currently supported options: ('CLASSIFICATION', 'DETECTION', 'INSTANCE_SEGMENTATION', 'SEGMENTATION', 'ACTION_CLASSIFICATION', 'ACTION_DETECTION', 'ANOMALY_CLASSIFICATION', 'ANOMALY_DETECTION', 'ANOMALY_SEGMENTATION'). --train-type TRAIN_TYPE The currently supported options: dict_keys(['INCREMENTAL', 'SEMISUPERVISED', 'SELFSUPERVISED']). - --work-dir WORK_DIR Location where the workspace. + --workspace WORKSPACE Location where the workspace. --model MODEL Enter the name of the model you want to use. (Ex. EfficientNet-B0). --backbone BACKBONE Available Backbone Type can be found using 'otx find --backbone {framework}'. If there is an already created backbone configuration yaml file, enter the corresponding path. @@ -150,7 +150,7 @@ Training - ``weights.pth`` - a model snapshot - ``label_schema.json`` - a label schema used in training, created from a dataset -The results will be saved in ``./model`` folder by default. The output folder can be modified by ``--save-model-to`` option. These files are used by other commands: ``export``, ``eval``, ``demo``, etc. +The results will be saved in ``./model`` folder by default. The output folder can be modified by ``--output`` option. These files are used by other commands: ``export``, ``eval``, ``demo``, etc. ``otx train`` receives ``template`` as a positional argument. ``template`` can be a path to the specific ``template.yaml`` file, template name or template ID. Also, the path to train and val data root should be passed to the CLI to start training. @@ -160,7 +160,7 @@ However, if you created a workspace with ``otx build``, the training process can otx train --help usage: otx train [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--unlabeled-data-roots UNLABELED_DATA_ROOTS] [--unlabeled-file-list UNLABELED_FILE_LIST] - [--load-weights LOAD_WEIGHTS] [--resume-from RESUME_FROM] [--save-model-to SAVE_MODEL_TO] [--work-dir WORK_DIR] [--enable-hpo] [--hpo-time-ratio HPO_TIME_RATIO] [--gpus GPUS] + [--load-weights LOAD_WEIGHTS] [--resume-from RESUME_FROM] [--output OUTPUT] [--workspace WORKSPACE] [--enable-hpo] [--hpo-time-ratio HPO_TIME_RATIO] [--gpus GPUS] [--rdzv-endpoint RDZV_ENDPOINT] [--base-rank BASE_RANK] [--world-size WORLD_SIZE] [--mem-cache-size PARAMS.ALGO_BACKEND.MEM_CACHE_SIZE] [--data DATA] [template] {params} ... @@ -186,9 +186,9 @@ However, if you created a workspace with ``otx build``, the training process can Load model weights from previously saved checkpoint. --resume-from RESUME_FROM Resume training from previously saved checkpoint - --save-model-to SAVE_MODEL_TO + --output OUTPUT Location where trained model will be stored. - --work-dir WORK_DIR Location where the intermediate output of the training will be stored. + --workspace WORKSPACE Location where the intermediate output of the training will be stored. --enable-hpo Execute hyper parameters optimization (HPO) before training. --hpo-time-ratio HPO_TIME_RATIO Expected ratio of total time to run HPO to time taken for full fine-tuning. @@ -261,7 +261,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx export --help - usage: otx export [-h] [--load-weights LOAD_WEIGHTS] [--save-model-to SAVE_MODEL_TO] [--work-dir WORK_DIR] [--dump-features] [--half-precision] [template] + usage: otx export [-h] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] [--workspace WORKSPACE] [--dump-features] [--half-precision] [template] positional arguments: template Enter the path or ID or name of the template file. @@ -271,9 +271,9 @@ With the ``--help`` command, you can list additional information, such as its pa -h, --help show this help message and exit --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint. - --save-model-to SAVE_MODEL_TO + --output OUTPUT Location where exported model will be stored. - --work-dir WORK_DIR Location where the intermediate output of the export will be stored. + --workspace WORKSPACE Location where the intermediate output of the export will be stored. --dump-features Whether to return feature vector and saliency map for explanation purposes. --half-precision This flag indicated if model is exported in half precision (FP16). @@ -282,7 +282,7 @@ The command below performs exporting to the ``outputs/openvino`` path. .. code-block:: - (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --save-model-to outputs/openvino + (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --output outputs/openvino The command results in ``openvino.xml``, ``openvino.bin`` and ``label_schema.json`` @@ -290,7 +290,7 @@ To use the exported model as an input for ``otx explain``, please dump additiona .. code-block:: - (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --save-model-to outputs/openvino/with_features --dump-features + (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --output outputs/openvino/with_features --dump-features ************ @@ -306,8 +306,8 @@ With the ``--help`` command, you can list additional information: .. code-block:: - usage: otx optimize [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--save-model-to SAVE_MODEL_TO] [--save-performance SAVE_PERFORMANCE] - [--work-dir WORK_DIR] + usage: otx optimize [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] + [--workspace WORKSPACE] [template] {params} ... positional arguments: @@ -324,11 +324,9 @@ With the ``--help`` command, you can list additional information: Comma-separated paths to validation data folders. --load-weights LOAD_WEIGHTS Load weights of trained model - --save-model-to SAVE_MODEL_TO - Location where trained model will be stored. - --save-performance SAVE_PERFORMANCE - Path to a json file where computed performance will be stored. - --work-dir WORK_DIR Location where the intermediate output of the task will be stored. + --output OUTPUT + Location where optimized model will be stored. + --workspace WORKSPACE Location where the intermediate output of the task will be stored. Command example for optimizing a PyTorch model (.pth) with OpenVINO™ NNCF: @@ -337,7 +335,7 @@ Command example for optimizing a PyTorch model (.pth) with OpenVINO™ NNCF: (otx) ...$ otx optimize SSD --load-weights \ --train-data-roots \ --val-data-roots \ - --save-model-to outputs/nncf + --output outputs/nncf Command example for optimizing OpenVINO™ model (.xml) with OpenVINO™ POT: @@ -346,7 +344,7 @@ Command example for optimizing OpenVINO™ model (.xml) with OpenVINO™ POT: (otx) ...$ otx optimize SSD --load-weights \ --val-data-roots \ - --save-model-to outputs/pot + --output outputs/pot Thus, to use POT pass the path to exported IR (.xml) model, to use NNCF pass the path to the PyTorch (.pth) weights. @@ -363,7 +361,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx eval --help - usage: otx eval [-h] [--test-data-roots TEST_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--save-performance SAVE_PERFORMANCE] [--work-dir WORK_DIR] [template] {params} ... + usage: otx eval [-h] [--test-data-roots TEST_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] [--workspace WORKSPACE] [template] {params} ... positional arguments: template Enter the path or ID or name of the template file. @@ -377,9 +375,9 @@ With the ``--help`` command, you can list additional information, such as its pa Comma-separated paths to test data folders. --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint.It could be a trained/optimized model (POT only) or exported model. - --save-performance SAVE_PERFORMANCE - Path to a json file where computed performance will be stored. - --work-dir WORK_DIR Location where the intermediate output of the task will be stored. + --output OUTPUT + Location where the intermediate output of the task will be stored. + --workspace WORKSPACE Path to the workspace where the command will run. The command below will evaluate the trained model on the provided dataset: @@ -388,7 +386,7 @@ The command below will evaluate the trained model on the provided dataset: (otx) ...$ otx eval SSD --test-data-roots \ --load-weights \ - --save-performance outputs/performance.json + --output .. note:: @@ -447,7 +445,7 @@ By default, the model is exported to the OpenVINO™ IR format without extra fea .. code-block:: (otx) ...$ otx export SSD --load-weights \ - --save-model-to outputs/openvino/with_features \ + --output outputs/openvino/with_features \ --dump-features (otx) ...$ otx explain SSD --explain-data-roots \ --load-weights outputs/openvino/with_features \ @@ -521,7 +519,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx deploy --help - usage: otx deploy [-h] [--load-weights LOAD_WEIGHTS] [--save-model-to SAVE_MODEL_TO] [template] + usage: otx deploy [-h] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] [template] positional arguments: template Enter the path or ID or name of the template file. @@ -531,7 +529,7 @@ With the ``--help`` command, you can list additional information, such as its pa -h, --help show this help message and exit --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint. - --save-model-to SAVE_MODEL_TO + --output OUTPUT Location where openvino.zip will be stored. @@ -540,5 +538,5 @@ Command example: .. code-block:: (otx) ...$ otx deploy SSD --load-weights \ - --save-model-to outputs/deploy + --output outputs/deploy diff --git a/docs/source/guide/tutorials/advanced/self_sl.rst b/docs/source/guide/tutorials/advanced/self_sl.rst index 96de2beb42c..a540d183f61 100644 --- a/docs/source/guide/tutorials/advanced/self_sl.rst +++ b/docs/source/guide/tutorials/advanced/self_sl.rst @@ -64,7 +64,7 @@ for **self-supervised learning** by running the following command: .. code-block:: - (otx) ...$ otx build --train-data-roots data/flower_photos --model MobileNet-V3-large-1x --train-type SELFSUPERVISED --work-dir otx-workspace-CLASSIFICATION-SELFSUPERVISED + (otx) ...$ otx build --train-data-roots data/flower_photos --model MobileNet-V3-large-1x --train-type SELFSUPERVISED --workspace otx-workspace-CLASSIFICATION-SELFSUPERVISED [*] Workspace Path: otx-workspace-CLASSIFICATION-SELFSUPERVISED [*] Load Model Template ID: Custom_Image_Classification_MobileNet-V3-large-1x @@ -82,7 +82,7 @@ for **self-supervised learning** by running the following command: 1. add ``--train-type SELFSUPERVISED`` in the command to get the training components for self-supervised learning, 2. update the path set as ``train-data-roots``, - 3. and add ``--work-dir`` to distinguish self-supervised learning workspace from supervised learning workspace. + 3. and add ``--workspace`` to distinguish self-supervised learning workspace from supervised learning workspace. After the workspace creation, the workspace structure is as follows: diff --git a/docs/source/guide/tutorials/advanced/semi_sl.rst b/docs/source/guide/tutorials/advanced/semi_sl.rst index cce334631e9..129d4fdf527 100644 --- a/docs/source/guide/tutorials/advanced/semi_sl.rst +++ b/docs/source/guide/tutorials/advanced/semi_sl.rst @@ -150,7 +150,7 @@ In the train log, you can check that the train type is set to **SEMISUPERVISED** ... -After training ends, a trained model is saved in the ``models`` sub-directory in the workspace named ``otx-workspace-CLASSIFICATION`` by default. +After training ends, a trained model is saved in the ``latest`` sub-directory in the workspace named ``otx-workspace-CLASSIFICATION`` by default. *************************** @@ -159,7 +159,7 @@ Validation In the same manner with `the normal validation <../base/how_to_train/classification.html#validation>`__, we can evaluate the trained model with auto-splitted validation dataset in the workspace and -save results to ``performance.json`` by the following command: +save results to ``outputs/performance.json`` by the following command: .. code-block:: @@ -167,4 +167,4 @@ save results to ``performance.json`` by the following command: (otx) ...$ otx eval otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml \ --test-data-roots splitted_dataset/val \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs diff --git a/docs/source/guide/tutorials/base/deploy.rst b/docs/source/guide/tutorials/base/deploy.rst index ee16dbc8d43..dfba9bda762 100644 --- a/docs/source/guide/tutorials/base/deploy.rst +++ b/docs/source/guide/tutorials/base/deploy.rst @@ -45,7 +45,7 @@ using the command below: (otx) ...$ otx deploy otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml \ --load-weights outputs/openvino/openvino.xml \ - --save-model-to outputs/deploy + --output outputs/deploy 2023-01-20 09:30:40,938 | INFO : Loading OpenVINO OTXDetectionTask 2023-01-20 09:30:41,736 | INFO : OpenVINO task initialization completed diff --git a/docs/source/guide/tutorials/base/how_to_train/action_classification.rst b/docs/source/guide/tutorials/base/how_to_train/action_classification.rst index f41f6051fb7..cf23f1b30fc 100644 --- a/docs/source/guide/tutorials/base/how_to_train/action_classification.rst +++ b/docs/source/guide/tutorials/base/how_to_train/action_classification.rst @@ -186,13 +186,13 @@ Keep in mind that ``label_schema.json`` file contains meta information about the ``otx eval`` will output a frame-wise accuracy for action classification. Note, that top-1 accuracy during training is video-wise accuracy. 2. The command below will run validation on the dataset -and save performance results in ``performance.json`` file: +and save performance results in ``outputs/performance.json`` file: .. code-block:: (otx) ...$ otx eval --test-data-roots ../data/hmdb51/CVAT/valid \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs You will get a similar validation output: @@ -220,7 +220,7 @@ and save the exported model to the ``openvino_models`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_models + --output openvino_models ... 2023-02-21 22:54:32,518 - mmaction - INFO - Model architecture: X3D @@ -242,7 +242,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots ../data/hmdb51/CVAT/valid \ --load-weights openvino_models/openvino.xml \ - --save-performance openvino_models/performance.json + --output outputs/openvino_models ... @@ -263,7 +263,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_models/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/action_detection.rst b/docs/source/guide/tutorials/base/how_to_train/action_detection.rst index 3340e012d11..80d980d3457 100644 --- a/docs/source/guide/tutorials/base/how_to_train/action_detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/action_detection.rst @@ -128,13 +128,13 @@ Please note, ``label_schema.json`` file contains meta information about the data ``otx eval`` will output a mAP score for spatio-temporal action detection. 2. The command below will run validation on our dataset -and save performance results in ``performance.json`` file: +and save performance results in ``outputs/performance.json`` file: .. code-block:: (otx) ...$ otx eval --test-data-roots ../data/JHMDB_5%/test \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs We will get a similar to this validation output after some validation time (about 2 minutes): diff --git a/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst b/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst index 561a8136a80..6e476a6294e 100644 --- a/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst @@ -138,7 +138,7 @@ Now we have trained the model, let's see how it performs on a specific dataset. (otx) ...$ otx eval ote_anomaly_detection_padim \ --test-data-roots datasets/MVTec/bottle/test \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --save-performance otx-workspace-ANOMALY_DETECTION/performance.json + --output otx-workspace-ANOMALY_DETECTION/outputs You should see an output similar to the following:: @@ -165,7 +165,7 @@ and save the exported model to the ``openvino_models`` folder: otx export ote_anomaly_detection_padim \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --save-model-to otx-workspace-ANOMALY_DETECTION/openvino_models + --output otx-workspace-ANOMALY_DETECTION/openvino_models You will see the outputs similar to the following: @@ -188,7 +188,7 @@ Now that we have the exported model, let's check its performance using ``otx eva otx eval ote_anomaly_detection_padim \ --test-data-roots datasets/MVTec/bottle/test \ --load-weights otx-workspace-ANOMALY_DETECTION/openvino_models/openvino.xml \ - --save-performance otx-workspace-ANOMALY_DETECTION/openvino_models/performance.json + --output otx-workspace-ANOMALY_DETECTION/openvino_models This gives the following results: @@ -211,7 +211,7 @@ optimization. otx optimize ote_anomaly_detection_padim \ --train-data-roots datasets/MVTec/bottle/train \ --load-weights otx-workspace-ANOMALY_DETECTION/openvino_models/openvino.xml \ - --save-model-to otx-workspace-ANOMALY_DETECTION/pot_model + --output otx-workspace-ANOMALY_DETECTION/pot_model This command generates the following files that can be used to run :doc:`otx demo <../demo>`: @@ -231,7 +231,7 @@ weights to the ``opitmize`` command: otx optimize ote_anomaly_detection_padim \ --train-data-roots datasets/MVTec/bottle/train \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --save-model-to otx-workspace-ANOMALY_DETECTION/nncf_model + --output otx-workspace-ANOMALY_DETECTION/nncf_model Similar to POT optimization, it generates the following files: diff --git a/docs/source/guide/tutorials/base/how_to_train/classification.rst b/docs/source/guide/tutorials/base/how_to_train/classification.rst index ff66d4b7f39..6f1f2bb03d2 100644 --- a/docs/source/guide/tutorials/base/how_to_train/classification.rst +++ b/docs/source/guide/tutorials/base/how_to_train/classification.rst @@ -136,7 +136,7 @@ The training time highly relies on the hardware characteristics, for example on After that, you have the PyTorch classification model trained with OpenVINO™ Training Extensions, which you can use for evaluation, export, optimization and deployment. .. note:: - If you specified ``--work-dir``, you also can visualize the training using ``Tensorboard`` as these logs are located in ``/tf_logs``. + If you specified ``--workspace``, you also can visualize the training using ``Tensorboard`` as these logs are located in ``/tf_logs``. *********** Validation @@ -157,7 +157,7 @@ and save performance results in ``performance.json`` file: (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs You will get a similar validation output: @@ -184,7 +184,7 @@ and save the exported model to the ``openvino_model`` folder: .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_model + --output openvino_model ... @@ -199,7 +199,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights openvino_model/openvino.xml \ - --save-performance openvino_model/performance.json + --output openvino_model ... @@ -220,7 +220,7 @@ a PyTorch model (`.pth`) with OpenVINO™ NNCF. .. code-block:: - (otx) ...$ otx optimize --load-weights models/weights.pth --save-model-to nncf_model + (otx) ...$ otx optimize --load-weights models/weights.pth --output nncf_model ... @@ -239,7 +239,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_model/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/detection.rst b/docs/source/guide/tutorials/base/how_to_train/detection.rst index 1e6a82c693e..1b48806b7bd 100644 --- a/docs/source/guide/tutorials/base/how_to_train/detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/detection.rst @@ -225,7 +225,7 @@ detection model on the first GPU on WGISD dataset: .. code-block:: (otx) ...$ cd otx-workspace-DETECTION/ - (otx) ...$ otx train --save-model-to ../outputs --work-dir ../outputs/logs --gpus 1 + (otx) ...$ otx train --output ../outputs --workspace ../outputs/logs --gpus 1 To start multi-gpu training, list the indexes of GPUs you want to train on or omit `gpus` parameter, so training will run on all available GPUs. @@ -288,7 +288,7 @@ folder on WGISD dataset and save results to ``outputs/performance``: (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights ../outputs/weights.pth \ - --save-performance ../outputs/performance.json + --output ../outputs/ 3. The output of ``../outputs/performance.json`` consists of @@ -309,7 +309,7 @@ Please note, by default, the optimal confidence threshold is detected based on v (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights ../outputs/weights.pth \ - --save-performance ../outputs/performance.json + --output ../outputs params \ --postprocessing.confidence_threshold 0.5 \ --postprocessing.result_based_confidence_threshold false @@ -332,7 +332,7 @@ from the previous section and save the exported model to the ``../outputs/openvi .. code-block:: (otx) ...$ otx export --load-weights ../outputs/weights.pth \ - --save-model-to ../outputs/openvino/ + --output ../outputs/openvino/ ... @@ -347,7 +347,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights ../outputs/openvino/openvino.xml \ - --save-performance ../outputs/performance.json + --output ../outputs ... 2023-01-10 06:24:50,382 | INFO : Start OpenVINO inference @@ -384,8 +384,8 @@ with OpenVINO NNCF. .. code-block:: (otx) ...$ otx optimize --load-weights ../outputs/weights.pth \ - --save-model-to ../outputs/nncf \ - --save-performance ../outputs/nncf/performance.json + --output ../outputs/nncf \ + --output ../outputs/nncf ... @@ -403,8 +403,8 @@ with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights ../outputs/openvino/openvino.xml \ - --save-model-to ../outputs/pot \ - --save-performance ../outputs/pot/performance.json + --output ../outputs/pot \ + --output ../outputs/pot ... diff --git a/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst b/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst index a9399d5ca24..84012c69999 100644 --- a/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst +++ b/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst @@ -217,13 +217,13 @@ Please note, ``label_schema.json`` file contains meta information about the data ``otx eval`` will output a F-measure for instance segmentation. 2. The command below will run validation on our dataset -and save performance results in ``performance.json`` file: +and save performance results in ``outputs/performance.json`` file: .. code-block:: (otx) ...$ otx eval --test-data-roots otx-workspace-INSTANCE_SEGMENTATION/splitted_dataset/car_tree_bug \ --load-weights models/weights.pth \ - --save-performance performance.json + --outputs outputs We will get a similar to this validation output: @@ -243,7 +243,7 @@ We will get a similar to this validation output: Also, if you're inside a workspace and ``weights.pth`` exists in ``models`` dir, you can omit ``--load-weights`` as well, assuming those weights are the default as ``models/weights.pth``. - If you omit ``--save-performance``, it will create a ``performance.json`` in the folder for those weights. + If you omit ``--output``, it will create a ``performance.json`` in the folder for those weights. The output of ``./outputs/performance.json`` consists of a dict with target metric name and its value. @@ -267,7 +267,7 @@ and save the exported model to the ``openvino_model`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_model + --output openvino_model ... [ SUCCESS ] Generated IR version 11 model. @@ -287,7 +287,7 @@ You can use ``otx train`` directly without ``otx build``. It will be required to (otx) ...$ otx eval --test-data-roots otx-workspace-INSTANCE_SEGMENTATION/splitted_dataset/car_tree_bug \ --load-weights openvino_model/openvino.xml \ - --save-performance openvino_model/performance.json + --output openvino_model ... @@ -313,7 +313,7 @@ a PyTorch model (`.pth`) with OpenVINO™ NNCF. .. code-block:: - (otx) ...$ otx optimize --load-weights models/weights.pth --save-model-to nncf_model + (otx) ...$ otx optimize --load-weights models/weights.pth --output nncf_model ... @@ -332,7 +332,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_model/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst b/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst index 4302190dcab..4cef182727c 100644 --- a/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst +++ b/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst @@ -151,7 +151,7 @@ By running this example command, the performance results evaluated by our splitt (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs Finally, we get the validation output: @@ -165,7 +165,7 @@ Finally, we get the validation output: 2023-02-21 18:09:58,508 | INFO : mDice after evaluation: 0.9659400544959128 Performance(score: 0.9659400544959128, dashboard: (1 metric groups)) -In ``performance.json`` file, the validation output score is saved as: +In ``outputs/performance.json`` file, the validation output score is saved as: .. code-block:: @@ -185,7 +185,7 @@ and save the exported model to the ``openvino_model`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_model + --output openvino_model ... @@ -200,7 +200,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights openvino_model/openvino.xml \ - --save-performance openvino_model/performance.json + --output openvino_model ... @@ -220,7 +220,7 @@ a PyTorch model (`.pth`) with OpenVINO™ NNCF. .. code-block:: - (otx) ...$ otx optimize --load-weights models/weights.pth --save-model-to nncf_model + (otx) ...$ otx optimize --load-weights models/weights.pth --output nncf_model ... @@ -239,7 +239,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_model/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/otx/algorithms/common/tasks/training_base.py b/otx/algorithms/common/tasks/training_base.py index 3016685fd4e..1e55ca3cd79 100644 --- a/otx/algorithms/common/tasks/training_base.py +++ b/otx/algorithms/common/tasks/training_base.py @@ -190,6 +190,11 @@ def project_path(self): """Return output path with logs.""" return self._output_path + @property + def config(self): + """Return output configs used in task.""" + return self._recipe_cfg + @property def model_name(self): """Name of Model Template.""" diff --git a/otx/cli/manager/config_manager.py b/otx/cli/manager/config_manager.py index 906f1d77e20..b17f57dfad7 100644 --- a/otx/cli/manager/config_manager.py +++ b/otx/cli/manager/config_manager.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # import shutil +from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional @@ -88,6 +89,7 @@ def __init__(self, args, workspace_root: Optional[str] = None, mode: str = "trai self.workspace_root = Path(workspace_root) if workspace_root else Path(".") self.mode = mode self.rebuild: bool = False + self.create_date: str = datetime.now().strftime("%Y%m%d_%H%M%S") self.args = args self.template = args.template @@ -115,6 +117,20 @@ def data_config_file_path(self) -> Path: raise FileNotFoundError(f"Not found: {self.args.data}") return self.workspace_root / "data.yaml" + @property + def output_path(self) -> Path: + """The path of output directory for workspace. + + Returns: + Path: Path of output directory. + """ + if "output" in self.args and self.args.output: + output_path = Path(self.args.output) + else: + output_path = self.workspace_root / "outputs" / self.create_date + output_path.mkdir(exist_ok=True, parents=True) + return output_path + def check_workspace(self) -> bool: """Check that the class's workspace_root is an actual workspace folder. diff --git a/otx/cli/tools/build.py b/otx/cli/tools/build.py index 0c5a4b3761f..16d018ae207 100644 --- a/otx/cli/tools/build.py +++ b/otx/cli/tools/build.py @@ -68,8 +68,8 @@ def get_args(): default="incremental", ) parser.add_argument( - "--work-dir", - help="Location where the workspace.", + "--workspace", + help="Path to the workspace where the command will run.", default=None, ) parser.add_argument( @@ -91,13 +91,13 @@ def main(): config_manager = ConfigManager(args, mode="build") if args.task: config_manager.task_type = args.task.upper() - if args.work_dir: - config_manager.workspace_root = Path(args.work_dir) + if args.workspace: + config_manager.workspace_root = Path(args.workspace) # Auto-Configuration for model template config_manager.configure_template(model=args.model) - config_manager.build_workspace(new_workspace_path=args.work_dir) + config_manager.build_workspace(new_workspace_path=args.workspace) # Auto-Configuration for Dataset configuration config_manager.configure_data_config() diff --git a/otx/cli/tools/deploy.py b/otx/cli/tools/deploy.py index 5e3ff06b416..9f2e99d50c7 100644 --- a/otx/cli/tools/deploy.py +++ b/otx/cli/tools/deploy.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions # and limitations under the License. -import os from pathlib import Path from otx.api.configuration.helper import create @@ -35,7 +34,8 @@ def get_args(): help="Load model weights from previously saved checkpoint.", ) parser.add_argument( - "--save-model-to", + "-o", + "--output", help="Location where openvino.zip will be stored.", ) @@ -59,9 +59,9 @@ def main(): assert hyper_parameters if not args.load_weights and config_manager.check_workspace(): - exported_weight_path = config_manager.workspace_root / "models-exported/openvino.xml" + exported_weight_path = config_manager.workspace_root / "latest/openvino_models/openvino.xml" if not exported_weight_path.exists(): - raise RuntimeError("OpenVINO-exported models are supported.") + raise RuntimeError("No appropriate OpenVINO exported model was found.") args.load_weights = str(exported_weight_path) # Get classes for Task, ConfigurableParameters and Dataset. @@ -82,11 +82,10 @@ def main(): deployed_model = ModelEntity(None, environment.get_model_configuration()) - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "model-deployed") - os.makedirs(args.save_model_to, exist_ok=True) + output_path = Path(args.output) if args.output else config_manager.output_path + output_path.mkdir(exist_ok=True, parents=True) task.deploy(deployed_model) - with open(Path(args.save_model_to) / "openvino.zip", "wb") as write_file: + with open(output_path / "openvino.zip", "wb") as write_file: write_file.write(deployed_model.exportable_code) return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index 0c9f0e16100..9b2b1d4ac7a 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -48,12 +48,13 @@ def get_args(): "It could be a trained/optimized model (POT only) or exported model.", ) parser.add_argument( - "--save-performance", - help="Path to a json file where computed performance will be stored.", + "-o", + "--output", + help="Location where the intermediate output of the task will be stored.", ) parser.add_argument( - "--work-dir", - help="Location where the intermediate output of the task will be stored.", + "--workspace", + help="Path to the workspace where the command will run.", default=None, ) @@ -82,12 +83,12 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.work_dir, mode="eval") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="eval") # Auto-Configuration for model template config_manager.configure_template() if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "models/weights.pth") + args.load_weights = str(config_manager.workspace_root / "latest" / "weights.pth") # Update Hyper Parameter Configs hyper_parameters = config_manager.get_hyparams_config(override_param) @@ -136,9 +137,8 @@ def main(): assert resultset.performance is not None print(resultset.performance) - if not args.save_performance: - args.save_performance = str(Path(args.load_weights).parent / "performance.json") - with open(args.save_performance, "w", encoding="UTF-8") as write_file: + output_path = Path(args.output) if args.output else config_manager.output_path + with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file: json.dump( {resultset.performance.score.name: resultset.performance.score.value}, write_file, diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index fdd2295a5a5..bb8733f6a79 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -37,12 +37,13 @@ def get_args(): help="Load model weights from previously saved checkpoint.", ) parser.add_argument( - "--save-model-to", + "-o", + "--output", help="Location where exported model will be stored.", ) parser.add_argument( - "--work-dir", - help="Location where the intermediate output of the export will be stored.", + "--workspace", + help="Path to the workspace where the command will run.", default=None, ) parser.add_argument( @@ -62,7 +63,7 @@ def get_args(): def main(): """Main function that is used for model exporting.""" args = get_args() - config_manager = ConfigManager(args, mode="eval", workspace_root=args.work_dir) + config_manager = ConfigManager(args, mode="eval", workspace_root=args.workspace) # Auto-Configuration for model template config_manager.configure_template() @@ -71,7 +72,8 @@ def main(): # Get class for Task. if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "models/weights.pth") + args.load_weights = str(config_manager.workspace_root / "latest" / "weights.pth") + is_nncf = is_checkpoint_nncf(args.load_weights) task_class = get_impl_class(template.entrypoints.nncf if is_nncf else template.entrypoints.base) @@ -95,17 +97,23 @@ def main(): ) environment.model = model - task = task_class(task_environment=environment, output_path=args.work_dir) + (config_manager.output_path / "logs").mkdir(exist_ok=True, parents=True) + task = task_class(task_environment=environment, output_path=str(config_manager.output_path / "logs")) exported_model = ModelEntity(None, environment.get_model_configuration()) export_precision = ModelPrecision.FP16 if args.half_precision else ModelPrecision.FP32 task.export(ExportType.OPENVINO, exported_model, export_precision, args.dump_features) - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "model-exported") - Path(args.save_model_to).mkdir(exist_ok=True, parents=True) - save_model_data(exported_model, args.save_model_to) + output_path = config_manager.output_path / "openvino_models" + output_path.mkdir(exist_ok=True, parents=True) + save_model_data(exported_model, str(output_path)) + + # Softlink to weights & openvino_models + pre_weight_path = Path(args.load_weights).resolve().parent / "openvino_models" + if pre_weight_path.exists(): + pre_weight_path.unlink() + pre_weight_path.symlink_to(output_path.resolve()) return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/optimize.py b/otx/cli/tools/optimize.py index 9d8def31357..5a6cb7b9168 100644 --- a/otx/cli/tools/optimize.py +++ b/otx/cli/tools/optimize.py @@ -15,6 +15,7 @@ # and limitations under the License. import json +from pathlib import Path from otx.api.entities.inference_parameters import InferenceParameters from otx.api.entities.model import ModelEntity @@ -55,15 +56,12 @@ def get_args(): help="Load weights of trained model", ) parser.add_argument( - "--save-model-to", - help="Location where trained model will be stored.", + "-o", + "--output", + help="Location where optimized model will be stored.", ) parser.add_argument( - "--save-performance", - help="Path to a json file where computed performance will be stored.", - ) - parser.add_argument( - "--work-dir", + "--workspace", help="Location where the intermediate output of the task will be stored.", default=None, ) @@ -80,13 +78,13 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.work_dir, mode="train") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="train") # Auto-Configuration for model template config_manager.configure_template() # The default in the workspace is the model weight of the OTX train. if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "models/weights.pth") + args.load_weights = str(config_manager.workspace_root / "latest" / "weights.pth") is_pot = False if args.load_weights.endswith(".bin") or args.load_weights.endswith(".xml"): @@ -128,9 +126,20 @@ def main(): OptimizationParameters(), ) - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "model-optimized") - save_model_data(output_model, args.save_model_to) + opt_method = "pot" if is_pot else "nncf" + if not args.output: + output_path = config_manager.output_path + output_path = output_path / opt_method + else: + output_path = Path(args.output) + output_path.mkdir(exist_ok=True, parents=True) + save_model_data(output_model, output_path) + + # Softlink to weights & optimized models + pre_weight_path = Path(args.load_weights).resolve().parent / opt_method + if pre_weight_path.exists(): + pre_weight_path.unlink() + pre_weight_path.symlink_to(output_path.resolve()) validation_dataset = dataset.get_subset(Subset.VALIDATION) predicted_validation_dataset = task.infer( @@ -147,12 +156,12 @@ def main(): assert resultset.performance is not None print(resultset.performance) - if args.save_performance: - with open(args.save_performance, "w", encoding="UTF-8") as write_file: - json.dump( - {resultset.performance.score.name: resultset.performance.score.value}, - write_file, - ) + performance_file_path = config_manager.output_path / f"{opt_method}_performance.json" + with open(performance_file_path, "w", encoding="UTF-8") as write_file: + json.dump( + {resultset.performance.score.name: resultset.performance.score.value}, + write_file, + ) return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/train.py b/otx/cli/tools/train.py index 164210366ba..402cb27d740 100644 --- a/otx/cli/tools/train.py +++ b/otx/cli/tools/train.py @@ -76,11 +76,12 @@ def get_args(): help="Resume training from previously saved checkpoint", ) parser.add_argument( - "--save-model-to", - help="Location where trained model will be stored.", + "-o", + "--output", + help="Location where outputs (model & logs) will be stored.", ) parser.add_argument( - "--work-dir", + "--workspace", help="Location where the intermediate output of the training will be stored.", default=None, ) @@ -157,13 +158,13 @@ def main(): # pylint: disable=too-many-branches """Main function that is used for model training.""" args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.work_dir, mode="train") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="train") # Auto-Configuration for model template config_manager.configure_template() # Creates a workspace if it doesn't exist. if not config_manager.check_workspace(): - config_manager.build_workspace(new_workspace_path=args.work_dir) + config_manager.build_workspace(new_workspace_path=args.workspace) # Auto-Configuration for Dataset configuration config_manager.configure_data_config(update_data_yaml=config_manager.check_workspace()) @@ -203,13 +204,12 @@ def main(): # pylint: disable=too-many-branches model_adapters=model_adapters, ) - # FIXME: Need to align output results & Current HPO use save_model_to.parent - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "models") if args.enable_hpo: + args.output = str(config_manager.output_path) environment = run_hpo(args, environment, dataset, config_manager.data_config) - task = task_class(task_environment=environment, output_path=args.work_dir) + (config_manager.output_path / "logs").mkdir(exist_ok=True, parents=True) + task = task_class(task_environment=environment, output_path=str(config_manager.output_path / "logs")) if args.gpus: multigpu_manager = MultiGPUManager(main, args.gpus, args.rdzv_endpoint, args.base_rank, args.world_size) @@ -225,8 +225,11 @@ def main(): # pylint: disable=too-many-branches task.train(dataset, output_model, train_parameters=TrainParameters()) - save_model_data(output_model, args.save_model_to) - print(f"[*] Save Model to: {args.save_model_to}") + save_model_data(output_model, str(config_manager.output_path / "models")) + # Latest model folder symbolic link to models + if (config_manager.workspace_root / "latest").exists(): + (config_manager.workspace_root / "latest").unlink() + (config_manager.workspace_root / "latest").symlink_to((config_manager.output_path / "models").resolve()) if config_manager.data_config["val_subset"]["data_root"]: validation_dataset = dataset.get_subset(Subset.VALIDATION) diff --git a/otx/cli/utils/hpo.py b/otx/cli/utils/hpo.py index 61a8ec18da0..6577434a442 100644 --- a/otx/cli/utils/hpo.py +++ b/otx/cli/utils/hpo.py @@ -554,7 +554,7 @@ def run_hpo( ) return None - hpo_save_path = (Path(args.save_model_to).parent / "hpo").absolute() + hpo_save_path = (Path(args.output) / "hpo").absolute() hpo_runner = HpoRunner( environment, len(dataset.get_subset(Subset.TRAINING)), diff --git a/otx/cli/utils/multi_gpu.py b/otx/cli/utils/multi_gpu.py index cea63cbc1b6..4334ec0df18 100644 --- a/otx/cli/utils/multi_gpu.py +++ b/otx/cli/utils/multi_gpu.py @@ -238,7 +238,7 @@ def run_child_process( sys.argv.pop(gpus_arg_idx) if "--enable-hpo" in sys.argv: sys.argv.remove("--enable-hpo") - set_arguments_to_argv("--work-dir", output_path) + set_arguments_to_argv("--workspace", output_path) set_arguments_to_argv("--rdzv-endpoint", rdzv_endpoint) MultiGPUManager.initialize_multigpu_train(rdzv_endpoint, rank, local_rank, gpu_ids, world_size) diff --git a/tests/conftest.py b/tests/conftest.py index d609c3a2fbc..4ae77116996 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ def pytest_addoption(parser): @pytest.fixture(scope="session") def tmp_dir_path(request) -> Generator[Path, None, None]: - prefix = request.config.getoption("--test-work-dir") + prefix = request.config.getoption("--test-workspace") with TemporaryDirectory(prefix=prefix) as tmp_dir: yield Path(tmp_dir) diff --git a/tests/e2e/cli/classification/test_classification.py b/tests/e2e/cli/classification/test_classification.py index a3278f0eb13..6370617daec 100644 --- a/tests/e2e/cli/classification/test_classification.py +++ b/tests/e2e/cli/classification/test_classification.py @@ -114,7 +114,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -126,7 +126,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -367,7 +369,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0_m) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args_m) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -379,7 +381,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0_m) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -568,7 +572,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args_h) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args_h) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -580,7 +584,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args_h) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -770,7 +776,7 @@ def test_otx_selfsl_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path_1, otx_dir, args_selfsl) template_work_dir = get_template_dir(template, tmp_dir_path_1) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" tmp_dir_path_2 = tmp_dir_path / "multi_class_cls/test_selfsl_sl" otx_train_testing(template, tmp_dir_path_2, otx_dir, args1) diff --git a/tests/e2e/cli/detection/test_detection.py b/tests/e2e/cli/detection/test_detection.py index 24c5d5b6274..4c3d5e6ea74 100644 --- a/tests/e2e/cli/detection/test_detection.py +++ b/tests/e2e/cli/detection/test_detection.py @@ -104,7 +104,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -116,7 +116,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/detection/test_instance_segmentation.py b/tests/e2e/cli/detection/test_instance_segmentation.py index ac7b3bdaeaf..4e7199c23fa 100644 --- a/tests/e2e/cli/detection/test_instance_segmentation.py +++ b/tests/e2e/cli/detection/test_instance_segmentation.py @@ -87,7 +87,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -99,7 +99,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/detection/test_tiling_detection.py b/tests/e2e/cli/detection/test_tiling_detection.py index 697256b9c68..c303bdde682 100644 --- a/tests/e2e/cli/detection/test_tiling_detection.py +++ b/tests/e2e/cli/detection/test_tiling_detection.py @@ -93,7 +93,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/detection/test_tiling_instseg.py b/tests/e2e/cli/detection/test_tiling_instseg.py index 4a11e139266..30f97ba7152 100644 --- a/tests/e2e/cli/detection/test_tiling_instseg.py +++ b/tests/e2e/cli/detection/test_tiling_instseg.py @@ -93,7 +93,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/segmentation/test_segmentation.py b/tests/e2e/cli/segmentation/test_segmentation.py index 96086fa25eb..a815d9d6c06 100644 --- a/tests/e2e/cli/segmentation/test_segmentation.py +++ b/tests/e2e/cli/segmentation/test_segmentation.py @@ -87,7 +87,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -99,7 +99,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -330,7 +332,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path_1, otx_dir, args_selfsl) template_work_dir = get_template_dir(template, tmp_dir_path_1) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" tmp_dir_path_2 = tmp_dir_path / "segmentation/test_selfsl_sl" otx_train_testing(template, tmp_dir_path_2, otx_dir, args1) diff --git a/tests/integration/cli/classification/test_classification.py b/tests/integration/cli/classification/test_classification.py index ee26e5cbc35..3560a7aa694 100644 --- a/tests/integration/cli/classification/test_classification.py +++ b/tests/integration/cli/classification/test_classification.py @@ -108,7 +108,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/detection/test_detection.py b/tests/integration/cli/detection/test_detection.py index 7e071a1ee35..8373984f353 100644 --- a/tests/integration/cli/detection/test_detection.py +++ b/tests/integration/cli/detection/test_detection.py @@ -89,7 +89,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/detection/test_instance_segmentation.py b/tests/integration/cli/detection/test_instance_segmentation.py index 13b74482d82..3c1035d8e5d 100644 --- a/tests/integration/cli/detection/test_instance_segmentation.py +++ b/tests/integration/cli/detection/test_instance_segmentation.py @@ -73,7 +73,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/segmentation/test_segmentation.py b/tests/integration/cli/segmentation/test_segmentation.py index be027226fb6..ae74d2c32e5 100644 --- a/tests/integration/cli/segmentation/test_segmentation.py +++ b/tests/integration/cli/segmentation/test_segmentation.py @@ -113,7 +113,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/regression/classification/test_classification.py b/tests/regression/classification/test_classification.py index 49a0593f014..86349cf77fc 100644 --- a/tests/regression/classification/test_classification.py +++ b/tests/regression/classification/test_classification.py @@ -114,7 +114,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "multi_class_cls_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() @@ -242,7 +244,7 @@ def test_otx_train_selfsl(self, template, tmp_dir_path): args_selfsl["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] args_selfsl["--val-data-roots"] = multi_class_data_args["--val-data-roots"] args_selfsl["--test-data-roots"] = multi_class_data_args["--test-data-roots"] - args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, new_tmp_dir_path, otx_dir, args_selfsl) # Evaluation with self + supervised training model @@ -460,7 +462,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "multi_label_cls_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() diff --git a/tests/regression/detection/test_detection.py b/tests/regression/detection/test_detection.py index 5837682e381..967b4fa1357 100644 --- a/tests/regression/detection/test_detection.py +++ b/tests/regression/detection/test_detection.py @@ -114,7 +114,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "det_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() diff --git a/tests/regression/detection/test_instnace_segmentation.py b/tests/regression/detection/test_instnace_segmentation.py index 731283315bd..9bddcc0d19f 100644 --- a/tests/regression/detection/test_instnace_segmentation.py +++ b/tests/regression/detection/test_instnace_segmentation.py @@ -114,7 +114,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "inst_seg_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() diff --git a/tests/regression/segmentation/test_segmentation.py b/tests/regression/segmentation/test_segmentation.py index e179559ce2c..68ea6dcf66a 100644 --- a/tests/regression/segmentation/test_segmentation.py +++ b/tests/regression/segmentation/test_segmentation.py @@ -115,7 +115,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "seg_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() @@ -234,7 +236,7 @@ def test_otx_train_selfsl(self, template, tmp_dir_path): args_selfsl["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] args_selfsl["--val-data-roots"] = segmentation_data_args["--val-data-roots"] args_selfsl["--test-data-roots"] = segmentation_data_args["--test-data-roots"] - args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, new_tmp_dir_path, otx_dir, args_selfsl) # Evaluation with self + supervised training model diff --git a/tests/test_suite/pytest_insertions.py b/tests/test_suite/pytest_insertions.py index 94fe699e8d8..b88060c8763 100644 --- a/tests/test_suite/pytest_insertions.py +++ b/tests/test_suite/pytest_insertions.py @@ -83,7 +83,7 @@ def otx_pytest_addoption_insertion(parser): ) parser.addoption( - "--test-work-dir", + "--test-workspace", type=str, default=None, help="OTX test requires a certain amount of storage in the test work directory. " diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index f385ce47382..374ac2359be 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -130,8 +130,8 @@ def otx_train_testing(template, root, otx_dir, args): arg_value = args.get(arg, None) if arg_value: command_line.extend([arg, os.path.join(otx_dir, arg_value)]) - command_line.extend(["--save-model-to", f"{template_work_dir}/trained_{template.model_template_id}"]) - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--output", f"{template_work_dir}/trained_{template.model_template_id}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) if "--load-weights" in args: command_line.extend(["--load-weights", args["--load-weights"]]) if "--gpus" in args: @@ -141,8 +141,8 @@ def otx_train_testing(template, root, otx_dir, args): if "train_params" in args: command_line.extend(args["train_params"]) check_run(command_line) - assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/weights.pth") - assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/label_schema.json") + assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth") + assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/models/label_schema.json") def otx_resume_testing(template, root, otx_dir, args): @@ -164,12 +164,14 @@ def otx_resume_testing(template, root, otx_dir, args): if option in args: command_line.extend([option, f"{os.path.join(otx_dir, args[option])}"]) - command_line.extend(["--save-model-to", f"{template_work_dir}/trained_for_resume_{template.model_template_id}"]) - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--output", f"{template_work_dir}/trained_for_resume_{template.model_template_id}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) - assert os.path.exists(f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth") - assert os.path.exists(f"{template_work_dir}/trained_for_resume_{template.model_template_id}/label_schema.json") + assert os.path.exists(f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth") + assert os.path.exists( + f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/label_schema.json" + ) def otx_hpo_testing(template, root, otx_dir, args): @@ -183,14 +185,17 @@ def otx_hpo_testing(template, root, otx_dir, args): arg_value = args.get(arg, None) if arg_value: command_line.extend([arg, os.path.join(otx_dir, arg_value)]) - command_line.extend(["--save-model-to", f"{template_work_dir}/hpo_trained_{template.model_template_id}"]) - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--output", f"{template_work_dir}/hpo_trained_{template.model_template_id}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(["--enable-hpo", "--hpo-time-ratio", "1"]) command_line.extend(args["train_params"]) check_run(command_line) trials_json = list( - filter(lambda x: x.name.split(".")[0].isnumeric(), Path(f"{template_work_dir}/hpo/").rglob("*.json")) + filter( + lambda x: x.name.split(".")[0].isnumeric(), + Path(f"{template_work_dir}/hpo_trained_{template.model_template_id}/hpo/").rglob("*.json"), + ) ) assert trials_json for trial_json in trials_json: @@ -198,8 +203,8 @@ def otx_hpo_testing(template, root, otx_dir, args): trial_result = json.load(f) assert trial_result.get("score") - assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/weights.pth") - assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/label_schema.json") + assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/models/weights.pth") + assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/models/label_schema.json") def otx_export_testing(template, root): @@ -209,8 +214,8 @@ def otx_export_testing(template, root): "export", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-model-to", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", f"{template_work_dir}/exported_{template.model_template_id}", ] check_run(command_line) @@ -226,8 +231,8 @@ def otx_export_testing_w_features(template, root): "export", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-model-to", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", f"{template_work_dir}/exported_{template.model_template_id}_w_features", "--dump-features", ] @@ -253,11 +258,11 @@ def otx_eval_testing(template, root, otx_dir, args): "--test-data-roots", f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-performance", - f"{template_work_dir}/trained_{template.model_template_id}/performance.json", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", + f"{template_work_dir}/trained_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args.get("eval_params", [])) check_run(command_line) assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/performance.json") @@ -275,10 +280,10 @@ def otx_eval_openvino_testing( f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", - "--save-performance", - f"{template_work_dir}/exported_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/exported_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/exported_{template.model_template_id}/performance.json") with open(f"{template_work_dir}/trained_{template.model_template_id}/performance.json") as read_file: @@ -310,7 +315,7 @@ def otx_demo_testing(template, root, otx_dir, args): "demo", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", "--input", os.path.join(otx_dir, args["--input"]), "--delay", @@ -344,7 +349,7 @@ def otx_deploy_openvino_testing(template, root, otx_dir, args): template.model_template_path, "--load-weights", f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", - "--save-model-to", + "--output", deployment_dir, ] check_run(command_line) @@ -403,10 +408,10 @@ def otx_eval_deployment_testing( f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/deployed_{template.model_template_id}/openvino.zip", - "--save-performance", - f"{template_work_dir}/deployed_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/deployed_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/deployed_{template.model_template_id}/performance.json") with open(f"{template_work_dir}/exported_{template.model_template_id}/performance.json") as read_file: @@ -458,10 +463,10 @@ def pot_optimize_testing(template, root, otx_dir, args): f'{os.path.join(otx_dir, args["--val-data-roots"])}', "--load-weights", f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", - "--save-model-to", + "--output", f"{template_work_dir}/pot_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml") assert os.path.exists(f"{template_work_dir}/pot_{template.model_template_id}/openvino.bin") @@ -497,10 +502,10 @@ def pot_eval_testing(template, root, otx_dir, args, criteria=None, reg_threshold f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml", - "--save-performance", - f"{template_work_dir}/pot_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/pot_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/pot_{template.model_template_id}/performance.json") @@ -530,13 +535,11 @@ def nncf_optimize_testing(template, root, otx_dir, args): "--val-data-roots", f'{os.path.join(otx_dir, args["--val-data-roots"])}', "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-model-to", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", f"{template_work_dir}/nncf_{template.model_template_id}", - "--save-performance", - f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth") @@ -551,7 +554,7 @@ def nncf_export_testing(template, root): template.model_template_path, "--load-weights", f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth", - "--save-model-to", + "--output", f"{template_work_dir}/exported_nncf_{template.model_template_id}", ] check_run(command_line) @@ -587,13 +590,13 @@ def nncf_eval_testing( f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth", - "--save-performance", - f"{template_work_dir}/nncf_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/nncf_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/nncf_{template.model_template_id}/performance.json") - with open(f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json") as read_file: + with open(f"{template_work_dir}/nncf_{template.model_template_id}/nncf_performance.json") as read_file: trained_performance = json.load(read_file) with open(f"{template_work_dir}/nncf_{template.model_template_id}/performance.json") as read_file: evaluated_performance = json.load(read_file) @@ -624,10 +627,10 @@ def nncf_eval_openvino_testing(template, root, otx_dir, args): f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml", - "--save-performance", - f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/exported_nncf_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json") @@ -671,7 +674,7 @@ def otx_explain_testing(template, root, otx_dir, args): "explain", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", "--explain-data-root", os.path.join(otx_dir, args["--input"]), "--save-explanation-to", @@ -753,7 +756,7 @@ def otx_build_task_testing(root, task): "build", "--task", task, - "--work-dir", + "--workspace", os.path.join(root, f"otx-workspace-{task}"), ] check_run(command_line) @@ -775,7 +778,7 @@ def otx_build_backbone_testing(root, backbone_args): "build", "--task", f"{task}", - "--work-dir", + "--workspace", task_workspace, ] check_run(command_line) @@ -787,7 +790,7 @@ def otx_build_backbone_testing(root, backbone_args): "build", "--backbone", backbone, - "--work-dir", + "--workspace", task_workspace, ] check_run(command_line) @@ -803,7 +806,7 @@ def otx_build_backbone_testing(root, backbone_args): def otx_build_testing(root, args: Dict[str, str], expected: Dict[str, str]): workspace_root = os.path.join(root, "otx-workspace") - command_line = ["otx", "build", "--work-dir", workspace_root] + command_line = ["otx", "build", "--workspace", workspace_root] for option, value in args.items(): command_line.extend([option, value]) check_run(command_line) @@ -819,7 +822,7 @@ def otx_build_testing(root, args: Dict[str, str], expected: Dict[str, str]): def otx_build_auto_config(root, otx_dir: str, args: Dict[str, str]): workspace_root = os.path.join(root, "otx-workspace") - command_line = ["otx", "build", "--work-dir", workspace_root] + command_line = ["otx", "build", "--workspace", workspace_root] for option, value in args.items(): if option in ["--train-data-roots", "--val-data-roots"]: @@ -838,8 +841,8 @@ def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str]): command_line.extend([args[option]]) elif option in ["--train-data-roots", "--val-data-roots"]: command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) - command_line.extend(["--save-model-to", f"{work_dir}"]) - command_line.extend(["--work-dir", f"{work_dir}"]) + command_line.extend(["--output", f"{work_dir}"]) + command_line.extend(["--workspace", f"{work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) @@ -862,11 +865,11 @@ def otx_eval_compare( "--test-data-roots", f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-performance", - f"{template_work_dir}/trained_{template.model_template_id}/performance.json", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", + f"{template_work_dir}/trained_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args.get("eval_params", [])) check_run(command_line) @@ -885,7 +888,7 @@ def otx_eval_compare( ), f"Current model performance: ({trained_performance[k]}) < criteria: ({modified_criteria})." result_dict["Model size (MB)"] = round( - os.path.getsize(f"{template_work_dir}/trained_{template.model_template_id}/weights.pth") / 1e6, 2 + os.path.getsize(f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth") / 1e6, 2 ) diff --git a/tests/unit/cli/tools/test_build.py b/tests/unit/cli/tools/test_build.py index d54652c58bf..49b4b6656e7 100644 --- a/tests/unit/cli/tools/test_build.py +++ b/tests/unit/cli/tools/test_build.py @@ -17,7 +17,7 @@ def test_get_args(mocker): "--unlabeled-file-list": "unlabeled/file/list", "--task": "detection", "--train-type": "SEMISUPERVISED", - "--work-dir": "work/dir/path", + "--workspace": "work/dir/path", "--model": "SSD", "--backbone": "torchvision.resnet18", } @@ -35,7 +35,7 @@ def test_get_args(mocker): assert parsed_args.test_data_roots == "test/data/root" assert parsed_args.unlabeled_data_roots == "unlabeled/data/root" assert parsed_args.unlabeled_file_list == "unlabeled/file/list" - assert parsed_args.work_dir == "work/dir/path" + assert parsed_args.workspace == "work/dir/path" assert parsed_args.task == "detection" assert parsed_args.train_type == "SEMISUPERVISED" assert parsed_args.model == "SSD" @@ -52,7 +52,7 @@ def mock_args(mocker, tmp_path): mock_args.unlabeled_file_list = None mock_args.task = "" mock_args.train_type = "incremental" - mock_args.work_dir = tmp_path / "work_dir" + mock_args.workspace = tmp_path / "work_dir" mock_args.model = "" mock_args.backbone = "torchvision.resnet18" diff --git a/tests/unit/cli/tools/test_deploy.py b/tests/unit/cli/tools/test_deploy.py index 844838af8e2..323294fd043 100644 --- a/tests/unit/cli/tools/test_deploy.py +++ b/tests/unit/cli/tools/test_deploy.py @@ -10,7 +10,7 @@ @e2e_pytest_unit def test_get_args(mocker): - mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--save-model-to", "save_model_to"]) + mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--output", "output"]) mocker.patch.object( target_package, "get_parser_and_hprams_data", return_value=[argparse.ArgumentParser(), "fake", "fake"] ) @@ -18,14 +18,14 @@ def test_get_args(mocker): parsed_args = get_args() assert parsed_args.load_weights == "load_weights" - assert parsed_args.save_model_to == "save_model_to" + assert parsed_args.output == "output" @pytest.fixture def mock_args(mocker, tmp_dir): mock_args = mocker.MagicMock() mock_args.load_weights = "fake.bin" - mock_args.save_model_to = tmp_dir + mock_args.output = tmp_dir def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_eval.py b/tests/unit/cli/tools/test_eval.py index 76eee501f89..ecbd4297ec2 100644 --- a/tests/unit/cli/tools/test_eval.py +++ b/tests/unit/cli/tools/test_eval.py @@ -12,8 +12,8 @@ def test_get_args(mocker): mock_options = { "--test-data-roots": "test/data/root", "--load-weights": "weight/path", - "--save-performance": "save/path", - "--work-dir": "work/dir/path", + "--output": "save/path", + "--workspace": "work/dir/path", } mock_command = ["otx"] for key, value in mock_options.items(): @@ -29,8 +29,7 @@ def test_get_args(mocker): assert parsed_args.test_data_roots == "test/data/root" assert parsed_args.load_weights == "weight/path" - assert parsed_args.save_performance == "save/path" - assert parsed_args.work_dir == "work/dir/path" + assert parsed_args.workspace == "work/dir/path" @pytest.fixture @@ -38,8 +37,7 @@ def mock_args(mocker, tmp_path): mock_args = mocker.MagicMock() mock_args.test_data_roots = "fake_test_data_root" mock_args.load_weights = "fake_load_weights.xml" - mock_args.save_performance = tmp_path / "save/performance.json" - mock_args.work_dir = tmp_path / "work_dir" + mock_args.workspace = tmp_path / "work_dir" def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_export.py b/tests/unit/cli/tools/test_export.py index 06aa79b2613..f32d9d872b1 100644 --- a/tests/unit/cli/tools/test_export.py +++ b/tests/unit/cli/tools/test_export.py @@ -10,7 +10,7 @@ @e2e_pytest_unit def test_get_args(mocker): - mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--save-model-to", "save_model_to"]) + mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--output", "output"]) mocker.patch.object( target_package, "get_parser_and_hprams_data", return_value=[argparse.ArgumentParser(), "fake", "fake"] ) @@ -18,14 +18,14 @@ def test_get_args(mocker): parsed_args = get_args() assert parsed_args.load_weights == "load_weights" - assert parsed_args.save_model_to == "save_model_to" + assert parsed_args.output == "output" @pytest.fixture def mock_args(mocker, tmp_dir): mock_args = mocker.MagicMock() mock_args.load_weights = "fake.bin" - mock_args.save_model_to = tmp_dir + mock_args.output = tmp_dir def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_optimize.py b/tests/unit/cli/tools/test_optimize.py index fe60e6c0d0f..93ad23bb179 100644 --- a/tests/unit/cli/tools/test_optimize.py +++ b/tests/unit/cli/tools/test_optimize.py @@ -13,9 +13,8 @@ def test_get_args(mocker): "--train-data-roots": "train_data_roots_path", "--val-data-roots": "val_data_roots_path", "--load-weights": "load_weights_path", - "--save-model-to": "save_model_path", - "--save-performance": "save_performance_path", - "--work-dir": "work_dir_path", + "--output": "output", + "--workspace": "work_dir_path", } mock_command = ["otx"] for key, value in mock_options.items(): @@ -32,9 +31,8 @@ def test_get_args(mocker): assert parsed_args.train_data_roots == "train_data_roots_path" assert parsed_args.val_data_roots == "val_data_roots_path" assert parsed_args.load_weights == "load_weights_path" - assert parsed_args.save_model_to == "save_model_path" - assert parsed_args.save_performance == "save_performance_path" - assert parsed_args.work_dir == "work_dir_path" + assert parsed_args.output == "output" + assert parsed_args.workspace == "work_dir_path" @pytest.fixture @@ -43,9 +41,8 @@ def mock_args(mocker, tmp_path): mock_args.train_data_roots = "fake_train_data_roots_path" mock_args.val_data_roots = "fake_val_data_roots_path" mock_args.load_weights = "fake_load_weights_path" - mock_args.save_model_to = tmp_path / "save/model" - mock_args.save_performance = tmp_path / "save/performance.json" - mock_args.work_dir = tmp_path / "work_dir_path" + mock_args.output = tmp_path / "save/model" + mock_args.workspace = tmp_path / "work_dir_path" def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_train.py b/tests/unit/cli/tools/test_train.py index 1db77cb3d2f..81b84844c91 100644 --- a/tests/unit/cli/tools/test_train.py +++ b/tests/unit/cli/tools/test_train.py @@ -16,8 +16,8 @@ def test_get_args(mocker): "--unlabeled-file-list": "unlabeled/file/list", "--load-weights": "weight/path", "--resume-from": "resume/path", - "--save-model-to": "save/path", - "--work-dir": "work/dir/path", + "--output": "save/path", + "--workspace": "work/dir/path", "--hpo-time-ratio": "2", "--gpus": "0,1", "--rdzv-endpoint": "localhost:1", @@ -43,8 +43,8 @@ def test_get_args(mocker): assert parsed_args.unlabeled_file_list == "unlabeled/file/list" assert parsed_args.load_weights == "weight/path" assert parsed_args.resume_from == "resume/path" - assert parsed_args.save_model_to == "save/path" - assert parsed_args.work_dir == "work/dir/path" + assert parsed_args.output == "save/path" + assert parsed_args.workspace == "work/dir/path" assert parsed_args.hpo_time_ratio == 2.0 assert parsed_args.gpus == "0,1" assert parsed_args.rdzv_endpoint == "localhost:1" @@ -60,8 +60,8 @@ def mock_args(mocker, tmp_path): mock_args.val_data_roots = "fake_val_data_root" mock_args.load_weights = "fake_load_weights" mock_args.resume_from = None - mock_args.save_model_to = tmp_path / "models" - mock_args.work_dir = tmp_path / "work_dir" + mock_args.output = tmp_path / "models" + mock_args.workspace = tmp_path / "work_dir" mock_args.enable_hpo = False mock_args.hpo_time_ratio = 4 mock_args.gpus = None @@ -120,6 +120,7 @@ def mock_task(mocker): def test_main(mocker, mock_args, mock_config_manager, mock_dataset_adapter, mock_task): mocker.patch.object(target_package, "read_label_schema") mocker.patch.object(target_package, "read_binary") + mocker.patch("otx.cli.tools.train.Path.symlink_to") mocker.patch.object( target_package, "run_hpo", diff --git a/tests/unit/cli/utils/test_hpo.py b/tests/unit/cli/utils/test_hpo.py index a71d48a0cbb..53f52e2fcd5 100644 --- a/tests/unit/cli/utils/test_hpo.py +++ b/tests/unit/cli/utils/test_hpo.py @@ -633,7 +633,7 @@ def mock_read_model(args1, path, arg2): mock_args = mocker.MagicMock() mock_args.hpo_time_ratio = "4" - mock_args.save_model_to = save_model_to_path + mock_args.output = save_model_to_path mock_environment.model_template.task_type = TaskType.CLASSIFICATION diff --git a/tests/unit/cli/utils/test_multi_gpu.py b/tests/unit/cli/utils/test_multi_gpu.py index a584860a6d9..fe6fde55ba4 100644 --- a/tests/unit/cli/utils/test_multi_gpu.py +++ b/tests/unit/cli/utils/test_multi_gpu.py @@ -375,8 +375,8 @@ def test_run_child_process(self, mocker): # check assert mock_set_start_method.call_args.kwargs["method"] is None assert "--gpus" not in mock_sys.argv - assert "--work-dir" in mock_sys.argv - assert mock_sys.argv[mock_sys.argv.index("--work-dir") + 1] == output_path + assert "--workspace" in mock_sys.argv + assert mock_sys.argv[mock_sys.argv.index("--workspace") + 1] == output_path assert "--rdzv-endpoint" in mock_sys.argv assert mock_sys.argv[mock_sys.argv.index("--rdzv-endpoint") + 1] == rdzv_endpoint mock_initialize_multigpu_train.assert_called_once() From 78116afefc715877cd6b434993d3a732b3f61218 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 11:05:12 +0900 Subject: [PATCH 02/17] Fix --output in export --- otx/cli/tools/export.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index bb8733f6a79..690a7683012 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -105,7 +105,11 @@ def main(): export_precision = ModelPrecision.FP16 if args.half_precision else ModelPrecision.FP32 task.export(ExportType.OPENVINO, exported_model, export_precision, args.dump_features) - output_path = config_manager.output_path / "openvino_models" + if not args.output: + output_path = config_manager.output_path + output_path = output_path / "openvino_models" + else: + output_path = Path(args.output) output_path.mkdir(exist_ok=True, parents=True) save_model_data(exported_model, str(output_path)) From e1cd778de877398523ab7feccd28b943dc9e9b40 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 13:36:23 +0900 Subject: [PATCH 03/17] Fix e2e segmentation weights path --- tests/e2e/cli/segmentation/test_segmentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/cli/segmentation/test_segmentation.py b/tests/e2e/cli/segmentation/test_segmentation.py index a815d9d6c06..43b8c2db4a8 100644 --- a/tests/e2e/cli/segmentation/test_segmentation.py +++ b/tests/e2e/cli/segmentation/test_segmentation.py @@ -101,7 +101,7 @@ def test_otx_resume(self, template, tmp_dir_path): args1["train_params"] = resume_params args1[ "--resume-from" - ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/models/weights.pth" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component From 4ddad6ac2a6fff1d3e7b5a3e1c0fa62bf63bfe9f Mon Sep 17 00:00:00 2001 From: eunwoosh Date: Tue, 28 Mar 2023 14:33:03 +0900 Subject: [PATCH 04/17] update --- otx/cli/tools/train.py | 9 ++++++--- otx/cli/utils/hpo.py | 9 +++++---- otx/cli/utils/multi_gpu.py | 2 +- tests/unit/cli/utils/test_hpo.py | 13 ++++++------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/otx/cli/tools/train.py b/otx/cli/tools/train.py index 402cb27d740..b5276a27c6e 100644 --- a/otx/cli/tools/train.py +++ b/otx/cli/tools/train.py @@ -205,8 +205,9 @@ def main(): # pylint: disable=too-many-branches ) if args.enable_hpo: - args.output = str(config_manager.output_path) - environment = run_hpo(args, environment, dataset, config_manager.data_config) + environment = run_hpo( + args.hpo_time_ratio, config_manager.output_path, environment, dataset, config_manager.data_config + ) (config_manager.output_path / "logs").mkdir(exist_ok=True, parents=True) task = task_class(task_environment=environment, output_path=str(config_manager.output_path / "logs")) @@ -219,7 +220,9 @@ def main(): # pylint: disable=too-many-branches multigpu_manager.is_available() and not template.task_type.is_anomaly # anomaly tasks don't use this way for multi-GPU training ): - multigpu_manager.setup_multi_gpu_train(task.project_path, hyper_parameters if args.enable_hpo else None) + multigpu_manager.setup_multi_gpu_train( + str(config_manager.output_path), hyper_parameters if args.enable_hpo else None + ) output_model = ModelEntity(dataset, environment.get_model_configuration()) diff --git a/otx/cli/utils/hpo.py b/otx/cli/utils/hpo.py index 6577434a442..ae4eb088688 100644 --- a/otx/cli/utils/hpo.py +++ b/otx/cli/utils/hpo.py @@ -536,12 +536,13 @@ def _get_initial_model_weight_path(self): def run_hpo( - args, environment: TaskEnvironment, dataset: DatasetEntity, data_roots: Dict[str, str] + hpo_time_ratio: int, output: Path, environment: TaskEnvironment, dataset: DatasetEntity, data_roots: Dict[str, str] ) -> Optional[TaskEnvironment]: """Run HPO and load optimized hyper parameter and best HPO model weight. Args: - args: arguments passed to otx train + hpo_time_ratio(int): expected ratio of total time to run HPO to time taken for full fine-tuning + output(Path): directory where HPO output is saved environment (TaskEnvironment): otx task environment dataset (DatasetEntity): dataset to use for training data_roots (Dict[str, str]): dataset path of each dataset type @@ -554,13 +555,13 @@ def run_hpo( ) return None - hpo_save_path = (Path(args.output) / "hpo").absolute() + hpo_save_path = (output / "hpo").absolute() hpo_runner = HpoRunner( environment, len(dataset.get_subset(Subset.TRAINING)), len(dataset.get_subset(Subset.VALIDATION)), hpo_save_path, - args.hpo_time_ratio, + hpo_time_ratio, ) logger.info("started hyper-parameter optimization") diff --git a/otx/cli/utils/multi_gpu.py b/otx/cli/utils/multi_gpu.py index 4334ec0df18..ab57c2b4953 100644 --- a/otx/cli/utils/multi_gpu.py +++ b/otx/cli/utils/multi_gpu.py @@ -238,7 +238,7 @@ def run_child_process( sys.argv.pop(gpus_arg_idx) if "--enable-hpo" in sys.argv: sys.argv.remove("--enable-hpo") - set_arguments_to_argv("--workspace", output_path) + set_arguments_to_argv("--output", output_path) set_arguments_to_argv("--rdzv-endpoint", rdzv_endpoint) MultiGPUManager.initialize_multigpu_train(rdzv_endpoint, rank, local_rank, gpu_ids, world_size) diff --git a/tests/unit/cli/utils/test_hpo.py b/tests/unit/cli/utils/test_hpo.py index 53f52e2fcd5..d839efca5d8 100644 --- a/tests/unit/cli/utils/test_hpo.py +++ b/tests/unit/cli/utils/test_hpo.py @@ -613,8 +613,7 @@ def test_getitem_before_get_subset(self): def test_run_hpo(mocker, mock_environment): with TemporaryDirectory() as tmp_dir: # prepare - save_model_to_path = Path(tmp_dir) / "fake" - + output = Path(tmp_dir) / "fake" mock_get_best_hpo_weight = mocker.patch("otx.cli.utils.hpo.get_best_hpo_weight") mock_get_best_hpo_weight.return_value = "mock_best_weight_path" @@ -631,14 +630,12 @@ def mock_read_model(args1, path, arg2): mocker.patch("otx.cli.utils.hpo.read_model", mock_read_model) - mock_args = mocker.MagicMock() - mock_args.hpo_time_ratio = "4" - mock_args.output = save_model_to_path + hpo_time_ratio = "4" mock_environment.model_template.task_type = TaskType.CLASSIFICATION # run - environment = run_hpo(mock_args, mock_environment, mocker.MagicMock(), mocker.MagicMock()) + environment = run_hpo(hpo_time_ratio, output, mock_environment, mocker.MagicMock(), mocker.MagicMock()) # check mock_hpo_runner_instance.run_hpo.assert_called() # Check that HpoRunner.run_hpo is called @@ -653,8 +650,10 @@ def test_run_hpo_not_supported_task(mocker, action_task_env): mock_hpo_runner_instance = mocker.MagicMock() mock_hpo_runner_class = mocker.patch("otx.cli.utils.hpo.HpoRunner") mock_hpo_runner_class.return_value = mock_hpo_runner_instance + hpo_time_ratio = "4" + output = "fake" - run_hpo(mocker.MagicMock(), action_task_env, mocker.MagicMock(), mocker.MagicMock()) + run_hpo(hpo_time_ratio, output, action_task_env, mocker.MagicMock(), mocker.MagicMock()) mock_hpo_runner_instance.run_hpo.assert_not_called() From 04891560c7cc6d43b49241d3e24121cfd0305a56 Mon Sep 17 00:00:00 2001 From: eunwoosh Date: Tue, 28 Mar 2023 14:51:58 +0900 Subject: [PATCH 05/17] consider -o argument --- otx/cli/utils/multi_gpu.py | 35 +++++++++++++++----------- tests/unit/cli/utils/test_hpo.py | 1 - tests/unit/cli/utils/test_multi_gpu.py | 13 ++++++++-- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/otx/cli/utils/multi_gpu.py b/otx/cli/utils/multi_gpu.py index ab57c2b4953..57bfef0ee5b 100644 --- a/otx/cli/utils/multi_gpu.py +++ b/otx/cli/utils/multi_gpu.py @@ -22,7 +22,7 @@ import threading import time from contextlib import closing -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Union import psutil import torch @@ -74,29 +74,34 @@ def get_gpu_ids(gpus: str) -> List[int]: return gpu_ids -def set_arguments_to_argv(key: str, value: Optional[str] = None, after_params: bool = False): +def set_arguments_to_argv(keys: Union[str, List[str]], value: Optional[str] = None, after_params: bool = False): """Add arguments at proper position in `sys.argv`. Args: - key (str): arguement key. + keys (str or List[str]): arguement keys. value (str or None): argument value. after_params (bool): whether argument should be after `param` or not. """ - if key in sys.argv: + if not isinstance(keys, list): + keys = [keys] + for key in keys: + if key in sys.argv: + if value is not None: + sys.argv[sys.argv.index(key) + 1] = value + return + + key = keys[0] + if not after_params and "params" in sys.argv: + sys.argv.insert(sys.argv.index("params"), key) if value is not None: - sys.argv[sys.argv.index(key) + 1] = value + sys.argv.insert(sys.argv.index("params"), value) else: - if not after_params and "params" in sys.argv: - sys.argv.insert(sys.argv.index("params"), key) - if value is not None: - sys.argv.insert(sys.argv.index("params"), value) + if after_params and "params" not in sys.argv: + sys.argv.append("params") + if value is not None: + sys.argv.extend([key, value]) else: - if after_params and "params" not in sys.argv: - sys.argv.append("params") - if value is not None: - sys.argv.extend([key, value]) - else: - sys.argv.append(key) + sys.argv.append(key) def is_multigpu_child_process(): diff --git a/tests/unit/cli/utils/test_hpo.py b/tests/unit/cli/utils/test_hpo.py index d839efca5d8..f810c0acd45 100644 --- a/tests/unit/cli/utils/test_hpo.py +++ b/tests/unit/cli/utils/test_hpo.py @@ -631,7 +631,6 @@ def mock_read_model(args1, path, arg2): mocker.patch("otx.cli.utils.hpo.read_model", mock_read_model) hpo_time_ratio = "4" - mock_environment.model_template.task_type = TaskType.CLASSIFICATION # run diff --git a/tests/unit/cli/utils/test_multi_gpu.py b/tests/unit/cli/utils/test_multi_gpu.py index fe6fde55ba4..d40d5c68872 100644 --- a/tests/unit/cli/utils/test_multi_gpu.py +++ b/tests/unit/cli/utils/test_multi_gpu.py @@ -67,6 +67,15 @@ def test_set_arguments_to_argv_key_exist(mock_argv_without_params): assert mock_argv_without_params[1] == other_val +@e2e_pytest_unit +def test_set_arguments_to_argv_keys_exist(mock_argv_without_params): + """Test a case where key already exists and value exists.""" + other_val = "other_val" + set_arguments_to_argv(["--a_key", "-a"], other_val) + + assert mock_argv_without_params[1] == other_val + + @e2e_pytest_unit def test_set_arguments_to_argv_key_exist_none_val(mock_argv_without_params): """Test a case where key already exists in argv and value doesn't exists.""" @@ -375,8 +384,8 @@ def test_run_child_process(self, mocker): # check assert mock_set_start_method.call_args.kwargs["method"] is None assert "--gpus" not in mock_sys.argv - assert "--workspace" in mock_sys.argv - assert mock_sys.argv[mock_sys.argv.index("--workspace") + 1] == output_path + assert "--output" in mock_sys.argv + assert mock_sys.argv[mock_sys.argv.index("--output") + 1] == output_path assert "--rdzv-endpoint" in mock_sys.argv assert mock_sys.argv[mock_sys.argv.index("--rdzv-endpoint") + 1] == rdzv_endpoint mock_initialize_multigpu_train.assert_called_once() From 7a7a903d2f3fd4f9a2a6a484ee940ce692a61849 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 14:57:58 +0900 Subject: [PATCH 06/17] Fix doc output args --- .../quick_start_guide/cli_commands.rst | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/source/guide/get_started/quick_start_guide/cli_commands.rst b/docs/source/guide/get_started/quick_start_guide/cli_commands.rst index f5bf7cf3efc..220fc5d88e8 100644 --- a/docs/source/guide/get_started/quick_start_guide/cli_commands.rst +++ b/docs/source/guide/get_started/quick_start_guide/cli_commands.rst @@ -150,7 +150,7 @@ Training - ``weights.pth`` - a model snapshot - ``label_schema.json`` - a label schema used in training, created from a dataset -The results will be saved in ``./model`` folder by default. The output folder can be modified by ``--output`` option. These files are used by other commands: ``export``, ``eval``, ``demo``, etc. +The results will be saved in ``./outputs/`` folder by default. The output folder can be modified by ``--output`` option. These files are used by other commands: ``export``, ``eval``, ``demo``, etc. ``otx train`` receives ``template`` as a positional argument. ``template`` can be a path to the specific ``template.yaml`` file, template name or template ID. Also, the path to train and val data root should be passed to the CLI to start training. @@ -160,7 +160,7 @@ However, if you created a workspace with ``otx build``, the training process can otx train --help usage: otx train [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--unlabeled-data-roots UNLABELED_DATA_ROOTS] [--unlabeled-file-list UNLABELED_FILE_LIST] - [--load-weights LOAD_WEIGHTS] [--resume-from RESUME_FROM] [--output OUTPUT] [--workspace WORKSPACE] [--enable-hpo] [--hpo-time-ratio HPO_TIME_RATIO] [--gpus GPUS] + [--load-weights LOAD_WEIGHTS] [--resume-from RESUME_FROM] [-o OUTPUT] [--workspace WORKSPACE] [--enable-hpo] [--hpo-time-ratio HPO_TIME_RATIO] [--gpus GPUS] [--rdzv-endpoint RDZV_ENDPOINT] [--base-rank BASE_RANK] [--world-size WORLD_SIZE] [--mem-cache-size PARAMS.ALGO_BACKEND.MEM_CACHE_SIZE] [--data DATA] [template] {params} ... @@ -186,7 +186,7 @@ However, if you created a workspace with ``otx build``, the training process can Load model weights from previously saved checkpoint. --resume-from RESUME_FROM Resume training from previously saved checkpoint - --output OUTPUT + -o OUTPUT, --output OUTPUT Location where trained model will be stored. --workspace WORKSPACE Location where the intermediate output of the training will be stored. --enable-hpo Execute hyper parameters optimization (HPO) before training. @@ -261,7 +261,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx export --help - usage: otx export [-h] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] [--workspace WORKSPACE] [--dump-features] [--half-precision] [template] + usage: otx export [-h] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [--workspace WORKSPACE] [--dump-features] [--half-precision] [template] positional arguments: template Enter the path or ID or name of the template file. @@ -271,7 +271,7 @@ With the ``--help`` command, you can list additional information, such as its pa -h, --help show this help message and exit --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint. - --output OUTPUT + -o OUTPUT, --output OUTPUT Location where exported model will be stored. --workspace WORKSPACE Location where the intermediate output of the export will be stored. --dump-features Whether to return feature vector and saliency map for explanation purposes. @@ -306,7 +306,7 @@ With the ``--help`` command, you can list additional information: .. code-block:: - usage: otx optimize [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] + usage: otx optimize [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [--workspace WORKSPACE] [template] {params} ... @@ -324,7 +324,7 @@ With the ``--help`` command, you can list additional information: Comma-separated paths to validation data folders. --load-weights LOAD_WEIGHTS Load weights of trained model - --output OUTPUT + -o OUTPUT, --output OUTPUT Location where optimized model will be stored. --workspace WORKSPACE Location where the intermediate output of the task will be stored. @@ -361,7 +361,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx eval --help - usage: otx eval [-h] [--test-data-roots TEST_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] [--workspace WORKSPACE] [template] {params} ... + usage: otx eval [-h] [--test-data-roots TEST_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [--workspace WORKSPACE] [template] {params} ... positional arguments: template Enter the path or ID or name of the template file. @@ -375,7 +375,7 @@ With the ``--help`` command, you can list additional information, such as its pa Comma-separated paths to test data folders. --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint.It could be a trained/optimized model (POT only) or exported model. - --output OUTPUT + -o OUTPUT, --output OUTPUT Location where the intermediate output of the task will be stored. --workspace WORKSPACE Path to the workspace where the command will run. @@ -519,7 +519,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx deploy --help - usage: otx deploy [-h] [--load-weights LOAD_WEIGHTS] [--output OUTPUT] [template] + usage: otx deploy [-h] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [template] positional arguments: template Enter the path or ID or name of the template file. @@ -529,7 +529,7 @@ With the ``--help`` command, you can list additional information, such as its pa -h, --help show this help message and exit --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint. - --output OUTPUT + -o OUTPUT, --output OUTPUT Location where openvino.zip will be stored. From 2cb8133579f0f6a36f78f4a32ef2a5287d1008e8 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 15:40:02 +0900 Subject: [PATCH 07/17] Reflect some comments --- otx/cli/manager/config_manager.py | 3 ++- otx/cli/tools/build.py | 6 +----- otx/cli/tools/deploy.py | 5 +---- otx/cli/tools/eval.py | 3 +-- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/otx/cli/manager/config_manager.py b/otx/cli/manager/config_manager.py index b17f57dfad7..0fc93c96b04 100644 --- a/otx/cli/manager/config_manager.py +++ b/otx/cli/manager/config_manager.py @@ -128,7 +128,8 @@ def output_path(self) -> Path: output_path = Path(self.args.output) else: output_path = self.workspace_root / "outputs" / self.create_date - output_path.mkdir(exist_ok=True, parents=True) + if not output_path.exists(): + output_path.mkdir(exist_ok=True, parents=True) return output_path def check_workspace(self) -> bool: diff --git a/otx/cli/tools/build.py b/otx/cli/tools/build.py index 16d018ae207..fbf850ddf45 100644 --- a/otx/cli/tools/build.py +++ b/otx/cli/tools/build.py @@ -17,8 +17,6 @@ # See the License for the specific language governing permissions # and limitations under the License. -from pathlib import Path - from otx.cli.manager.config_manager import TASK_TYPE_TO_SUB_DIR_NAME, ConfigManager from otx.cli.utils.parser import get_parser_and_hprams_data @@ -88,11 +86,9 @@ def main(): """Main function for model or backbone or task building.""" args = get_args() - config_manager = ConfigManager(args, mode="build") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="build") if args.task: config_manager.task_type = args.task.upper() - if args.workspace: - config_manager.workspace_root = Path(args.workspace) # Auto-Configuration for model template config_manager.configure_template(model=args.model) diff --git a/otx/cli/tools/deploy.py b/otx/cli/tools/deploy.py index 9f2e99d50c7..73b83e78357 100644 --- a/otx/cli/tools/deploy.py +++ b/otx/cli/tools/deploy.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions # and limitations under the License. -from pathlib import Path - from otx.api.configuration.helper import create from otx.api.entities.model import ModelEntity from otx.api.entities.task_environment import TaskEnvironment @@ -82,8 +80,7 @@ def main(): deployed_model = ModelEntity(None, environment.get_model_configuration()) - output_path = Path(args.output) if args.output else config_manager.output_path - output_path.mkdir(exist_ok=True, parents=True) + output_path = config_manager.output_path task.deploy(deployed_model) with open(output_path / "openvino.zip", "wb") as write_file: write_file.write(deployed_model.exportable_code) diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index 9b2b1d4ac7a..1052662536c 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -15,7 +15,6 @@ # and limitations under the License. import json -from pathlib import Path from otx.api.entities.inference_parameters import InferenceParameters from otx.api.entities.resultset import ResultSetEntity @@ -137,7 +136,7 @@ def main(): assert resultset.performance is not None print(resultset.performance) - output_path = Path(args.output) if args.output else config_manager.output_path + output_path = config_manager.output_path with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file: json.dump( {resultset.performance.score.name: resultset.performance.score.value}, From ecbdd8a9babb8983b863eb7190c27601172f160c Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 16:20:49 +0900 Subject: [PATCH 08/17] Fix unit-test's statement --- tests/unit/cli/tools/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cli/tools/test_build.py b/tests/unit/cli/tools/test_build.py index 49b4b6656e7..55eaf1ae5e9 100644 --- a/tests/unit/cli/tools/test_build.py +++ b/tests/unit/cli/tools/test_build.py @@ -92,7 +92,7 @@ def test_main(mocker, mock_config_manager, mock_args): assert result == {"retcode": 0, "task_type": ""} # Check ConfigManager constructor call - mock_config_manager.assert_called_once_with(mock_args, mode="build") + mock_config_manager.assert_called_once() # Check ConfigManager method calls mock_config_manager.return_value.configure_template.assert_called_once_with(model="") From 1f542416dec19021f925158b9c869a80b7219f80 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 17:06:36 +0900 Subject: [PATCH 09/17] Revert some comments --- otx/cli/tools/deploy.py | 5 ++++- otx/cli/tools/eval.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/otx/cli/tools/deploy.py b/otx/cli/tools/deploy.py index 73b83e78357..9f2e99d50c7 100644 --- a/otx/cli/tools/deploy.py +++ b/otx/cli/tools/deploy.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions # and limitations under the License. +from pathlib import Path + from otx.api.configuration.helper import create from otx.api.entities.model import ModelEntity from otx.api.entities.task_environment import TaskEnvironment @@ -80,7 +82,8 @@ def main(): deployed_model = ModelEntity(None, environment.get_model_configuration()) - output_path = config_manager.output_path + output_path = Path(args.output) if args.output else config_manager.output_path + output_path.mkdir(exist_ok=True, parents=True) task.deploy(deployed_model) with open(output_path / "openvino.zip", "wb") as write_file: write_file.write(deployed_model.exportable_code) diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index 1052662536c..43d6c53a93b 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -136,7 +136,7 @@ def main(): assert resultset.performance is not None print(resultset.performance) - output_path = config_manager.output_path + output_path = Path(args.output) if args.output else config_manager.output_path with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file: json.dump( {resultset.performance.score.name: resultset.performance.score.value}, From e5c05f6510c7fd104d6f95326b0bf9eba6699c9a Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Tue, 28 Mar 2023 17:28:13 +0900 Subject: [PATCH 10/17] Fix pre-commit --- otx/cli/tools/eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index 43d6c53a93b..9b2b1d4ac7a 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -15,6 +15,7 @@ # and limitations under the License. import json +from pathlib import Path from otx.api.entities.inference_parameters import InferenceParameters from otx.api.entities.resultset import ResultSetEntity From b4b8c271528d4b5990572e309a32f6da08e4f9e8 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Wed, 29 Mar 2023 09:28:41 +0900 Subject: [PATCH 11/17] Move latest to outputs/latest --- otx/cli/tools/deploy.py | 2 +- otx/cli/tools/eval.py | 3 ++- otx/cli/tools/export.py | 3 ++- otx/cli/tools/optimize.py | 3 ++- otx/cli/tools/train.py | 9 ++++++--- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/otx/cli/tools/deploy.py b/otx/cli/tools/deploy.py index 9f2e99d50c7..6ccf81302c6 100644 --- a/otx/cli/tools/deploy.py +++ b/otx/cli/tools/deploy.py @@ -59,7 +59,7 @@ def main(): assert hyper_parameters if not args.load_weights and config_manager.check_workspace(): - exported_weight_path = config_manager.workspace_root / "latest/openvino_models/openvino.xml" + exported_weight_path = config_manager.workspace_root / "outputs" / "latest" / "openvino_models" / "openvino.xml" if not exported_weight_path.exists(): raise RuntimeError("No appropriate OpenVINO exported model was found.") args.load_weights = str(exported_weight_path) diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index 9b2b1d4ac7a..fe591e82dbe 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -88,7 +88,8 @@ def main(): config_manager.configure_template() if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "latest" / "weights.pth") + latest_model_path = config_manager.workspace_root / "outputs" / "latest" / "weights.pth" + args.load_weights = str(latest_model_path) # Update Hyper Parameter Configs hyper_parameters = config_manager.get_hyparams_config(override_param) diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index 690a7683012..519ec625e5c 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -72,7 +72,8 @@ def main(): # Get class for Task. if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "latest" / "weights.pth") + latest_model_path = config_manager.workspace_root / "outputs" / "latest" / "weights.pth" + args.load_weights = str(latest_model_path) is_nncf = is_checkpoint_nncf(args.load_weights) task_class = get_impl_class(template.entrypoints.nncf if is_nncf else template.entrypoints.base) diff --git a/otx/cli/tools/optimize.py b/otx/cli/tools/optimize.py index 5a6cb7b9168..0369e71af20 100644 --- a/otx/cli/tools/optimize.py +++ b/otx/cli/tools/optimize.py @@ -84,7 +84,8 @@ def main(): # The default in the workspace is the model weight of the OTX train. if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "latest" / "weights.pth") + latest_model_path = config_manager.workspace_root / "outputs" / "latest" / "weights.pth" + args.load_weights = str(latest_model_path) is_pot = False if args.load_weights.endswith(".bin") or args.load_weights.endswith(".xml"): diff --git a/otx/cli/tools/train.py b/otx/cli/tools/train.py index b5276a27c6e..3ffe81612ea 100644 --- a/otx/cli/tools/train.py +++ b/otx/cli/tools/train.py @@ -230,9 +230,12 @@ def main(): # pylint: disable=too-many-branches save_model_data(output_model, str(config_manager.output_path / "models")) # Latest model folder symbolic link to models - if (config_manager.workspace_root / "latest").exists(): - (config_manager.workspace_root / "latest").unlink() - (config_manager.workspace_root / "latest").symlink_to((config_manager.output_path / "models").resolve()) + latest_path = config_manager.workspace_root / "outputs" / "latest" + if latest_path.exists(): + (config_manager.workspace_root / "outputs/latest").unlink() + elif not latest_path.parent.exists(): + latest_path.parent.mkdir(exist_ok=True, parents=True) + latest_path.symlink_to((config_manager.output_path / "models").resolve()) if config_manager.data_config["val_subset"]["data_root"]: validation_dataset = dataset.get_subset(Subset.VALIDATION) From 673f268f937df627cb15aa06f4153461c965d882 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Wed, 29 Mar 2023 14:08:23 +0900 Subject: [PATCH 12/17] Add test cases into integration test --- tests/integration/cli/test_cli.py | 119 +++++++++++++++++++++++++++ tests/test_suite/run_test_command.py | 5 +- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 825a811750b..4b998edbb76 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -12,6 +12,7 @@ from otx.cli.tools import cli from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( + check_run, otx_build_auto_config, otx_build_backbone_testing, otx_build_testing, @@ -120,6 +121,124 @@ def test_otx_train(self, case, tmp_dir_path): tmp_dir_path = tmp_dir_path / "test_train_auto_config" / case train_auto_config_args[case]["train_params"] = train_params otx_train_auto_config(root=tmp_dir_path, otx_dir=otx_dir, args=train_auto_config_args[case]) + # check output (use --workspace & --output) + output_path = os.path.join(tmp_dir_path, "otx-workspace") + assert os.path.exists(os.path.join(output_path, "outputs")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest", "weights.pth")) + assert os.path.exists(os.path.join(output_path, "models")) + assert os.path.exists(os.path.join(output_path, "models", "weights.pth")) + + @e2e_pytest_component + def test_otx_train_wo_output_args(self, tmp_dir_path): + otx_dir = os.getcwd() + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_train_auto_config_wo_output" / case + train_auto_config_args[case]["train_params"] = train_params + otx_train_auto_config(root=tmp_dir_path, otx_dir=otx_dir, args=train_auto_config_args[case], use_output=False) + + # check output (without --output -> Default outputs) + output_path = os.path.join(tmp_dir_path, "otx-workspace", "outputs") + assert os.path.exists(output_path) + file_list = sorted(os.listdir(output_path)) + assert len(file_list) == 2 + assert os.path.exists(os.path.join(output_path, "latest")) + assert os.path.exists(os.path.join(output_path, "latest", "weights.pth")) + assert os.path.exists(os.path.join(output_path, "latest", "label_schema.json")) + file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + assert os.path.exists(os.path.join(output_path, file_list[-1])) + assert os.path.exists(os.path.join(output_path, file_list[-1], "models")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "models", "weights.pth")) + + @e2e_pytest_component + def test_otx_export_wo_output_args(self, tmp_dir_path): + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_train_auto_config_wo_output" / case + workspace_path = os.path.join(tmp_dir_path, "otx-workspace") + command_line = [ + "otx", + "export", + "--workspace", + os.path.join(tmp_dir_path, "otx-workspace"), + ] + check_run(command_line) + + # check output (without --output -> Default outputs) + output_path = os.path.join(workspace_path, "outputs") + assert os.path.exists(output_path) + file_list = sorted(os.listdir(output_path)) + assert len(file_list) == 3 + assert os.path.exists(os.path.join(output_path, "latest")) + assert os.path.exists(os.path.join(output_path, "latest", "openvino_models")) + assert os.path.exists(os.path.join(output_path, "latest", "openvino_models", "openvino.xml")) + assert os.path.exists(os.path.join(output_path, "latest", "openvino_models", "openvino.bin")) + assert os.path.exists(os.path.join(output_path, "latest", "openvino_models", "label_schema.json")) + file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + assert os.path.exists(os.path.join(output_path, file_list[-1])) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models", "openvino.xml")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models", "openvino.bin")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models", "label_schema.json")) + + @e2e_pytest_component + def test_otx_optimize_wo_output_args(self, tmp_dir_path): + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_train_auto_config_wo_output" / case + workspace_path = os.path.join(tmp_dir_path, "otx-workspace") + command_line = [ + "otx", + "optimize", + "--workspace", + os.path.join(tmp_dir_path, "otx-workspace"), + ] + check_run(command_line) + + # check output (without --output -> Default outputs) + output_path = os.path.join(workspace_path, "outputs") + assert os.path.exists(output_path) + file_list = sorted(os.listdir(output_path)) + assert len(file_list) == 4 + assert os.path.exists(os.path.join(output_path, "latest")) + assert os.path.exists(os.path.join(output_path, "latest", "nncf")) + assert os.path.exists(os.path.join(output_path, "latest", "nncf", "weights.pth")) + assert os.path.exists(os.path.join(output_path, "latest", "nncf", "label_schema.json")) + file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + assert os.path.exists(os.path.join(output_path, file_list[-1])) + assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "weights.pth")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "label_schema.json")) + + # @e2e_pytest_component + # def test_otx_train_wo_workspace_and_output_args(self, tmp_dir_path): + # otx_dir = os.getcwd() + # case = list(train_auto_config_args.keys())[0] + # tmp_dir_path = tmp_dir_path / "test_otx_train_wo_workspace_and_output_args" + # tmp_dir_path.mkdir(exist_ok=True) + # expected_workspace_path = os.path.join(tmp_dir_path, f"otx-workspace-{case.upper()}") + # command_line = [ + # "otx", + # "train", + # ] + # args = train_auto_config_args[case] + # for option, value in args.items(): + # if option in ["--train-data-roots", "--val-data-roots"]: + # command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) + # command_line.extend(train_params) + # check_run(command_line, cwd=tmp_dir_path) + + # # check output (without --output -> Default outputs) + # assert os.path.exists(expected_workspace_path) + # expected_output_path = os.path.join(expected_workspace_path, "outputs") + # assert os.path.exists(expected_output_path) + # file_list = sorted(os.listdir(expected_output_path)) + # assert len(file_list) == 2 + # assert os.path.exists(os.path.join(expected_output_path, "latest")) + # assert os.path.exists(os.path.join(expected_output_path, "latest", "weights.pth")) + # assert os.path.exists(os.path.join(expected_output_path, "latest", "label_schema.json")) + # file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + # assert os.path.exists(os.path.join(expected_output_path, file_list[-1])) + # assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models")) + # assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "weights.pth")) class TestTelemetryIntegration: diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index 4afdaea98b1..a1d11e1cf49 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -847,7 +847,7 @@ def otx_build_auto_config(root, otx_dir: str, args: Dict[str, str]): check_run(command_line) -def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str]): +def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str], use_output: bool = True): work_dir = os.path.join(root, "otx-workspace") command_line = ["otx", "train"] @@ -856,7 +856,8 @@ def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str]): command_line.extend([args[option]]) elif option in ["--train-data-roots", "--val-data-roots"]: command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) - command_line.extend(["--output", f"{work_dir}"]) + if use_output: + command_line.extend(["--output", f"{work_dir}"]) command_line.extend(["--workspace", f"{work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) From 16e33f1bd9406b237be7a0bcd28ad9058c56e5e0 Mon Sep 17 00:00:00 2001 From: "Kang, Harim" Date: Wed, 29 Mar 2023 16:42:08 +0900 Subject: [PATCH 13/17] Reflect some feedbacks --- .../guide/tutorials/advanced/semi_sl.rst | 2 +- .../how_to_train/action_classification.rst | 10 +- .../base/how_to_train/action_detection.rst | 4 +- .../base/how_to_train/anomaly_detection.rst | 10 +- otx/cli/manager/config_manager.py | 14 ++- otx/cli/tools/demo.py | 2 +- otx/cli/tools/deploy.py | 7 +- otx/cli/tools/eval.py | 4 +- otx/cli/tools/explain.py | 2 +- otx/cli/tools/export.py | 14 +-- otx/cli/tools/optimize.py | 12 +-- otx/cli/tools/train.py | 6 +- tests/integration/cli/test_cli.py | 96 +++++++++---------- 13 files changed, 88 insertions(+), 95 deletions(-) diff --git a/docs/source/guide/tutorials/advanced/semi_sl.rst b/docs/source/guide/tutorials/advanced/semi_sl.rst index 66664229fd9..081d86d69ae 100644 --- a/docs/source/guide/tutorials/advanced/semi_sl.rst +++ b/docs/source/guide/tutorials/advanced/semi_sl.rst @@ -150,7 +150,7 @@ In the train log, you can check that the train type is set to **Semisupervised** ... -After training ends, a trained model is saved in the ``latest`` sub-directory in the workspace named ``otx-workspace-CLASSIFICATION`` by default. +After training ends, a trained model is saved in the ``latest_trained_model`` sub-directory in the workspace named ``otx-workspace-CLASSIFICATION`` by default. *************************** diff --git a/docs/source/guide/tutorials/base/how_to_train/action_classification.rst b/docs/source/guide/tutorials/base/how_to_train/action_classification.rst index 3eb4254a31c..e6418b8904d 100644 --- a/docs/source/guide/tutorials/base/how_to_train/action_classification.rst +++ b/docs/source/guide/tutorials/base/how_to_train/action_classification.rst @@ -215,12 +215,12 @@ Export It allows running the model on the Intel hardware much more efficiently, especially on the CPU. Also, the resulting IR model is required to run POT optimization. IR model consists of two files: ``openvino.xml`` for weights and ``openvino.bin`` for architecture. 2. Run the command line below to export the trained model -and save the exported model to the ``openvino_models`` folder. +and save the exported model to the ``openvino`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --output openvino_models + --output openvino ... 2023-02-21 22:54:32,518 - mmaction - INFO - Model architecture: X3D @@ -241,8 +241,8 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param .. code-block:: (otx) ...$ otx eval --test-data-roots ../data/hmdb51/CVAT/valid \ - --load-weights openvino_models/openvino.xml \ - --output outputs/openvino_models + --load-weights openvino/openvino.xml \ + --output outputs/openvino ... @@ -262,7 +262,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: - (otx) ...$ otx optimize --load-weights openvino_models/openvino.xml \ + (otx) ...$ otx optimize --load-weights openvino/openvino.xml \ --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/action_detection.rst b/docs/source/guide/tutorials/base/how_to_train/action_detection.rst index 8f47c228984..c4e73292145 100644 --- a/docs/source/guide/tutorials/base/how_to_train/action_detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/action_detection.rst @@ -163,7 +163,7 @@ Export It allows running the model on the Intel hardware much more efficiently, especially on the CPU. Also, the resulting IR model is required to run POT optimization. IR model consists of two files: ``openvino.xml`` for weights and ``openvino.bin`` for architecture. 2. Run the command line below to export the trained model -and save the exported model to the ``openvino_models`` folder. +and save the exported model to the ``openvino`` folder. .. code-block:: @@ -213,7 +213,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: - (otx) ...$ otx optimize --load-weights openvino_models/openvino.xml \ + (otx) ...$ otx optimize --load-weights openvino/openvino.xml \ --save-model-to pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst b/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst index 6e476a6294e..2118343b128 100644 --- a/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst @@ -159,13 +159,13 @@ Export It allows running the model on the Intel hardware much more efficient, especially on the CPU. Also, the resulting IR model is required to run POT optimization. IR model consists of 2 files: ``openvino.xml`` for weights and ``openvino.bin`` for architecture. 2. We can run the below command line to export the trained model -and save the exported model to the ``openvino_models`` folder: +and save the exported model to the ``openvino`` folder: .. code-block:: otx export ote_anomaly_detection_padim \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --output otx-workspace-ANOMALY_DETECTION/openvino_models + --output otx-workspace-ANOMALY_DETECTION/openvino You will see the outputs similar to the following: @@ -187,8 +187,8 @@ Now that we have the exported model, let's check its performance using ``otx eva otx eval ote_anomaly_detection_padim \ --test-data-roots datasets/MVTec/bottle/test \ - --load-weights otx-workspace-ANOMALY_DETECTION/openvino_models/openvino.xml \ - --output otx-workspace-ANOMALY_DETECTION/openvino_models + --load-weights otx-workspace-ANOMALY_DETECTION/openvino/openvino.xml \ + --output otx-workspace-ANOMALY_DETECTION/openvino This gives the following results: @@ -210,7 +210,7 @@ optimization. otx optimize ote_anomaly_detection_padim \ --train-data-roots datasets/MVTec/bottle/train \ - --load-weights otx-workspace-ANOMALY_DETECTION/openvino_models/openvino.xml \ + --load-weights otx-workspace-ANOMALY_DETECTION/openvino/openvino.xml \ --output otx-workspace-ANOMALY_DETECTION/pot_model This command generates the following files that can be used to run :doc:`otx demo <../demo>`: diff --git a/otx/cli/manager/config_manager.py b/otx/cli/manager/config_manager.py index 5e365eb02f8..ecd52dd879a 100644 --- a/otx/cli/manager/config_manager.py +++ b/otx/cli/manager/config_manager.py @@ -133,7 +133,7 @@ def output_path(self) -> Path: if "output" in self.args and self.args.output: output_path = Path(self.args.output) else: - output_path = self.workspace_root / "outputs" / self.create_date + output_path = self.workspace_root / "outputs" / f"{self.create_date}_{self.mode}" if not output_path.exists(): output_path.mkdir(exist_ok=True, parents=True) return output_path @@ -190,7 +190,7 @@ def configure_data_config(self, update_data_yaml: bool = True) -> None: """Configure data_config according to the situation and create data.yaml.""" data_yaml_path = self.data_config_file_path data_yaml = configure_dataset(self.args, data_yaml_path=data_yaml_path) - if self.mode in ("train", "build"): + if self.mode in ("train", "build", "optimize"): use_auto_split = data_yaml["data"]["train"]["data-roots"] and not data_yaml["data"]["val"]["data-roots"] # FIXME: Hardcoded for Self-Supervised Learning if use_auto_split and str(self.train_type).upper() != "SELFSUPERVISED": @@ -211,7 +211,11 @@ def _get_train_type(self, ignore_args: bool = False) -> str: if arg_algo_backend: train_type = arg_algo_backend.get("train_type", {"value": "Incremental"}) # type: ignore return train_type.get("value", "Incremental") - if hasattr(self.args, "train_type") and self.mode in ("build", "train") and self.args.train_type: + if ( + hasattr(self.args, "train_type") + and self.mode in ("build", "train", "optimize") + and self.args.train_type + ): self.train_type = self.args.train_type if self.train_type not in TASK_TYPE_TO_SUB_DIR_NAME: raise NotSupportedError(f"{self.train_type} is not currently supported by otx.") @@ -277,7 +281,7 @@ def _get_arg_data_yaml(self): # TODO: This should modify data yaml format to data_config format. """Save the splitted dataset and data.yaml to the workspace.""" data_yaml = self._create_empty_data_cfg() - if self.mode == "train": + if self.mode in ("train", "optimize"): if self.args.train_data_roots: data_yaml["data"]["train"]["data-roots"] = self.args.train_data_roots if self.args.val_data_roots: @@ -380,7 +384,7 @@ def update_data_config(self, data_yaml: dict) -> None: "file_list": data_yaml["data"]["unlabeled"]["file-list"], } # FIXME: Hardcoded for Self-Supervised Learning - if self.mode == "train" and str(self.train_type).upper() == "SELFSUPERVISED": + if self.mode in ("train", "optimize") and str(self.train_type).upper() == "SELFSUPERVISED": self.data_config["val_subset"] = {"data_root": None} def _get_template(self, task_type: str, model: Optional[str] = None) -> ModelTemplate: diff --git a/otx/cli/tools/demo.py b/otx/cli/tools/demo.py index 48efda739e6..ea6547322e8 100644 --- a/otx/cli/tools/demo.py +++ b/otx/cli/tools/demo.py @@ -106,7 +106,7 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, mode="eval") + config_manager = ConfigManager(args, mode="demo") # Auto-Configuration for model template config_manager.configure_template() diff --git a/otx/cli/tools/deploy.py b/otx/cli/tools/deploy.py index 6ccf81302c6..0a69bccbad8 100644 --- a/otx/cli/tools/deploy.py +++ b/otx/cli/tools/deploy.py @@ -58,11 +58,8 @@ def main(): hyper_parameters = template.hyper_parameters.data assert hyper_parameters - if not args.load_weights and config_manager.check_workspace(): - exported_weight_path = config_manager.workspace_root / "outputs" / "latest" / "openvino_models" / "openvino.xml" - if not exported_weight_path.exists(): - raise RuntimeError("No appropriate OpenVINO exported model was found.") - args.load_weights = str(exported_weight_path) + if not args.load_weights: + raise RuntimeError("No appropriate OpenVINO exported model was found.") # Get classes for Task, ConfigurableParameters and Dataset. if not args.load_weights.endswith(".bin") and not args.load_weights.endswith(".xml"): diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index fe591e82dbe..da89cd2d174 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -88,7 +88,9 @@ def main(): config_manager.configure_template() if not args.load_weights and config_manager.check_workspace(): - latest_model_path = config_manager.workspace_root / "outputs" / "latest" / "weights.pth" + latest_model_path = ( + config_manager.workspace_root / "outputs" / "latest_trained_model" / "models" / "weights.pth" + ) args.load_weights = str(latest_model_path) # Update Hyper Parameter Configs diff --git a/otx/cli/tools/explain.py b/otx/cli/tools/explain.py index c9b4ff7632d..0defc1a87a8 100644 --- a/otx/cli/tools/explain.py +++ b/otx/cli/tools/explain.py @@ -82,7 +82,7 @@ def main(): args, override_param = get_args() - config_manager = ConfigManager(args, mode="eval") + config_manager = ConfigManager(args, mode="explain") # Auto-Configuration for model template config_manager.configure_template() diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index 519ec625e5c..2546ef9b864 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -63,7 +63,7 @@ def get_args(): def main(): """Main function that is used for model exporting.""" args = get_args() - config_manager = ConfigManager(args, mode="eval", workspace_root=args.workspace) + config_manager = ConfigManager(args, mode="export", workspace_root=args.workspace) # Auto-Configuration for model template config_manager.configure_template() @@ -72,7 +72,9 @@ def main(): # Get class for Task. if not args.load_weights and config_manager.check_workspace(): - latest_model_path = config_manager.workspace_root / "outputs" / "latest" / "weights.pth" + latest_model_path = ( + config_manager.workspace_root / "outputs" / "latest_trained_model" / "models" / "weights.pth" + ) args.load_weights = str(latest_model_path) is_nncf = is_checkpoint_nncf(args.load_weights) @@ -108,18 +110,12 @@ def main(): if not args.output: output_path = config_manager.output_path - output_path = output_path / "openvino_models" + output_path = output_path / "openvino" else: output_path = Path(args.output) output_path.mkdir(exist_ok=True, parents=True) save_model_data(exported_model, str(output_path)) - # Softlink to weights & openvino_models - pre_weight_path = Path(args.load_weights).resolve().parent / "openvino_models" - if pre_weight_path.exists(): - pre_weight_path.unlink() - pre_weight_path.symlink_to(output_path.resolve()) - return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/optimize.py b/otx/cli/tools/optimize.py index 0369e71af20..eaa9cc2e7c3 100644 --- a/otx/cli/tools/optimize.py +++ b/otx/cli/tools/optimize.py @@ -78,13 +78,15 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.workspace, mode="train") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="optimize") # Auto-Configuration for model template config_manager.configure_template() # The default in the workspace is the model weight of the OTX train. if not args.load_weights and config_manager.check_workspace(): - latest_model_path = config_manager.workspace_root / "outputs" / "latest" / "weights.pth" + latest_model_path = ( + config_manager.workspace_root / "outputs" / "latest_trained_model" / "models" / "weights.pth" + ) args.load_weights = str(latest_model_path) is_pot = False @@ -136,12 +138,6 @@ def main(): output_path.mkdir(exist_ok=True, parents=True) save_model_data(output_model, output_path) - # Softlink to weights & optimized models - pre_weight_path = Path(args.load_weights).resolve().parent / opt_method - if pre_weight_path.exists(): - pre_weight_path.unlink() - pre_weight_path.symlink_to(output_path.resolve()) - validation_dataset = dataset.get_subset(Subset.VALIDATION) predicted_validation_dataset = task.infer( validation_dataset.with_empty_annotations(), diff --git a/otx/cli/tools/train.py b/otx/cli/tools/train.py index 305b76255c0..ee88c6a5986 100644 --- a/otx/cli/tools/train.py +++ b/otx/cli/tools/train.py @@ -230,12 +230,12 @@ def main(): # pylint: disable=too-many-branches save_model_data(output_model, str(config_manager.output_path / "models")) # Latest model folder symbolic link to models - latest_path = config_manager.workspace_root / "outputs" / "latest" + latest_path = config_manager.workspace_root / "outputs" / "latest_trained_model" if latest_path.exists(): - (config_manager.workspace_root / "outputs/latest").unlink() + latest_path.unlink() elif not latest_path.parent.exists(): latest_path.parent.mkdir(exist_ok=True, parents=True) - latest_path.symlink_to((config_manager.output_path / "models").resolve()) + latest_path.symlink_to(config_manager.output_path.resolve()) if config_manager.data_config["val_subset"]["data_root"]: validation_dataset = dataset.get_subset(Subset.VALIDATION) diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 4b998edbb76..53c21c11e3d 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -124,8 +124,12 @@ def test_otx_train(self, case, tmp_dir_path): # check output (use --workspace & --output) output_path = os.path.join(tmp_dir_path, "otx-workspace") assert os.path.exists(os.path.join(output_path, "outputs")) - assert os.path.exists(os.path.join(output_path, "outputs", "latest")) - assert os.path.exists(os.path.join(output_path, "outputs", "latest", "weights.pth")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest_trained_model")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest_trained_model", "models")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest_trained_model", "models", "weights.pth")) + assert os.path.exists( + os.path.join(output_path, "outputs", "latest_trained_model", "models", "label_schema.json") + ) assert os.path.exists(os.path.join(output_path, "models")) assert os.path.exists(os.path.join(output_path, "models", "weights.pth")) @@ -142,9 +146,10 @@ def test_otx_train_wo_output_args(self, tmp_dir_path): assert os.path.exists(output_path) file_list = sorted(os.listdir(output_path)) assert len(file_list) == 2 - assert os.path.exists(os.path.join(output_path, "latest")) - assert os.path.exists(os.path.join(output_path, "latest", "weights.pth")) - assert os.path.exists(os.path.join(output_path, "latest", "label_schema.json")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models", "weights.pth")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models", "label_schema.json")) file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest assert os.path.exists(os.path.join(output_path, file_list[-1])) assert os.path.exists(os.path.join(output_path, file_list[-1], "models")) @@ -168,17 +173,12 @@ def test_otx_export_wo_output_args(self, tmp_dir_path): assert os.path.exists(output_path) file_list = sorted(os.listdir(output_path)) assert len(file_list) == 3 - assert os.path.exists(os.path.join(output_path, "latest")) - assert os.path.exists(os.path.join(output_path, "latest", "openvino_models")) - assert os.path.exists(os.path.join(output_path, "latest", "openvino_models", "openvino.xml")) - assert os.path.exists(os.path.join(output_path, "latest", "openvino_models", "openvino.bin")) - assert os.path.exists(os.path.join(output_path, "latest", "openvino_models", "label_schema.json")) file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest assert os.path.exists(os.path.join(output_path, file_list[-1])) - assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models")) - assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models", "openvino.xml")) - assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models", "openvino.bin")) - assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino_models", "label_schema.json")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "openvino.xml")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "openvino.bin")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "label_schema.json")) @e2e_pytest_component def test_otx_optimize_wo_output_args(self, tmp_dir_path): @@ -198,47 +198,45 @@ def test_otx_optimize_wo_output_args(self, tmp_dir_path): assert os.path.exists(output_path) file_list = sorted(os.listdir(output_path)) assert len(file_list) == 4 - assert os.path.exists(os.path.join(output_path, "latest")) - assert os.path.exists(os.path.join(output_path, "latest", "nncf")) - assert os.path.exists(os.path.join(output_path, "latest", "nncf", "weights.pth")) - assert os.path.exists(os.path.join(output_path, "latest", "nncf", "label_schema.json")) file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest assert os.path.exists(os.path.join(output_path, file_list[-1])) assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf")) assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "weights.pth")) assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "label_schema.json")) - # @e2e_pytest_component - # def test_otx_train_wo_workspace_and_output_args(self, tmp_dir_path): - # otx_dir = os.getcwd() - # case = list(train_auto_config_args.keys())[0] - # tmp_dir_path = tmp_dir_path / "test_otx_train_wo_workspace_and_output_args" - # tmp_dir_path.mkdir(exist_ok=True) - # expected_workspace_path = os.path.join(tmp_dir_path, f"otx-workspace-{case.upper()}") - # command_line = [ - # "otx", - # "train", - # ] - # args = train_auto_config_args[case] - # for option, value in args.items(): - # if option in ["--train-data-roots", "--val-data-roots"]: - # command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) - # command_line.extend(train_params) - # check_run(command_line, cwd=tmp_dir_path) - - # # check output (without --output -> Default outputs) - # assert os.path.exists(expected_workspace_path) - # expected_output_path = os.path.join(expected_workspace_path, "outputs") - # assert os.path.exists(expected_output_path) - # file_list = sorted(os.listdir(expected_output_path)) - # assert len(file_list) == 2 - # assert os.path.exists(os.path.join(expected_output_path, "latest")) - # assert os.path.exists(os.path.join(expected_output_path, "latest", "weights.pth")) - # assert os.path.exists(os.path.join(expected_output_path, "latest", "label_schema.json")) - # file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest - # assert os.path.exists(os.path.join(expected_output_path, file_list[-1])) - # assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models")) - # assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "weights.pth")) + @e2e_pytest_component + def test_otx_train_wo_workspace_and_output_args(self, tmp_dir_path): + otx_dir = os.getcwd() + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_otx_train_wo_workspace_and_output_args" + tmp_dir_path.mkdir(exist_ok=True) + expected_workspace_path = os.path.join(tmp_dir_path, f"otx-workspace-{case.upper()}") + command_line = [ + "otx", + "train", + ] + args = train_auto_config_args[case] + for option, value in args.items(): + if option in ["--train-data-roots", "--val-data-roots"]: + command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) + command_line.extend(train_params) + check_run(command_line, cwd=tmp_dir_path) + + # check output (without --output -> Default outputs) + assert os.path.exists(expected_workspace_path) + expected_output_path = os.path.join(expected_workspace_path, "outputs") + assert os.path.exists(expected_output_path) + file_list = sorted(os.listdir(expected_output_path)) + assert len(file_list) == 2 + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model")) + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models")) + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models", "weights.pth")) + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models", "label_schema.json")) + file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + assert os.path.exists(os.path.join(expected_output_path, file_list[-1])) + assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models")) + assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "weights.pth")) + assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "label_schema.json")) class TestTelemetryIntegration: From bb6f355a792ffa3e1b485050f1adc162ef76eb37 Mon Sep 17 00:00:00 2001 From: Harim Kang Date: Thu, 30 Mar 2023 11:16:18 +0900 Subject: [PATCH 14/17] Update tests/integration/cli/test_cli.py Co-authored-by: Eunwoo Shin --- tests/integration/cli/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 53c21c11e3d..f1b996ea24b 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -150,7 +150,7 @@ def test_otx_train_wo_output_args(self, tmp_dir_path): assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models")) assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models", "weights.pth")) assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models", "label_schema.json")) - file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + file_list.remove("latest_trained_model") assert os.path.exists(os.path.join(output_path, file_list[-1])) assert os.path.exists(os.path.join(output_path, file_list[-1], "models")) assert os.path.exists(os.path.join(output_path, file_list[-1], "models", "weights.pth")) From 0623001d709a890d81341260cc32dff3cce52444 Mon Sep 17 00:00:00 2001 From: Harim Kang Date: Thu, 30 Mar 2023 11:16:28 +0900 Subject: [PATCH 15/17] Update tests/integration/cli/test_cli.py Co-authored-by: Eunwoo Shin --- tests/integration/cli/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index f1b996ea24b..5a37d778a4c 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -173,7 +173,7 @@ def test_otx_export_wo_output_args(self, tmp_dir_path): assert os.path.exists(output_path) file_list = sorted(os.listdir(output_path)) assert len(file_list) == 3 - file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + file_list.remove("latest_trained_model") assert os.path.exists(os.path.join(output_path, file_list[-1])) assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino")) assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "openvino.xml")) From 28ab1f55b29bfc0eed198d76edddf18d8bc307b3 Mon Sep 17 00:00:00 2001 From: Harim Kang Date: Thu, 30 Mar 2023 11:16:36 +0900 Subject: [PATCH 16/17] Update tests/integration/cli/test_cli.py Co-authored-by: Eunwoo Shin --- tests/integration/cli/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 5a37d778a4c..080493ef3af 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -198,7 +198,7 @@ def test_otx_optimize_wo_output_args(self, tmp_dir_path): assert os.path.exists(output_path) file_list = sorted(os.listdir(output_path)) assert len(file_list) == 4 - file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + file_list.remove("latest_trained_model") assert os.path.exists(os.path.join(output_path, file_list[-1])) assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf")) assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "weights.pth")) From 652af52521d8697dcb5a0ab82bcc905c82d38ec5 Mon Sep 17 00:00:00 2001 From: Harim Kang Date: Thu, 30 Mar 2023 11:16:44 +0900 Subject: [PATCH 17/17] Update tests/integration/cli/test_cli.py Co-authored-by: Eunwoo Shin --- tests/integration/cli/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 080493ef3af..00ab5dd9322 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -232,7 +232,7 @@ def test_otx_train_wo_workspace_and_output_args(self, tmp_dir_path): assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models")) assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models", "weights.pth")) assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models", "label_schema.json")) - file_list = file_list[:-1] # Remove latest from list, then file_list[-1] is latest + file_list.remove("latest_trained_model") assert os.path.exists(os.path.join(expected_output_path, file_list[-1])) assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models")) assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "weights.pth"))