diff --git a/docs/source/guide/get_started/quick_start_guide/cli_commands.rst b/docs/source/guide/get_started/quick_start_guide/cli_commands.rst index f7878dce927..23a371bd101 100644 --- a/docs/source/guide/get_started/quick_start_guide/cli_commands.rst +++ b/docs/source/guide/get_started/quick_start_guide/cli_commands.rst @@ -71,7 +71,7 @@ Building workspace folder (otx) ...$ otx build --help usage: otx build [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--test-data-roots TEST_DATA_ROOTS] [--unlabeled-data-roots UNLABELED_DATA_ROOTS] - [--unlabeled-file-list UNLABELED_FILE_LIST] [--task TASK] [--train-type TRAIN_TYPE] [--work-dir WORK_DIR] [--model MODEL] [--backbone BACKBONE] + [--unlabeled-file-list UNLABELED_FILE_LIST] [--task TASK] [--train-type TRAIN_TYPE] [--workspace WORKSPACE] [--model MODEL] [--backbone BACKBONE] [template] positional arguments: @@ -93,7 +93,7 @@ Building workspace folder --task TASK The currently supported options: ('CLASSIFICATION', 'DETECTION', 'INSTANCE_SEGMENTATION', 'SEGMENTATION', 'ACTION_CLASSIFICATION', 'ACTION_DETECTION', 'ANOMALY_CLASSIFICATION', 'ANOMALY_DETECTION', 'ANOMALY_SEGMENTATION'). --train-type TRAIN_TYPE The currently supported options: dict_keys(['Incremental', 'Semisupervised', 'Selfsupervised']). - --work-dir WORK_DIR Location where the workspace. + --workspace WORKSPACE Location where the workspace. --model MODEL Enter the name of the model you want to use. (Ex. EfficientNet-B0). --backbone BACKBONE Available Backbone Type can be found using 'otx find --backbone {framework}'. If there is an already created backbone configuration yaml file, enter the corresponding path. @@ -150,7 +150,7 @@ Training - ``weights.pth`` - a model snapshot - ``label_schema.json`` - a label schema used in training, created from a dataset -The results will be saved in ``./model`` folder by default. The output folder can be modified by ``--save-model-to`` option. These files are used by other commands: ``export``, ``eval``, ``demo``, etc. +The results will be saved in ``./outputs/`` folder by default. The output folder can be modified by ``--output`` option. These files are used by other commands: ``export``, ``eval``, ``demo``, etc. ``otx train`` receives ``template`` as a positional argument. ``template`` can be a path to the specific ``template.yaml`` file, template name or template ID. Also, the path to train and val data root should be passed to the CLI to start training. @@ -160,7 +160,7 @@ However, if you created a workspace with ``otx build``, the training process can otx train --help usage: otx train [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--unlabeled-data-roots UNLABELED_DATA_ROOTS] [--unlabeled-file-list UNLABELED_FILE_LIST] - [--load-weights LOAD_WEIGHTS] [--resume-from RESUME_FROM] [--save-model-to SAVE_MODEL_TO] [--work-dir WORK_DIR] [--enable-hpo] [--hpo-time-ratio HPO_TIME_RATIO] [--gpus GPUS] + [--load-weights LOAD_WEIGHTS] [--resume-from RESUME_FROM] [-o OUTPUT] [--workspace WORKSPACE] [--enable-hpo] [--hpo-time-ratio HPO_TIME_RATIO] [--gpus GPUS] [--rdzv-endpoint RDZV_ENDPOINT] [--base-rank BASE_RANK] [--world-size WORLD_SIZE] [--mem-cache-size PARAMS.ALGO_BACKEND.MEM_CACHE_SIZE] [--data DATA] [template] {params} ... @@ -186,9 +186,9 @@ However, if you created a workspace with ``otx build``, the training process can Load model weights from previously saved checkpoint. --resume-from RESUME_FROM Resume training from previously saved checkpoint - --save-model-to SAVE_MODEL_TO + -o OUTPUT, --output OUTPUT Location where trained model will be stored. - --work-dir WORK_DIR Location where the intermediate output of the training will be stored. + --workspace WORKSPACE Location where the intermediate output of the training will be stored. --enable-hpo Execute hyper parameters optimization (HPO) before training. --hpo-time-ratio HPO_TIME_RATIO Expected ratio of total time to run HPO to time taken for full fine-tuning. @@ -261,7 +261,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx export --help - usage: otx export [-h] [--load-weights LOAD_WEIGHTS] [--save-model-to SAVE_MODEL_TO] [--work-dir WORK_DIR] [--dump-features] [--half-precision] [template] + usage: otx export [-h] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [--workspace WORKSPACE] [--dump-features] [--half-precision] [template] positional arguments: template Enter the path or ID or name of the template file. @@ -271,9 +271,9 @@ With the ``--help`` command, you can list additional information, such as its pa -h, --help show this help message and exit --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint. - --save-model-to SAVE_MODEL_TO + -o OUTPUT, --output OUTPUT Location where exported model will be stored. - --work-dir WORK_DIR Location where the intermediate output of the export will be stored. + --workspace WORKSPACE Location where the intermediate output of the export will be stored. --dump-features Whether to return feature vector and saliency map for explanation purposes. --half-precision This flag indicated if model is exported in half precision (FP16). @@ -282,7 +282,7 @@ The command below performs exporting to the ``outputs/openvino`` path. .. code-block:: - (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --save-model-to outputs/openvino + (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --output outputs/openvino The command results in ``openvino.xml``, ``openvino.bin`` and ``label_schema.json`` @@ -290,7 +290,7 @@ To use the exported model as an input for ``otx explain``, please dump additiona .. code-block:: - (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --save-model-to outputs/openvino/with_features --dump-features + (otx) ...$ otx export Custom_Object_Detection_Gen3_SSD --load-weights --output outputs/openvino/with_features --dump-features ************ @@ -306,8 +306,8 @@ With the ``--help`` command, you can list additional information: .. code-block:: - usage: otx optimize [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--save-model-to SAVE_MODEL_TO] [--save-performance SAVE_PERFORMANCE] - [--work-dir WORK_DIR] + usage: otx optimize [-h] [--train-data-roots TRAIN_DATA_ROOTS] [--val-data-roots VAL_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] + [--workspace WORKSPACE] [template] {params} ... positional arguments: @@ -324,11 +324,9 @@ With the ``--help`` command, you can list additional information: Comma-separated paths to validation data folders. --load-weights LOAD_WEIGHTS Load weights of trained model - --save-model-to SAVE_MODEL_TO - Location where trained model will be stored. - --save-performance SAVE_PERFORMANCE - Path to a json file where computed performance will be stored. - --work-dir WORK_DIR Location where the intermediate output of the task will be stored. + -o OUTPUT, --output OUTPUT + Location where optimized model will be stored. + --workspace WORKSPACE Location where the intermediate output of the task will be stored. Command example for optimizing a PyTorch model (.pth) with OpenVINO™ NNCF: @@ -337,7 +335,7 @@ Command example for optimizing a PyTorch model (.pth) with OpenVINO™ NNCF: (otx) ...$ otx optimize SSD --load-weights \ --train-data-roots \ --val-data-roots \ - --save-model-to outputs/nncf + --output outputs/nncf Command example for optimizing OpenVINO™ model (.xml) with OpenVINO™ POT: @@ -346,7 +344,7 @@ Command example for optimizing OpenVINO™ model (.xml) with OpenVINO™ POT: (otx) ...$ otx optimize SSD --load-weights \ --val-data-roots \ - --save-model-to outputs/pot + --output outputs/pot Thus, to use POT pass the path to exported IR (.xml) model, to use NNCF pass the path to the PyTorch (.pth) weights. @@ -363,7 +361,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx eval --help - usage: otx eval [-h] [--test-data-roots TEST_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [--save-performance SAVE_PERFORMANCE] [--work-dir WORK_DIR] [template] {params} ... + usage: otx eval [-h] [--test-data-roots TEST_DATA_ROOTS] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [--workspace WORKSPACE] [template] {params} ... positional arguments: template Enter the path or ID or name of the template file. @@ -377,9 +375,9 @@ With the ``--help`` command, you can list additional information, such as its pa Comma-separated paths to test data folders. --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint.It could be a trained/optimized model (POT only) or exported model. - --save-performance SAVE_PERFORMANCE - Path to a json file where computed performance will be stored. - --work-dir WORK_DIR Location where the intermediate output of the task will be stored. + -o OUTPUT, --output OUTPUT + Location where the intermediate output of the task will be stored. + --workspace WORKSPACE Path to the workspace where the command will run. The command below will evaluate the trained model on the provided dataset: @@ -388,7 +386,7 @@ The command below will evaluate the trained model on the provided dataset: (otx) ...$ otx eval SSD --test-data-roots \ --load-weights \ - --save-performance outputs/performance.json + --output .. note:: @@ -447,7 +445,7 @@ By default, the model is exported to the OpenVINO™ IR format without extra fea .. code-block:: (otx) ...$ otx export SSD --load-weights \ - --save-model-to outputs/openvino/with_features \ + --output outputs/openvino/with_features \ --dump-features (otx) ...$ otx explain SSD --explain-data-roots \ --load-weights outputs/openvino/with_features \ @@ -521,7 +519,7 @@ With the ``--help`` command, you can list additional information, such as its pa .. code-block:: (otx) ...$ otx deploy --help - usage: otx deploy [-h] [--load-weights LOAD_WEIGHTS] [--save-model-to SAVE_MODEL_TO] [template] + usage: otx deploy [-h] [--load-weights LOAD_WEIGHTS] [-o OUTPUT] [template] positional arguments: template Enter the path or ID or name of the template file. @@ -531,7 +529,7 @@ With the ``--help`` command, you can list additional information, such as its pa -h, --help show this help message and exit --load-weights LOAD_WEIGHTS Load model weights from previously saved checkpoint. - --save-model-to SAVE_MODEL_TO + -o OUTPUT, --output OUTPUT Location where openvino.zip will be stored. @@ -540,5 +538,5 @@ Command example: .. code-block:: (otx) ...$ otx deploy SSD --load-weights \ - --save-model-to outputs/deploy + --output outputs/deploy diff --git a/docs/source/guide/tutorials/advanced/self_sl.rst b/docs/source/guide/tutorials/advanced/self_sl.rst index e474fe0c0bc..0c027d19cf7 100644 --- a/docs/source/guide/tutorials/advanced/self_sl.rst +++ b/docs/source/guide/tutorials/advanced/self_sl.rst @@ -64,7 +64,7 @@ for **self-supervised learning** by running the following command: .. code-block:: - (otx) ...$ otx build --train-data-roots data/flower_photos --model MobileNet-V3-large-1x --train-type Selfsupervised --work-dir otx-workspace-CLASSIFICATION-Selfsupervised + (otx) ...$ otx build --train-data-roots data/flower_photos --model MobileNet-V3-large-1x --train-type SELFSUPERVISED --workspace otx-workspace-CLASSIFICATION-Selfsupervised [*] Workspace Path: otx-workspace-CLASSIFICATION-Selfsupervised [*] Load Model Template ID: Custom_Image_Classification_MobileNet-V3-large-1x @@ -82,7 +82,7 @@ for **self-supervised learning** by running the following command: 1. add ``--train-type Selfsupervised`` in the command to get the training components for self-supervised learning, 2. update the path set as ``train-data-roots``, - 3. and add ``--work-dir`` to distinguish self-supervised learning workspace from supervised learning workspace. + 3. and add ``--workspace`` to distinguish self-supervised learning workspace from supervised learning workspace. After the workspace creation, the workspace structure is as follows: diff --git a/docs/source/guide/tutorials/advanced/semi_sl.rst b/docs/source/guide/tutorials/advanced/semi_sl.rst index fa866675526..081d86d69ae 100644 --- a/docs/source/guide/tutorials/advanced/semi_sl.rst +++ b/docs/source/guide/tutorials/advanced/semi_sl.rst @@ -150,7 +150,7 @@ In the train log, you can check that the train type is set to **Semisupervised** ... -After training ends, a trained model is saved in the ``models`` sub-directory in the workspace named ``otx-workspace-CLASSIFICATION`` by default. +After training ends, a trained model is saved in the ``latest_trained_model`` sub-directory in the workspace named ``otx-workspace-CLASSIFICATION`` by default. *************************** @@ -159,7 +159,7 @@ Validation In the same manner with `the normal validation <../base/how_to_train/classification.html#validation>`__, we can evaluate the trained model with auto-splitted validation dataset in the workspace and -save results to ``performance.json`` by the following command: +save results to ``outputs/performance.json`` by the following command: .. code-block:: @@ -167,4 +167,4 @@ save results to ``performance.json`` by the following command: (otx) ...$ otx eval otx/algorithms/classification/configs/mobilenet_v3_large_1_cls_incr/template.yaml \ --test-data-roots splitted_dataset/val \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs diff --git a/docs/source/guide/tutorials/base/deploy.rst b/docs/source/guide/tutorials/base/deploy.rst index ee16dbc8d43..dfba9bda762 100644 --- a/docs/source/guide/tutorials/base/deploy.rst +++ b/docs/source/guide/tutorials/base/deploy.rst @@ -45,7 +45,7 @@ using the command below: (otx) ...$ otx deploy otx/algorithms/detection/configs/detection/mobilenetv2_atss/template.yaml \ --load-weights outputs/openvino/openvino.xml \ - --save-model-to outputs/deploy + --output outputs/deploy 2023-01-20 09:30:40,938 | INFO : Loading OpenVINO OTXDetectionTask 2023-01-20 09:30:41,736 | INFO : OpenVINO task initialization completed diff --git a/docs/source/guide/tutorials/base/how_to_train/action_classification.rst b/docs/source/guide/tutorials/base/how_to_train/action_classification.rst index c81d5946b56..e6418b8904d 100644 --- a/docs/source/guide/tutorials/base/how_to_train/action_classification.rst +++ b/docs/source/guide/tutorials/base/how_to_train/action_classification.rst @@ -186,13 +186,13 @@ Keep in mind that ``label_schema.json`` file contains meta information about the ``otx eval`` will output a frame-wise accuracy for action classification. Note, that top-1 accuracy during training is video-wise accuracy. 2. The command below will run validation on the dataset -and save performance results in ``performance.json`` file: +and save performance results in ``outputs/performance.json`` file: .. code-block:: (otx) ...$ otx eval --test-data-roots ../data/hmdb51/CVAT/valid \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs You will get a similar validation output: @@ -215,12 +215,12 @@ Export It allows running the model on the Intel hardware much more efficiently, especially on the CPU. Also, the resulting IR model is required to run POT optimization. IR model consists of two files: ``openvino.xml`` for weights and ``openvino.bin`` for architecture. 2. Run the command line below to export the trained model -and save the exported model to the ``openvino_models`` folder. +and save the exported model to the ``openvino`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_models + --output openvino ... 2023-02-21 22:54:32,518 - mmaction - INFO - Model architecture: X3D @@ -241,8 +241,8 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param .. code-block:: (otx) ...$ otx eval --test-data-roots ../data/hmdb51/CVAT/valid \ - --load-weights openvino_models/openvino.xml \ - --save-performance openvino_models/performance.json + --load-weights openvino/openvino.xml \ + --output outputs/openvino ... @@ -262,8 +262,8 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: - (otx) ...$ otx optimize --load-weights openvino_models/openvino.xml \ - --save-model-to pot_model + (otx) ...$ otx optimize --load-weights openvino/openvino.xml \ + --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/action_detection.rst b/docs/source/guide/tutorials/base/how_to_train/action_detection.rst index f19349921b4..c4e73292145 100644 --- a/docs/source/guide/tutorials/base/how_to_train/action_detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/action_detection.rst @@ -128,13 +128,13 @@ Please note, ``label_schema.json`` file contains meta information about the data ``otx eval`` will output a mAP score for spatio-temporal action detection. 2. The command below will run validation on our dataset -and save performance results in ``performance.json`` file: +and save performance results in ``outputs/performance.json`` file: .. code-block:: (otx) ...$ otx eval --test-data-roots ../data/JHMDB_5%/test \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs We will get a similar to this validation output after some validation time (about 2 minutes): @@ -163,7 +163,7 @@ Export It allows running the model on the Intel hardware much more efficiently, especially on the CPU. Also, the resulting IR model is required to run POT optimization. IR model consists of two files: ``openvino.xml`` for weights and ``openvino.bin`` for architecture. 2. Run the command line below to export the trained model -and save the exported model to the ``openvino_models`` folder. +and save the exported model to the ``openvino`` folder. .. code-block:: @@ -213,7 +213,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: - (otx) ...$ otx optimize --load-weights openvino_models/openvino.xml \ + (otx) ...$ otx optimize --load-weights openvino/openvino.xml \ --save-model-to pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst b/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst index 561a8136a80..2118343b128 100644 --- a/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/anomaly_detection.rst @@ -138,7 +138,7 @@ Now we have trained the model, let's see how it performs on a specific dataset. (otx) ...$ otx eval ote_anomaly_detection_padim \ --test-data-roots datasets/MVTec/bottle/test \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --save-performance otx-workspace-ANOMALY_DETECTION/performance.json + --output otx-workspace-ANOMALY_DETECTION/outputs You should see an output similar to the following:: @@ -159,13 +159,13 @@ Export It allows running the model on the Intel hardware much more efficient, especially on the CPU. Also, the resulting IR model is required to run POT optimization. IR model consists of 2 files: ``openvino.xml`` for weights and ``openvino.bin`` for architecture. 2. We can run the below command line to export the trained model -and save the exported model to the ``openvino_models`` folder: +and save the exported model to the ``openvino`` folder: .. code-block:: otx export ote_anomaly_detection_padim \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --save-model-to otx-workspace-ANOMALY_DETECTION/openvino_models + --output otx-workspace-ANOMALY_DETECTION/openvino You will see the outputs similar to the following: @@ -187,8 +187,8 @@ Now that we have the exported model, let's check its performance using ``otx eva otx eval ote_anomaly_detection_padim \ --test-data-roots datasets/MVTec/bottle/test \ - --load-weights otx-workspace-ANOMALY_DETECTION/openvino_models/openvino.xml \ - --save-performance otx-workspace-ANOMALY_DETECTION/openvino_models/performance.json + --load-weights otx-workspace-ANOMALY_DETECTION/openvino/openvino.xml \ + --output otx-workspace-ANOMALY_DETECTION/openvino This gives the following results: @@ -210,8 +210,8 @@ optimization. otx optimize ote_anomaly_detection_padim \ --train-data-roots datasets/MVTec/bottle/train \ - --load-weights otx-workspace-ANOMALY_DETECTION/openvino_models/openvino.xml \ - --save-model-to otx-workspace-ANOMALY_DETECTION/pot_model + --load-weights otx-workspace-ANOMALY_DETECTION/openvino/openvino.xml \ + --output otx-workspace-ANOMALY_DETECTION/pot_model This command generates the following files that can be used to run :doc:`otx demo <../demo>`: @@ -231,7 +231,7 @@ weights to the ``opitmize`` command: otx optimize ote_anomaly_detection_padim \ --train-data-roots datasets/MVTec/bottle/train \ --load-weights otx-workspace-ANOMALY_DETECTION/models/weights.pth \ - --save-model-to otx-workspace-ANOMALY_DETECTION/nncf_model + --output otx-workspace-ANOMALY_DETECTION/nncf_model Similar to POT optimization, it generates the following files: diff --git a/docs/source/guide/tutorials/base/how_to_train/classification.rst b/docs/source/guide/tutorials/base/how_to_train/classification.rst index d645d9ec0ee..e1ad99e9b66 100644 --- a/docs/source/guide/tutorials/base/how_to_train/classification.rst +++ b/docs/source/guide/tutorials/base/how_to_train/classification.rst @@ -137,7 +137,7 @@ The training time highly relies on the hardware characteristics, for example on After that, you have the PyTorch classification model trained with OpenVINO™ Training Extensions, which you can use for evaluation, export, optimization and deployment. .. note:: - If you specified ``--work-dir``, you also can visualize the training using ``Tensorboard`` as these logs are located in ``/tf_logs``. + If you specified ``--workspace``, you also can visualize the training using ``Tensorboard`` as these logs are located in ``/tf_logs``. *********** Validation @@ -158,7 +158,7 @@ and save performance results in ``performance.json`` file: (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs You will get a similar validation output: @@ -185,7 +185,7 @@ and save the exported model to the ``openvino_model`` folder: .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_model + --output openvino_model ... @@ -200,7 +200,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights openvino_model/openvino.xml \ - --save-performance openvino_model/performance.json + --output openvino_model ... @@ -221,7 +221,7 @@ a PyTorch model (`.pth`) with OpenVINO™ NNCF. .. code-block:: - (otx) ...$ otx optimize --load-weights models/weights.pth --save-model-to nncf_model + (otx) ...$ otx optimize --load-weights models/weights.pth --output nncf_model ... @@ -240,7 +240,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_model/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/detection.rst b/docs/source/guide/tutorials/base/how_to_train/detection.rst index f55d1419af8..8fbeddc8449 100644 --- a/docs/source/guide/tutorials/base/how_to_train/detection.rst +++ b/docs/source/guide/tutorials/base/how_to_train/detection.rst @@ -227,7 +227,7 @@ detection model on the first GPU on WGISD dataset: .. code-block:: (otx) ...$ cd otx-workspace-DETECTION/ - (otx) ...$ otx train --save-model-to ../outputs --work-dir ../outputs/logs --gpus 1 + (otx) ...$ otx train --output ../outputs --workspace ../outputs/logs --gpus 1 To start multi-gpu training, list the indexes of GPUs you want to train on or omit `gpus` parameter, so training will run on all available GPUs. @@ -290,7 +290,7 @@ folder on WGISD dataset and save results to ``outputs/performance``: (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights ../outputs/weights.pth \ - --save-performance ../outputs/performance.json + --output ../outputs/ 3. The output of ``../outputs/performance.json`` consists of @@ -311,7 +311,7 @@ Please note, by default, the optimal confidence threshold is detected based on v (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights ../outputs/weights.pth \ - --save-performance ../outputs/performance.json + --output ../outputs params \ --postprocessing.confidence_threshold 0.5 \ --postprocessing.result_based_confidence_threshold false @@ -334,7 +334,7 @@ from the previous section and save the exported model to the ``../outputs/openvi .. code-block:: (otx) ...$ otx export --load-weights ../outputs/weights.pth \ - --save-model-to ../outputs/openvino/ + --output ../outputs/openvino/ ... @@ -349,7 +349,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights ../outputs/openvino/openvino.xml \ - --save-performance ../outputs/performance.json + --output ../outputs ... 2023-01-10 06:24:50,382 | INFO : Start OpenVINO inference @@ -386,8 +386,8 @@ with OpenVINO NNCF. .. code-block:: (otx) ...$ otx optimize --load-weights ../outputs/weights.pth \ - --save-model-to ../outputs/nncf \ - --save-performance ../outputs/nncf/performance.json + --output ../outputs/nncf \ + --output ../outputs/nncf ... @@ -405,8 +405,8 @@ with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights ../outputs/openvino/openvino.xml \ - --save-model-to ../outputs/pot \ - --save-performance ../outputs/pot/performance.json + --output ../outputs/pot \ + --output ../outputs/pot ... diff --git a/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst b/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst index a9399d5ca24..84012c69999 100644 --- a/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst +++ b/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst @@ -217,13 +217,13 @@ Please note, ``label_schema.json`` file contains meta information about the data ``otx eval`` will output a F-measure for instance segmentation. 2. The command below will run validation on our dataset -and save performance results in ``performance.json`` file: +and save performance results in ``outputs/performance.json`` file: .. code-block:: (otx) ...$ otx eval --test-data-roots otx-workspace-INSTANCE_SEGMENTATION/splitted_dataset/car_tree_bug \ --load-weights models/weights.pth \ - --save-performance performance.json + --outputs outputs We will get a similar to this validation output: @@ -243,7 +243,7 @@ We will get a similar to this validation output: Also, if you're inside a workspace and ``weights.pth`` exists in ``models`` dir, you can omit ``--load-weights`` as well, assuming those weights are the default as ``models/weights.pth``. - If you omit ``--save-performance``, it will create a ``performance.json`` in the folder for those weights. + If you omit ``--output``, it will create a ``performance.json`` in the folder for those weights. The output of ``./outputs/performance.json`` consists of a dict with target metric name and its value. @@ -267,7 +267,7 @@ and save the exported model to the ``openvino_model`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_model + --output openvino_model ... [ SUCCESS ] Generated IR version 11 model. @@ -287,7 +287,7 @@ You can use ``otx train`` directly without ``otx build``. It will be required to (otx) ...$ otx eval --test-data-roots otx-workspace-INSTANCE_SEGMENTATION/splitted_dataset/car_tree_bug \ --load-weights openvino_model/openvino.xml \ - --save-performance openvino_model/performance.json + --output openvino_model ... @@ -313,7 +313,7 @@ a PyTorch model (`.pth`) with OpenVINO™ NNCF. .. code-block:: - (otx) ...$ otx optimize --load-weights models/weights.pth --save-model-to nncf_model + (otx) ...$ otx optimize --load-weights models/weights.pth --output nncf_model ... @@ -332,7 +332,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_model/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst b/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst index 4302190dcab..4cef182727c 100644 --- a/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst +++ b/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst @@ -151,7 +151,7 @@ By running this example command, the performance results evaluated by our splitt (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights models/weights.pth \ - --save-performance performance.json + --output outputs Finally, we get the validation output: @@ -165,7 +165,7 @@ Finally, we get the validation output: 2023-02-21 18:09:58,508 | INFO : mDice after evaluation: 0.9659400544959128 Performance(score: 0.9659400544959128, dashboard: (1 metric groups)) -In ``performance.json`` file, the validation output score is saved as: +In ``outputs/performance.json`` file, the validation output score is saved as: .. code-block:: @@ -185,7 +185,7 @@ and save the exported model to the ``openvino_model`` folder. .. code-block:: (otx) ...$ otx export --load-weights models/weights.pth \ - --save-model-to openvino_model + --output openvino_model ... @@ -200,7 +200,7 @@ using ``otx eval`` and passing the IR model path to the ``--load-weights`` param (otx) ...$ otx eval --test-data-roots splitted_dataset/val \ --load-weights openvino_model/openvino.xml \ - --save-performance openvino_model/performance.json + --output openvino_model ... @@ -220,7 +220,7 @@ a PyTorch model (`.pth`) with OpenVINO™ NNCF. .. code-block:: - (otx) ...$ otx optimize --load-weights models/weights.pth --save-model-to nncf_model + (otx) ...$ otx optimize --load-weights models/weights.pth --output nncf_model ... @@ -239,7 +239,7 @@ OpenVINO™ model (.xml) with OpenVINO™ POT. .. code-block:: (otx) ...$ otx optimize --load-weights openvino_model/openvino.xml \ - --save-model-to pot_model + --output pot_model ... diff --git a/otx/algorithms/common/tasks/training_base.py b/otx/algorithms/common/tasks/training_base.py index 59333cb4a50..e9cd5deef7a 100644 --- a/otx/algorithms/common/tasks/training_base.py +++ b/otx/algorithms/common/tasks/training_base.py @@ -190,6 +190,11 @@ def project_path(self): """Return output path with logs.""" return self._output_path + @property + def config(self): + """Return output configs used in task.""" + return self._recipe_cfg + @property def model_name(self): """Name of Model Template.""" diff --git a/otx/cli/manager/config_manager.py b/otx/cli/manager/config_manager.py index d82e98f6bb5..ecd52dd879a 100644 --- a/otx/cli/manager/config_manager.py +++ b/otx/cli/manager/config_manager.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # import shutil +from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional @@ -94,6 +95,7 @@ def __init__(self, args, workspace_root: Optional[str] = None, mode: str = "trai self.workspace_root = Path(workspace_root) if workspace_root else Path(".") self.mode = mode self.rebuild: bool = False + self.create_date: str = datetime.now().strftime("%Y%m%d_%H%M%S") self.args = args self.template = args.template @@ -121,6 +123,21 @@ def data_config_file_path(self) -> Path: raise FileNotExistError(f"Not found: {self.args.data}") return self.workspace_root / "data.yaml" + @property + def output_path(self) -> Path: + """The path of output directory for workspace. + + Returns: + Path: Path of output directory. + """ + if "output" in self.args and self.args.output: + output_path = Path(self.args.output) + else: + output_path = self.workspace_root / "outputs" / f"{self.create_date}_{self.mode}" + if not output_path.exists(): + output_path.mkdir(exist_ok=True, parents=True) + return output_path + def check_workspace(self) -> bool: """Check that the class's workspace_root is an actual workspace folder. @@ -173,7 +190,7 @@ def configure_data_config(self, update_data_yaml: bool = True) -> None: """Configure data_config according to the situation and create data.yaml.""" data_yaml_path = self.data_config_file_path data_yaml = configure_dataset(self.args, data_yaml_path=data_yaml_path) - if self.mode in ("train", "build"): + if self.mode in ("train", "build", "optimize"): use_auto_split = data_yaml["data"]["train"]["data-roots"] and not data_yaml["data"]["val"]["data-roots"] # FIXME: Hardcoded for Self-Supervised Learning if use_auto_split and str(self.train_type).upper() != "SELFSUPERVISED": @@ -194,7 +211,11 @@ def _get_train_type(self, ignore_args: bool = False) -> str: if arg_algo_backend: train_type = arg_algo_backend.get("train_type", {"value": "Incremental"}) # type: ignore return train_type.get("value", "Incremental") - if hasattr(self.args, "train_type") and self.mode in ("build", "train") and self.args.train_type: + if ( + hasattr(self.args, "train_type") + and self.mode in ("build", "train", "optimize") + and self.args.train_type + ): self.train_type = self.args.train_type if self.train_type not in TASK_TYPE_TO_SUB_DIR_NAME: raise NotSupportedError(f"{self.train_type} is not currently supported by otx.") @@ -260,7 +281,7 @@ def _get_arg_data_yaml(self): # TODO: This should modify data yaml format to data_config format. """Save the splitted dataset and data.yaml to the workspace.""" data_yaml = self._create_empty_data_cfg() - if self.mode == "train": + if self.mode in ("train", "optimize"): if self.args.train_data_roots: data_yaml["data"]["train"]["data-roots"] = self.args.train_data_roots if self.args.val_data_roots: @@ -363,7 +384,7 @@ def update_data_config(self, data_yaml: dict) -> None: "file_list": data_yaml["data"]["unlabeled"]["file-list"], } # FIXME: Hardcoded for Self-Supervised Learning - if self.mode == "train" and str(self.train_type).upper() == "SELFSUPERVISED": + if self.mode in ("train", "optimize") and str(self.train_type).upper() == "SELFSUPERVISED": self.data_config["val_subset"] = {"data_root": None} def _get_template(self, task_type: str, model: Optional[str] = None) -> ModelTemplate: diff --git a/otx/cli/tools/build.py b/otx/cli/tools/build.py index 246b2efb954..403d6310174 100644 --- a/otx/cli/tools/build.py +++ b/otx/cli/tools/build.py @@ -17,8 +17,6 @@ # See the License for the specific language governing permissions # and limitations under the License. -from pathlib import Path - from otx.cli.manager.config_manager import TASK_TYPE_TO_SUB_DIR_NAME, ConfigManager from otx.cli.utils.parser import get_parser_and_hprams_data @@ -68,8 +66,8 @@ def get_args(): default="Incremental", ) parser.add_argument( - "--work-dir", - help="Location where the workspace.", + "--workspace", + help="Path to the workspace where the command will run.", default=None, ) parser.add_argument( @@ -88,16 +86,14 @@ def main(): """Main function for model or backbone or task building.""" args = get_args() - config_manager = ConfigManager(args, mode="build") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="build") if args.task: config_manager.task_type = args.task.upper() - if args.work_dir: - config_manager.workspace_root = Path(args.work_dir) # Auto-Configuration for model template config_manager.configure_template(model=args.model) - config_manager.build_workspace(new_workspace_path=args.work_dir) + config_manager.build_workspace(new_workspace_path=args.workspace) # Auto-Configuration for Dataset configuration config_manager.configure_data_config() diff --git a/otx/cli/tools/demo.py b/otx/cli/tools/demo.py index 48efda739e6..ea6547322e8 100644 --- a/otx/cli/tools/demo.py +++ b/otx/cli/tools/demo.py @@ -106,7 +106,7 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, mode="eval") + config_manager = ConfigManager(args, mode="demo") # Auto-Configuration for model template config_manager.configure_template() diff --git a/otx/cli/tools/deploy.py b/otx/cli/tools/deploy.py index 5e3ff06b416..0a69bccbad8 100644 --- a/otx/cli/tools/deploy.py +++ b/otx/cli/tools/deploy.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions # and limitations under the License. -import os from pathlib import Path from otx.api.configuration.helper import create @@ -35,7 +34,8 @@ def get_args(): help="Load model weights from previously saved checkpoint.", ) parser.add_argument( - "--save-model-to", + "-o", + "--output", help="Location where openvino.zip will be stored.", ) @@ -58,11 +58,8 @@ def main(): hyper_parameters = template.hyper_parameters.data assert hyper_parameters - if not args.load_weights and config_manager.check_workspace(): - exported_weight_path = config_manager.workspace_root / "models-exported/openvino.xml" - if not exported_weight_path.exists(): - raise RuntimeError("OpenVINO-exported models are supported.") - args.load_weights = str(exported_weight_path) + if not args.load_weights: + raise RuntimeError("No appropriate OpenVINO exported model was found.") # Get classes for Task, ConfigurableParameters and Dataset. if not args.load_weights.endswith(".bin") and not args.load_weights.endswith(".xml"): @@ -82,11 +79,10 @@ def main(): deployed_model = ModelEntity(None, environment.get_model_configuration()) - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "model-deployed") - os.makedirs(args.save_model_to, exist_ok=True) + output_path = Path(args.output) if args.output else config_manager.output_path + output_path.mkdir(exist_ok=True, parents=True) task.deploy(deployed_model) - with open(Path(args.save_model_to) / "openvino.zip", "wb") as write_file: + with open(output_path / "openvino.zip", "wb") as write_file: write_file.write(deployed_model.exportable_code) return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/eval.py b/otx/cli/tools/eval.py index 0c9f0e16100..da89cd2d174 100644 --- a/otx/cli/tools/eval.py +++ b/otx/cli/tools/eval.py @@ -48,12 +48,13 @@ def get_args(): "It could be a trained/optimized model (POT only) or exported model.", ) parser.add_argument( - "--save-performance", - help="Path to a json file where computed performance will be stored.", + "-o", + "--output", + help="Location where the intermediate output of the task will be stored.", ) parser.add_argument( - "--work-dir", - help="Location where the intermediate output of the task will be stored.", + "--workspace", + help="Path to the workspace where the command will run.", default=None, ) @@ -82,12 +83,15 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.work_dir, mode="eval") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="eval") # Auto-Configuration for model template config_manager.configure_template() if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "models/weights.pth") + latest_model_path = ( + config_manager.workspace_root / "outputs" / "latest_trained_model" / "models" / "weights.pth" + ) + args.load_weights = str(latest_model_path) # Update Hyper Parameter Configs hyper_parameters = config_manager.get_hyparams_config(override_param) @@ -136,9 +140,8 @@ def main(): assert resultset.performance is not None print(resultset.performance) - if not args.save_performance: - args.save_performance = str(Path(args.load_weights).parent / "performance.json") - with open(args.save_performance, "w", encoding="UTF-8") as write_file: + output_path = Path(args.output) if args.output else config_manager.output_path + with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file: json.dump( {resultset.performance.score.name: resultset.performance.score.value}, write_file, diff --git a/otx/cli/tools/explain.py b/otx/cli/tools/explain.py index c9b4ff7632d..0defc1a87a8 100644 --- a/otx/cli/tools/explain.py +++ b/otx/cli/tools/explain.py @@ -82,7 +82,7 @@ def main(): args, override_param = get_args() - config_manager = ConfigManager(args, mode="eval") + config_manager = ConfigManager(args, mode="explain") # Auto-Configuration for model template config_manager.configure_template() diff --git a/otx/cli/tools/export.py b/otx/cli/tools/export.py index fdd2295a5a5..2546ef9b864 100644 --- a/otx/cli/tools/export.py +++ b/otx/cli/tools/export.py @@ -37,12 +37,13 @@ def get_args(): help="Load model weights from previously saved checkpoint.", ) parser.add_argument( - "--save-model-to", + "-o", + "--output", help="Location where exported model will be stored.", ) parser.add_argument( - "--work-dir", - help="Location where the intermediate output of the export will be stored.", + "--workspace", + help="Path to the workspace where the command will run.", default=None, ) parser.add_argument( @@ -62,7 +63,7 @@ def get_args(): def main(): """Main function that is used for model exporting.""" args = get_args() - config_manager = ConfigManager(args, mode="eval", workspace_root=args.work_dir) + config_manager = ConfigManager(args, mode="export", workspace_root=args.workspace) # Auto-Configuration for model template config_manager.configure_template() @@ -71,7 +72,11 @@ def main(): # Get class for Task. if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "models/weights.pth") + latest_model_path = ( + config_manager.workspace_root / "outputs" / "latest_trained_model" / "models" / "weights.pth" + ) + args.load_weights = str(latest_model_path) + is_nncf = is_checkpoint_nncf(args.load_weights) task_class = get_impl_class(template.entrypoints.nncf if is_nncf else template.entrypoints.base) @@ -95,17 +100,21 @@ def main(): ) environment.model = model - task = task_class(task_environment=environment, output_path=args.work_dir) + (config_manager.output_path / "logs").mkdir(exist_ok=True, parents=True) + task = task_class(task_environment=environment, output_path=str(config_manager.output_path / "logs")) exported_model = ModelEntity(None, environment.get_model_configuration()) export_precision = ModelPrecision.FP16 if args.half_precision else ModelPrecision.FP32 task.export(ExportType.OPENVINO, exported_model, export_precision, args.dump_features) - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "model-exported") - Path(args.save_model_to).mkdir(exist_ok=True, parents=True) - save_model_data(exported_model, args.save_model_to) + if not args.output: + output_path = config_manager.output_path + output_path = output_path / "openvino" + else: + output_path = Path(args.output) + output_path.mkdir(exist_ok=True, parents=True) + save_model_data(exported_model, str(output_path)) return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/optimize.py b/otx/cli/tools/optimize.py index 9d8def31357..eaa9cc2e7c3 100644 --- a/otx/cli/tools/optimize.py +++ b/otx/cli/tools/optimize.py @@ -15,6 +15,7 @@ # and limitations under the License. import json +from pathlib import Path from otx.api.entities.inference_parameters import InferenceParameters from otx.api.entities.model import ModelEntity @@ -55,15 +56,12 @@ def get_args(): help="Load weights of trained model", ) parser.add_argument( - "--save-model-to", - help="Location where trained model will be stored.", + "-o", + "--output", + help="Location where optimized model will be stored.", ) parser.add_argument( - "--save-performance", - help="Path to a json file where computed performance will be stored.", - ) - parser.add_argument( - "--work-dir", + "--workspace", help="Location where the intermediate output of the task will be stored.", default=None, ) @@ -80,13 +78,16 @@ def main(): # Dynamically create an argument parser based on override parameters. args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.work_dir, mode="train") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="optimize") # Auto-Configuration for model template config_manager.configure_template() # The default in the workspace is the model weight of the OTX train. if not args.load_weights and config_manager.check_workspace(): - args.load_weights = str(config_manager.workspace_root / "models/weights.pth") + latest_model_path = ( + config_manager.workspace_root / "outputs" / "latest_trained_model" / "models" / "weights.pth" + ) + args.load_weights = str(latest_model_path) is_pot = False if args.load_weights.endswith(".bin") or args.load_weights.endswith(".xml"): @@ -128,9 +129,14 @@ def main(): OptimizationParameters(), ) - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "model-optimized") - save_model_data(output_model, args.save_model_to) + opt_method = "pot" if is_pot else "nncf" + if not args.output: + output_path = config_manager.output_path + output_path = output_path / opt_method + else: + output_path = Path(args.output) + output_path.mkdir(exist_ok=True, parents=True) + save_model_data(output_model, output_path) validation_dataset = dataset.get_subset(Subset.VALIDATION) predicted_validation_dataset = task.infer( @@ -147,12 +153,12 @@ def main(): assert resultset.performance is not None print(resultset.performance) - if args.save_performance: - with open(args.save_performance, "w", encoding="UTF-8") as write_file: - json.dump( - {resultset.performance.score.name: resultset.performance.score.value}, - write_file, - ) + performance_file_path = config_manager.output_path / f"{opt_method}_performance.json" + with open(performance_file_path, "w", encoding="UTF-8") as write_file: + json.dump( + {resultset.performance.score.name: resultset.performance.score.value}, + write_file, + ) return dict(retcode=0, template=template.name) diff --git a/otx/cli/tools/train.py b/otx/cli/tools/train.py index faffff85635..ee88c6a5986 100644 --- a/otx/cli/tools/train.py +++ b/otx/cli/tools/train.py @@ -76,11 +76,12 @@ def get_args(): help="Resume training from previously saved checkpoint", ) parser.add_argument( - "--save-model-to", - help="Location where trained model will be stored.", + "-o", + "--output", + help="Location where outputs (model & logs) will be stored.", ) parser.add_argument( - "--work-dir", + "--workspace", help="Location where the intermediate output of the training will be stored.", default=None, ) @@ -157,13 +158,13 @@ def main(): # pylint: disable=too-many-branches """Main function that is used for model training.""" args, override_param = get_args() - config_manager = ConfigManager(args, workspace_root=args.work_dir, mode="train") + config_manager = ConfigManager(args, workspace_root=args.workspace, mode="train") # Auto-Configuration for model template config_manager.configure_template() # Creates a workspace if it doesn't exist. if not config_manager.check_workspace(): - config_manager.build_workspace(new_workspace_path=args.work_dir) + config_manager.build_workspace(new_workspace_path=args.workspace) # Auto-Configuration for Dataset configuration config_manager.configure_data_config(update_data_yaml=config_manager.check_workspace()) @@ -203,13 +204,13 @@ def main(): # pylint: disable=too-many-branches model_adapters=model_adapters, ) - # FIXME: Need to align output results & Current HPO use save_model_to.parent - if "save_model_to" not in args or not args.save_model_to: - args.save_model_to = str(config_manager.workspace_root / "models") if args.enable_hpo: - environment = run_hpo(args, environment, dataset, config_manager.data_config) + environment = run_hpo( + args.hpo_time_ratio, config_manager.output_path, environment, dataset, config_manager.data_config + ) - task = task_class(task_environment=environment, output_path=args.work_dir) + (config_manager.output_path / "logs").mkdir(exist_ok=True, parents=True) + task = task_class(task_environment=environment, output_path=str(config_manager.output_path / "logs")) if args.gpus: multigpu_manager = MultiGPUManager(main, args.gpus, args.rdzv_endpoint, args.base_rank, args.world_size) @@ -219,14 +220,22 @@ def main(): # pylint: disable=too-many-branches multigpu_manager.is_available() and not template.task_type.is_anomaly # anomaly tasks don't use this way for multi-GPU training ): - multigpu_manager.setup_multi_gpu_train(task.project_path, hyper_parameters if args.enable_hpo else None) + multigpu_manager.setup_multi_gpu_train( + str(config_manager.output_path), hyper_parameters if args.enable_hpo else None + ) output_model = ModelEntity(dataset, environment.get_model_configuration()) task.train(dataset, output_model, train_parameters=TrainParameters()) - save_model_data(output_model, args.save_model_to) - print(f"[*] Save Model to: {args.save_model_to}") + save_model_data(output_model, str(config_manager.output_path / "models")) + # Latest model folder symbolic link to models + latest_path = config_manager.workspace_root / "outputs" / "latest_trained_model" + if latest_path.exists(): + latest_path.unlink() + elif not latest_path.parent.exists(): + latest_path.parent.mkdir(exist_ok=True, parents=True) + latest_path.symlink_to(config_manager.output_path.resolve()) if config_manager.data_config["val_subset"]["data_root"]: validation_dataset = dataset.get_subset(Subset.VALIDATION) diff --git a/otx/cli/utils/hpo.py b/otx/cli/utils/hpo.py index 61a8ec18da0..ae4eb088688 100644 --- a/otx/cli/utils/hpo.py +++ b/otx/cli/utils/hpo.py @@ -536,12 +536,13 @@ def _get_initial_model_weight_path(self): def run_hpo( - args, environment: TaskEnvironment, dataset: DatasetEntity, data_roots: Dict[str, str] + hpo_time_ratio: int, output: Path, environment: TaskEnvironment, dataset: DatasetEntity, data_roots: Dict[str, str] ) -> Optional[TaskEnvironment]: """Run HPO and load optimized hyper parameter and best HPO model weight. Args: - args: arguments passed to otx train + hpo_time_ratio(int): expected ratio of total time to run HPO to time taken for full fine-tuning + output(Path): directory where HPO output is saved environment (TaskEnvironment): otx task environment dataset (DatasetEntity): dataset to use for training data_roots (Dict[str, str]): dataset path of each dataset type @@ -554,13 +555,13 @@ def run_hpo( ) return None - hpo_save_path = (Path(args.save_model_to).parent / "hpo").absolute() + hpo_save_path = (output / "hpo").absolute() hpo_runner = HpoRunner( environment, len(dataset.get_subset(Subset.TRAINING)), len(dataset.get_subset(Subset.VALIDATION)), hpo_save_path, - args.hpo_time_ratio, + hpo_time_ratio, ) logger.info("started hyper-parameter optimization") diff --git a/otx/cli/utils/multi_gpu.py b/otx/cli/utils/multi_gpu.py index cea63cbc1b6..57bfef0ee5b 100644 --- a/otx/cli/utils/multi_gpu.py +++ b/otx/cli/utils/multi_gpu.py @@ -22,7 +22,7 @@ import threading import time from contextlib import closing -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Union import psutil import torch @@ -74,29 +74,34 @@ def get_gpu_ids(gpus: str) -> List[int]: return gpu_ids -def set_arguments_to_argv(key: str, value: Optional[str] = None, after_params: bool = False): +def set_arguments_to_argv(keys: Union[str, List[str]], value: Optional[str] = None, after_params: bool = False): """Add arguments at proper position in `sys.argv`. Args: - key (str): arguement key. + keys (str or List[str]): arguement keys. value (str or None): argument value. after_params (bool): whether argument should be after `param` or not. """ - if key in sys.argv: + if not isinstance(keys, list): + keys = [keys] + for key in keys: + if key in sys.argv: + if value is not None: + sys.argv[sys.argv.index(key) + 1] = value + return + + key = keys[0] + if not after_params and "params" in sys.argv: + sys.argv.insert(sys.argv.index("params"), key) if value is not None: - sys.argv[sys.argv.index(key) + 1] = value + sys.argv.insert(sys.argv.index("params"), value) else: - if not after_params and "params" in sys.argv: - sys.argv.insert(sys.argv.index("params"), key) - if value is not None: - sys.argv.insert(sys.argv.index("params"), value) + if after_params and "params" not in sys.argv: + sys.argv.append("params") + if value is not None: + sys.argv.extend([key, value]) else: - if after_params and "params" not in sys.argv: - sys.argv.append("params") - if value is not None: - sys.argv.extend([key, value]) - else: - sys.argv.append(key) + sys.argv.append(key) def is_multigpu_child_process(): @@ -238,7 +243,7 @@ def run_child_process( sys.argv.pop(gpus_arg_idx) if "--enable-hpo" in sys.argv: sys.argv.remove("--enable-hpo") - set_arguments_to_argv("--work-dir", output_path) + set_arguments_to_argv("--output", output_path) set_arguments_to_argv("--rdzv-endpoint", rdzv_endpoint) MultiGPUManager.initialize_multigpu_train(rdzv_endpoint, rank, local_rank, gpu_ids, world_size) diff --git a/tests/conftest.py b/tests/conftest.py index d609c3a2fbc..4ae77116996 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ def pytest_addoption(parser): @pytest.fixture(scope="session") def tmp_dir_path(request) -> Generator[Path, None, None]: - prefix = request.config.getoption("--test-work-dir") + prefix = request.config.getoption("--test-workspace") with TemporaryDirectory(prefix=prefix) as tmp_dir: yield Path(tmp_dir) diff --git a/tests/e2e/cli/classification/test_classification.py b/tests/e2e/cli/classification/test_classification.py index 610fb398245..e12877be78c 100644 --- a/tests/e2e/cli/classification/test_classification.py +++ b/tests/e2e/cli/classification/test_classification.py @@ -113,7 +113,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -125,7 +125,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -368,7 +370,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0_m) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args_m) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -380,7 +382,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0_m) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -563,7 +567,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args_h) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args_h) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -575,7 +579,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args_h) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -760,7 +766,7 @@ def test_otx_selfsl_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path_1, otx_dir, args_selfsl) template_work_dir = get_template_dir(template, tmp_dir_path_1) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" tmp_dir_path_2 = tmp_dir_path / "multi_class_cls/test_selfsl_sl" otx_train_testing(template, tmp_dir_path_2, otx_dir, args1) diff --git a/tests/e2e/cli/detection/test_detection.py b/tests/e2e/cli/detection/test_detection.py index 28b786afed6..ce2aa12651c 100644 --- a/tests/e2e/cli/detection/test_detection.py +++ b/tests/e2e/cli/detection/test_detection.py @@ -103,7 +103,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -115,7 +115,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/detection/test_instance_segmentation.py b/tests/e2e/cli/detection/test_instance_segmentation.py index efe79e9587c..9268af9ad65 100644 --- a/tests/e2e/cli/detection/test_instance_segmentation.py +++ b/tests/e2e/cli/detection/test_instance_segmentation.py @@ -86,7 +86,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args0) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -98,7 +98,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args0) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/detection/test_tiling_detection.py b/tests/e2e/cli/detection/test_tiling_detection.py index 329cdb235e1..ced6fd2cb35 100644 --- a/tests/e2e/cli/detection/test_tiling_detection.py +++ b/tests/e2e/cli/detection/test_tiling_detection.py @@ -92,7 +92,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/detection/test_tiling_instseg.py b/tests/e2e/cli/detection/test_tiling_instseg.py index 68668130080..48b3204d9e8 100644 --- a/tests/e2e/cli/detection/test_tiling_instseg.py +++ b/tests/e2e/cli/detection/test_tiling_instseg.py @@ -92,7 +92,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/e2e/cli/segmentation/test_segmentation.py b/tests/e2e/cli/segmentation/test_segmentation.py index 6aa445d4261..b1d0cd4ad8c 100644 --- a/tests/e2e/cli/segmentation/test_segmentation.py +++ b/tests/e2e/cli/segmentation/test_segmentation.py @@ -86,7 +86,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path, otx_dir, args) template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -98,7 +98,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component @@ -331,7 +333,7 @@ def test_otx_train(self, template, tmp_dir_path): otx_train_testing(template, tmp_dir_path_1, otx_dir, args_selfsl) template_work_dir = get_template_dir(template, tmp_dir_path_1) args1 = copy.deepcopy(args) - args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args1["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" tmp_dir_path_2 = tmp_dir_path / "segmentation/test_selfsl_sl" otx_train_testing(template, tmp_dir_path_2, otx_dir, args1) diff --git a/tests/integration/cli/classification/test_classification.py b/tests/integration/cli/classification/test_classification.py index 95bbc96020d..2fb4f05a5b2 100644 --- a/tests/integration/cli/classification/test_classification.py +++ b/tests/integration/cli/classification/test_classification.py @@ -107,7 +107,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/detection/test_detection.py b/tests/integration/cli/detection/test_detection.py index 25ed0b15639..9e9c13d1639 100644 --- a/tests/integration/cli/detection/test_detection.py +++ b/tests/integration/cli/detection/test_detection.py @@ -88,7 +88,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/detection/test_instance_segmentation.py b/tests/integration/cli/detection/test_instance_segmentation.py index 0ac0a41d4da..edea7906f30 100644 --- a/tests/integration/cli/detection/test_instance_segmentation.py +++ b/tests/integration/cli/detection/test_instance_segmentation.py @@ -72,7 +72,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/segmentation/test_segmentation.py b/tests/integration/cli/segmentation/test_segmentation.py index 4aac8920411..2026cd3d8a4 100644 --- a/tests/integration/cli/segmentation/test_segmentation.py +++ b/tests/integration/cli/segmentation/test_segmentation.py @@ -112,7 +112,9 @@ def test_otx_resume(self, template, tmp_dir_path): template_work_dir = get_template_dir(template, tmp_dir_path) args1 = copy.deepcopy(args) args1["train_params"] = resume_params - args1["--resume-from"] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth" + args1[ + "--resume-from" + ] = f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth" otx_resume_testing(template, tmp_dir_path, otx_dir, args1) @e2e_pytest_component diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 825a811750b..00ab5dd9322 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -12,6 +12,7 @@ from otx.cli.tools import cli from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( + check_run, otx_build_auto_config, otx_build_backbone_testing, otx_build_testing, @@ -120,6 +121,122 @@ def test_otx_train(self, case, tmp_dir_path): tmp_dir_path = tmp_dir_path / "test_train_auto_config" / case train_auto_config_args[case]["train_params"] = train_params otx_train_auto_config(root=tmp_dir_path, otx_dir=otx_dir, args=train_auto_config_args[case]) + # check output (use --workspace & --output) + output_path = os.path.join(tmp_dir_path, "otx-workspace") + assert os.path.exists(os.path.join(output_path, "outputs")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest_trained_model")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest_trained_model", "models")) + assert os.path.exists(os.path.join(output_path, "outputs", "latest_trained_model", "models", "weights.pth")) + assert os.path.exists( + os.path.join(output_path, "outputs", "latest_trained_model", "models", "label_schema.json") + ) + assert os.path.exists(os.path.join(output_path, "models")) + assert os.path.exists(os.path.join(output_path, "models", "weights.pth")) + + @e2e_pytest_component + def test_otx_train_wo_output_args(self, tmp_dir_path): + otx_dir = os.getcwd() + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_train_auto_config_wo_output" / case + train_auto_config_args[case]["train_params"] = train_params + otx_train_auto_config(root=tmp_dir_path, otx_dir=otx_dir, args=train_auto_config_args[case], use_output=False) + + # check output (without --output -> Default outputs) + output_path = os.path.join(tmp_dir_path, "otx-workspace", "outputs") + assert os.path.exists(output_path) + file_list = sorted(os.listdir(output_path)) + assert len(file_list) == 2 + assert os.path.exists(os.path.join(output_path, "latest_trained_model")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models", "weights.pth")) + assert os.path.exists(os.path.join(output_path, "latest_trained_model", "models", "label_schema.json")) + file_list.remove("latest_trained_model") + assert os.path.exists(os.path.join(output_path, file_list[-1])) + assert os.path.exists(os.path.join(output_path, file_list[-1], "models")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "models", "weights.pth")) + + @e2e_pytest_component + def test_otx_export_wo_output_args(self, tmp_dir_path): + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_train_auto_config_wo_output" / case + workspace_path = os.path.join(tmp_dir_path, "otx-workspace") + command_line = [ + "otx", + "export", + "--workspace", + os.path.join(tmp_dir_path, "otx-workspace"), + ] + check_run(command_line) + + # check output (without --output -> Default outputs) + output_path = os.path.join(workspace_path, "outputs") + assert os.path.exists(output_path) + file_list = sorted(os.listdir(output_path)) + assert len(file_list) == 3 + file_list.remove("latest_trained_model") + assert os.path.exists(os.path.join(output_path, file_list[-1])) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "openvino.xml")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "openvino.bin")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "openvino", "label_schema.json")) + + @e2e_pytest_component + def test_otx_optimize_wo_output_args(self, tmp_dir_path): + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_train_auto_config_wo_output" / case + workspace_path = os.path.join(tmp_dir_path, "otx-workspace") + command_line = [ + "otx", + "optimize", + "--workspace", + os.path.join(tmp_dir_path, "otx-workspace"), + ] + check_run(command_line) + + # check output (without --output -> Default outputs) + output_path = os.path.join(workspace_path, "outputs") + assert os.path.exists(output_path) + file_list = sorted(os.listdir(output_path)) + assert len(file_list) == 4 + file_list.remove("latest_trained_model") + assert os.path.exists(os.path.join(output_path, file_list[-1])) + assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "weights.pth")) + assert os.path.exists(os.path.join(output_path, file_list[-1], "nncf", "label_schema.json")) + + @e2e_pytest_component + def test_otx_train_wo_workspace_and_output_args(self, tmp_dir_path): + otx_dir = os.getcwd() + case = list(train_auto_config_args.keys())[0] + tmp_dir_path = tmp_dir_path / "test_otx_train_wo_workspace_and_output_args" + tmp_dir_path.mkdir(exist_ok=True) + expected_workspace_path = os.path.join(tmp_dir_path, f"otx-workspace-{case.upper()}") + command_line = [ + "otx", + "train", + ] + args = train_auto_config_args[case] + for option, value in args.items(): + if option in ["--train-data-roots", "--val-data-roots"]: + command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) + command_line.extend(train_params) + check_run(command_line, cwd=tmp_dir_path) + + # check output (without --output -> Default outputs) + assert os.path.exists(expected_workspace_path) + expected_output_path = os.path.join(expected_workspace_path, "outputs") + assert os.path.exists(expected_output_path) + file_list = sorted(os.listdir(expected_output_path)) + assert len(file_list) == 2 + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model")) + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models")) + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models", "weights.pth")) + assert os.path.exists(os.path.join(expected_output_path, "latest_trained_model", "models", "label_schema.json")) + file_list.remove("latest_trained_model") + assert os.path.exists(os.path.join(expected_output_path, file_list[-1])) + assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models")) + assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "weights.pth")) + assert os.path.exists(os.path.join(expected_output_path, file_list[-1], "models", "label_schema.json")) class TestTelemetryIntegration: diff --git a/tests/regression/classification/test_classification.py b/tests/regression/classification/test_classification.py index 1e0f73a1a94..5d2c00fde68 100644 --- a/tests/regression/classification/test_classification.py +++ b/tests/regression/classification/test_classification.py @@ -114,7 +114,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "multi_class_cls_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() @@ -242,7 +244,7 @@ def test_otx_train_selfsl(self, template, tmp_dir_path): args_selfsl["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] args_selfsl["--val-data-roots"] = multi_class_data_args["--val-data-roots"] args_selfsl["--test-data-roots"] = multi_class_data_args["--test-data-roots"] - args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, new_tmp_dir_path, otx_dir, args_selfsl) # Evaluation with self + supervised training model @@ -460,7 +462,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "multi_label_cls_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() diff --git a/tests/regression/detection/test_detection.py b/tests/regression/detection/test_detection.py index ff0cb17c0ed..ae7253c2ca2 100644 --- a/tests/regression/detection/test_detection.py +++ b/tests/regression/detection/test_detection.py @@ -114,7 +114,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "det_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() diff --git a/tests/regression/detection/test_instnace_segmentation.py b/tests/regression/detection/test_instnace_segmentation.py index fe732f735b7..3517accd47c 100644 --- a/tests/regression/detection/test_instnace_segmentation.py +++ b/tests/regression/detection/test_instnace_segmentation.py @@ -114,7 +114,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "inst_seg_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() diff --git a/tests/regression/segmentation/test_segmentation.py b/tests/regression/segmentation/test_segmentation.py index d800110d734..3a4104dc65c 100644 --- a/tests/regression/segmentation/test_segmentation.py +++ b/tests/regression/segmentation/test_segmentation.py @@ -115,7 +115,9 @@ def test_otx_train_cls_incr(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "seg_incr" config_cls_incr = load_regression_configuration(otx_dir, TASK_TYPE, "class_incr", self.label_type) args_cls_incr = config_cls_incr["data_path"] - args_cls_incr["--load-weights"] = f"{sl_template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_cls_incr[ + "--load-weights" + ] = f"{sl_template_work_dir}/trained_{template.model_template_id}/models/weights.pth" args_cls_incr["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] train_start_time = timer() @@ -234,7 +236,7 @@ def test_otx_train_selfsl(self, template, tmp_dir_path): args_selfsl["train_params"] = ["params", "--learning_parameters.num_iters", REGRESSION_TEST_EPOCHS] args_selfsl["--val-data-roots"] = segmentation_data_args["--val-data-roots"] args_selfsl["--test-data-roots"] = segmentation_data_args["--test-data-roots"] - args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + args_selfsl["--load-weights"] = f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth" otx_train_testing(template, new_tmp_dir_path, otx_dir, args_selfsl) # Evaluation with self + supervised training model diff --git a/tests/test_suite/pytest_insertions.py b/tests/test_suite/pytest_insertions.py index 94fe699e8d8..b88060c8763 100644 --- a/tests/test_suite/pytest_insertions.py +++ b/tests/test_suite/pytest_insertions.py @@ -83,7 +83,7 @@ def otx_pytest_addoption_insertion(parser): ) parser.addoption( - "--test-work-dir", + "--test-workspace", type=str, default=None, help="OTX test requires a certain amount of storage in the test work directory. " diff --git a/tests/test_suite/run_test_command.py b/tests/test_suite/run_test_command.py index 7a9d7b78c7c..a1d11e1cf49 100644 --- a/tests/test_suite/run_test_command.py +++ b/tests/test_suite/run_test_command.py @@ -130,8 +130,8 @@ def otx_train_testing(template, root, otx_dir, args): arg_value = args.get(arg, None) if arg_value: command_line.extend([arg, os.path.join(otx_dir, arg_value)]) - command_line.extend(["--save-model-to", f"{template_work_dir}/trained_{template.model_template_id}"]) - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--output", f"{template_work_dir}/trained_{template.model_template_id}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) if "--load-weights" in args: command_line.extend(["--load-weights", args["--load-weights"]]) if "--gpus" in args: @@ -141,8 +141,8 @@ def otx_train_testing(template, root, otx_dir, args): if "train_params" in args: command_line.extend(args["train_params"]) check_run(command_line) - assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/weights.pth") - assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/label_schema.json") + assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth") + assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/models/label_schema.json") def otx_resume_testing(template, root, otx_dir, args): @@ -164,12 +164,14 @@ def otx_resume_testing(template, root, otx_dir, args): if option in args: command_line.extend([option, f"{os.path.join(otx_dir, args[option])}"]) - command_line.extend(["--save-model-to", f"{template_work_dir}/trained_for_resume_{template.model_template_id}"]) - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--output", f"{template_work_dir}/trained_for_resume_{template.model_template_id}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) - assert os.path.exists(f"{template_work_dir}/trained_for_resume_{template.model_template_id}/weights.pth") - assert os.path.exists(f"{template_work_dir}/trained_for_resume_{template.model_template_id}/label_schema.json") + assert os.path.exists(f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/weights.pth") + assert os.path.exists( + f"{template_work_dir}/trained_for_resume_{template.model_template_id}/models/label_schema.json" + ) def otx_hpo_testing(template, root, otx_dir, args): @@ -183,14 +185,17 @@ def otx_hpo_testing(template, root, otx_dir, args): arg_value = args.get(arg, None) if arg_value: command_line.extend([arg, os.path.join(otx_dir, arg_value)]) - command_line.extend(["--save-model-to", f"{template_work_dir}/hpo_trained_{template.model_template_id}"]) - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--output", f"{template_work_dir}/hpo_trained_{template.model_template_id}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(["--enable-hpo", "--hpo-time-ratio", "1"]) command_line.extend(args["train_params"]) check_run(command_line) trials_json = list( - filter(lambda x: x.name.split(".")[0].isnumeric(), Path(f"{template_work_dir}/hpo/").rglob("*.json")) + filter( + lambda x: x.name.split(".")[0].isnumeric(), + Path(f"{template_work_dir}/hpo_trained_{template.model_template_id}/hpo/").rglob("*.json"), + ) ) assert trials_json for trial_json in trials_json: @@ -198,8 +203,8 @@ def otx_hpo_testing(template, root, otx_dir, args): trial_result = json.load(f) assert trial_result.get("score") - assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/weights.pth") - assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/label_schema.json") + assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/models/weights.pth") + assert os.path.exists(f"{template_work_dir}/hpo_trained_{template.model_template_id}/models/label_schema.json") def otx_export_testing(template, root, dump_features=False, half_precision=False): @@ -210,8 +215,8 @@ def otx_export_testing(template, root, dump_features=False, half_precision=False "export", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-model-to", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", save_path, ] @@ -251,11 +256,11 @@ def otx_eval_testing(template, root, otx_dir, args): "--test-data-roots", f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-performance", - f"{template_work_dir}/trained_{template.model_template_id}/performance.json", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", + f"{template_work_dir}/trained_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args.get("eval_params", [])) check_run(command_line) assert os.path.exists(f"{template_work_dir}/trained_{template.model_template_id}/performance.json") @@ -274,10 +279,12 @@ def otx_eval_openvino_testing( ): template_work_dir = get_template_dir(template, root) weights_path = f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml" + output_path = f"{template_work_dir}/exported_{template.model_template_id}" perf_path = f"{template_work_dir}/exported_{template.model_template_id}/performance.json" if half_precision: weights_path = f"{template_work_dir}/exported_{template.model_template_id}_fp16/openvino.xml" + output_path = f"{template_work_dir}/exported_{template.model_template_id}_fp16" perf_path = f"{template_work_dir}/exported_{template.model_template_id}_fp16/performance.json" command_line = [ @@ -288,10 +295,10 @@ def otx_eval_openvino_testing( f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", weights_path, - "--save-performance", - perf_path, + "--output", + output_path, ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(perf_path) with open(f"{template_work_dir}/trained_{template.model_template_id}/performance.json") as read_file: @@ -323,7 +330,7 @@ def otx_demo_testing(template, root, otx_dir, args): "demo", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", "--input", os.path.join(otx_dir, args["--input"]), "--delay", @@ -357,7 +364,7 @@ def otx_deploy_openvino_testing(template, root, otx_dir, args): template.model_template_path, "--load-weights", f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", - "--save-model-to", + "--output", deployment_dir, ] check_run(command_line) @@ -416,10 +423,10 @@ def otx_eval_deployment_testing( f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/deployed_{template.model_template_id}/openvino.zip", - "--save-performance", - f"{template_work_dir}/deployed_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/deployed_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/deployed_{template.model_template_id}/performance.json") with open(f"{template_work_dir}/exported_{template.model_template_id}/performance.json") as read_file: @@ -471,10 +478,10 @@ def pot_optimize_testing(template, root, otx_dir, args): f'{os.path.join(otx_dir, args["--val-data-roots"])}', "--load-weights", f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", - "--save-model-to", + "--output", f"{template_work_dir}/pot_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml") assert os.path.exists(f"{template_work_dir}/pot_{template.model_template_id}/openvino.bin") @@ -510,10 +517,10 @@ def pot_eval_testing(template, root, otx_dir, args, criteria=None, reg_threshold f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml", - "--save-performance", - f"{template_work_dir}/pot_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/pot_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/pot_{template.model_template_id}/performance.json") @@ -543,13 +550,11 @@ def nncf_optimize_testing(template, root, otx_dir, args): "--val-data-roots", f'{os.path.join(otx_dir, args["--val-data-roots"])}', "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-model-to", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", f"{template_work_dir}/nncf_{template.model_template_id}", - "--save-performance", - f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth") @@ -564,7 +569,7 @@ def nncf_export_testing(template, root): template.model_template_path, "--load-weights", f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth", - "--save-model-to", + "--output", f"{template_work_dir}/exported_nncf_{template.model_template_id}", ] check_run(command_line) @@ -600,13 +605,13 @@ def nncf_eval_testing( f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth", - "--save-performance", - f"{template_work_dir}/nncf_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/nncf_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/nncf_{template.model_template_id}/performance.json") - with open(f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json") as read_file: + with open(f"{template_work_dir}/nncf_{template.model_template_id}/nncf_performance.json") as read_file: trained_performance = json.load(read_file) with open(f"{template_work_dir}/nncf_{template.model_template_id}/performance.json") as read_file: evaluated_performance = json.load(read_file) @@ -637,10 +642,10 @@ def nncf_eval_openvino_testing(template, root, otx_dir, args): f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml", - "--save-performance", - f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json", + "--output", + f"{template_work_dir}/exported_nncf_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) check_run(command_line) assert os.path.exists(f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json") @@ -684,7 +689,7 @@ def otx_explain_testing(template, root, otx_dir, args): "explain", template.model_template_path, "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", "--explain-data-root", os.path.join(otx_dir, args["--input"]), "--save-explanation-to", @@ -766,7 +771,7 @@ def otx_build_task_testing(root, task): "build", "--task", task, - "--work-dir", + "--workspace", os.path.join(root, f"otx-workspace-{task}"), ] check_run(command_line) @@ -788,7 +793,7 @@ def otx_build_backbone_testing(root, backbone_args): "build", "--task", f"{task}", - "--work-dir", + "--workspace", task_workspace, ] check_run(command_line) @@ -800,7 +805,7 @@ def otx_build_backbone_testing(root, backbone_args): "build", "--backbone", backbone, - "--work-dir", + "--workspace", task_workspace, ] check_run(command_line) @@ -816,7 +821,7 @@ def otx_build_backbone_testing(root, backbone_args): def otx_build_testing(root, args: Dict[str, str], expected: Dict[str, str]): workspace_root = os.path.join(root, "otx-workspace") - command_line = ["otx", "build", "--work-dir", workspace_root] + command_line = ["otx", "build", "--workspace", workspace_root] for option, value in args.items(): command_line.extend([option, value]) check_run(command_line) @@ -832,7 +837,7 @@ def otx_build_testing(root, args: Dict[str, str], expected: Dict[str, str]): def otx_build_auto_config(root, otx_dir: str, args: Dict[str, str]): workspace_root = os.path.join(root, "otx-workspace") - command_line = ["otx", "build", "--work-dir", workspace_root] + command_line = ["otx", "build", "--workspace", workspace_root] for option, value in args.items(): if option in ["--train-data-roots", "--val-data-roots"]: @@ -842,7 +847,7 @@ def otx_build_auto_config(root, otx_dir: str, args: Dict[str, str]): check_run(command_line) -def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str]): +def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str], use_output: bool = True): work_dir = os.path.join(root, "otx-workspace") command_line = ["otx", "train"] @@ -851,8 +856,9 @@ def otx_train_auto_config(root, otx_dir: str, args: Dict[str, str]): command_line.extend([args[option]]) elif option in ["--train-data-roots", "--val-data-roots"]: command_line.extend([option, f"{os.path.join(otx_dir, value)}"]) - command_line.extend(["--save-model-to", f"{work_dir}"]) - command_line.extend(["--work-dir", f"{work_dir}"]) + if use_output: + command_line.extend(["--output", f"{work_dir}"]) + command_line.extend(["--workspace", f"{work_dir}"]) command_line.extend(args["train_params"]) check_run(command_line) @@ -875,11 +881,11 @@ def otx_eval_compare( "--test-data-roots", f'{os.path.join(otx_dir, args["--test-data-roots"])}', "--load-weights", - f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", - "--save-performance", - f"{template_work_dir}/trained_{template.model_template_id}/performance.json", + f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth", + "--output", + f"{template_work_dir}/trained_{template.model_template_id}", ] - command_line.extend(["--work-dir", f"{template_work_dir}"]) + command_line.extend(["--workspace", f"{template_work_dir}"]) command_line.extend(args.get("eval_params", [])) check_run(command_line) @@ -898,7 +904,7 @@ def otx_eval_compare( ), f"Current model performance: ({trained_performance[k]}) < criteria: ({modified_criteria})." result_dict["Model size (MB)"] = round( - os.path.getsize(f"{template_work_dir}/trained_{template.model_template_id}/weights.pth") / 1e6, 2 + os.path.getsize(f"{template_work_dir}/trained_{template.model_template_id}/models/weights.pth") / 1e6, 2 ) diff --git a/tests/unit/cli/tools/test_build.py b/tests/unit/cli/tools/test_build.py index 3ddb60689bc..7d43a77f54a 100644 --- a/tests/unit/cli/tools/test_build.py +++ b/tests/unit/cli/tools/test_build.py @@ -17,7 +17,7 @@ def test_get_args(mocker): "--unlabeled-file-list": "unlabeled/file/list", "--task": "detection", "--train-type": "Semisupervised", - "--work-dir": "work/dir/path", + "--workspace": "work/dir/path", "--model": "SSD", "--backbone": "torchvision.resnet18", } @@ -35,7 +35,7 @@ def test_get_args(mocker): assert parsed_args.test_data_roots == "test/data/root" assert parsed_args.unlabeled_data_roots == "unlabeled/data/root" assert parsed_args.unlabeled_file_list == "unlabeled/file/list" - assert parsed_args.work_dir == "work/dir/path" + assert parsed_args.workspace == "work/dir/path" assert parsed_args.task == "detection" assert parsed_args.train_type == "Semisupervised" assert parsed_args.model == "SSD" @@ -52,7 +52,7 @@ def mock_args(mocker, tmp_path): mock_args.unlabeled_file_list = None mock_args.task = "" mock_args.train_type = "incremental" - mock_args.work_dir = tmp_path / "work_dir" + mock_args.workspace = tmp_path / "work_dir" mock_args.model = "" mock_args.backbone = "torchvision.resnet18" @@ -92,7 +92,7 @@ def test_main(mocker, mock_config_manager, mock_args): assert result == {"retcode": 0, "task_type": ""} # Check ConfigManager constructor call - mock_config_manager.assert_called_once_with(mock_args, mode="build") + mock_config_manager.assert_called_once() # Check ConfigManager method calls mock_config_manager.return_value.configure_template.assert_called_once_with(model="") diff --git a/tests/unit/cli/tools/test_deploy.py b/tests/unit/cli/tools/test_deploy.py index 844838af8e2..323294fd043 100644 --- a/tests/unit/cli/tools/test_deploy.py +++ b/tests/unit/cli/tools/test_deploy.py @@ -10,7 +10,7 @@ @e2e_pytest_unit def test_get_args(mocker): - mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--save-model-to", "save_model_to"]) + mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--output", "output"]) mocker.patch.object( target_package, "get_parser_and_hprams_data", return_value=[argparse.ArgumentParser(), "fake", "fake"] ) @@ -18,14 +18,14 @@ def test_get_args(mocker): parsed_args = get_args() assert parsed_args.load_weights == "load_weights" - assert parsed_args.save_model_to == "save_model_to" + assert parsed_args.output == "output" @pytest.fixture def mock_args(mocker, tmp_dir): mock_args = mocker.MagicMock() mock_args.load_weights = "fake.bin" - mock_args.save_model_to = tmp_dir + mock_args.output = tmp_dir def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_eval.py b/tests/unit/cli/tools/test_eval.py index 76eee501f89..ecbd4297ec2 100644 --- a/tests/unit/cli/tools/test_eval.py +++ b/tests/unit/cli/tools/test_eval.py @@ -12,8 +12,8 @@ def test_get_args(mocker): mock_options = { "--test-data-roots": "test/data/root", "--load-weights": "weight/path", - "--save-performance": "save/path", - "--work-dir": "work/dir/path", + "--output": "save/path", + "--workspace": "work/dir/path", } mock_command = ["otx"] for key, value in mock_options.items(): @@ -29,8 +29,7 @@ def test_get_args(mocker): assert parsed_args.test_data_roots == "test/data/root" assert parsed_args.load_weights == "weight/path" - assert parsed_args.save_performance == "save/path" - assert parsed_args.work_dir == "work/dir/path" + assert parsed_args.workspace == "work/dir/path" @pytest.fixture @@ -38,8 +37,7 @@ def mock_args(mocker, tmp_path): mock_args = mocker.MagicMock() mock_args.test_data_roots = "fake_test_data_root" mock_args.load_weights = "fake_load_weights.xml" - mock_args.save_performance = tmp_path / "save/performance.json" - mock_args.work_dir = tmp_path / "work_dir" + mock_args.workspace = tmp_path / "work_dir" def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_export.py b/tests/unit/cli/tools/test_export.py index 06aa79b2613..f32d9d872b1 100644 --- a/tests/unit/cli/tools/test_export.py +++ b/tests/unit/cli/tools/test_export.py @@ -10,7 +10,7 @@ @e2e_pytest_unit def test_get_args(mocker): - mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--save-model-to", "save_model_to"]) + mocker.patch("sys.argv", ["otx", "--load-weights", "load_weights", "--output", "output"]) mocker.patch.object( target_package, "get_parser_and_hprams_data", return_value=[argparse.ArgumentParser(), "fake", "fake"] ) @@ -18,14 +18,14 @@ def test_get_args(mocker): parsed_args = get_args() assert parsed_args.load_weights == "load_weights" - assert parsed_args.save_model_to == "save_model_to" + assert parsed_args.output == "output" @pytest.fixture def mock_args(mocker, tmp_dir): mock_args = mocker.MagicMock() mock_args.load_weights = "fake.bin" - mock_args.save_model_to = tmp_dir + mock_args.output = tmp_dir def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_optimize.py b/tests/unit/cli/tools/test_optimize.py index fe60e6c0d0f..93ad23bb179 100644 --- a/tests/unit/cli/tools/test_optimize.py +++ b/tests/unit/cli/tools/test_optimize.py @@ -13,9 +13,8 @@ def test_get_args(mocker): "--train-data-roots": "train_data_roots_path", "--val-data-roots": "val_data_roots_path", "--load-weights": "load_weights_path", - "--save-model-to": "save_model_path", - "--save-performance": "save_performance_path", - "--work-dir": "work_dir_path", + "--output": "output", + "--workspace": "work_dir_path", } mock_command = ["otx"] for key, value in mock_options.items(): @@ -32,9 +31,8 @@ def test_get_args(mocker): assert parsed_args.train_data_roots == "train_data_roots_path" assert parsed_args.val_data_roots == "val_data_roots_path" assert parsed_args.load_weights == "load_weights_path" - assert parsed_args.save_model_to == "save_model_path" - assert parsed_args.save_performance == "save_performance_path" - assert parsed_args.work_dir == "work_dir_path" + assert parsed_args.output == "output" + assert parsed_args.workspace == "work_dir_path" @pytest.fixture @@ -43,9 +41,8 @@ def mock_args(mocker, tmp_path): mock_args.train_data_roots = "fake_train_data_roots_path" mock_args.val_data_roots = "fake_val_data_roots_path" mock_args.load_weights = "fake_load_weights_path" - mock_args.save_model_to = tmp_path / "save/model" - mock_args.save_performance = tmp_path / "save/performance.json" - mock_args.work_dir = tmp_path / "work_dir_path" + mock_args.output = tmp_path / "save/model" + mock_args.workspace = tmp_path / "work_dir_path" def mock_contains(self, val): return val in self.__dict__ diff --git a/tests/unit/cli/tools/test_train.py b/tests/unit/cli/tools/test_train.py index 1db77cb3d2f..81b84844c91 100644 --- a/tests/unit/cli/tools/test_train.py +++ b/tests/unit/cli/tools/test_train.py @@ -16,8 +16,8 @@ def test_get_args(mocker): "--unlabeled-file-list": "unlabeled/file/list", "--load-weights": "weight/path", "--resume-from": "resume/path", - "--save-model-to": "save/path", - "--work-dir": "work/dir/path", + "--output": "save/path", + "--workspace": "work/dir/path", "--hpo-time-ratio": "2", "--gpus": "0,1", "--rdzv-endpoint": "localhost:1", @@ -43,8 +43,8 @@ def test_get_args(mocker): assert parsed_args.unlabeled_file_list == "unlabeled/file/list" assert parsed_args.load_weights == "weight/path" assert parsed_args.resume_from == "resume/path" - assert parsed_args.save_model_to == "save/path" - assert parsed_args.work_dir == "work/dir/path" + assert parsed_args.output == "save/path" + assert parsed_args.workspace == "work/dir/path" assert parsed_args.hpo_time_ratio == 2.0 assert parsed_args.gpus == "0,1" assert parsed_args.rdzv_endpoint == "localhost:1" @@ -60,8 +60,8 @@ def mock_args(mocker, tmp_path): mock_args.val_data_roots = "fake_val_data_root" mock_args.load_weights = "fake_load_weights" mock_args.resume_from = None - mock_args.save_model_to = tmp_path / "models" - mock_args.work_dir = tmp_path / "work_dir" + mock_args.output = tmp_path / "models" + mock_args.workspace = tmp_path / "work_dir" mock_args.enable_hpo = False mock_args.hpo_time_ratio = 4 mock_args.gpus = None @@ -120,6 +120,7 @@ def mock_task(mocker): def test_main(mocker, mock_args, mock_config_manager, mock_dataset_adapter, mock_task): mocker.patch.object(target_package, "read_label_schema") mocker.patch.object(target_package, "read_binary") + mocker.patch("otx.cli.tools.train.Path.symlink_to") mocker.patch.object( target_package, "run_hpo", diff --git a/tests/unit/cli/utils/test_hpo.py b/tests/unit/cli/utils/test_hpo.py index a71d48a0cbb..f810c0acd45 100644 --- a/tests/unit/cli/utils/test_hpo.py +++ b/tests/unit/cli/utils/test_hpo.py @@ -613,8 +613,7 @@ def test_getitem_before_get_subset(self): def test_run_hpo(mocker, mock_environment): with TemporaryDirectory() as tmp_dir: # prepare - save_model_to_path = Path(tmp_dir) / "fake" - + output = Path(tmp_dir) / "fake" mock_get_best_hpo_weight = mocker.patch("otx.cli.utils.hpo.get_best_hpo_weight") mock_get_best_hpo_weight.return_value = "mock_best_weight_path" @@ -631,14 +630,11 @@ def mock_read_model(args1, path, arg2): mocker.patch("otx.cli.utils.hpo.read_model", mock_read_model) - mock_args = mocker.MagicMock() - mock_args.hpo_time_ratio = "4" - mock_args.save_model_to = save_model_to_path - + hpo_time_ratio = "4" mock_environment.model_template.task_type = TaskType.CLASSIFICATION # run - environment = run_hpo(mock_args, mock_environment, mocker.MagicMock(), mocker.MagicMock()) + environment = run_hpo(hpo_time_ratio, output, mock_environment, mocker.MagicMock(), mocker.MagicMock()) # check mock_hpo_runner_instance.run_hpo.assert_called() # Check that HpoRunner.run_hpo is called @@ -653,8 +649,10 @@ def test_run_hpo_not_supported_task(mocker, action_task_env): mock_hpo_runner_instance = mocker.MagicMock() mock_hpo_runner_class = mocker.patch("otx.cli.utils.hpo.HpoRunner") mock_hpo_runner_class.return_value = mock_hpo_runner_instance + hpo_time_ratio = "4" + output = "fake" - run_hpo(mocker.MagicMock(), action_task_env, mocker.MagicMock(), mocker.MagicMock()) + run_hpo(hpo_time_ratio, output, action_task_env, mocker.MagicMock(), mocker.MagicMock()) mock_hpo_runner_instance.run_hpo.assert_not_called() diff --git a/tests/unit/cli/utils/test_multi_gpu.py b/tests/unit/cli/utils/test_multi_gpu.py index a584860a6d9..d40d5c68872 100644 --- a/tests/unit/cli/utils/test_multi_gpu.py +++ b/tests/unit/cli/utils/test_multi_gpu.py @@ -67,6 +67,15 @@ def test_set_arguments_to_argv_key_exist(mock_argv_without_params): assert mock_argv_without_params[1] == other_val +@e2e_pytest_unit +def test_set_arguments_to_argv_keys_exist(mock_argv_without_params): + """Test a case where key already exists and value exists.""" + other_val = "other_val" + set_arguments_to_argv(["--a_key", "-a"], other_val) + + assert mock_argv_without_params[1] == other_val + + @e2e_pytest_unit def test_set_arguments_to_argv_key_exist_none_val(mock_argv_without_params): """Test a case where key already exists in argv and value doesn't exists.""" @@ -375,8 +384,8 @@ def test_run_child_process(self, mocker): # check assert mock_set_start_method.call_args.kwargs["method"] is None assert "--gpus" not in mock_sys.argv - assert "--work-dir" in mock_sys.argv - assert mock_sys.argv[mock_sys.argv.index("--work-dir") + 1] == output_path + assert "--output" in mock_sys.argv + assert mock_sys.argv[mock_sys.argv.index("--output") + 1] == output_path assert "--rdzv-endpoint" in mock_sys.argv assert mock_sys.argv[mock_sys.argv.index("--rdzv-endpoint") + 1] == rdzv_endpoint mock_initialize_multigpu_train.assert_called_once()