From 0f4faac0f50949eee2289f651dce160bdad49c4b Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Tue, 20 Dec 2022 11:03:57 +0900 Subject: [PATCH 1/7] Fix CLI test run for better error message Signed-off-by: Songki Choi --- ote_cli/ote_cli/utils/tests.py | 122 +++++++++++++++------------------ 1 file changed, 56 insertions(+), 66 deletions(-) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index cacd4c5cea2..8dd0be29065 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -73,6 +73,11 @@ def collect_env_vars(work_dir): return vars +def check_run(cmd, **kwargs): + result = run(cmd, capture_output=True, **kwargs) + assert result.returncode == 0, result.stderr.decode("utf=8") + + def ote_train_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) command_line = [ @@ -95,7 +100,7 @@ def ote_train_testing(template, root, ote_dir, args): ["--load-weights", f'{os.path.join(ote_dir, args["--load-weights"])}'] ) command_line.extend(args["train_params"]) - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" ) @@ -127,7 +132,7 @@ def ote_hpo_testing(template, root, ote_dir, args): "1", ] command_line.extend(args["train_params"]) - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists(f"{template_work_dir}/hpo/hpopt_status.json") with open(f"{template_work_dir}/hpo/hpopt_status.json", "r") as f: assert json.load(f).get("best_config_id", None) is not None @@ -150,7 +155,7 @@ def ote_export_testing(template, root): "--save-model-to", f"{template_work_dir}/exported_{template.model_template_id}", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml" ) @@ -177,7 +182,7 @@ def ote_eval_testing(template, root, ote_dir, args): "--save-performance", f"{template_work_dir}/trained_{template.model_template_id}/performance.json", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/trained_{template.model_template_id}/performance.json" ) @@ -198,7 +203,7 @@ def ote_eval_openvino_testing(template, root, ote_dir, args, threshold): "--save-performance", f"{template_work_dir}/exported_{template.model_template_id}/performance.json", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/exported_{template.model_template_id}/performance.json" ) @@ -233,7 +238,7 @@ def ote_demo_testing(template, root, ote_dir, args): "--delay", "-1", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) def ote_demo_openvino_testing(template, root, ote_dir, args): @@ -249,7 +254,7 @@ def ote_demo_openvino_testing(template, root, ote_dir, args): "--delay", "-1", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) def ote_deploy_openvino_testing(template, root, ote_dir, args): @@ -264,63 +269,48 @@ def ote_deploy_openvino_testing(template, root, ote_dir, args): "--save-model-to", deployment_dir, ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert run(["unzip", "-o", "openvino.zip"], cwd=deployment_dir).returncode == 0 - assert ( - run( - ["python3", "-m", "venv", "venv"], - cwd=os.path.join(deployment_dir, "python"), - ).returncode - == 0 + check_run(command_line, env=collect_env_vars(work_dir)) + check_run(["unzip", "-o", "openvino.zip"], cwd=deployment_dir) + check_run( + ["python3", "-m", "venv", "venv"], + cwd=os.path.join(deployment_dir, "python"), ) - assert ( - run( - ["python3", "-m", "pip", "install", "wheel"], - cwd=os.path.join(deployment_dir, "python"), - env=collect_env_vars(os.path.join(deployment_dir, "python")), - ).returncode - == 0 + check_run( + ["python3", "-m", "pip", "install", "wheel"], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), ) - assert ( - run( - ["python3", "-m", "pip", "install", "pip", "--upgrade"], - cwd=os.path.join(deployment_dir, "python"), - env=collect_env_vars(os.path.join(deployment_dir, "python")), - ).returncode - == 0 + check_run( + ["python3", "-m", "pip", "install", "pip", "--upgrade"], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), ) - assert ( - run( - [ - "python3", - "-m", - "pip", - "install", - "-r", - os.path.join(deployment_dir, "python", "requirements.txt"), - ], - cwd=os.path.join(deployment_dir, "python"), - env=collect_env_vars(os.path.join(deployment_dir, "python")), - ).returncode - == 0 + check_run( + [ + "python3", + "-m", + "pip", + "install", + "-r", + os.path.join(deployment_dir, "python", "requirements.txt"), + ], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), ) - assert ( - run( - [ - "python3", - "demo.py", - "-m", - "../model", - "-i", - os.path.join(ote_dir, args["--input"]), - "--no_show", - ], - cwd=os.path.join(deployment_dir, "python"), - env=collect_env_vars(os.path.join(deployment_dir, "python")), - ).returncode - == 0 + check_run( + [ + "python3", + "demo.py", + "-m", + "../model", + "-i", + os.path.join(ote_dir, args["--input"]), + "--no_show", + ], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), ) @@ -339,7 +329,7 @@ def ote_eval_deployment_testing(template, root, ote_dir, args, threshold): "--save-performance", f"{template_work_dir}/deployed_{template.model_template_id}/performance.json", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/deployed_{template.model_template_id}/performance.json" ) @@ -373,7 +363,7 @@ def ote_demo_deployment_testing(template, root, ote_dir, args): "--delay", "-1", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) def pot_optimize_testing(template, root, ote_dir, args): @@ -395,7 +385,7 @@ def pot_optimize_testing(template, root, ote_dir, args): "--save-model-to", f"{template_work_dir}/pot_{template.model_template_id}", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml" ) @@ -422,7 +412,7 @@ def pot_eval_testing(template, root, ote_dir, args): "--save-performance", f"{template_work_dir}/pot_{template.model_template_id}/performance.json", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/pot_{template.model_template_id}/performance.json" ) @@ -450,7 +440,7 @@ def nncf_optimize_testing(template, root, ote_dir, args): f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json", ] command_line.extend(args["train_params"]) - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth" ) @@ -470,7 +460,7 @@ def nncf_export_testing(template, root): "--save-model-to", f"{template_work_dir}/exported_nncf_{template.model_template_id}", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml" ) @@ -506,7 +496,7 @@ def nncf_eval_testing(template, root, ote_dir, args, threshold): "--save-performance", f"{template_work_dir}/nncf_{template.model_template_id}/performance.json", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/nncf_{template.model_template_id}/performance.json" ) @@ -543,7 +533,7 @@ def nncf_eval_openvino_testing(template, root, ote_dir, args): "--save-performance", f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json", ] - assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 + check_run(command_line, env=collect_env_vars(work_dir)) assert os.path.exists( f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json" ) From 7de151456f98c66038b2d92260a7c7acfa903d06 Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Tue, 20 Dec 2022 15:12:13 +0900 Subject: [PATCH 2/7] Numpy constraint for deprecated np.bool error Signed-off-by: Songki Choi --- external/anomaly/requirements.txt | 1 + external/deep-object-reid/requirements.txt | 1 + external/mmsegmentation/requirements.txt | 1 + ote_cli/requirements.txt | 2 +- ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt | 3 ++- ote_sdk/requirements.txt | 2 +- 6 files changed, 7 insertions(+), 3 deletions(-) diff --git a/external/anomaly/requirements.txt b/external/anomaly/requirements.txt index 8d52d94cade..4462d1005e5 100644 --- a/external/anomaly/requirements.txt +++ b/external/anomaly/requirements.txt @@ -5,3 +5,4 @@ openmodelzoo-modelapi @ git+https://github.com/openvinotoolkit/open_model_zoo/@r openvino==2022.1.0 openvino-dev==2022.1.0 pytorch-lightning>=1.6.0,<1.7.0 +numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime diff --git a/external/deep-object-reid/requirements.txt b/external/deep-object-reid/requirements.txt index e43c59b2357..1f76e1199c7 100644 --- a/external/deep-object-reid/requirements.txt +++ b/external/deep-object-reid/requirements.txt @@ -3,3 +3,4 @@ openvino-dev==2022.1.0 openmodelzoo-modelapi @ git+https://github.com/openvinotoolkit/open_model_zoo/@releases/2022/SCv1.1#egg=openmodelzoo-modelapi&subdirectory=demos/common/python nncf@ git+https://github.com/openvinotoolkit/nncf@e85a695da95b202b4579ea11f880f1b14bfcda2e#egg=nncf networkx<=2.8.0 +numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime diff --git a/external/mmsegmentation/requirements.txt b/external/mmsegmentation/requirements.txt index aad70cde18f..309e7d3ff6b 100644 --- a/external/mmsegmentation/requirements.txt +++ b/external/mmsegmentation/requirements.txt @@ -4,3 +4,4 @@ openmodelzoo-modelapi @ git+https://github.com/openvinotoolkit/open_model_zoo/@r nncf@ git+https://github.com/openvinotoolkit/nncf@e85a695da95b202b4579ea11f880f1b14bfcda2e#egg=nncf networkx<=2.8.0 protobuf<3.21.0 +numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime diff --git a/ote_cli/requirements.txt b/ote_cli/requirements.txt index ba2ec969b67..35145b49a17 100644 --- a/ote_cli/requirements.txt +++ b/ote_cli/requirements.txt @@ -1,6 +1,6 @@ pyyaml opencv-python -numpy +numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime nbmake pytest pytest-ordering diff --git a/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt b/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt index 9be2994cd4e..a08537432da 100644 --- a/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt +++ b/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt @@ -1,3 +1,4 @@ openvino==2022.1.0 openmodelzoo-modelapi @ git+https://github.com/openvinotoolkit/open_model_zoo/@releases/2022/SCv1.1#egg=openmodelzoo-modelapi&subdirectory=demos/common/python -ote-sdk @ git+https://github.com/openvinotoolkit/training_extensions/@c29f1b385d5eabcf07bc9a6e9c340fd0dbf67c8d#egg=ote-sdk&subdirectory=ote_sdk \ No newline at end of file +ote-sdk @ git+https://github.com/openvinotoolkit/training_extensions/@c29f1b385d5eabcf07bc9a6e9c340fd0dbf67c8d#egg=ote-sdk&subdirectory=ote_sdk +numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime diff --git a/ote_sdk/requirements.txt b/ote_sdk/requirements.txt index 35c569cbdfe..c43cc3b3a97 100644 --- a/ote_sdk/requirements.txt +++ b/ote_sdk/requirements.txt @@ -1,4 +1,4 @@ -numpy>=1.16.4 +numpy>=1.16.4,<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime scikit-learn==0.24.* Shapely>=1.7.1,<=1.8.0 networkx>=2.6,<2.8.1rc1 From a6afa044dbdd17af24ed26a6b5c930cf00947823 Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Tue, 20 Dec 2022 20:29:35 +0900 Subject: [PATCH 3/7] Update OTE_SDK commit Signed-off-by: Songki Choi --- ote_cli/ote_cli/utils/tests.py | 5 +++-- .../ote_sdk/usecases/exportable_code/demo/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index 8dd0be29065..98e6b5555e6 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -74,8 +74,9 @@ def collect_env_vars(work_dir): def check_run(cmd, **kwargs): - result = run(cmd, capture_output=True, **kwargs) - assert result.returncode == 0, result.stderr.decode("utf=8") + #result = run(cmd, capture_output=True, **kwargs) + #assert result.returncode == 0, result.stderr.decode("utf=8") + run(cmd, check=True, **kwargs) def ote_train_testing(template, root, ote_dir, args): diff --git a/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt b/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt index a08537432da..0c9197b3d60 100644 --- a/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt +++ b/ote_sdk/ote_sdk/usecases/exportable_code/demo/requirements.txt @@ -1,4 +1,4 @@ openvino==2022.1.0 openmodelzoo-modelapi @ git+https://github.com/openvinotoolkit/open_model_zoo/@releases/2022/SCv1.1#egg=openmodelzoo-modelapi&subdirectory=demos/common/python -ote-sdk @ git+https://github.com/openvinotoolkit/training_extensions/@c29f1b385d5eabcf07bc9a6e9c340fd0dbf67c8d#egg=ote-sdk&subdirectory=ote_sdk +ote-sdk @ git+https://github.com/openvinotoolkit/training_extensions/@565fbf682cc8a182ce67a69688f0e7b48a991696#egg=ote-sdk&subdirectory=ote_sdk numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime From cabdec3737bf5d2d66577868b2299ca418fa3006 Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Wed, 21 Dec 2022 09:52:55 +0900 Subject: [PATCH 4/7] Capture stderr only Signed-off-by: Songki Choi --- CHANGELOG.md | 1 + ote_cli/ote_cli/utils/tests.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec9169c4d25..f125a427e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ All notable changes to this project will be documented in this file. ### Fixed - Neural Network Compression Framework (NNCF) + - Fix CUDA OOM for NNCF optimization model MaskRCNN-EfficientNetB2B () - Model Preparation Algorithm (MPA) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index 98e6b5555e6..e4fb15fcca7 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -15,7 +15,7 @@ import json import os import shutil -from subprocess import run # nosec +import subprocess # nosec import pytest @@ -74,9 +74,8 @@ def collect_env_vars(work_dir): def check_run(cmd, **kwargs): - #result = run(cmd, capture_output=True, **kwargs) - #assert result.returncode == 0, result.stderr.decode("utf=8") - run(cmd, check=True, **kwargs) + result = subprocess.run(cmd, stderr=subprocess.PIPE, **kwargs) + assert result.returncode == 0, result.stderr.decode("utf=8") def ote_train_testing(template, root, ote_dir, args): From 677e50284d1c5311826422ba610688e3845cb73e Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Wed, 21 Dec 2022 14:19:50 +0900 Subject: [PATCH 5/7] Fix missing part Signed-off-by: Songki Choi --- ote_cli/ote_cli/utils/tests.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index e4fb15fcca7..ecb92abdc1a 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -36,13 +36,8 @@ def get_some_vars(template, root): def create_venv(algo_backend_dir, work_dir): venv_dir = f"{work_dir}/venv" if not os.path.exists(venv_dir): - assert run([f"./{algo_backend_dir}/init_venv.sh", venv_dir]).returncode == 0 - assert ( - run( - [f"{work_dir}/venv/bin/python", "-m", "pip", "install", "-e", "ote_cli"] - ).returncode - == 0 - ) + check_run([f"./{algo_backend_dir}/init_venv.sh", venv_dir]) + check_run([f"{work_dir}/venv/bin/python", "-m", "pip", "install", "-e", "ote_cli"]) def extract_export_vars(path): From ee96380dfd2ee1ff360cadee7660511aff523ecf Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Fri, 23 Dec 2022 10:46:48 +0900 Subject: [PATCH 6/7] Fix precommit issue Signed-off-by: Songki Choi --- ote_cli/ote_cli/utils/tests.py | 4 +++- tests/ote_cli/misc/test_code_checks.py | 4 ++-- tests/run_model_templates_tests.py | 9 +++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index ecb92abdc1a..6097a016721 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -37,7 +37,9 @@ def create_venv(algo_backend_dir, work_dir): venv_dir = f"{work_dir}/venv" if not os.path.exists(venv_dir): check_run([f"./{algo_backend_dir}/init_venv.sh", venv_dir]) - check_run([f"{work_dir}/venv/bin/python", "-m", "pip", "install", "-e", "ote_cli"]) + check_run( + [f"{work_dir}/venv/bin/python", "-m", "pip", "install", "-e", "ote_cli"] + ) def extract_export_vars(path): diff --git a/tests/ote_cli/misc/test_code_checks.py b/tests/ote_cli/misc/test_code_checks.py index a42eb7ed770..d4d1b13176e 100644 --- a/tests/ote_cli/misc/test_code_checks.py +++ b/tests/ote_cli/misc/test_code_checks.py @@ -13,12 +13,12 @@ # and limitations under the License. import os -from subprocess import run from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component +from ote_cli.utils.tests import check_run class TestCodeChecks: @e2e_pytest_component def test_code_checks(self): wd = os.path.join(os.path.dirname(__file__), "..", "..", "..") - assert run(["./tests/run_code_checks.sh"], cwd=wd, check=True).returncode == 0 + check_run(["./tests/run_code_checks.sh"], cwd=wd) diff --git a/tests/run_model_templates_tests.py b/tests/run_model_templates_tests.py index 0f070314d02..bd72968579e 100644 --- a/tests/run_model_templates_tests.py +++ b/tests/run_model_templates_tests.py @@ -2,9 +2,8 @@ import os import sys -from subprocess import run -from ote_cli.utils.tests import collect_env_vars +from ote_cli.utils.tests import collect_env_vars, check_run ALGO_ROOT_DIR = "external" ALGO_DIRS = [ @@ -59,7 +58,8 @@ def test(run_algo_tests): success = True command = ["pytest", os.path.join("tests", "ote_cli", "misc"), "-v"] try: - res = run(command, env=collect_env_vars(wd), check=True).returncode == 0 + check_run(command, env=collect_env_vars(wd)) + res = True except: res = False passed["misc"] = res @@ -68,7 +68,8 @@ def test(run_algo_tests): if run_algo_tests[algo_dir]: command = ["pytest", os.path.join(algo_dir, "tests", "ote_cli"), "-v", "-rxXs", "--durations=10"] try: - res = run(command, env=collect_env_vars(wd), check=True).returncode == 0 + check_run(command, env=collect_env_vars(wd)) + res = True except: res = False passed[algo_dir] = res From 99165d7ca0ec5d70bff725009d7b2d0ff3c337ce Mon Sep 17 00:00:00 2001 From: Songki Choi Date: Fri, 23 Dec 2022 11:22:48 +0900 Subject: [PATCH 7/7] Added numpy constraint to MPA Signed-off-by: Songki Choi --- external/model-preparation-algorithm/constraints.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/external/model-preparation-algorithm/constraints.txt b/external/model-preparation-algorithm/constraints.txt index 3bc76e52a30..966eca912d3 100644 --- a/external/model-preparation-algorithm/constraints.txt +++ b/external/model-preparation-algorithm/constraints.txt @@ -1 +1,2 @@ optuna==2.10.1 # remedy for fixed optuna version incompatible in OTE CI +numpy<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime