From a1926225dc325e8c68dbcf40b79967d5904bbdba Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 12:58:04 +0300 Subject: [PATCH 01/16] moved common test utils --- tests/ote_cli/common.py => ote_cli/ote_cli/utils/tests.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/ote_cli/common.py => ote_cli/ote_cli/utils/tests.py (100%) diff --git a/tests/ote_cli/common.py b/ote_cli/ote_cli/utils/tests.py similarity index 100% rename from tests/ote_cli/common.py rename to ote_cli/ote_cli/utils/tests.py From 5a2b1c01f047c7e831bfca3ee1b69016c28419d1 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 13:16:45 +0300 Subject: [PATCH 02/16] updated tests --- QUICK_START_GUIDE.md | 14 +- ote_cli/ote_cli/utils/tests.py | 695 +++++++++++------- ...st_ote_cli_tools_anomaly_classification.py | 6 +- ...test_ote_cli_tools_anomaly_segmentation.py | 6 +- .../test_ote_cli_tools_classification.py | 6 +- .../test_ote_cli_tools_detection.py | 6 +- ...est_ote_cli_tools_instance_segmentation.py | 6 +- .../test_ote_cli_tools_rotated_detection.py | 6 +- .../test_ote_cli_tools_segmentation.py | 6 +- tests/run_model_templates_tests.py | 2 +- tests/run_model_templates_tests.sh | 4 +- 11 files changed, 457 insertions(+), 300 deletions(-) diff --git a/QUICK_START_GUIDE.md b/QUICK_START_GUIDE.md index 56c2aab5c03..9bf7ddbdac6 100644 --- a/QUICK_START_GUIDE.md +++ b/QUICK_START_GUIDE.md @@ -14,17 +14,13 @@ git checkout -b develop origin/develop git submodule update --init --recursive ``` -2. Export `OTE_SDK_PATH` environment variable to use it inside our scripts: - ``` - export OTE_SDK_PATH=`pwd`/ote_sdk - ``` -3. Install prerequisites by running the following: +2. Install prerequisites by running the following: ``` sudo apt-get install python3-pip python3-venv ``` -4. Search for available scripts that create python virtual environments for different task types: +3. Search for available scripts that create python virtual environments for different task types: ```bash find external/ -name init_venv.sh ``` @@ -38,7 +34,7 @@ Each line in the output gives an `init_venv.sh` script that creates a virtual environment for the corresponding task type. -5. Let's choose a task type. +4. Let's choose a task type. Let it be `external/mmdetection` for Object Detection task. ```bash TASK_ALGO_DIR=./external/mmdetection/ @@ -46,7 +42,7 @@ Note that we will not use the variable `TASK_ALGO_DIR` inside our scripts, we set it just to simplify this guide. -6. Let's create, activate virtual environment for the chosen task, and install `ote_cli`. +5. Let's create, activate virtual environment for the chosen task, and install `ote_cli`. Note that the virtual environment folder may be created in any place in your system, but we will create it in the folder `./cur_task_venv` for convenience. ```bash @@ -62,7 +58,7 @@ from the chosen task folder is used to avoid breaking constraints for the OTE task. -7. As soon as `ote_cli` is installed in the virtual environment, you can use +6. As soon as `ote_cli` is installed in the virtual environment, you can use `ote` command line interface described below to run train/eval/export/other action for templates related to the chosen task type. diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index 3c29dd5a32b..f217f69a570 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -24,18 +24,23 @@ def get_template_rel_dir(template): def get_some_vars(template, root): template_dir = get_template_rel_dir(template) - algo_backend_dir = '/'.join(template_dir.split('/')[:2]) + algo_backend_dir = "/".join(template_dir.split("/")[:2]) work_dir = os.path.join(root, os.path.basename(algo_backend_dir)) template_work_dir = os.path.join(work_dir, template_dir) os.makedirs(template_work_dir, exist_ok=True) return work_dir, template_work_dir, algo_backend_dir -def create_venv(algo_backend_dir, work_dir, template_work_dir): - venv_dir = f'{work_dir}/venv' +def create_venv(algo_backend_dir, work_dir): + venv_dir = f"{work_dir}/venv" if not os.path.exists(venv_dir): - assert run([f'./{algo_backend_dir}/init_venv.sh', venv_dir]).returncode == 0 - assert run([f'{work_dir}/venv/bin/python', '-m', 'pip', 'install', '-e', 'ote_cli']).returncode == 0 + assert run([f"./{algo_backend_dir}/init_venv.sh", venv_dir]).returncode == 0 + assert ( + run( + [f"{work_dir}/venv/bin/python", "-m", "pip", "install", "-e", "ote_cli"] + ).returncode + == 0 + ) def extract_export_vars(path): @@ -43,24 +48,24 @@ def extract_export_vars(path): with open(path) as f: for line in f: line = line.strip() - if line.startswith('export ') and '=' in line: - line = line.replace('export ', '').split('=') + if line.startswith("export ") and "=" in line: + line = line.replace("export ", "").split("=") assert len(line) == 2 vars[line[0].strip()] = line[1].strip() return vars def collect_env_vars(work_dir): - vars = extract_export_vars(f'{work_dir}/venv/bin/activate') - vars.update({'PATH':f'{work_dir}/venv/bin/:' + os.environ['PATH']}) - if 'HTTP_PROXY' in os.environ: - vars.update({'HTTP_PROXY': os.environ['HTTP_PROXY']}) - if 'HTTPS_PROXY' in os.environ: - vars.update({'HTTPS_PROXY': os.environ['HTTPS_PROXY']}) - if 'NO_PROXY' in os.environ: - vars.update({'NO_PROXY': os.environ['NO_PROXY']}) - if 'OTE_SDK_PATH' in os.environ: - vars.update({'OTE_SDK_PATH': os.environ['OTE_SDK_PATH']}) + vars = extract_export_vars(f"{work_dir}/venv/bin/activate") + vars.update({"PATH": f"{work_dir}/venv/bin/:" + os.environ["PATH"]}) + if "HTTP_PROXY" in os.environ: + vars.update({"HTTP_PROXY": os.environ["HTTP_PROXY"]}) + if "HTTPS_PROXY" in os.environ: + vars.update({"HTTPS_PROXY": os.environ["HTTPS_PROXY"]}) + if "NO_PROXY" in os.environ: + vars.update({"NO_PROXY": os.environ["NO_PROXY"]}) + if "OTE_SDK_PATH" in os.environ: + vars.update({"OTE_SDK_PATH": os.environ["OTE_SDK_PATH"]}) return vars @@ -69,123 +74,155 @@ def patch_demo_py(src_path, dst_path): content = [line for line in read_file] replaced = False for i, line in enumerate(content): - if 'visualizer = Visualizer(media_type)' in line: - content[i] = line.rstrip() + '; visualizer.show = show\n' + if "visualizer = Visualizer(media_type)" in line: + content[i] = line.rstrip() + "; visualizer.show = show\n" replaced = True assert replaced - content = ['def show(self):\n', ' pass\n\n'] + content - with open(dst_path, 'w') as write_file: - write_file.write(''.join(content)) + content = ["def show(self):\n", " pass\n\n"] + content + with open(dst_path, "w") as write_file: + write_file.write("".join(content)) def remove_ote_sdk_from_requirements(path): - with open(path, encoding='UTF-8') as read_file: - content = ''.join([line for line in read_file if 'ote_sdk' not in line]) + with open(path, encoding="UTF-8") as read_file: + content = "".join([line for line in read_file if "ote_sdk" not in line]) - with open(path, 'w', encoding='UTF-8') as write_file: + with open(path, "w", encoding="UTF-8") as write_file: write_file.write(content) def ote_train_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'train', - template.model_template_id, - '--train-ann-file', - f'{os.path.join(ote_dir, args["--train-ann-file"])}', - '--train-data-roots', - f'{os.path.join(ote_dir, args["--train-data-roots"])}', - '--val-ann-file', - f'{os.path.join(ote_dir, args["--val-ann-file"])}', - '--val-data-roots', - f'{os.path.join(ote_dir, args["--val-data-roots"])}', - '--save-model-to', - f'{template_work_dir}/trained_{template.model_template_id}'] - command_line.extend(args['train_params']) + command_line = [ + "ote", + "train", + template.model_template_id, + "--train-ann-file", + f'{os.path.join(ote_dir, args["--train-ann-file"])}', + "--train-data-roots", + f'{os.path.join(ote_dir, args["--train-data-roots"])}', + "--val-ann-file", + f'{os.path.join(ote_dir, args["--val-ann-file"])}', + "--val-data-roots", + f'{os.path.join(ote_dir, args["--val-data-roots"])}', + "--save-model-to", + f"{template_work_dir}/trained_{template.model_template_id}", + ] + command_line.extend(args["train_params"]) assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/trained_{template.model_template_id}/weights.pth') - assert os.path.exists(f'{template_work_dir}/trained_{template.model_template_id}/label_schema.json') + assert os.path.exists( + f"{template_work_dir}/trained_{template.model_template_id}/weights.pth" + ) + assert os.path.exists( + f"{template_work_dir}/trained_{template.model_template_id}/label_schema.json" + ) def ote_hpo_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) if os.path.exists(f"{template_work_dir}/hpo"): shutil.rmtree(f"{template_work_dir}/hpo") - command_line = ['ote', - 'train', - template.model_template_id, - '--train-ann-file', - f'{os.path.join(ote_dir, args["--train-ann-file"])}', - '--train-data-roots', - f'{os.path.join(ote_dir, args["--train-data-roots"])}', - '--val-ann-file', - f'{os.path.join(ote_dir, args["--val-ann-file"])}', - '--val-data-roots', - f'{os.path.join(ote_dir, args["--val-data-roots"])}', - '--save-model-to', - f'{template_work_dir}/hpo_trained_{template.model_template_id}', - '--enable-hpo', - '--hpo-time-ratio', - '1'] - command_line.extend(args['train_params']) + command_line = [ + "ote", + "train", + template.model_template_id, + "--train-ann-file", + f'{os.path.join(ote_dir, args["--train-ann-file"])}', + "--train-data-roots", + f'{os.path.join(ote_dir, args["--train-data-roots"])}', + "--val-ann-file", + f'{os.path.join(ote_dir, args["--val-ann-file"])}', + "--val-data-roots", + f'{os.path.join(ote_dir, args["--val-data-roots"])}', + "--save-model-to", + f"{template_work_dir}/hpo_trained_{template.model_template_id}", + "--enable-hpo", + "--hpo-time-ratio", + "1", + ] + command_line.extend(args["train_params"]) assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 assert os.path.exists(f"{template_work_dir}/hpo/hpopt_status.json") with open(f"{template_work_dir}/hpo/hpopt_status.json", "r") as f: - assert json.load(f).get('best_config_id', None) is not None - assert os.path.exists(f'{template_work_dir}/hpo_trained_{template.model_template_id}/weights.pth') - assert os.path.exists(f'{template_work_dir}/hpo_trained_{template.model_template_id}/label_schema.json') + assert json.load(f).get("best_config_id", None) is not None + assert os.path.exists( + f"{template_work_dir}/hpo_trained_{template.model_template_id}/weights.pth" + ) + assert os.path.exists( + f"{template_work_dir}/hpo_trained_{template.model_template_id}/label_schema.json" + ) def ote_export_testing(template, root): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'export', - template.model_template_id, - '--load-weights', - f'{template_work_dir}/trained_{template.model_template_id}/weights.pth', - f'--save-model-to', - f'{template_work_dir}/exported_{template.model_template_id}'] + command_line = [ + "ote", + "export", + template.model_template_id, + "--load-weights", + f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + "--save-model-to", + f"{template_work_dir}/exported_{template.model_template_id}", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml') - assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/openvino.bin') - assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/label_schema.json') + assert os.path.exists( + f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml" + ) + assert os.path.exists( + f"{template_work_dir}/exported_{template.model_template_id}/openvino.bin" + ) + assert os.path.exists( + f"{template_work_dir}/exported_{template.model_template_id}/label_schema.json" + ) def ote_eval_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'eval', - template.model_template_id, - '--test-ann-file', - f'{os.path.join(ote_dir, args["--test-ann-files"])}', - '--test-data-roots', - f'{os.path.join(ote_dir, args["--test-data-roots"])}', - '--load-weights', - f'{template_work_dir}/trained_{template.model_template_id}/weights.pth', - '--save-performance', - f'{template_work_dir}/trained_{template.model_template_id}/performance.json'] + command_line = [ + "ote", + "eval", + template.model_template_id, + "--test-ann-file", + f'{os.path.join(ote_dir, args["--test-ann-files"])}', + "--test-data-roots", + f'{os.path.join(ote_dir, args["--test-data-roots"])}', + "--load-weights", + f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + "--save-performance", + f"{template_work_dir}/trained_{template.model_template_id}/performance.json", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/trained_{template.model_template_id}/performance.json') + assert os.path.exists( + f"{template_work_dir}/trained_{template.model_template_id}/performance.json" + ) def ote_eval_openvino_testing(template, root, ote_dir, args, threshold): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'eval', - template.model_template_id, - '--test-ann-file', - f'{os.path.join(ote_dir, args["--test-ann-files"])}', - '--test-data-roots', - f'{os.path.join(ote_dir, args["--test-data-roots"])}', - '--load-weights', - f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml', - '--save-performance', - f'{template_work_dir}/exported_{template.model_template_id}/performance.json'] + command_line = [ + "ote", + "eval", + template.model_template_id, + "--test-ann-file", + f'{os.path.join(ote_dir, args["--test-ann-files"])}', + "--test-data-roots", + f'{os.path.join(ote_dir, args["--test-data-roots"])}', + "--load-weights", + f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", + "--save-performance", + f"{template_work_dir}/exported_{template.model_template_id}/performance.json", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/performance.json') - with open(f'{template_work_dir}/trained_{template.model_template_id}/performance.json') as read_file: + assert os.path.exists( + f"{template_work_dir}/exported_{template.model_template_id}/performance.json" + ) + with open( + f"{template_work_dir}/trained_{template.model_template_id}/performance.json" + ) as read_file: trained_performance = json.load(read_file) - with open(f'{template_work_dir}/exported_{template.model_template_id}/performance.json') as read_file: + with open( + f"{template_work_dir}/exported_{template.model_template_id}/performance.json" + ) as read_file: exported_performance = json.load(read_file) for k in trained_performance.keys(): @@ -198,95 +235,177 @@ def ote_eval_openvino_testing(template, root, ote_dir, args, threshold): def ote_demo_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'demo', - template.model_template_id, - '--load-weights', - f'{template_work_dir}/trained_{template.model_template_id}/weights.pth', - '--input', - os.path.join(ote_dir, args['--input']), - '--delay', - '-1'] + command_line = [ + "ote", + "demo", + template.model_template_id, + "--load-weights", + f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + "--input", + os.path.join(ote_dir, args["--input"]), + "--delay", + "-1", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 def ote_demo_openvino_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'demo', - template.model_template_id, - '--load-weights', - f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml', - '--input', - os.path.join(ote_dir, args['--input']), - '--delay', - '-1'] + command_line = [ + "ote", + "demo", + template.model_template_id, + "--load-weights", + f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", + "--input", + os.path.join(ote_dir, args["--input"]), + "--delay", + "-1", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 def ote_deploy_openvino_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - deployment_dir = f'{template_work_dir}/deployed_{template.model_template_id}' - command_line = ['ote', - 'deploy', - template.model_template_id, - '--load-weights', - f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml', - f'--save-model-to', - deployment_dir] + deployment_dir = f"{template_work_dir}/deployed_{template.model_template_id}" + command_line = [ + "ote", + "deploy", + template.model_template_id, + "--load-weights", + f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", + "--save-model-to", + deployment_dir, + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert run(['unzip', 'openvino.zip'], - cwd=deployment_dir).returncode == 0 - assert run(['python3', '-m', 'venv', 'venv'], - cwd=os.path.join(deployment_dir, 'python')).returncode == 0 - assert run(['python3', '-m', 'pip', 'install', 'wheel'], - cwd=os.path.join(deployment_dir, 'python'), - env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0 - - # Remove ote_sdk from requirements.txt, since merge commit (that is created on CI) is not pushed to github and that's why cannot be cloned. + assert run(["unzip", "openvino.zip"], cwd=deployment_dir).returncode == 0 + assert ( + run( + ["python3", "-m", "venv", "venv"], + cwd=os.path.join(deployment_dir, "python"), + ).returncode + == 0 + ) + assert ( + run( + ["python3", "-m", "pip", "install", "wheel"], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), + ).returncode + == 0 + ) + + # Remove ote_sdk from requirements.txt, since merge commit (that is created on CI) + # is not pushed to github and that's why cannot be cloned. # Install ote_sdk from local folder instead. - # Install the demo_package with --no-deps since, requirements.txt has been embedded to the demo_package during creation. - remove_ote_sdk_from_requirements(os.path.join(deployment_dir, 'python', 'requirements.txt')) - assert run(['python3', '-m', 'pip', 'install', 'pip', '--upgrade'], - cwd=os.path.join(deployment_dir, 'python'), - env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0 - assert run(['python3', '-m', 'pip', 'install', '-e', os.path.join(os.path.dirname(__file__), '..', '..', 'ote_sdk')], - cwd=os.path.join(deployment_dir, 'python'), - env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0 - assert run(['python3', '-m', 'pip', 'install', '-r', os.path.join(deployment_dir, 'python', 'requirements.txt')], - cwd=os.path.join(deployment_dir, 'python'), - env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0 - assert run(['python3', '-m', 'pip', 'install', 'demo_package-0.0-py3-none-any.whl', '--no-deps'], - cwd=os.path.join(deployment_dir, 'python'), - env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0 + # Install the demo_package with --no-deps since, requirements.txt + # has been embedded to the demo_package during creation. + remove_ote_sdk_from_requirements( + os.path.join(deployment_dir, "python", "requirements.txt") + ) + assert ( + run( + ["python3", "-m", "pip", "install", "pip", "--upgrade"], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), + ).returncode + == 0 + ) + assert ( + run( + [ + "python3", + "-m", + "pip", + "install", + "-e", + os.path.join(os.path.dirname(__file__), "..", "..", "ote_sdk"), + ], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), + ).returncode + == 0 + ) + assert ( + run( + [ + "python3", + "-m", + "pip", + "install", + "-r", + os.path.join(deployment_dir, "python", "requirements.txt"), + ], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), + ).returncode + == 0 + ) + assert ( + run( + [ + "python3", + "-m", + "pip", + "install", + "demo_package-0.0-py3-none-any.whl", + "--no-deps", + ], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), + ).returncode + == 0 + ) # Patch demo since we are not able to run cv2.imshow on CI. - patch_demo_py(os.path.join(deployment_dir, 'python', 'demo.py'), - os.path.join(deployment_dir, 'python', 'demo_patched.py')) - - assert run(['python3', 'demo_patched.py', '-m', '../model/model.xml', '-i', os.path.join(ote_dir, args['--input'])], - cwd=os.path.join(deployment_dir, 'python'), - env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0 + patch_demo_py( + os.path.join(deployment_dir, "python", "demo.py"), + os.path.join(deployment_dir, "python", "demo_patched.py"), + ) + + assert ( + run( + [ + "python3", + "demo_patched.py", + "-m", + "../model/model.xml", + "-i", + os.path.join(ote_dir, args["--input"]), + ], + cwd=os.path.join(deployment_dir, "python"), + env=collect_env_vars(os.path.join(deployment_dir, "python")), + ).returncode + == 0 + ) def ote_eval_deployment_testing(template, root, ote_dir, args, threshold): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'eval', - template.model_template_id, - '--test-ann-file', - f'{os.path.join(ote_dir, args["--test-ann-files"])}', - '--test-data-roots', - f'{os.path.join(ote_dir, args["--test-data-roots"])}', - '--load-weights', - f'{template_work_dir}/deployed_{template.model_template_id}/openvino.zip', - '--save-performance', - f'{template_work_dir}/deployed_{template.model_template_id}/performance.json'] + command_line = [ + "ote", + "eval", + template.model_template_id, + "--test-ann-file", + f'{os.path.join(ote_dir, args["--test-ann-files"])}', + "--test-data-roots", + f'{os.path.join(ote_dir, args["--test-data-roots"])}', + "--load-weights", + f"{template_work_dir}/deployed_{template.model_template_id}/openvino.zip", + "--save-performance", + f"{template_work_dir}/deployed_{template.model_template_id}/performance.json", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/deployed_{template.model_template_id}/performance.json') - with open(f'{template_work_dir}/exported_{template.model_template_id}/performance.json') as read_file: + assert os.path.exists( + f"{template_work_dir}/deployed_{template.model_template_id}/performance.json" + ) + with open( + f"{template_work_dir}/exported_{template.model_template_id}/performance.json" + ) as read_file: exported_performance = json.load(read_file) - with open(f'{template_work_dir}/deployed_{template.model_template_id}/performance.json') as read_file: + with open( + f"{template_work_dir}/deployed_{template.model_template_id}/performance.json" + ) as read_file: deployed_performance = json.load(read_file) for k in exported_performance.keys(): @@ -299,121 +418,161 @@ def ote_eval_deployment_testing(template, root, ote_dir, args, threshold): def ote_demo_deployment_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'demo', - template.model_template_id, - '--load-weights', - f'{template_work_dir}/deployed_{template.model_template_id}/openvino.zip', - '--input', - os.path.join(ote_dir, args['--input']), - '--delay', - '-1'] + command_line = [ + "ote", + "demo", + template.model_template_id, + "--load-weights", + f"{template_work_dir}/deployed_{template.model_template_id}/openvino.zip", + "--input", + os.path.join(ote_dir, args["--input"]), + "--delay", + "-1", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 def pot_optimize_testing(template, root, ote_dir, args): work_dir, template_work_dir, algo_backend_dir = get_some_vars(template, root) - command_line = ['ote', - 'optimize', - template.model_template_id, - '--train-ann-file', - f'{os.path.join(ote_dir, args["--train-ann-file"])}', - '--train-data-roots', - f'{os.path.join(ote_dir, args["--train-data-roots"])}', - '--val-ann-file', - f'{os.path.join(ote_dir, args["--val-ann-file"])}', - '--val-data-roots', - f'{os.path.join(ote_dir, args["--val-data-roots"])}', - '--load-weights', - f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml', - '--save-model-to', - f'{template_work_dir}/pot_{template.model_template_id}', - ] + command_line = [ + "ote", + "optimize", + template.model_template_id, + "--train-ann-file", + f'{os.path.join(ote_dir, args["--train-ann-file"])}', + "--train-data-roots", + f'{os.path.join(ote_dir, args["--train-data-roots"])}', + "--val-ann-file", + f'{os.path.join(ote_dir, args["--val-ann-file"])}', + "--val-data-roots", + f'{os.path.join(ote_dir, args["--val-data-roots"])}', + "--load-weights", + f"{template_work_dir}/exported_{template.model_template_id}/openvino.xml", + "--save-model-to", + f"{template_work_dir}/pot_{template.model_template_id}", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/pot_{template.model_template_id}/openvino.xml') - assert os.path.exists(f'{template_work_dir}/pot_{template.model_template_id}/openvino.bin') - assert os.path.exists(f'{template_work_dir}/pot_{template.model_template_id}/label_schema.json') + assert os.path.exists( + f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml" + ) + assert os.path.exists( + f"{template_work_dir}/pot_{template.model_template_id}/openvino.bin" + ) + assert os.path.exists( + f"{template_work_dir}/pot_{template.model_template_id}/label_schema.json" + ) def pot_eval_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'eval', - template.model_template_id, - '--test-ann-file', - f'{os.path.join(ote_dir, args["--test-ann-files"])}', - '--test-data-roots', - f'{os.path.join(ote_dir, args["--test-data-roots"])}', - '--load-weights', - f'{template_work_dir}/pot_{template.model_template_id}/openvino.xml', - '--save-performance', - f'{template_work_dir}/pot_{template.model_template_id}/performance.json'] + command_line = [ + "ote", + "eval", + template.model_template_id, + "--test-ann-file", + f'{os.path.join(ote_dir, args["--test-ann-files"])}', + "--test-data-roots", + f'{os.path.join(ote_dir, args["--test-data-roots"])}', + "--load-weights", + f"{template_work_dir}/pot_{template.model_template_id}/openvino.xml", + "--save-performance", + f"{template_work_dir}/pot_{template.model_template_id}/performance.json", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/pot_{template.model_template_id}/performance.json') + assert os.path.exists( + f"{template_work_dir}/pot_{template.model_template_id}/performance.json" + ) def nncf_optimize_testing(template, root, ote_dir, args): work_dir, template_work_dir, algo_backend_dir = get_some_vars(template, root) - command_line = ['ote', - 'optimize', - template.model_template_id, - '--train-ann-file', - f'{os.path.join(ote_dir, args["--train-ann-file"])}', - '--train-data-roots', - f'{os.path.join(ote_dir, args["--train-data-roots"])}', - '--val-ann-file', - f'{os.path.join(ote_dir, args["--val-ann-file"])}', - '--val-data-roots', - f'{os.path.join(ote_dir, args["--val-data-roots"])}', - '--load-weights', - f'{template_work_dir}/trained_{template.model_template_id}/weights.pth', - '--save-model-to', - f'{template_work_dir}/nncf_{template.model_template_id}', - '--save-performance', - f'{template_work_dir}/nncf_{template.model_template_id}/train_performance.json', - ] - command_line.extend(args['train_params']) + command_line = [ + "ote", + "optimize", + template.model_template_id, + "--train-ann-file", + f'{os.path.join(ote_dir, args["--train-ann-file"])}', + "--train-data-roots", + f'{os.path.join(ote_dir, args["--train-data-roots"])}', + "--val-ann-file", + f'{os.path.join(ote_dir, args["--val-ann-file"])}', + "--val-data-roots", + f'{os.path.join(ote_dir, args["--val-data-roots"])}', + "--load-weights", + f"{template_work_dir}/trained_{template.model_template_id}/weights.pth", + "--save-model-to", + f"{template_work_dir}/nncf_{template.model_template_id}", + "--save-performance", + f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json", + ] + command_line.extend(args["train_params"]) assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/nncf_{template.model_template_id}/weights.pth') - assert os.path.exists(f'{template_work_dir}/nncf_{template.model_template_id}/label_schema.json') + assert os.path.exists( + f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth" + ) + assert os.path.exists( + f"{template_work_dir}/nncf_{template.model_template_id}/label_schema.json" + ) def nncf_export_testing(template, root): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'export', - template.model_template_id, - '--load-weights', - f'{template_work_dir}/nncf_{template.model_template_id}/weights.pth', - f'--save-model-to', - f'{template_work_dir}/exported_nncf_{template.model_template_id}'] + command_line = [ + "ote", + "export", + template.model_template_id, + "--load-weights", + f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth", + "--save-model-to", + f"{template_work_dir}/exported_nncf_{template.model_template_id}", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml') - assert os.path.exists(f'{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.bin') - assert os.path.exists(f'{template_work_dir}/exported_nncf_{template.model_template_id}/label_schema.json') - original_bin_size = os.path.getsize(f'{template_work_dir}/exported_{template.model_template_id}/openvino.bin') - compressed_bin_size = os.path.getsize(f'{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.bin') - assert compressed_bin_size < original_bin_size, f"{compressed_bin_size=}, {original_bin_size=}" + assert os.path.exists( + f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml" + ) + assert os.path.exists( + f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.bin" + ) + assert os.path.exists( + f"{template_work_dir}/exported_nncf_{template.model_template_id}/label_schema.json" + ) + original_bin_size = os.path.getsize( + f"{template_work_dir}/exported_{template.model_template_id}/openvino.bin" + ) + compressed_bin_size = os.path.getsize( + f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.bin" + ) + assert ( + compressed_bin_size < original_bin_size + ), f"{compressed_bin_size=}, {original_bin_size=}" def nncf_eval_testing(template, root, ote_dir, args, threshold): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'eval', - template.model_template_id, - '--test-ann-file', - f'{os.path.join(ote_dir, args["--test-ann-files"])}', - '--test-data-roots', - f'{os.path.join(ote_dir, args["--test-data-roots"])}', - '--load-weights', - f'{template_work_dir}/nncf_{template.model_template_id}/weights.pth', - '--save-performance', - f'{template_work_dir}/nncf_{template.model_template_id}/performance.json'] + command_line = [ + "ote", + "eval", + template.model_template_id, + "--test-ann-file", + f'{os.path.join(ote_dir, args["--test-ann-files"])}', + "--test-data-roots", + f'{os.path.join(ote_dir, args["--test-data-roots"])}', + "--load-weights", + f"{template_work_dir}/nncf_{template.model_template_id}/weights.pth", + "--save-performance", + f"{template_work_dir}/nncf_{template.model_template_id}/performance.json", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/nncf_{template.model_template_id}/performance.json') - with open(f'{template_work_dir}/nncf_{template.model_template_id}/train_performance.json') as read_file: + assert os.path.exists( + f"{template_work_dir}/nncf_{template.model_template_id}/performance.json" + ) + with open( + f"{template_work_dir}/nncf_{template.model_template_id}/train_performance.json" + ) as read_file: trained_performance = json.load(read_file) - with open(f'{template_work_dir}/nncf_{template.model_template_id}/performance.json') as read_file: + with open( + f"{template_work_dir}/nncf_{template.model_template_id}/performance.json" + ) as read_file: evaluated_performance = json.load(read_file) for k in trained_performance.keys(): @@ -426,16 +585,20 @@ def nncf_eval_testing(template, root, ote_dir, args, threshold): def nncf_eval_openvino_testing(template, root, ote_dir, args): work_dir, template_work_dir, _ = get_some_vars(template, root) - command_line = ['ote', - 'eval', - template.model_template_id, - '--test-ann-file', - f'{os.path.join(ote_dir, args["--test-ann-files"])}', - '--test-data-roots', - f'{os.path.join(ote_dir, args["--test-data-roots"])}', - '--load-weights', - f'{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml', - '--save-performance', - f'{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json'] + command_line = [ + "ote", + "eval", + template.model_template_id, + "--test-ann-file", + f'{os.path.join(ote_dir, args["--test-ann-files"])}', + "--test-data-roots", + f'{os.path.join(ote_dir, args["--test-data-roots"])}', + "--load-weights", + f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml", + "--save-performance", + f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json", + ] assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0 - assert os.path.exists(f'{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json') + assert os.path.exists( + f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json" + ) diff --git a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_classification.py b/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_classification.py index 7e49d819218..d741c0b1ab6 100644 --- a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_classification.py +++ b/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_classification.py @@ -20,7 +20,7 @@ from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, ote_demo_deployment_testing, @@ -62,8 +62,8 @@ class TestToolsAnomalyClassification: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_segmentation.py b/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_segmentation.py index 284f3ec6c29..0a612270d97 100644 --- a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_segmentation.py +++ b/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_segmentation.py @@ -20,7 +20,7 @@ from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, ote_demo_deployment_testing, @@ -62,8 +62,8 @@ class TestToolsAnomalySegmentation: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/ote_cli/external/deep-object-reid/test_ote_cli_tools_classification.py b/tests/ote_cli/external/deep-object-reid/test_ote_cli_tools_classification.py index fbeb07716bb..bce27f1bbc1 100644 --- a/tests/ote_cli/external/deep-object-reid/test_ote_cli_tools_classification.py +++ b/tests/ote_cli/external/deep-object-reid/test_ote_cli_tools_classification.py @@ -20,7 +20,7 @@ from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, ote_demo_deployment_testing, @@ -69,8 +69,8 @@ class TestToolsClassification: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_detection.py b/tests/ote_cli/external/mmdetection/test_ote_cli_tools_detection.py index 01d1be249db..0ee0abdec49 100644 --- a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_detection.py +++ b/tests/ote_cli/external/mmdetection/test_ote_cli_tools_detection.py @@ -21,7 +21,7 @@ from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( collect_env_vars, create_venv, get_some_vars, @@ -71,8 +71,8 @@ class TestToolsDetection: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_instance_segmentation.py b/tests/ote_cli/external/mmdetection/test_ote_cli_tools_instance_segmentation.py index b20deb3f0e9..34254d1ad0a 100644 --- a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_instance_segmentation.py +++ b/tests/ote_cli/external/mmdetection/test_ote_cli_tools_instance_segmentation.py @@ -22,7 +22,7 @@ from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, ote_demo_deployment_testing, @@ -64,8 +64,8 @@ class TestToolsInstanceSegmentation: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_rotated_detection.py b/tests/ote_cli/external/mmdetection/test_ote_cli_tools_rotated_detection.py index c3efb5dc7f1..661d5989c78 100644 --- a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_rotated_detection.py +++ b/tests/ote_cli/external/mmdetection/test_ote_cli_tools_rotated_detection.py @@ -22,7 +22,7 @@ from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, ote_demo_deployment_testing, @@ -64,8 +64,8 @@ class TestToolsRotatedDetection: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/ote_cli/external/mmsegmentation/test_ote_cli_tools_segmentation.py b/tests/ote_cli/external/mmsegmentation/test_ote_cli_tools_segmentation.py index 33e3ea48807..bcf3e49c1a9 100644 --- a/tests/ote_cli/external/mmsegmentation/test_ote_cli_tools_segmentation.py +++ b/tests/ote_cli/external/mmsegmentation/test_ote_cli_tools_segmentation.py @@ -22,7 +22,7 @@ from ote_cli.registry import Registry -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, ote_demo_deployment_testing, @@ -75,8 +75,8 @@ class TestToolsSegmentation: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/run_model_templates_tests.py b/tests/run_model_templates_tests.py index 568a501678a..9197b06f9e2 100644 --- a/tests/run_model_templates_tests.py +++ b/tests/run_model_templates_tests.py @@ -4,7 +4,7 @@ import sys from subprocess import run -from tests.ote_cli.common import collect_env_vars +from ote_cli.utils.tests import collect_env_vars ALGO_ROOT_DIR = "external" ALGO_DIRS = [ diff --git a/tests/run_model_templates_tests.sh b/tests/run_model_templates_tests.sh index 9b892f29275..212fe42c75b 100755 --- a/tests/run_model_templates_tests.sh +++ b/tests/run_model_templates_tests.sh @@ -4,8 +4,6 @@ python3 -m venv venv || exit 1 . venv/bin/activate || exit 1 pip install --upgrade pip || exit 1 pip install -e ote_cli || exit 1 -pip install -e $OTE_SDK_PATH || exit 1 - -export PYTHONPATH=${PYTHONPATH}:`pwd` +pip install -e ote_sdk || exit 1 python tests/run_model_templates_tests.py `pwd` $@ || exit 1 From 961f1a15827c4a8c3c785d3ef6e8f770d3477fbb Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 13:28:48 +0300 Subject: [PATCH 03/16] moved --- .../tests/ote_cli}/test_ote_cli_tools_anomaly_classification.py | 0 .../tests/ote_cli}/test_ote_cli_tools_anomaly_detection.py | 0 .../tests/ote_cli}/test_ote_cli_tools_anomaly_segmentation.py | 0 .../tests/ote_cli}/test_ote_cli_tools_classification.py | 0 .../mmdetection/tests/ote_cli}/test_ote_cli_tools_detection.py | 0 .../tests/ote_cli}/test_ote_cli_tools_instance_segmentation.py | 0 .../tests/ote_cli}/test_ote_cli_tools_rotated_detection.py | 0 .../tests/ote_cli}/test_ote_cli_tools_segmentation.py | 0 tests/run_model_templates_tests.py | 2 +- 9 files changed, 1 insertion(+), 1 deletion(-) rename {tests/ote_cli/external/anomaly => external/anomaly/tests/ote_cli}/test_ote_cli_tools_anomaly_classification.py (100%) rename {tests/ote_cli/external/anomaly => external/anomaly/tests/ote_cli}/test_ote_cli_tools_anomaly_detection.py (100%) rename {tests/ote_cli/external/anomaly => external/anomaly/tests/ote_cli}/test_ote_cli_tools_anomaly_segmentation.py (100%) rename {tests/ote_cli/external/deep-object-reid => external/deep-object-reid/tests/ote_cli}/test_ote_cli_tools_classification.py (100%) rename {tests/ote_cli/external/mmdetection => external/mmdetection/tests/ote_cli}/test_ote_cli_tools_detection.py (100%) rename {tests/ote_cli/external/mmdetection => external/mmdetection/tests/ote_cli}/test_ote_cli_tools_instance_segmentation.py (100%) rename {tests/ote_cli/external/mmdetection => external/mmdetection/tests/ote_cli}/test_ote_cli_tools_rotated_detection.py (100%) rename {tests/ote_cli/external/mmsegmentation => external/mmsegmentation/tests/ote_cli}/test_ote_cli_tools_segmentation.py (100%) diff --git a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_classification.py b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py similarity index 100% rename from tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_classification.py rename to external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py diff --git a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_detection.py b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_detection.py similarity index 100% rename from tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_detection.py rename to external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_detection.py diff --git a/tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_segmentation.py b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py similarity index 100% rename from tests/ote_cli/external/anomaly/test_ote_cli_tools_anomaly_segmentation.py rename to external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py diff --git a/tests/ote_cli/external/deep-object-reid/test_ote_cli_tools_classification.py b/external/deep-object-reid/tests/ote_cli/test_ote_cli_tools_classification.py similarity index 100% rename from tests/ote_cli/external/deep-object-reid/test_ote_cli_tools_classification.py rename to external/deep-object-reid/tests/ote_cli/test_ote_cli_tools_classification.py diff --git a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_detection.py b/external/mmdetection/tests/ote_cli/test_ote_cli_tools_detection.py similarity index 100% rename from tests/ote_cli/external/mmdetection/test_ote_cli_tools_detection.py rename to external/mmdetection/tests/ote_cli/test_ote_cli_tools_detection.py diff --git a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_instance_segmentation.py b/external/mmdetection/tests/ote_cli/test_ote_cli_tools_instance_segmentation.py similarity index 100% rename from tests/ote_cli/external/mmdetection/test_ote_cli_tools_instance_segmentation.py rename to external/mmdetection/tests/ote_cli/test_ote_cli_tools_instance_segmentation.py diff --git a/tests/ote_cli/external/mmdetection/test_ote_cli_tools_rotated_detection.py b/external/mmdetection/tests/ote_cli/test_ote_cli_tools_rotated_detection.py similarity index 100% rename from tests/ote_cli/external/mmdetection/test_ote_cli_tools_rotated_detection.py rename to external/mmdetection/tests/ote_cli/test_ote_cli_tools_rotated_detection.py diff --git a/tests/ote_cli/external/mmsegmentation/test_ote_cli_tools_segmentation.py b/external/mmsegmentation/tests/ote_cli/test_ote_cli_tools_segmentation.py similarity index 100% rename from tests/ote_cli/external/mmsegmentation/test_ote_cli_tools_segmentation.py rename to external/mmsegmentation/tests/ote_cli/test_ote_cli_tools_segmentation.py diff --git a/tests/run_model_templates_tests.py b/tests/run_model_templates_tests.py index 9197b06f9e2..eb654533040 100644 --- a/tests/run_model_templates_tests.py +++ b/tests/run_model_templates_tests.py @@ -66,7 +66,7 @@ def test(run_algo_tests): success *= res for algo_dir in ALGO_DIRS: if run_algo_tests[algo_dir]: - command = ["pytest", os.path.join("tests", "ote_cli", algo_dir), "-v", "--durations=10"] + command = ["pytest", os.path.join(algo_dir, "tests", "ote_cli"), "-v", "--durations=10"] try: res = run(command, env=collect_env_vars(wd), check=True).returncode == 0 except: From c5d71cd008be1b470e71e433161fa3c771e67aeb Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 13:32:39 +0300 Subject: [PATCH 04/16] minor --- ote_cli/ote_cli/utils/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index f217f69a570..e48dd544a47 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -433,7 +433,7 @@ def ote_demo_deployment_testing(template, root, ote_dir, args): def pot_optimize_testing(template, root, ote_dir, args): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(template, root) + work_dir, template_work_dir, _ = get_some_vars(template, root) command_line = [ "ote", "optimize", @@ -485,7 +485,7 @@ def pot_eval_testing(template, root, ote_dir, args): def nncf_optimize_testing(template, root, ote_dir, args): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(template, root) + work_dir, template_work_dir, _ = get_some_vars(template, root) command_line = [ "ote", "optimize", From d35532cbc9730d409ca9264217a63764d58241f0 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 13:39:15 +0300 Subject: [PATCH 05/16] minor --- ...st_ote_cli_tools_anomaly_classification.py | 20 +++++++++---------- ...test_ote_cli_tools_anomaly_segmentation.py | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py index d741c0b1ab6..76fa1cfbc94 100644 --- a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py +++ b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py @@ -42,20 +42,20 @@ args = { - '--train-ann-file': 'data/anomaly/classification/train.json', - '--train-data-roots': 'data/anomaly/shapes', - '--val-ann-file': 'data/anomaly/classification/val.json', - '--val-data-roots': 'data/anomaly/shapes', - '--test-ann-files': 'data/anomaly/classification/test.json', - '--test-data-roots': 'data/anomaly/shapes', - '--input': 'data/anomaly/shapes/test/hexagon', - 'train_params': [], + "--train-ann-file": "data/anomaly/classification/train.json", + "--train-data-roots": "data/anomaly/shapes", + "--val-ann-file": "data/anomaly/classification/val.json", + "--val-data-roots": "data/anomaly/shapes", + "--test-ann-files": "data/anomaly/classification/test.json", + "--test-data-roots": "data/anomaly/shapes", + "--input": "data/anomaly/shapes/test/hexagon", + "train_params": [], } -root = '/tmp/ote_cli/' +root = "/tmp/ote_cli/" ote_dir = os.getcwd() -templates = Registry('external').filter(task_type='ANOMALY_CLASSIFICATION').templates +templates = Registry("external").filter(task_type="ANOMALY_CLASSIFICATION").templates templates_ids = [template.model_template_id for template in templates] diff --git a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py index 0a612270d97..0789816fc0b 100644 --- a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py +++ b/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py @@ -42,20 +42,20 @@ args = { - '--train-ann-file': 'data/anomaly/segmentation/train.json', - '--train-data-roots': 'data/anomaly/shapes', - '--val-ann-file': 'data/anomaly/segmentation/val.json', - '--val-data-roots': 'data/anomaly/shapes', - '--test-ann-files': 'data/anomaly/segmentation/test.json', - '--test-data-roots': 'data/anomaly/shapes', - '--input': 'data/anomaly/shapes/test/hexagon', - 'train_params': [], + "--train-ann-file": "data/anomaly/segmentation/train.json", + "--train-data-roots": "data/anomaly/shapes", + "--val-ann-file": "data/anomaly/segmentation/val.json", + "--val-data-roots": "data/anomaly/shapes", + "--test-ann-files": "data/anomaly/segmentation/test.json", + "--test-data-roots": "data/anomaly/shapes", + "--input": "data/anomaly/shapes/test/hexagon", + "train_params": [], } -root = '/tmp/ote_cli/' +root = "/tmp/ote_cli/" ote_dir = os.getcwd() -templates = Registry('external').filter(task_type='ANOMALY_SEGMENTATION').templates +templates = Registry("external").filter(task_type="ANOMALY_SEGMENTATION").templates templates_ids = [template.model_template_id for template in templates] From 15fd72f340ab744c0d8d1309e58365de1731197d Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 13:41:16 +0300 Subject: [PATCH 06/16] renamed --- ...s_anomaly_classification.py => test_anomaly_classification.py} | 0 ...tools_anomaly_segmentation.py => test_anomaly_segmentation.py} | 0 ...est_ote_cli_tools_classification.py => test_classification.py} | 0 .../{test_ote_cli_tools_detection.py => test_detection.py} | 0 ...ols_instance_segmentation.py => test_instance_segmentation.py} | 0 ...e_cli_tools_rotated_detection.py => test_rotated_detection.py} | 0 .../{test_ote_cli_tools_segmentation.py => test_segmentation.py} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename external/anomaly/tests/ote_cli/{test_ote_cli_tools_anomaly_classification.py => test_anomaly_classification.py} (100%) rename external/anomaly/tests/ote_cli/{test_ote_cli_tools_anomaly_segmentation.py => test_anomaly_segmentation.py} (100%) rename external/deep-object-reid/tests/ote_cli/{test_ote_cli_tools_classification.py => test_classification.py} (100%) rename external/mmdetection/tests/ote_cli/{test_ote_cli_tools_detection.py => test_detection.py} (100%) rename external/mmdetection/tests/ote_cli/{test_ote_cli_tools_instance_segmentation.py => test_instance_segmentation.py} (100%) rename external/mmdetection/tests/ote_cli/{test_ote_cli_tools_rotated_detection.py => test_rotated_detection.py} (100%) rename external/mmsegmentation/tests/ote_cli/{test_ote_cli_tools_segmentation.py => test_segmentation.py} (100%) diff --git a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py b/external/anomaly/tests/ote_cli/test_anomaly_classification.py similarity index 100% rename from external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_classification.py rename to external/anomaly/tests/ote_cli/test_anomaly_classification.py diff --git a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py b/external/anomaly/tests/ote_cli/test_anomaly_segmentation.py similarity index 100% rename from external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_segmentation.py rename to external/anomaly/tests/ote_cli/test_anomaly_segmentation.py diff --git a/external/deep-object-reid/tests/ote_cli/test_ote_cli_tools_classification.py b/external/deep-object-reid/tests/ote_cli/test_classification.py similarity index 100% rename from external/deep-object-reid/tests/ote_cli/test_ote_cli_tools_classification.py rename to external/deep-object-reid/tests/ote_cli/test_classification.py diff --git a/external/mmdetection/tests/ote_cli/test_ote_cli_tools_detection.py b/external/mmdetection/tests/ote_cli/test_detection.py similarity index 100% rename from external/mmdetection/tests/ote_cli/test_ote_cli_tools_detection.py rename to external/mmdetection/tests/ote_cli/test_detection.py diff --git a/external/mmdetection/tests/ote_cli/test_ote_cli_tools_instance_segmentation.py b/external/mmdetection/tests/ote_cli/test_instance_segmentation.py similarity index 100% rename from external/mmdetection/tests/ote_cli/test_ote_cli_tools_instance_segmentation.py rename to external/mmdetection/tests/ote_cli/test_instance_segmentation.py diff --git a/external/mmdetection/tests/ote_cli/test_ote_cli_tools_rotated_detection.py b/external/mmdetection/tests/ote_cli/test_rotated_detection.py similarity index 100% rename from external/mmdetection/tests/ote_cli/test_ote_cli_tools_rotated_detection.py rename to external/mmdetection/tests/ote_cli/test_rotated_detection.py diff --git a/external/mmsegmentation/tests/ote_cli/test_ote_cli_tools_segmentation.py b/external/mmsegmentation/tests/ote_cli/test_segmentation.py similarity index 100% rename from external/mmsegmentation/tests/ote_cli/test_ote_cli_tools_segmentation.py rename to external/mmsegmentation/tests/ote_cli/test_segmentation.py From 50f5961daf1e28e48087565f14a9a67b5277a487 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 13:41:30 +0300 Subject: [PATCH 07/16] added symlinks --- tests/ote_cli/external/anomaly/ote_cli | 1 + tests/ote_cli/external/deep-object-reid/ote_cli | 1 + tests/ote_cli/external/mmdetection/ote_cli | 1 + tests/ote_cli/external/mmsegmentation/ote_cli | 1 + 4 files changed, 4 insertions(+) create mode 120000 tests/ote_cli/external/anomaly/ote_cli create mode 120000 tests/ote_cli/external/deep-object-reid/ote_cli create mode 120000 tests/ote_cli/external/mmdetection/ote_cli create mode 120000 tests/ote_cli/external/mmsegmentation/ote_cli diff --git a/tests/ote_cli/external/anomaly/ote_cli b/tests/ote_cli/external/anomaly/ote_cli new file mode 120000 index 00000000000..b7a8e6b55c1 --- /dev/null +++ b/tests/ote_cli/external/anomaly/ote_cli @@ -0,0 +1 @@ +../../../../external/anomaly/tests/ote_cli \ No newline at end of file diff --git a/tests/ote_cli/external/deep-object-reid/ote_cli b/tests/ote_cli/external/deep-object-reid/ote_cli new file mode 120000 index 00000000000..ff593f72ea6 --- /dev/null +++ b/tests/ote_cli/external/deep-object-reid/ote_cli @@ -0,0 +1 @@ +../../../../external/deep-object-reid/tests/ote_cli \ No newline at end of file diff --git a/tests/ote_cli/external/mmdetection/ote_cli b/tests/ote_cli/external/mmdetection/ote_cli new file mode 120000 index 00000000000..7c90d654753 --- /dev/null +++ b/tests/ote_cli/external/mmdetection/ote_cli @@ -0,0 +1 @@ +../../../../external/mmdetection/tests/ote_cli \ No newline at end of file diff --git a/tests/ote_cli/external/mmsegmentation/ote_cli b/tests/ote_cli/external/mmsegmentation/ote_cli new file mode 120000 index 00000000000..898e3d4a498 --- /dev/null +++ b/tests/ote_cli/external/mmsegmentation/ote_cli @@ -0,0 +1 @@ +../../../../external/mmsegmentation/tests/ote_cli \ No newline at end of file From b5689a0dcbf35f683f99c2b3098ad588b2cb5c6b Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 14:05:42 +0300 Subject: [PATCH 08/16] fix --- external/anomaly/init_venv.sh | 8 ++------ ote_cli/ote_cli/utils/tests.py | 2 -- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/external/anomaly/init_venv.sh b/external/anomaly/init_venv.sh index 015b677c8c2..0081c4fce0f 100755 --- a/external/anomaly/init_venv.sh +++ b/external/anomaly/init_venv.sh @@ -26,11 +26,6 @@ if [[ $PYTHON_VERSION != "3.8" && $PYTHON_VERSION != "3.9" ]]; then exit 1 fi -if [[ -z $OTE_SDK_PATH ]]; then - echo "The environment variable OTE_SDK_PATH is not set -- it is required for creating virtual environment" - exit 1 -fi - cd ${work_dir} if [[ -e ${venv_dir} ]]; then @@ -112,7 +107,8 @@ fi pip install -r requirements.txt pip install -e . -pip install -e $OTE_SDK_PATH || exit 1 +# Install OTE SDK +pip install -e ../../ote_sdk/ || exit 1 deactivate diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index e48dd544a47..89361757cb2 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -64,8 +64,6 @@ def collect_env_vars(work_dir): vars.update({"HTTPS_PROXY": os.environ["HTTPS_PROXY"]}) if "NO_PROXY" in os.environ: vars.update({"NO_PROXY": os.environ["NO_PROXY"]}) - if "OTE_SDK_PATH" in os.environ: - vars.update({"OTE_SDK_PATH": os.environ["OTE_SDK_PATH"]}) return vars From 77dbbae289ea53e77277a046213d0ec5a39f0121 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 15:28:42 +0300 Subject: [PATCH 09/16] xfail --- ote_cli/ote_cli/utils/tests.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index 89361757cb2..07d361df273 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -17,6 +17,8 @@ import shutil from subprocess import run # nosec +import pytest + def get_template_rel_dir(template): return os.path.dirname(os.path.relpath(template.model_template_path)) @@ -600,3 +602,24 @@ def nncf_eval_openvino_testing(template, root, ote_dir, args): assert os.path.exists( f"{template_work_dir}/exported_nncf_{template.model_template_id}/performance.json" ) + + +def xfail_templates(templates, xfail_template_ids_reasons): + xfailed_templates = [] + for template in templates: + reasons = [ + reason + for template_id, reason in xfail_template_ids_reasons + if template_id == template.model_template_id + ] + if len(reasons) == 0: + xfailed_templates.append(template) + elif len(reasons) == 1: + xfailed_templates.append( + pytest.param(template, marks=pytest.mark.xfail(reason=reasons[0])) + ) + else: + raise RuntimeError( + "More than one reason for template. If you have more than one Jira tickets, list them in one reason." + ) + return xfailed_templates From cc5bfc2c12c4b317e2210ebf93f7cd097ba5c720 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 15:34:33 +0300 Subject: [PATCH 10/16] moved --- ote_cli/ote_cli/utils/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ote_cli/ote_cli/utils/tests.py b/ote_cli/ote_cli/utils/tests.py index 07d361df273..d53e4a4c861 100644 --- a/ote_cli/ote_cli/utils/tests.py +++ b/ote_cli/ote_cli/utils/tests.py @@ -319,7 +319,7 @@ def ote_deploy_openvino_testing(template, root, ote_dir, args): "pip", "install", "-e", - os.path.join(os.path.dirname(__file__), "..", "..", "ote_sdk"), + os.path.join(os.path.dirname(__file__), "..", "..", "..", "ote_sdk"), ], cwd=os.path.join(deployment_dir, "python"), env=collect_env_vars(os.path.join(deployment_dir, "python")), From 36694889e5274dd4c89b31b1810ae64c25dee8dd Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 22:37:35 +0300 Subject: [PATCH 11/16] fix --- external/deep-object-reid/tests/conftest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/external/deep-object-reid/tests/conftest.py b/external/deep-object-reid/tests/conftest.py index 24f5e6883da..5e7aad70cfa 100644 --- a/external/deep-object-reid/tests/conftest.py +++ b/external/deep-object-reid/tests/conftest.py @@ -5,15 +5,14 @@ from e2e.conftest_utils import pytest_addoption as _e2e_pytest_addoption # noqa from e2e import config # noqa from e2e.utils import get_plugins_from_packages - from ote_sdk.test_suite.pytest_insertions import * - from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT pytest_plugins = get_plugins_from_packages([e2e]) except ImportError: _e2e_pytest_addoption = None pass - import config - +import pytest +from ote_sdk.test_suite.pytest_insertions import * +from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT pytest_plugins = get_pytest_plugins_from_ote() From 6237601522e6fc7c935900eecd9ac1ef6b16c9c1 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Thu, 24 Mar 2022 22:44:17 +0300 Subject: [PATCH 12/16] minor --- external/mmdetection/tests/ote_cli/test_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/mmdetection/tests/ote_cli/test_detection.py b/external/mmdetection/tests/ote_cli/test_detection.py index 0ee0abdec49..964b1a1bed0 100644 --- a/external/mmdetection/tests/ote_cli/test_detection.py +++ b/external/mmdetection/tests/ote_cli/test_detection.py @@ -92,7 +92,7 @@ def test_ote_eval(self, template): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_ote_eval_openvino(self, template): - ote_eval_openvino_testing(template, root, ote_dir, args, threshold=0.1) + ote_eval_openvino_testing(template, root, ote_dir, args, threshold=0.2) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) From ce8d4e2ef92fa649f244a73e722836024df9affe Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Fri, 25 Mar 2022 09:41:42 +0300 Subject: [PATCH 13/16] minor --- ...tools_anomaly_detection.py => test_anomaly_detection.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename external/anomaly/tests/ote_cli/{test_ote_cli_tools_anomaly_detection.py => test_anomaly_detection.py} (96%) diff --git a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_detection.py b/external/anomaly/tests/ote_cli/test_anomaly_detection.py similarity index 96% rename from external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_detection.py rename to external/anomaly/tests/ote_cli/test_anomaly_detection.py index d36ec2dd45d..db68c75c231 100644 --- a/external/anomaly/tests/ote_cli/test_ote_cli_tools_anomaly_detection.py +++ b/external/anomaly/tests/ote_cli/test_anomaly_detection.py @@ -17,7 +17,7 @@ import os import pytest -from common import ( +from ote_cli.utils.tests import ( create_venv, get_some_vars, nncf_eval_openvino_testing, @@ -61,8 +61,8 @@ class TestToolsAnomalyDetection: @e2e_pytest_component def test_create_venv(self): - work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root) - create_venv(algo_backend_dir, work_dir, template_work_dir) + work_dir, _, algo_backend_dir = get_some_vars(templates[0], root) + create_venv(algo_backend_dir, work_dir) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) From a632052d5738d1d5156fe31a1fcaba67ea325287 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Fri, 25 Mar 2022 09:52:59 +0300 Subject: [PATCH 14/16] minor --- .../tests/ote_cli/test_anomaly_detection.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/external/anomaly/tests/ote_cli/test_anomaly_detection.py b/external/anomaly/tests/ote_cli/test_anomaly_detection.py index db68c75c231..e6412b169db 100644 --- a/external/anomaly/tests/ote_cli/test_anomaly_detection.py +++ b/external/anomaly/tests/ote_cli/test_anomaly_detection.py @@ -41,20 +41,20 @@ from ote_cli.registry import Registry args = { - '--train-ann-file': 'data/anomaly/detection/train.json', - '--train-data-roots': 'data/anomaly/shapes', - '--val-ann-file': 'data/anomaly/detection/val.json', - '--val-data-roots': 'data/anomaly/shapes', - '--test-ann-files': 'data/anomaly/detection/test.json', - '--test-data-roots': 'data/anomaly/shapes', - '--input': 'data/anomaly/shapes/test/hexagon', - 'train_params': [], + "--train-ann-file": "data/anomaly/detection/train.json", + "--train-data-roots": "data/anomaly/shapes", + "--val-ann-file": "data/anomaly/detection/val.json", + "--val-data-roots": "data/anomaly/shapes", + "--test-ann-files": "data/anomaly/detection/test.json", + "--test-data-roots": "data/anomaly/shapes", + "--input": "data/anomaly/shapes/test/hexagon", + "train_params": [], } -root = '/tmp/ote_cli/' +root = "/tmp/ote_cli/" ote_dir = os.getcwd() -templates = Registry('external').filter(task_type='ANOMALY_DETECTION').templates +templates = Registry("external").filter(task_type="ANOMALY_DETECTION").templates templates_ids = [template.model_template_id for template in templates] From 3e2a810b6cc1ecd22c5938dd6ad4b2cbfacb8ab1 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Fri, 25 Mar 2022 10:11:39 +0300 Subject: [PATCH 15/16] Update init_venv.sh --- external/anomaly/init_venv.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/anomaly/init_venv.sh b/external/anomaly/init_venv.sh index 0081c4fce0f..dc5f06b4080 100755 --- a/external/anomaly/init_venv.sh +++ b/external/anomaly/init_venv.sh @@ -104,8 +104,8 @@ else echo torchvision==${TORCHVISION_VERSION}+cu${CUDA_VERSION_CODE} >> ${CONSTRAINTS_FILE} fi -pip install -r requirements.txt -pip install -e . +pip install -r requirements.txt || exit 1 +pip install -e . || exit 1 # Install OTE SDK pip install -e ../../ote_sdk/ || exit 1 From 7301d0df043b97f0816294d06f90e147a5bf8451 Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Fri, 25 Mar 2022 10:54:22 +0300 Subject: [PATCH 16/16] code checks as pytest test --- tests/ote_cli/misc/test_code_checks.py | 24 ++++++++++++++++++++++++ tests/run_model_templates_tests.sh | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/ote_cli/misc/test_code_checks.py diff --git a/tests/ote_cli/misc/test_code_checks.py b/tests/ote_cli/misc/test_code_checks.py new file mode 100644 index 00000000000..a42eb7ed770 --- /dev/null +++ b/tests/ote_cli/misc/test_code_checks.py @@ -0,0 +1,24 @@ +# Copyright (C) 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import os +from subprocess import run + +from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component + +class TestCodeChecks: + @e2e_pytest_component + def test_code_checks(self): + wd = os.path.join(os.path.dirname(__file__), "..", "..", "..") + assert run(["./tests/run_code_checks.sh"], cwd=wd, check=True).returncode == 0 diff --git a/tests/run_model_templates_tests.sh b/tests/run_model_templates_tests.sh index 212fe42c75b..9de5d74a3c4 100755 --- a/tests/run_model_templates_tests.sh +++ b/tests/run_model_templates_tests.sh @@ -1,5 +1,3 @@ -./tests/run_code_checks.sh || exit 1 - python3 -m venv venv || exit 1 . venv/bin/activate || exit 1 pip install --upgrade pip || exit 1