From 1dc21b2667e385cb4850a5780d60d44788c4d07d Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 19 Sep 2023 05:33:08 +0900 Subject: [PATCH 01/13] Update OV --- requirements/openvino.txt | 4 ++-- src/otx/api/usecases/exportable_code/demo/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/openvino.txt b/requirements/openvino.txt index 0324ed22f42..cc1d0eb70f0 100644 --- a/requirements/openvino.txt +++ b/requirements/openvino.txt @@ -3,6 +3,6 @@ nncf==2.5.0 onnx==1.13.0 openvino-model-api==0.1.5 -openvino==2023.0 -openvino-dev==2023.0 +openvino==2023.1.0 +openvino-dev==2023.1.0 openvino-telemetry>=2022.1.0 diff --git a/src/otx/api/usecases/exportable_code/demo/requirements.txt b/src/otx/api/usecases/exportable_code/demo/requirements.txt index 061a8546f4e..62b9d3910d1 100644 --- a/src/otx/api/usecases/exportable_code/demo/requirements.txt +++ b/src/otx/api/usecases/exportable_code/demo/requirements.txt @@ -1,4 +1,4 @@ -openvino==2023.0 +openvino==2023.1.0 openvino-model-api==0.1.5 otx @ git+https://github.com/sovrasov/openvino_training_extensions/@38a9d7b20e3359d6139616c7127eac98d304c9f6#egg=otx numpy>=1.21.0,<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime From 8eb0045b00ac76d320218c981510a366a3aab1a5 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 19 Sep 2023 09:37:26 +0900 Subject: [PATCH 02/13] Update NNCF --- requirements/openvino.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/openvino.txt b/requirements/openvino.txt index cc1d0eb70f0..4baf403a51a 100644 --- a/requirements/openvino.txt +++ b/requirements/openvino.txt @@ -1,6 +1,6 @@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # OpenVINO Requirements. # -nncf==2.5.0 +nncf==2.6.0 onnx==1.13.0 openvino-model-api==0.1.5 openvino==2023.1.0 From 39d274f2937482a783f326c3c980600649c373d0 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Tue, 19 Sep 2023 09:40:18 +0900 Subject: [PATCH 03/13] Update demo requirements --- src/otx/api/usecases/exportable_code/demo/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otx/api/usecases/exportable_code/demo/requirements.txt b/src/otx/api/usecases/exportable_code/demo/requirements.txt index 62b9d3910d1..dc552c92775 100644 --- a/src/otx/api/usecases/exportable_code/demo/requirements.txt +++ b/src/otx/api/usecases/exportable_code/demo/requirements.txt @@ -1,4 +1,4 @@ openvino==2023.1.0 openvino-model-api==0.1.5 -otx @ git+https://github.com/sovrasov/openvino_training_extensions/@38a9d7b20e3359d6139616c7127eac98d304c9f6#egg=otx +otx @ git+https://github.com/openvinotoolkit/training_extensions/@8eb0045b00ac76d320218c981510a366a3aab1a5#egg=otx numpy>=1.21.0,<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime From 7a58b2beb7486ddb001ee4681cb5f994f31416ca Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Wed, 20 Sep 2023 02:50:56 +0900 Subject: [PATCH 04/13] Use datumaro with updated OV --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4ffd4af053e..e65497d3064 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ natsort>=6.0.0 prettytable protobuf>=3.20.0 pyyaml -datumaro==1.5.0rc0 +datumaro @ git+https://github.com/sovrasov/datumaro/@d846b7cada13cc1f03592e2f64ecb07037da834b#egg=datumaro psutil scipy>=1.8 bayesian-optimization>=1.2.0 From c7e6c772a9761702a4ae7417075aac2b7c2bcc98 Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Wed, 20 Sep 2023 15:57:09 +0900 Subject: [PATCH 05/13] add rust toolchain installation step for ov2023.1 testing --- .github/workflows/pre_merge.yml | 12 ++++++++++++ .github/workflows/run_tests_in_tox.yml | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index 5f13ac2ce9d..960a734fe87 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -22,6 +22,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + - name: Installing Rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true - name: Set up Python uses: actions/setup-python@v4 with: @@ -52,6 +58,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + - name: Installing Rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/run_tests_in_tox.yml b/.github/workflows/run_tests_in_tox.yml index b0e10b4f4d4..62871b1fe94 100644 --- a/.github/workflows/run_tests_in_tox.yml +++ b/.github/workflows/run_tests_in_tox.yml @@ -39,6 +39,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + - name: Installing Rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true - name: Set up Python uses: actions/setup-python@v4 with: From 975c4875ced3ef4ba08bdbf33329000ac1fe3d67 Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Wed, 4 Oct 2023 13:45:40 +0900 Subject: [PATCH 06/13] change rust toolchain setup action --- .github/workflows/pre_merge.yml | 12 ++---------- .github/workflows/run_tests_in_tox.yml | 6 +----- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index 71503054ac5..5a7014a5c61 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -23,11 +23,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Installing Rust toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Set up Python uses: actions/setup-python@v4 with: @@ -59,11 +55,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Installing Rust toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/run_tests_in_tox.yml b/.github/workflows/run_tests_in_tox.yml index 3b036b592ba..cae970b35fb 100644 --- a/.github/workflows/run_tests_in_tox.yml +++ b/.github/workflows/run_tests_in_tox.yml @@ -43,11 +43,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Installing Rust toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Set up Python uses: actions/setup-python@v4 with: From ebb1be4f455c6e5e7a8c13a58d5bd82400aaede4 Mon Sep 17 00:00:00 2001 From: Vladislav Sovrasov Date: Fri, 6 Oct 2023 06:42:53 +0900 Subject: [PATCH 07/13] Lower accuracy threshold for MRCNN in e2e --- .../e2e/cli/instance_segmentation/test_instance_segmentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py b/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py index 9dcc64fd21d..a48d1e02c5d 100644 --- a/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py +++ b/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py @@ -152,7 +152,7 @@ def test_otx_eval(self, template, tmp_dir_path): @pytest.mark.parametrize("half_precision", [True, False]) def test_otx_eval_openvino(self, template, tmp_dir_path, half_precision): tmp_dir_path = tmp_dir_path / "ins_seg" - otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=0.2, half_precision=half_precision) + otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=0.1, half_precision=half_precision) @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") From 1e34480a96addd0674ddbf2771cbbfa6d447fbfb Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Fri, 6 Oct 2023 13:56:30 +0900 Subject: [PATCH 08/13] bump datumaro to 1.5.1rc1 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index e65497d3064..4bc2a30c0de 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ natsort>=6.0.0 prettytable protobuf>=3.20.0 pyyaml -datumaro @ git+https://github.com/sovrasov/datumaro/@d846b7cada13cc1f03592e2f64ecb07037da834b#egg=datumaro +datumaro~=1.5.1rc1 psutil scipy>=1.8 bayesian-optimization>=1.2.0 From 41e1dc5fc398d990d64e381f6a33e40b21645606 Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Fri, 6 Oct 2023 13:58:39 +0900 Subject: [PATCH 09/13] remove rust toolchain setup step from workflow --- .github/workflows/pre_merge.yml | 4 ---- .github/workflows/run_tests_in_tox.yml | 2 -- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index 5a7014a5c61..142b3b38781 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -22,8 +22,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Installing Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Set up Python uses: actions/setup-python@v4 with: @@ -54,8 +52,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Installing Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/run_tests_in_tox.yml b/.github/workflows/run_tests_in_tox.yml index cae970b35fb..aaa00970b74 100644 --- a/.github/workflows/run_tests_in_tox.yml +++ b/.github/workflows/run_tests_in_tox.yml @@ -42,8 +42,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Installing Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Set up Python uses: actions/setup-python@v4 with: From d90c80da19da8946c18094142c2cd1d33fed43e9 Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Fri, 6 Oct 2023 19:30:56 +0900 Subject: [PATCH 10/13] bump datumaro version to 1.6.0rc0 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4bc2a30c0de..859cc9f8ddf 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ natsort>=6.0.0 prettytable protobuf>=3.20.0 pyyaml -datumaro~=1.5.1rc1 +datumaro~=1.6.0rc0 psutil scipy>=1.8 bayesian-optimization>=1.2.0 From 82aed8234f3b0317f6cbca323874cde378fa6f8d Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Wed, 18 Oct 2023 22:40:05 +0900 Subject: [PATCH 11/13] Update OTX commit in demo --- src/otx/api/usecases/exportable_code/demo/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otx/api/usecases/exportable_code/demo/requirements.txt b/src/otx/api/usecases/exportable_code/demo/requirements.txt index 6dd9ac00e8d..95593e05160 100644 --- a/src/otx/api/usecases/exportable_code/demo/requirements.txt +++ b/src/otx/api/usecases/exportable_code/demo/requirements.txt @@ -1,4 +1,4 @@ openvino==2023.0 openvino-model-api==0.1.6 -otx @ git+https://github.com/openvinotoolkit/training_extensions/@4bfbf8a4873d3d39a369cb2c3f08cfb358feb21e#egg=otx +otx @ git+https://github.com/openvinotoolkit/training_extensions/@0e1200f4cc5343dcac597674fc97a898fc9829c2#egg=otx numpy>=1.21.0,<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime From de70e71c2d18595fe7f144f2119aef63ac7266c3 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Wed, 18 Oct 2023 23:36:14 +0900 Subject: [PATCH 12/13] Update IS tests --- .../cli/instance_segmentation/test_instance_segmentation.py | 2 +- tests/e2e/cli/instance_segmentation/test_tiling_instseg.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py b/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py index a48d1e02c5d..9dcc64fd21d 100644 --- a/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py +++ b/tests/e2e/cli/instance_segmentation/test_instance_segmentation.py @@ -152,7 +152,7 @@ def test_otx_eval(self, template, tmp_dir_path): @pytest.mark.parametrize("half_precision", [True, False]) def test_otx_eval_openvino(self, template, tmp_dir_path, half_precision): tmp_dir_path = tmp_dir_path / "ins_seg" - otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=0.1, half_precision=half_precision) + otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=0.2, half_precision=half_precision) @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") diff --git a/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py b/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py index dc8777bf543..4c808069c5e 100644 --- a/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py +++ b/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py @@ -124,7 +124,6 @@ def test_otx_eval(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") @pytest.mark.parametrize("half_precision", [True, False]) def test_otx_eval_openvino(self, template, tmp_dir_path, half_precision): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" @@ -140,7 +139,6 @@ def test_otx_explain(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") def test_otx_explain_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" otx_explain_openvino_testing(template, tmp_dir_path, otx_dir, args) @@ -155,7 +153,6 @@ def test_otx_demo(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") def test_otx_demo_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" otx_demo_openvino_testing(template, tmp_dir_path, otx_dir, args) @@ -163,7 +160,6 @@ def test_otx_demo_openvino(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") def test_otx_deploy_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" otx_deploy_openvino_testing(template, tmp_dir_path, otx_dir, args) @@ -171,7 +167,6 @@ def test_otx_deploy_openvino(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") def test_otx_eval_deployment(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" otx_eval_deployment_testing(template, tmp_dir_path, otx_dir, args, threshold=0.0) @@ -179,7 +174,6 @@ def test_otx_eval_deployment(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) - @pytest.mark.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") def test_otx_demo_deployment(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" otx_demo_deployment_testing(template, tmp_dir_path, otx_dir, args) From 2641fbd96b9ec49ce84dcd4cd8cb7b5202905cf5 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Thu, 19 Oct 2023 19:48:16 +0900 Subject: [PATCH 13/13] Update skips in iseg tiling tests --- .../cli/instance_segmentation/test_tiling_instseg.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py b/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py index 4c808069c5e..30e7aec80a2 100644 --- a/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py +++ b/tests/e2e/cli/instance_segmentation/test_tiling_instseg.py @@ -127,6 +127,8 @@ def test_otx_eval(self, template, tmp_dir_path): @pytest.mark.parametrize("half_precision", [True, False]) def test_otx_eval_openvino(self, template, tmp_dir_path, half_precision): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" + if "ResNet50" in template.name: + pytest.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=0.2, half_precision=half_precision) @e2e_pytest_component @@ -141,6 +143,8 @@ def test_otx_explain(self, template, tmp_dir_path): @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_explain_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" + if "ResNet50" in template.name: + pytest.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") otx_explain_openvino_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component @@ -155,12 +159,16 @@ def test_otx_demo(self, template, tmp_dir_path): @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_demo_openvino(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" + if "ResNet50" in template.name: + pytest.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") otx_demo_openvino_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component @pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS") @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_deploy_openvino(self, template, tmp_dir_path): + if "ResNet50" in template.name: + pytest.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") tmp_dir_path = tmp_dir_path / "tiling_ins_seg" otx_deploy_openvino_testing(template, tmp_dir_path, otx_dir, args) @@ -169,6 +177,8 @@ def test_otx_deploy_openvino(self, template, tmp_dir_path): @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_eval_deployment(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" + if "ResNet50" in template.name: + pytest.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") otx_eval_deployment_testing(template, tmp_dir_path, otx_dir, args, threshold=0.0) @e2e_pytest_component @@ -176,6 +186,8 @@ def test_otx_eval_deployment(self, template, tmp_dir_path): @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_demo_deployment(self, template, tmp_dir_path): tmp_dir_path = tmp_dir_path / "tiling_ins_seg" + if "ResNet50" in template.name: + pytest.skip(reason="Issue#2290: MaskRCNN shows degraded performance when inferencing in OpenVINO") otx_demo_deployment_testing(template, tmp_dir_path, otx_dir, args) @e2e_pytest_component