Skip to content

Commit

Permalink
Disable QAT for SegNexts (#2565)
Browse files Browse the repository at this point in the history 
* Disable NNCF QAT for SegNext

* Del obsolete pot configs

* Move NNCF skip marks to test commands to avoid duplication
  • Loading branch information
@sovrasov
sovrasov authored Oct 19, 2023
1 parent f512025 commit a5193b1
Show file tree
Hide file tree
Showing 11 changed files with 11 additions and 117 deletions.
14 changes: 0 additions & 14 deletions src/otx/algorithms/segmentation/configs/ham_segnext_b/pot_optimization_config.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion src/otx/algorithms/segmentation/configs/ham_segnext_b/template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ framework: OTXSegmentation v0.14.0
entrypoints:
base: otx.algorithms.segmentation.adapters.mmseg.task.MMSegmentationTask
openvino: otx.algorithms.segmentation.adapters.openvino.task.OpenVINOSegmentationTask
nncf: otx.algorithms.segmentation.adapters.mmseg.nncf.task.SegmentationNNCFTask

# Capabilities.
capabilities:
Expand Down
14 changes: 0 additions & 14 deletions src/otx/algorithms/segmentation/configs/ham_segnext_s/pot_optimization_config.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion src/otx/algorithms/segmentation/configs/ham_segnext_s/template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ framework: OTXSegmentation v0.14.0
entrypoints:
base: otx.algorithms.segmentation.adapters.mmseg.task.MMSegmentationTask
openvino: otx.algorithms.segmentation.adapters.openvino.task.OpenVINOSegmentationTask
nncf: otx.algorithms.segmentation.adapters.mmseg.nncf.task.SegmentationNNCFTask

# Capabilities.
capabilities:
Expand Down
14 changes: 0 additions & 14 deletions src/otx/algorithms/segmentation/configs/ham_segnext_t/pot_optimization_config.json
View file

This file was deleted.

1 change: 0 additions & 1 deletion src/otx/algorithms/segmentation/configs/ham_segnext_t/template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ framework: OTXSegmentation v0.14.0
entrypoints:
base: otx.algorithms.segmentation.adapters.mmseg.task.MMSegmentationTask
openvino: otx.algorithms.segmentation.adapters.openvino.task.OpenVINOSegmentationTask
nncf: otx.algorithms.segmentation.adapters.mmseg.nncf.task.SegmentationNNCFTask

# Capabilities.
capabilities:
Expand Down
45 changes: 0 additions & 45 deletions tests/e2e/cli/classification/test_classification.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,49 +229,34 @@ def test_otx_hpo(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_class_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_export(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_class_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_export_testing(template, tmp_dir_path)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_validate_fq(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_class_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_validate_fq_testing(template, tmp_dir_path, otx_dir, "classification", type(self).__name__)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_class_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_testing(template, tmp_dir_path, otx_dir, args, threshold=0.01)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval_openvino(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_class_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_openvino_testing(template, tmp_dir_path, otx_dir, args)

@e2e_pytest_component
Expand Down Expand Up @@ -452,49 +437,34 @@ def test_otx_hpo(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args_m)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_export(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_export_testing(template, tmp_dir_path)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_validate_fq(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_validate_fq_testing(template, tmp_dir_path, otx_dir, "classification", type(self).__name__)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_testing(template, tmp_dir_path, otx_dir, args_m, threshold=0.01)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval_openvino(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_openvino_testing(template, tmp_dir_path, otx_dir, args_m)

@e2e_pytest_component
Expand Down Expand Up @@ -630,49 +600,34 @@ def test_otx_hpo(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "h_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args_h)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_export(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "h_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_export_testing(template, tmp_dir_path)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "h_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_testing(template, tmp_dir_path, otx_dir, args_h, threshold=0.01)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_validate_fq(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "h_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_validate_fq_testing(template, tmp_dir_path, otx_dir, "classification", type(self).__name__)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval_openvino(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "h_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_openvino_testing(template, tmp_dir_path, otx_dir, args_h)

@e2e_pytest_component
Expand Down
15 changes: 0 additions & 15 deletions tests/e2e/cli/semantic_segmentation/test_segmentation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,49 +188,34 @@ def test_otx_hpo(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "segmentation"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_export(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "segmentation"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_export_testing(template, tmp_dir_path)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_validate_fq(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "segmentation"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_validate_fq_testing(template, tmp_dir_path, otx_dir, "semantic_segmentation", type(self).__name__)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "segmentation"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_testing(template, tmp_dir_path, otx_dir, args, threshold=0.01)

@e2e_pytest_component
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_eval_openvino(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "segmentation"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_eval_openvino_testing(template, tmp_dir_path, otx_dir, args)

@e2e_pytest_component
Expand Down
9 changes: 0 additions & 9 deletions tests/integration/cli/classification/test_classification.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,6 @@ def test_otx_hpo(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", templates, ids=templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_class_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args)

@e2e_pytest_component
Expand Down Expand Up @@ -436,9 +433,6 @@ def test_otx_eval_deployment(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", default_templates, ids=default_templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "multi_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args_m)

@e2e_pytest_component
Expand Down Expand Up @@ -576,7 +570,4 @@ def test_otx_eval_deployment(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", default_templates, ids=default_templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "h_label_cls"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args_h)
3 changes: 0 additions & 3 deletions tests/integration/cli/semantic_segmentation/test_segmentation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,9 +181,6 @@ def test_otx_hpo(self, template, tmp_dir_path):
@pytest.mark.parametrize("template", default_templates, ids=default_templates_ids)
def test_nncf_optimize(self, template, tmp_dir_path):
tmp_dir_path = tmp_dir_path / "segmentation"
if template.entrypoints.nncf is None:
pytest.skip("nncf entrypoint is none")

nncf_optimize_testing(template, tmp_dir_path, otx_dir, args)

@e2e_pytest_component
Expand Down
11 changes: 11 additions & 0 deletions tests/test_suite/run_test_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,9 @@ def ptq_eval_testing(template, root, otx_dir, args, is_visual_prompting=False):


def nncf_optimize_testing(template, root, otx_dir, args):
if template.entrypoints.nncf is None:
pytest.skip("NNCF QAT is disabled: entrypoints.nncf in template is not specified")

template_work_dir = get_template_dir(template, root)
command_line = [
"otx",
Expand All @@ -649,6 +652,8 @@ def nncf_optimize_testing(template, root, otx_dir, args):


def nncf_export_testing(template, root):
if template.entrypoints.nncf is None:
pytest.skip("NNCF QAT is disabled: entrypoints.nncf in template is not specified")
template_work_dir = get_template_dir(template, root)
command_line = [
"otx",
Expand Down Expand Up @@ -679,6 +684,8 @@ def nncf_export_testing(template, root):


def nncf_validate_fq_testing(template, root, otx_dir, task_type, test_name):
if template.entrypoints.nncf is None:
pytest.skip("NNCF QAT is disabled: entrypoints.nncf in template is not specified")
template_work_dir = get_template_dir(template, root)
xml_path = f"{template_work_dir}/exported_nncf_{template.model_template_id}/openvino.xml"
path_to_ref_data = os.path.join(
Expand All @@ -689,6 +696,8 @@ def nncf_validate_fq_testing(template, root, otx_dir, task_type, test_name):


def nncf_eval_testing(template, root, otx_dir, args, threshold=0.01):
if template.entrypoints.nncf is None:
pytest.skip("NNCF QAT is disabled: entrypoints.nncf in template is not specified")
template_work_dir = get_template_dir(template, root)
command_line = [
"otx",
Expand Down Expand Up @@ -717,6 +726,8 @@ def nncf_eval_testing(template, root, otx_dir, args, threshold=0.01):


def nncf_eval_openvino_testing(template, root, otx_dir, args):
if template.entrypoints.nncf is None:
pytest.skip("NNCF QAT is disabled: entrypoints.nncf in template is not specified")
template_work_dir = get_template_dir(template, root)
command_line = [
"otx",
Expand Down

0 comments on commit a5193b1

Please sign in to comment.