Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix some export issues. Remove EXPORTABLE_CODE as export parameter. #3577

Merged
merged 14 commits into from
Jun 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions src/otx/core/exporter/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
base_model_name: str = "exported_model",
export_format: OTXExportFormatType = OTXExportFormatType.OPENVINO,
precision: OTXPrecisionType = OTXPrecisionType.FP32,
to_exportable_code: bool = False,
kprokofi marked this conversation as resolved.
Show resolved Hide resolved
) -> Path:
"""Exports input model to the specified deployable format, such as OpenVINO IR or ONNX.

Expand All @@ -91,21 +92,22 @@
base_model_name (str, optional): exported model name
format (OTXExportFormatType): final format of the exported model
precision (OTXExportPrecisionType, optional): precision of the exported model's weights
to_exportable_code (bool, optional): whether to generate exportable code

Returns:
Path: path to the exported model
"""
if export_format == OTXExportFormatType.OPENVINO:
if to_exportable_code:
return self.to_exportable_code(

Check warning on line 102 in src/otx/core/exporter/base.py

View check run for this annotation

Codecov / codecov/patch

src/otx/core/exporter/base.py#L102

Added line #L102 was not covered by tests
model,
output_dir,
base_model_name,
precision,
)
return self.to_openvino(model, output_dir, base_model_name, precision)
if export_format == OTXExportFormatType.ONNX:
return self.to_onnx(model, output_dir, base_model_name, precision)
if export_format == OTXExportFormatType.EXPORTABLE_CODE:
return self.to_exportable_code(
model,
output_dir,
base_model_name,
precision,
)

msg = f"Unsupported export format: {export_format}"
raise ValueError(msg)
Expand Down
9 changes: 6 additions & 3 deletions src/otx/core/exporter/visual_prompting.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ def export( # type: ignore[override]
base_model_name: str = "exported_model",
export_format: OTXExportFormatType = OTXExportFormatType.OPENVINO,
precision: OTXPrecisionType = OTXPrecisionType.FP32,
to_exportable_code: bool = False,
) -> dict[str, Path]:
"""Exports input model to the specified deployable format, such as OpenVINO IR or ONNX.

Expand All @@ -41,6 +42,8 @@ def export( # type: ignore[override]
base_model_name (str, optional): exported model name
format (OTXExportFormatType): final format of the exported model
precision (OTXExportPrecisionType, optional): precision of the exported model's weights
to_exportable_code (bool, optional): whether to generate exportable code.
Currently not supported by Visual Promting task.

Returns:
dict[str, Path]: paths to the exported models
Expand All @@ -53,12 +56,12 @@ def export( # type: ignore[override]
}

if export_format == OTXExportFormatType.OPENVINO:
if to_exportable_code:
msg = "Exportable code option is not supported and will be ignored."
log.warning(msg)
fn = self.to_openvino
elif export_format == OTXExportFormatType.ONNX:
fn = self.to_onnx
elif export_format == OTXExportFormatType.EXPORTABLE_CODE:
msg = "exportable code will be supported soon."
raise NotImplementedError(msg)
else:
msg = f"Unsupported export format: {export_format}"
raise ValueError(msg)
Expand Down
42 changes: 41 additions & 1 deletion src/otx/core/model/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,7 @@
base_name: str,
export_format: OTXExportFormatType,
precision: OTXPrecisionType = OTXPrecisionType.FP32,
to_exportable_code: bool = False,
kprokofi marked this conversation as resolved.
Show resolved Hide resolved
) -> Path:
"""Export this model to the specified output directory.

Expand All @@ -630,6 +631,7 @@
base_name: (str): base name for the exported model file. Extension is defined by the target export format
export_format (OTXExportFormatType): format of the output model
precision (OTXExportPrecisionType): precision of the output model
to_exportable_code (bool): flag to export model in exportable code with demo package

Returns:
Path: path to the exported model.
Expand All @@ -649,6 +651,7 @@
base_name,
export_format,
precision,
to_exportable_code,
)
finally:
self.train(mode)
Expand Down Expand Up @@ -965,6 +968,40 @@

return output_model_path

def export(
self,
output_dir: Path,
base_name: str,
export_format: OTXExportFormatType,
precision: OTXPrecisionType = OTXPrecisionType.FP32,
to_exportable_code: bool = True,
) -> Path:
"""Export this model to the specified output directory.

Args:
output_dir (Path): directory for saving the exported model
base_name: (str): base name for the exported model file. Extension is defined by the target export format
export_format (OTXExportFormatType): format of the output model
precision (OTXExportPrecisionType): precision of the output model
to_exportable_code (bool): whether to generate exportable code with demo package.
OpenVINO model supports only exportable code option.

Returns:
Path: path to the exported model.
"""
if not to_exportable_code:
msg = "OpenVINO model can be exported only as exportable code with demo package."
raise RuntimeError(msg)

Check warning on line 994 in src/otx/core/model/base.py

View check run for this annotation

Codecov / codecov/patch

src/otx/core/model/base.py#L992-L994

Added lines #L992 - L994 were not covered by tests

return self._exporter.export(

Check warning on line 996 in src/otx/core/model/base.py

View check run for this annotation

Codecov / codecov/patch

src/otx/core/model/base.py#L996

Added line #L996 was not covered by tests
self.model,
output_dir,
base_name,
export_format,
precision,
to_exportable_code,
)

def transform_fn(self, data_batch: T_OTXBatchDataEntity) -> np.array:
"""Data transform function for PTQ."""
np_data = self._customize_inputs(data_batch)
Expand Down Expand Up @@ -1014,7 +1051,10 @@
@property
def _exporter(self) -> OTXNativeModelExporter:
"""Exporter of the OVModel for exportable code."""
return OTXNativeModelExporter(input_size=(1, 3, self.model.h, self.model.w), **self._export_parameters)
return OTXNativeModelExporter(

Check warning on line 1054 in src/otx/core/model/base.py

View check run for this annotation

Codecov / codecov/patch

src/otx/core/model/base.py#L1054

Added line #L1054 was not covered by tests
task_level_export_parameters=self._export_parameters,
input_size=(1, 3, self.model.h, self.model.w),
)

@property
def model_adapter_parameters(self) -> dict:
Expand Down
1 change: 0 additions & 1 deletion src/otx/core/types/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ class OTXExportFormatType(str, Enum):

ONNX = "ONNX"
OPENVINO = "OPENVINO"
EXPORTABLE_CODE = "EXPORTABLE_CODE"


@dataclass(frozen=True)
Expand Down
20 changes: 14 additions & 6 deletions src/otx/engine/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,7 @@
export_format: OTXExportFormatType = OTXExportFormatType.OPENVINO,
export_precision: OTXPrecisionType = OTXPrecisionType.FP32,
explain: bool = False,
export_demo_package: bool = False,
kprokofi marked this conversation as resolved.
Show resolved Hide resolved
) -> Path:
"""Export the trained model to OpenVINO Intermediate Representation (IR) or ONNX formats.

Expand All @@ -498,6 +499,8 @@
export_config (ExportConfig | None, optional): Config that allows to set export
format and precision. Defaults to None.
explain (bool): Whether to get "saliency_map" and "feature_vector" or not.
export_demo_package (bool): Whether to export demo package with the model.
Only OpenVINO model can be exported with demo package.

Returns:
Path: Path to the exported model.
Expand Down Expand Up @@ -530,14 +533,18 @@
msg = "To make export, checkpoint must be specified."
raise RuntimeError(msg)
is_ir_ckpt = Path(checkpoint).suffix in [".xml"]

if is_ir_ckpt and export_format != OTXExportFormatType.EXPORTABLE_CODE:
if export_demo_package and export_format == OTXExportFormatType.ONNX:
msg = (
"Export format is automatically changed to EXPORTABLE_CODE, "
"since openvino IR model is passed as a checkpoint."
"ONNX export is not supported in exportable code mode. "
"Exportable code parameter will be disregarded. "
)
warn(msg, stacklevel=1)
export_format = OTXExportFormatType.EXPORTABLE_CODE
export_demo_package = False

if is_ir_ckpt and not export_demo_package:
msg = "IR model is passed as a checkpoint, export automaticaly switched to exportable code."
warn(msg, stacklevel=1)
export_demo_package = True

Check warning on line 547 in src/otx/engine/engine.py

View check run for this annotation

Codecov / codecov/patch

src/otx/engine/engine.py#L545-L547

Added lines #L545 - L547 were not covered by tests

if is_ir_ckpt and not isinstance(self.model, OVModel):
# create OVModel
Expand All @@ -557,6 +564,7 @@
base_name=self._EXPORTED_MODEL_BASE_NAME,
export_format=export_format,
precision=export_precision,
to_exportable_code=export_demo_package,
)

self.model.explain_mode = False
Expand Down Expand Up @@ -636,7 +644,7 @@
tmp_model_path = model.optimize(Path(tmp_dir), optimize_datamodule, ptq_config)
return self.export(
checkpoint=tmp_model_path,
export_format=OTXExportFormatType.EXPORTABLE_CODE,
export_demo_package=True,
)

def explain(
Expand Down
23 changes: 12 additions & 11 deletions tests/unit/core/exporter/test_visual_prompting.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ def test_export_openvino(self, mocker, tmpdir, otx_visual_prompting_model_export
mocker_postprocess_openvino_model.assert_called()
mocker_openvino_save_model.assert_called()

otx_visual_prompting_model_exporter.export(
model=mock_model,
output_dir=tmpdir,
export_format=OTXExportFormatType.OPENVINO,
to_exportable_code=True,
)

# ensure that export to openvino model called
# and to_exportable_code was ignored
mocker_openvino_convert_model.assert_called()
mocker_openvino_save_model.assert_called()

def test_export_onnx(self, mocker, tmpdir, otx_visual_prompting_model_exporter) -> None:
"""Test export for ONNX."""
mocker_torch_onnx_export = mocker.patch("torch.onnx.export")
Expand All @@ -70,14 +82,3 @@ def test_export_onnx(self, mocker, tmpdir, otx_visual_prompting_model_exporter)
mocker_onnx_load.assert_called()
mocker_onnx_save.assert_called()
mocker_postprocess_onnx_model.assert_called()

def test_export_exportable_code(self, mocker, tmpdir, otx_visual_prompting_model_exporter) -> None:
"""Test export for EXPORTABLE_CODE."""
mock_model = mocker.MagicMock()

with pytest.raises(NotImplementedError):
otx_visual_prompting_model_exporter.export(
model=mock_model,
output_dir=tmpdir,
export_format=OTXExportFormatType.EXPORTABLE_CODE,
)
17 changes: 15 additions & 2 deletions tests/unit/engine/test_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ def test_exporting(self, fxt_engine, mocker) -> None:
base_name="exported_model",
export_format=OTXExportFormatType.OPENVINO,
precision=OTXPrecisionType.FP32,
to_exportable_code=False,
)

fxt_engine.export(export_precision=OTXPrecisionType.FP16)
Expand All @@ -208,6 +209,7 @@ def test_exporting(self, fxt_engine, mocker) -> None:
base_name="exported_model",
export_format=OTXExportFormatType.OPENVINO,
precision=OTXPrecisionType.FP16,
to_exportable_code=False,
)

fxt_engine.export(export_format=OTXExportFormatType.ONNX)
Expand All @@ -216,6 +218,16 @@ def test_exporting(self, fxt_engine, mocker) -> None:
base_name="exported_model",
export_format=OTXExportFormatType.ONNX,
precision=OTXPrecisionType.FP32,
to_exportable_code=False,
)

fxt_engine.export(export_format=OTXExportFormatType.ONNX, export_demo_package=True)
mock_export.assert_called_with(
output_dir=Path(fxt_engine.work_dir),
base_name="exported_model",
export_format=OTXExportFormatType.ONNX,
precision=OTXPrecisionType.FP32,
to_exportable_code=False,
)

# check exportable code with IR OpenVINO model
Expand All @@ -225,13 +237,14 @@ def test_exporting(self, fxt_engine, mocker) -> None:
"otx.engine.engine.AutoConfigurator.get_ov_model",
return_value=OVModel(model_name="efficientnet-b0-pytorch", model_type="classification"),
)
fxt_engine.export(export_format=OTXExportFormatType.EXPORTABLE_CODE, checkpoint="path/to/checkpoint.xml")
fxt_engine.export(checkpoint="path/to/checkpoint.xml", export_demo_package=True)
mock_get_ov_model.assert_called_once()
mock_export.assert_called_with(
output_dir=Path(fxt_engine.work_dir),
base_name="exported_model",
export_format=OTXExportFormatType.EXPORTABLE_CODE,
export_format=OTXExportFormatType.OPENVINO,
precision=OTXPrecisionType.FP32,
to_exportable_code=True,
)

def test_optimizing_model(self, fxt_engine, mocker) -> None:
Expand Down
Loading