From 2811c99818c372b784e5f14cf43440edc9dc52de Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Mon, 8 Jan 2024 23:13:05 +0800 Subject: [PATCH 1/9] [Core] implement BGR2RGB for postprocess --- .../core/preprocess/output_model_info.hpp | 11 +++ .../core/preprocess/postprocess_steps.hpp | 8 ++ src/core/src/preprocess/pre_post_process.cpp | 11 +++ src/core/src/preprocess/preprocess_impls.cpp | 4 + src/core/src/preprocess/preprocess_impls.hpp | 21 ++++- .../src/preprocess/preprocess_steps_impl.cpp | 74 +++++++++++++++++ .../src/preprocess/preprocess_steps_impl.hpp | 24 ++++-- src/core/tests/preprocess.cpp | 82 +++++++++++++++++++ .../subgraph_tests/preprocess.cpp | 9 +- .../subgraph/preprocess.hpp | 15 ++++ .../preprocess/preprocess_builders.hpp | 30 +++++++ 11 files changed, 278 insertions(+), 11 deletions(-) diff --git a/src/core/include/openvino/core/preprocess/output_model_info.hpp b/src/core/include/openvino/core/preprocess/output_model_info.hpp index 752985d724a64b..e5b96ee5134bb2 100644 --- a/src/core/include/openvino/core/preprocess/output_model_info.hpp +++ b/src/core/include/openvino/core/preprocess/output_model_info.hpp @@ -6,6 +6,7 @@ #include "openvino/core/core_visibility.hpp" #include "openvino/core/layout.hpp" +#include "openvino/core/preprocess/color_format.hpp" namespace ov { namespace preprocess { @@ -42,6 +43,16 @@ class OPENVINO_API OutputModelInfo final { /// /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner OutputModelInfo& set_layout(const ov::Layout& layout); + + /// \brief Set color format for model's output tensor + /// + /// \param format Color format for model's output tensor. + /// + /// \param sub_names Optional list of sub-names, not used, placeholder for future. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + OutputModelInfo& set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names = {}); }; } // namespace preprocess diff --git a/src/core/include/openvino/core/preprocess/postprocess_steps.hpp b/src/core/include/openvino/core/preprocess/postprocess_steps.hpp index ebb3e03ab17f83..d9e0cf75c74f47 100644 --- a/src/core/include/openvino/core/preprocess/postprocess_steps.hpp +++ b/src/core/include/openvino/core/preprocess/postprocess_steps.hpp @@ -93,6 +93,14 @@ class OPENVINO_API PostProcessSteps final { /// /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner PostProcessSteps& custom(const CustomPostprocessOp& postprocess_cb); + + /// \brief Converts color format for user's output tensor. Requires destinantion color format to be specified by + /// OutputTensorInfo::set_color_format. + /// + /// \param dst_format Destination color format of input image + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PostProcessSteps& convert_color(const ov::preprocess::ColorFormat& dst_format); }; } // namespace preprocess diff --git a/src/core/src/preprocess/pre_post_process.cpp b/src/core/src/preprocess/pre_post_process.cpp index c219a77d91a780..0350437299a3c1 100644 --- a/src/core/src/preprocess/pre_post_process.cpp +++ b/src/core/src/preprocess/pre_post_process.cpp @@ -361,6 +361,12 @@ OutputModelInfo& OutputModelInfo::set_layout(const Layout& layout) { return *this; } +OutputModelInfo& OutputModelInfo::set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names) { + m_impl->set_color_format(format); + return *this; +} + // --------------------- PostProcessSteps ------------------ PostProcessSteps::PostProcessSteps() : m_impl(std::unique_ptr(new PostProcessStepsImpl())) {} @@ -381,6 +387,11 @@ PostProcessSteps& PostProcessSteps::convert_layout(const std::vector& return *this; } +PostProcessSteps& PostProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) { + m_impl->add_convert_color_impl(dst_format); + return *this; +} + PostProcessSteps& PostProcessSteps::custom(const CustomPostprocessOp& postprocess_cb) { // 'true' indicates that custom postprocessing step will trigger validate_and_infer_types m_impl->actions().emplace_back( diff --git a/src/core/src/preprocess/preprocess_impls.cpp b/src/core/src/preprocess/preprocess_impls.cpp index 57a02fae606220..1888a4c62fb738 100644 --- a/src/core/src/preprocess/preprocess_impls.cpp +++ b/src/core/src/preprocess/preprocess_impls.cpp @@ -338,6 +338,10 @@ void OutputInfo::OutputInfoImpl::build(ov::ResultVector& results) { if (get_tensor_data()->is_element_type_set()) { context.target_element_type() = get_tensor_data()->get_element_type(); } + if (get_model_data()->is_color_format_set()) { + context.color_format() = get_model_data()->get_color_format(); + } + // Apply post-processing node = result->get_input_source_output(0); bool post_processing_applied = false; diff --git a/src/core/src/preprocess/preprocess_impls.hpp b/src/core/src/preprocess/preprocess_impls.hpp index 64ebfd736f4ff3..5e8e5c0153bb08 100644 --- a/src/core/src/preprocess/preprocess_impls.hpp +++ b/src/core/src/preprocess/preprocess_impls.hpp @@ -37,7 +37,26 @@ class ModelInfoImpl { class InputModelInfo::InputModelInfoImpl : public ModelInfoImpl {}; -class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl {}; +class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl { +public: + void set_color_format(const ColorFormat& color_format, const std::vector& sub_names = {}) { + OPENVINO_ASSERT (color_format == ColorFormat::RGB || color_format == ColorFormat::BGR); + m_color_format = color_format; + m_planes_sub_names = sub_names; + m_color_format_set = true; + } + bool is_color_format_set() const { + return m_color_format_set; + } + const ColorFormat& get_color_format() const { + return m_color_format; + } + +private: + ColorFormat m_color_format = ColorFormat::UNDEFINED; + std::vector m_planes_sub_names; + bool m_color_format_set = false; +}; /// \brief OutputInfoImpl - internal data structure struct OutputInfo::OutputInfoImpl { diff --git a/src/core/src/preprocess/preprocess_steps_impl.cpp b/src/core/src/preprocess/preprocess_steps_impl.cpp index e155f89f5ac88d..29a4bc084f057e 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.cpp +++ b/src/core/src/preprocess/preprocess_steps_impl.cpp @@ -695,5 +695,79 @@ void PostStepsList::add_convert_layout_impl(const std::vector& dims) { }, "convert layout " + vector_to_string(dims)); } + +void PostStepsList::add_convert_color_impl(const ColorFormat& dst_format) { + m_actions.emplace_back( + [dst_format](const Output& node, PostprocessingContext& context) { + if (context.color_format() == dst_format) { + return std::make_tuple(node, false); + } + if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) && + (dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) { + auto res = reverse_channels({node}, nullptr, context); + context.color_format() = dst_format; + return std::make_tuple(std::get<0>(res)[0], std::get<1>(res));; + } + OPENVINO_ASSERT(false, + "Source color format '", + color_format_name(context.color_format()), + "' is not convertible to '", + color_format_name(dst_format), + "'"); + }, + "convert color (" + color_format_name(dst_format) + ")"); +} + +std::tuple>, bool> PostStepsList::reverse_channels(const std::vector>& nodes, + const std::shared_ptr& function, + PostprocessingContext& context) { + OPENVINO_ASSERT(nodes.size() == 1, "Internal error: can't reverse channels for multi-plane inputs"); + OPENVINO_ASSERT(ov::layout::has_channels(context.layout()), + "Layout ", + context.layout().to_string(), + " doesn't have `channels` dimension"); + auto shape = nodes[0].get_partial_shape(); + if (shape.rank().is_static()) { + // This block of code is to preserve output shape if it contains dynamic dimensions + // Otherwise, dynamic version will transform shape {?,3,?,?} to {?,?,?,?} which is still ok but not desired + auto channels_idx = get_and_check_channels_idx(context.layout(), shape); + if (shape[channels_idx].is_static()) { + auto channels_count = shape[channels_idx].get_length(); + // Add range from constants + auto range_from = op::v0::Constant::create(element::i64, {}, {channels_count - 1}); + auto range_to = op::v0::Constant::create(element::i64, {}, {-1}); + auto range_step = op::v0::Constant::create(element::i64, {}, {-1}); + auto range = std::make_shared(range_from, range_to, range_step, element::i32); + + auto constant_axis = op::v0::Constant::create(element::i32, {1}, {channels_idx}); + auto convert = std::make_shared(nodes[0], range, constant_axis); + return std::make_tuple(std::vector>{convert}, false); + } + } + + auto channels_idx = ov::layout::channels_idx(context.layout()); + // Get shape of user's input tensor (e.g. Tensor[1, 3, 224, 224] -> {1, 3, 224, 224}) + auto shape_of = std::make_shared(nodes[0]); // E.g. {1, 3, 224, 224} + + auto constant_chan_idx = op::v0::Constant::create(element::i32, {}, {channels_idx}); // E.g. 1 + auto constant_chan_axis = op::v0::Constant::create(element::i32, {}, {0}); + // Gather will return scalar with number of channels (e.g. 3) + auto gather_channels_num = std::make_shared(shape_of, constant_chan_idx, constant_chan_axis); + + // Create Range from channels_num-1 to 0 (e.g. {2, 1, 0}) + auto const_minus1 = op::v0::Constant::create(element::i64, {}, {-1}); + auto channels_num_minus1 = std::make_shared(gather_channels_num, const_minus1); // E.g. 3-1=2 + // Add range + auto range_to = op::v0::Constant::create(element::i64, {}, {-1}); + auto range_step = op::v0::Constant::create(element::i64, {}, {-1}); + // E.g. {2, 1, 0} + auto range = std::make_shared(channels_num_minus1, range_to, range_step, element::i32); + + // Gather slices in reverse order (indexes are specified by 'range' operation) + auto constant_axis = op::v0::Constant::create(element::i32, {1}, {channels_idx}); + auto gather = std::make_shared(nodes[0], range, constant_axis); + return std::make_tuple(std::vector>{gather}, false); +} + } // namespace preprocess } // namespace ov diff --git a/src/core/src/preprocess/preprocess_steps_impl.hpp b/src/core/src/preprocess/preprocess_steps_impl.hpp index 16b5a43c6ff384..ab4ebbac2494b1 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.hpp +++ b/src/core/src/preprocess/preprocess_steps_impl.hpp @@ -92,10 +92,19 @@ class PrePostProcessingContextBase { return m_target_element_type; } + const ColorFormat& color_format() const { + return m_color_format; + } + + ColorFormat& color_format() { + return m_color_format; + } + protected: Layout m_layout; Layout m_target_layout; element::Type m_target_element_type; + ColorFormat m_color_format = ColorFormat::UNDEFINED; }; /// \brief Preprocessing context passed to each preprocessing operation. @@ -126,18 +135,9 @@ class PreprocessingContext : public PrePostProcessingContextBase { return model_shape()[model_width_idx].get_length(); } - const ColorFormat& color_format() const { - return m_color_format; - } - - ColorFormat& color_format() { - return m_color_format; - } - private: PartialShape m_model_shape; Layout m_model_layout; - ColorFormat m_color_format = ColorFormat::UNDEFINED; }; using InternalPreprocessOp = @@ -219,6 +219,7 @@ class PostStepsList { void add_convert_impl(const element::Type& type); void add_convert_layout_impl(const Layout& layout); void add_convert_layout_impl(const std::vector& dims); + void add_convert_color_impl(const ColorFormat& dst_format); const std::list& actions() const { return m_actions; @@ -227,6 +228,11 @@ class PostStepsList { return m_actions; } +private: + static std::tuple>, bool> reverse_channels(const std::vector>& nodes, + const std::shared_ptr& function, + PostprocessingContext& context); + private: std::list m_actions; }; diff --git a/src/core/tests/preprocess.cpp b/src/core/tests/preprocess.cpp index 0ce5d3a8f3590e..18832ac231e3c7 100644 --- a/src/core/tests/preprocess.cpp +++ b/src/core/tests/preprocess.cpp @@ -1782,6 +1782,88 @@ TEST(pre_post_process, postprocess_keep_friendly_names_compatibility_implicit) { EXPECT_NE(node_before_result_old->get_friendly_name(), node_name); } +// --- PostProcess - convert color format --- +TEST(pre_post_process, postprocess_convert_color_format_BGR_RGB) { + auto f = create_simple_function(element::f32, Shape{5, 30, 20, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::BGR); + p.output().postprocess().convert_color(ColorFormat::RGB); + f = p.build(); + + EXPECT_EQ(f->get_results().size(), 1); + EXPECT_EQ(f->get_result()->get_output_partial_shape(0), (PartialShape{5, 30, 20, 3})); +} + +TEST(pre_post_process, postprocess_convert_color_format_RGB_BGR) { + auto f = create_simple_function(element::f32, Shape{5, 30, 20, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::BGR); + f = p.build(); + + EXPECT_EQ(f->get_results().size(), 1); + EXPECT_EQ(f->get_result()->get_output_partial_shape(0), (PartialShape{5, 30, 20, 3})); +} + +TEST(pre_post_process, postprocess_convert_color_format_RGB_BGR_dynamic_batch) { + auto f = create_simple_function(element::f32, PartialShape{-1, 30, 20, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::BGR); + f = p.build(); + + EXPECT_EQ(f->get_results().size(), 1); + EXPECT_EQ(f->get_result()->get_output_partial_shape(0), (PartialShape{-1, 30, 20, 3})); +} + +TEST(pre_post_process, postprocess_convert_color_format_RGB_BGR_dynamic_shape) { + auto f = create_simple_function(element::f32, PartialShape{-1, -1, 20, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::BGR); + f = p.build(); + + EXPECT_EQ(f->get_results().size(), 1); + EXPECT_EQ(f->get_result()->get_output_partial_shape(0), (PartialShape{-1, -1, 20, 3})); +} + +TEST(pre_post_process, postprocess_convert_color_format_RGB_RGB) { + auto f = create_simple_function(element::f32, Shape{5, 30, 20, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::RGB); + f = p.build(); + + EXPECT_EQ(f->get_results().size(), 1); + EXPECT_EQ(f->get_result()->get_output_partial_shape(0), (PartialShape{5, 30, 20, 3})); +} + +TEST(pre_post_process, postprocess_convert_color_format_BGR_BGR) { + auto f = create_simple_function(element::f32, Shape{5, 30, 20, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::BGR); + p.output().postprocess().convert_color(ColorFormat::BGR); + f = p.build(); + + EXPECT_EQ(f->get_results().size(), 1); + EXPECT_EQ(f->get_result()->get_output_partial_shape(0), (PartialShape{5, 30, 20, 3})); +} + +TEST(pre_post_process, postprocess_convert_color_format_unsupported) { + auto f = create_simple_function(element::f32, Shape{5, 30, 20, 3}); + + EXPECT_THROW(auto p = PrePostProcessor(f); p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::GRAY); + f = p.build(), ov::AssertFailure); + + EXPECT_THROW(auto p = PrePostProcessor(f); p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::UNDEFINED); + f = p.build(), ov::AssertFailure); + EXPECT_THROW(auto p = PrePostProcessor(f); p.output().model().set_color_format(ColorFormat::UNDEFINED); + p.output().postprocess().convert_color(ColorFormat::BGR); + f = p.build(), ov::AssertFailure); +} + // Postprocessing - other TEST(pre_post_process, postprocess_preserve_rt_info) { diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp index b8379e24fbf093..9cf1bd337db9ea 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp @@ -9,8 +9,15 @@ using namespace ov::test; INSTANTIATE_TEST_SUITE_P( - smoke_PrePostProcess, + smoke_PreProcess, PrePostProcessTest, ::testing::Combine(::testing::ValuesIn(ov::builder::preprocess::generic_preprocess_functions()), ::testing::Values(ov::test::utils::DEVICE_CPU)), PrePostProcessTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_PostProcess, + PostProcessTest, + ::testing::Combine(::testing::ValuesIn(ov::builder::preprocess::generic_postprocess_functions()), + ::testing::Values(ov::test::utils::DEVICE_CPU)), + PostProcessTest::getTestCaseName); diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/preprocess.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/preprocess.hpp index 59a242990c5e2c..dfb218ad1a89b2 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/preprocess.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/preprocess.hpp @@ -29,6 +29,21 @@ class PrePostProcessTest : public testing::WithParamInterface; // Device name + +class PostProcessTest : public testing::WithParamInterface, + virtual public ov::test::SubgraphBaseTest { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj); + +protected: + void SetUp() override; + +public: + std::string func_name; +}; + } // namespace test } // namespace ov diff --git a/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp b/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp index 7367de93d4fad4..71f40ebac65b97 100644 --- a/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp +++ b/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp @@ -33,6 +33,9 @@ struct preprocess_func { inline std::vector generic_preprocess_functions(); +using postprocess_func = preprocess_func; +inline std::vector generic_postprocess_functions(); + /// -------- Functions --------------- inline std::shared_ptr create_preprocess_1input(element::Type type, const PartialShape& shape) { @@ -476,6 +479,33 @@ inline std::vector generic_preprocess_functions() { }; } +inline std::shared_ptr cvt_color_rgb_to_bgr() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 30, 3}); + auto p = PrePostProcessor(function); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::BGR); + function = p.build(); + return function; +} + +inline std::shared_ptr cvt_color_bgr_to_rgb() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 30, 3}); + auto p = PrePostProcessor(function); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::BGR); + p.output().postprocess().convert_color(ColorFormat::RGB); + function = p.build(); + return function; +} + +inline std::vector generic_postprocess_functions() { + return std::vector{ + postprocess_func(cvt_color_rgb_to_bgr, "convert_color_rgb_to_bgr", 0.01f), + postprocess_func(cvt_color_bgr_to_rgb, "convert_color_bgr_to_rgb", 0.01f), + }; +} + } // namespace preprocess } // namespace builder } // namespace ov From 7347604df5bac53349086aaa12456bafd9de17ca Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Mon, 8 Jan 2024 23:21:38 +0800 Subject: [PATCH 2/9] Fix clang format issue --- src/core/src/preprocess/preprocess_impls.hpp | 2 +- src/core/src/preprocess/preprocess_steps_impl.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/src/preprocess/preprocess_impls.hpp b/src/core/src/preprocess/preprocess_impls.hpp index 5e8e5c0153bb08..a63f207ad22e87 100644 --- a/src/core/src/preprocess/preprocess_impls.hpp +++ b/src/core/src/preprocess/preprocess_impls.hpp @@ -40,7 +40,7 @@ class InputModelInfo::InputModelInfoImpl : public ModelInfoImpl {}; class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl { public: void set_color_format(const ColorFormat& color_format, const std::vector& sub_names = {}) { - OPENVINO_ASSERT (color_format == ColorFormat::RGB || color_format == ColorFormat::BGR); + OPENVINO_ASSERT(color_format == ColorFormat::RGB || color_format == ColorFormat::BGR); m_color_format = color_format; m_planes_sub_names = sub_names; m_color_format_set = true; diff --git a/src/core/src/preprocess/preprocess_steps_impl.cpp b/src/core/src/preprocess/preprocess_steps_impl.cpp index 29a4bc084f057e..43ecc01d83c8d7 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.cpp +++ b/src/core/src/preprocess/preprocess_steps_impl.cpp @@ -706,7 +706,8 @@ void PostStepsList::add_convert_color_impl(const ColorFormat& dst_format) { (dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) { auto res = reverse_channels({node}, nullptr, context); context.color_format() = dst_format; - return std::make_tuple(std::get<0>(res)[0], std::get<1>(res));; + return std::make_tuple(std::get<0>(res)[0], std::get<1>(res)); + ; } OPENVINO_ASSERT(false, "Source color format '", From 279fe7eb62be02b4b84fd03bb945af7a4cb67d11 Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Tue, 9 Jan 2024 00:25:46 +0800 Subject: [PATCH 3/9] Add missing file --- .../src/subgraph/postprocess.cpp | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp diff --git a/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp b/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp new file mode 100644 index 00000000000000..73602774f88741 --- /dev/null +++ b/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp @@ -0,0 +1,50 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shared_test_classes/subgraph/preprocess.hpp" + +#include "openvino/core/preprocess/pre_post_process.hpp" +#include "ov_models/preprocess/preprocess_builders.hpp" + +using namespace ov; +using namespace ov::preprocess; +using namespace ov::builder::preprocess; + +namespace ov { +namespace test { + +std::string PostProcessTest::getTestCaseName(const testing::TestParamInfo& obj) { + std::string targetName; + postprocess_func func; + + std::tie(func, targetName) = obj.param; + + std::ostringstream result; + result << "Func=" << func.m_name << "_"; + result << "Device=" << targetName << ""; + return result.str(); +} + +void PostProcessTest::SetUp() { + postprocess_func func; + std::tie(func, targetDevice) = GetParam(); + function = func.m_function(); + rel_threshold = func.m_accuracy; + functionRefs = function->clone(); + abs_threshold = func.m_accuracy; + if (func.m_shapes.empty()) { + for (const auto& input : function->inputs()) { + func.m_shapes.push_back(input.get_shape()); + } + } + init_input_shapes(ov::test::static_shapes_to_test_representation(func.m_shapes)); + func_name = func.m_name; +} + +TEST_P(PostProcessTest, CompareWithRefs) { + run(); +} + +} // namespace test +} // namespace ov From 0f2d25a43d12dbf512ea7e4f23d9513bcd3e38ad Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Tue, 9 Jan 2024 16:08:40 +0800 Subject: [PATCH 4/9] Solve failure issue --- .../shared_tests_instances/subgraph_tests/preprocess.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp index 9cf1bd337db9ea..7a2b8c1ef62001 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp @@ -9,7 +9,7 @@ using namespace ov::test; INSTANTIATE_TEST_SUITE_P( - smoke_PreProcess, + smoke_PrePostProcess, PrePostProcessTest, ::testing::Combine(::testing::ValuesIn(ov::builder::preprocess::generic_preprocess_functions()), ::testing::Values(ov::test::utils::DEVICE_CPU)), From 40ab6282d0acc9dc96f88f1ddaaacecce4af58b5 Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Wed, 10 Jan 2024 12:18:12 +0800 Subject: [PATCH 5/9] Minor update --- .../src/preprocess/preprocess_steps_impl.cpp | 23 ++++++++----------- .../src/preprocess/preprocess_steps_impl.hpp | 4 +--- .../src/subgraph/postprocess.cpp | 3 +-- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/core/src/preprocess/preprocess_steps_impl.cpp b/src/core/src/preprocess/preprocess_steps_impl.cpp index 43ecc01d83c8d7..bf72be6fdccaec 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.cpp +++ b/src/core/src/preprocess/preprocess_steps_impl.cpp @@ -704,10 +704,9 @@ void PostStepsList::add_convert_color_impl(const ColorFormat& dst_format) { } if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) && (dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) { - auto res = reverse_channels({node}, nullptr, context); + auto res = reverse_channels({node}, context); context.color_format() = dst_format; - return std::make_tuple(std::get<0>(res)[0], std::get<1>(res)); - ; + return res; } OPENVINO_ASSERT(false, "Source color format '", @@ -719,15 +718,13 @@ void PostStepsList::add_convert_color_impl(const ColorFormat& dst_format) { "convert color (" + color_format_name(dst_format) + ")"); } -std::tuple>, bool> PostStepsList::reverse_channels(const std::vector>& nodes, - const std::shared_ptr& function, - PostprocessingContext& context) { - OPENVINO_ASSERT(nodes.size() == 1, "Internal error: can't reverse channels for multi-plane inputs"); +std::tuple, bool> PostStepsList::reverse_channels(const Output& node, + PostprocessingContext& context) { OPENVINO_ASSERT(ov::layout::has_channels(context.layout()), "Layout ", context.layout().to_string(), " doesn't have `channels` dimension"); - auto shape = nodes[0].get_partial_shape(); + auto shape = node.get_partial_shape(); if (shape.rank().is_static()) { // This block of code is to preserve output shape if it contains dynamic dimensions // Otherwise, dynamic version will transform shape {?,3,?,?} to {?,?,?,?} which is still ok but not desired @@ -741,14 +738,14 @@ std::tuple>, bool> PostStepsList::reverse_channels(cons auto range = std::make_shared(range_from, range_to, range_step, element::i32); auto constant_axis = op::v0::Constant::create(element::i32, {1}, {channels_idx}); - auto convert = std::make_shared(nodes[0], range, constant_axis); - return std::make_tuple(std::vector>{convert}, false); + auto convert = std::make_shared(node, range, constant_axis); + return std::make_tuple(convert, false); } } auto channels_idx = ov::layout::channels_idx(context.layout()); // Get shape of user's input tensor (e.g. Tensor[1, 3, 224, 224] -> {1, 3, 224, 224}) - auto shape_of = std::make_shared(nodes[0]); // E.g. {1, 3, 224, 224} + auto shape_of = std::make_shared(node); // E.g. {1, 3, 224, 224} auto constant_chan_idx = op::v0::Constant::create(element::i32, {}, {channels_idx}); // E.g. 1 auto constant_chan_axis = op::v0::Constant::create(element::i32, {}, {0}); @@ -766,8 +763,8 @@ std::tuple>, bool> PostStepsList::reverse_channels(cons // Gather slices in reverse order (indexes are specified by 'range' operation) auto constant_axis = op::v0::Constant::create(element::i32, {1}, {channels_idx}); - auto gather = std::make_shared(nodes[0], range, constant_axis); - return std::make_tuple(std::vector>{gather}, false); + auto gather = std::make_shared(node, range, constant_axis); + return std::make_tuple(gather, false); } } // namespace preprocess diff --git a/src/core/src/preprocess/preprocess_steps_impl.hpp b/src/core/src/preprocess/preprocess_steps_impl.hpp index ab4ebbac2494b1..79d6e0ef297888 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.hpp +++ b/src/core/src/preprocess/preprocess_steps_impl.hpp @@ -229,9 +229,7 @@ class PostStepsList { } private: - static std::tuple>, bool> reverse_channels(const std::vector>& nodes, - const std::shared_ptr& function, - PostprocessingContext& context); + static std::tuple, bool> reverse_channels(const Output& nodes, PostprocessingContext& context); private: std::list m_actions; diff --git a/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp b/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp index 73602774f88741..072edac1071070 100644 --- a/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp +++ b/src/tests/functional/shared_test_classes/src/subgraph/postprocess.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2023 Intel Corporation +// Copyright (C) 2018-2024 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -19,7 +19,6 @@ std::string PostProcessTest::getTestCaseName(const testing::TestParamInfo Date: Wed, 10 Jan 2024 16:04:42 +0800 Subject: [PATCH 6/9] Add test cases for GPU and TEMPLATE plugin --- .../subgraph_tests/preprocess.cpp | 7 ++++ .../subgraph_reference/preprocess.cpp | 34 +++++++++++++++++++ .../preprocess/preprocess_builders.hpp | 4 +-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp index 32ecadf49663b4..0d28022a7a915e 100644 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp +++ b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/preprocess.cpp @@ -49,4 +49,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_PrePostProcess_GPU, ::testing::Values(ov::test::utils::DEVICE_GPU)), PrePostProcessTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P( + smoke_PostProcess, + PostProcessTest, + ::testing::Combine(::testing::ValuesIn(ov::builder::preprocess::generic_postprocess_functions()), + ::testing::Values(ov::test::utils::DEVICE_GPU)), + PostProcessTest::getTestCaseName); + } // namespace diff --git a/src/plugins/template/tests/functional/subgraph_reference/preprocess.cpp b/src/plugins/template/tests/functional/subgraph_reference/preprocess.cpp index 865d8c3d92b2f9..306885bde3e519 100644 --- a/src/plugins/template/tests/functional/subgraph_reference/preprocess.cpp +++ b/src/plugins/template/tests/functional/subgraph_reference/preprocess.cpp @@ -1159,6 +1159,38 @@ static RefPreprocessParams post_convert_layout_by_dims_multi() { return res; } +static RefPreprocessParams post_convert_color_rgb_to_bgr() { + RefPreprocessParams res("post_convert_color_rgb_to_bgr"); + res.function = []() { + auto f = create_simple_function(element::f32, Shape{2, 1, 1, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); + p.output().postprocess().convert_color(ColorFormat::BGR); + p.build(); + return f; + }; + + res.inputs.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector{1, 2, 3, 4, 5, 6}); + res.expected.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector{3, 2, 1, 6, 5, 4}); + return res; +} + +static RefPreprocessParams post_convert_color_bgr_to_rgb() { + RefPreprocessParams res("post_convert_color_bgr_to_rgb"); + res.function = []() { + auto f = create_simple_function(element::f32, Shape{2, 1, 1, 3}); + auto p = PrePostProcessor(f); + p.output().model().set_layout("NHWC").set_color_format(ColorFormat::BGR); + p.output().postprocess().convert_color(ColorFormat::RGB); + p.build(); + return f; + }; + + res.inputs.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector{1, 2, 3, 4, 5, 6}); + res.expected.emplace_back(Shape{2, 3, 1, 1}, element::f32, std::vector{3, 2, 1, 6, 5, 4}); + return res; +} + static RefPreprocessParams pre_and_post_processing() { RefPreprocessParams res("pre_and_post_processing"); res.function = []() { @@ -1382,6 +1414,8 @@ std::vector allPreprocessTests() { postprocess_2_inputs_basic(), post_convert_layout_by_dims(), post_convert_layout_by_dims_multi(), + post_convert_color_rgb_to_bgr(), + post_convert_color_bgr_to_rgb(), pre_and_post_processing(), rgb_to_bgr(), bgr_to_rgb(), diff --git a/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp b/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp index 71f40ebac65b97..ac0e40e9975bd3 100644 --- a/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp +++ b/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp @@ -501,8 +501,8 @@ inline std::shared_ptr cvt_color_bgr_to_rgb() { inline std::vector generic_postprocess_functions() { return std::vector{ - postprocess_func(cvt_color_rgb_to_bgr, "convert_color_rgb_to_bgr", 0.01f), - postprocess_func(cvt_color_bgr_to_rgb, "convert_color_bgr_to_rgb", 0.01f), + postprocess_func(cvt_color_rgb_to_bgr, "convert_color_rgb_to_bgr", 1e-5f), + postprocess_func(cvt_color_bgr_to_rgb, "convert_color_bgr_to_rgb", 1e-5f), }; } From b96ec28190d981daee95dfb5b4472acd17da0f46 Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Wed, 10 Jan 2024 22:09:40 +0800 Subject: [PATCH 7/9] Fix GPU test case failure issue Failed: [GPU] Output tensor with name Abs_60 is not found --- .../include/ov_models/preprocess/preprocess_builders.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp b/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp index ac0e40e9975bd3..4dcb76c5c42589 100644 --- a/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp +++ b/src/tests/ov_helpers/ov_models/include/ov_models/preprocess/preprocess_builders.hpp @@ -44,6 +44,7 @@ inline std::shared_ptr create_preprocess_1input(element::Type type, const data1->output(0).get_tensor().set_names({"input1"}); std::shared_ptr res; auto op1 = std::make_shared(data1); + op1->set_friendly_name("abs1"); if (type == element::f32) { res = std::make_shared(op1); } else { From 33d457ef91d82cee0eff3c0fe2b8004fadf0827f Mon Sep 17 00:00:00 2001 From: "River.Li" Date: Wed, 10 Jan 2024 23:52:13 +0800 Subject: [PATCH 8/9] Update for code reviewing --- src/core/src/preprocess/preprocess_impls.hpp | 6 +++--- .../src/preprocess/preprocess_steps_impl.cpp | 19 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/core/src/preprocess/preprocess_impls.hpp b/src/core/src/preprocess/preprocess_impls.hpp index a63f207ad22e87..4d72a826a286e3 100644 --- a/src/core/src/preprocess/preprocess_impls.hpp +++ b/src/core/src/preprocess/preprocess_impls.hpp @@ -40,10 +40,10 @@ class InputModelInfo::InputModelInfoImpl : public ModelInfoImpl {}; class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl { public: void set_color_format(const ColorFormat& color_format, const std::vector& sub_names = {}) { - OPENVINO_ASSERT(color_format == ColorFormat::RGB || color_format == ColorFormat::BGR); + m_color_format_set = (color_format == ColorFormat::RGB) || (color_format == ColorFormat::BGR); + OPENVINO_ASSERT(m_color_format_set); m_color_format = color_format; m_planes_sub_names = sub_names; - m_color_format_set = true; } bool is_color_format_set() const { return m_color_format_set; @@ -54,7 +54,7 @@ class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl { private: ColorFormat m_color_format = ColorFormat::UNDEFINED; - std::vector m_planes_sub_names; + std::vector m_planes_sub_names{}; bool m_color_format_set = false; }; diff --git a/src/core/src/preprocess/preprocess_steps_impl.cpp b/src/core/src/preprocess/preprocess_steps_impl.cpp index bf72be6fdccaec..fadf5330c80fc1 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.cpp +++ b/src/core/src/preprocess/preprocess_steps_impl.cpp @@ -701,19 +701,18 @@ void PostStepsList::add_convert_color_impl(const ColorFormat& dst_format) { [dst_format](const Output& node, PostprocessingContext& context) { if (context.color_format() == dst_format) { return std::make_tuple(node, false); - } - if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) && - (dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) { + } else if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) && + (dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) { auto res = reverse_channels({node}, context); context.color_format() = dst_format; return res; + } else { + OPENVINO_THROW("Source color format '", + color_format_name(context.color_format()), + "' is not convertible to '", + color_format_name(dst_format), + "'"); } - OPENVINO_ASSERT(false, - "Source color format '", - color_format_name(context.color_format()), - "' is not convertible to '", - color_format_name(dst_format), - "'"); }, "convert color (" + color_format_name(dst_format) + ")"); } @@ -724,7 +723,7 @@ std::tuple, bool> PostStepsList::reverse_channels(const Output Date: Thu, 11 Jan 2024 16:18:39 +0800 Subject: [PATCH 9/9] Fix test failure issue of exception type --- src/core/tests/preprocess.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/tests/preprocess.cpp b/src/core/tests/preprocess.cpp index 18832ac231e3c7..8824abdc1b580e 100644 --- a/src/core/tests/preprocess.cpp +++ b/src/core/tests/preprocess.cpp @@ -1854,11 +1854,11 @@ TEST(pre_post_process, postprocess_convert_color_format_unsupported) { EXPECT_THROW(auto p = PrePostProcessor(f); p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); p.output().postprocess().convert_color(ColorFormat::GRAY); - f = p.build(), ov::AssertFailure); + f = p.build(), ov::Exception); EXPECT_THROW(auto p = PrePostProcessor(f); p.output().model().set_layout("NHWC").set_color_format(ColorFormat::RGB); p.output().postprocess().convert_color(ColorFormat::UNDEFINED); - f = p.build(), ov::AssertFailure); + f = p.build(), ov::Exception); EXPECT_THROW(auto p = PrePostProcessor(f); p.output().model().set_color_format(ColorFormat::UNDEFINED); p.output().postprocess().convert_color(ColorFormat::BGR); f = p.build(), ov::AssertFailure);