From c8b186c96594cc668033b65b700649599f46f28a Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 9 Aug 2023 09:34:51 -0700 Subject: [PATCH 1/3] Increase tolerance for ReduceProd test on x64 Windows --- .../test/providers/qnn/reduce_op_test.cc | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/providers/qnn/reduce_op_test.cc b/onnxruntime/test/providers/qnn/reduce_op_test.cc index e0357de3e52f1..9f28ca1729703 100644 --- a/onnxruntime/test/providers/qnn/reduce_op_test.cc +++ b/onnxruntime/test/providers/qnn/reduce_op_test.cc @@ -76,7 +76,8 @@ static void RunReduceOpCpuTest(const std::string& op_type, const std::vector& axes, bool keepdims, int opset, - ExpectedEPNodeAssignment expected_ep_assignment) { + ExpectedEPNodeAssignment expected_ep_assignment, + float fp32_abs_err = 1e-5f) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnCpu.dll"; @@ -92,7 +93,8 @@ static void RunReduceOpCpuTest(const std::string& op_type, false), // noop_with_empty_axes provider_options, opset, - expected_ep_assignment); + expected_ep_assignment, + fp32_abs_err); } // @@ -170,7 +172,12 @@ TEST_F(QnnCPUBackendTests, ReduceProdOpset18) { std::vector{0, 1}, true, // keepdims 18, - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, +#if defined(_WIN32) && !defined(__aarch64__) + 2e-5f); // x64 Windows requires a tolerance > 1.5e-5f +#else + 1e-5f); +#endif } // Test creates a graph with a ReduceProd node, and checks that all @@ -184,7 +191,12 @@ TEST_F(QnnCPUBackendTests, ReduceProdOpset13) { std::vector{0, 1}, true, // keepdims 13, - ExpectedEPNodeAssignment::All); + ExpectedEPNodeAssignment::All, +#if defined(_WIN32) && !defined(__aarch64__) + 2e-5f); // x64 Windows requires a tolerance > 1.5e-5f +#else + 1e-5f); +#endif } // From 60dc960966e5fd192bc928c33646667f1383c573 Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 9 Aug 2023 10:11:19 -0700 Subject: [PATCH 2/3] Create separate test for failing scenario --- .../test/providers/qnn/reduce_op_test.cc | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/onnxruntime/test/providers/qnn/reduce_op_test.cc b/onnxruntime/test/providers/qnn/reduce_op_test.cc index 9f28ca1729703..4311b6cc3b9af 100644 --- a/onnxruntime/test/providers/qnn/reduce_op_test.cc +++ b/onnxruntime/test/providers/qnn/reduce_op_test.cc @@ -168,13 +168,24 @@ TEST_F(QnnCPUBackendTests, ReduceSumOpset11_Float) { // - Uses opset 18, which has "axes" as an input. TEST_F(QnnCPUBackendTests, ReduceProdOpset18) { RunReduceOpCpuTest("ReduceProd", - TestInputDef({2, 2}, false, -10.0f, 10.0f), + TestInputDef({2, 2}, false, {-10.0f, -8.2f, 0.0f, 10.0f}), + std::vector{0, 1}, + true, // keepdims + 18, + ExpectedEPNodeAssignment::All); +} + +// TODO: Investigate slight inaccuracy. x64 Windows requires a slightly larger error tolerance greater than 1.5e-f. +// LOG: ... the value pair (208.881729, 208.881744) at index #0 don't match, which is 1.52588e-05 from 208.882 +TEST_F(QnnCPUBackendTests, ReduceProdOpset18_SlightlyInaccurate_WindowsX64) { + RunReduceOpCpuTest("ReduceProd", + TestInputDef({2, 2}, false, {3.21289f, -5.9981f, -1.72799f, 6.27263f}), std::vector{0, 1}, true, // keepdims 18, ExpectedEPNodeAssignment::All, #if defined(_WIN32) && !defined(__aarch64__) - 2e-5f); // x64 Windows requires a tolerance > 1.5e-5f + 2e-5f); // x64 Windows requires larger tolerance. #else 1e-5f); #endif @@ -187,16 +198,11 @@ TEST_F(QnnCPUBackendTests, ReduceProdOpset18) { // - Uses opset 13, which has "axes" as an attribute. TEST_F(QnnCPUBackendTests, ReduceProdOpset13) { RunReduceOpCpuTest("ReduceProd", - TestInputDef({2, 2}, false, -10.0f, 10.0f), + TestInputDef({2, 2}, false, {-10.0f, -8.2f, 0.0f, 10.0f}), std::vector{0, 1}, true, // keepdims 13, - ExpectedEPNodeAssignment::All, -#if defined(_WIN32) && !defined(__aarch64__) - 2e-5f); // x64 Windows requires a tolerance > 1.5e-5f -#else - 1e-5f); -#endif + ExpectedEPNodeAssignment::All); } // From f469cad3ef5195391cb78a78ce5ea1cc830cf9de Mon Sep 17 00:00:00 2001 From: adrianlizarraga Date: Wed, 9 Aug 2023 10:41:12 -0700 Subject: [PATCH 3/3] Failure also happens on linux x64 --- onnxruntime/test/providers/qnn/reduce_op_test.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/onnxruntime/test/providers/qnn/reduce_op_test.cc b/onnxruntime/test/providers/qnn/reduce_op_test.cc index 4311b6cc3b9af..b57483245c4cc 100644 --- a/onnxruntime/test/providers/qnn/reduce_op_test.cc +++ b/onnxruntime/test/providers/qnn/reduce_op_test.cc @@ -175,20 +175,16 @@ TEST_F(QnnCPUBackendTests, ReduceProdOpset18) { ExpectedEPNodeAssignment::All); } -// TODO: Investigate slight inaccuracy. x64 Windows requires a slightly larger error tolerance greater than 1.5e-f. +// TODO: Investigate slight inaccuracy. x64 Windows/Linux require a slightly larger error tolerance greater than 1.5e-5f. // LOG: ... the value pair (208.881729, 208.881744) at index #0 don't match, which is 1.52588e-05 from 208.882 -TEST_F(QnnCPUBackendTests, ReduceProdOpset18_SlightlyInaccurate_WindowsX64) { +TEST_F(QnnCPUBackendTests, ReduceProdOpset18_SlightlyInaccurate_WindowsLinuxX64) { RunReduceOpCpuTest("ReduceProd", TestInputDef({2, 2}, false, {3.21289f, -5.9981f, -1.72799f, 6.27263f}), std::vector{0, 1}, true, // keepdims 18, ExpectedEPNodeAssignment::All, -#if defined(_WIN32) && !defined(__aarch64__) - 2e-5f); // x64 Windows requires larger tolerance. -#else - 1e-5f); -#endif + 2e-5f); // x64 Linux & Windows require larger tolerance. } // Test creates a graph with a ReduceProd node, and checks that all