Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[QNN EP] Increase tolerance for ReduceProd test on x64 Windows #17078

Merged
merged 3 commits into from
Aug 9, 2023
Merged

[QNN EP] Increase tolerance for ReduceProd test on x64 Windows #17078

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c8b186c
Increase tolerance for ReduceProd test on x64 Windows
adrianlizarraga Aug 9, 2023
60dc960
Create separate test for failing scenario
adrianlizarraga Aug 9, 2023
f469cad
Failure also happens on linux x64
adrianlizarraga Aug 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions onnxruntime/test/providers/qnn/reduce_op_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ static void RunReduceOpCpuTest(const std::string& op_type,
const std::vector<int64_t>& axes,
bool keepdims,
int opset,
ExpectedEPNodeAssignment expected_ep_assignment) {
ExpectedEPNodeAssignment expected_ep_assignment,
float fp32_abs_err = 1e-5f) {
ProviderOptions provider_options;
#if defined(_WIN32)
provider_options["backend_path"] = "QnnCpu.dll";
Expand All @@ -92,7 +93,8 @@ static void RunReduceOpCpuTest(const std::string& op_type,
false), // noop_with_empty_axes
provider_options,
opset,
expected_ep_assignment);
expected_ep_assignment,
fp32_abs_err);
}

//
Expand Down Expand Up @@ -166,21 +168,33 @@ TEST_F(QnnCPUBackendTests, ReduceSumOpset11_Float) {
// - Uses opset 18, which has "axes" as an input.
TEST_F(QnnCPUBackendTests, ReduceProdOpset18) {
RunReduceOpCpuTest<float>("ReduceProd",
TestInputDef<float>({2, 2}, false, -10.0f, 10.0f),
TestInputDef<float>({2, 2}, false, {-10.0f, -8.2f, 0.0f, 10.0f}),
std::vector<int64_t>{0, 1},
true, // keepdims
18,
ExpectedEPNodeAssignment::All);
}

// TODO: Investigate slight inaccuracy. x64 Windows/Linux require a slightly larger error tolerance greater than 1.5e-5f.
// LOG: ... the value pair (208.881729, 208.881744) at index #0 don't match, which is 1.52588e-05 from 208.882
TEST_F(QnnCPUBackendTests, ReduceProdOpset18_SlightlyInaccurate_WindowsLinuxX64) {
RunReduceOpCpuTest<float>("ReduceProd",
TestInputDef<float>({2, 2}, false, {3.21289f, -5.9981f, -1.72799f, 6.27263f}),
std::vector<int64_t>{0, 1},
true, // keepdims
18,
ExpectedEPNodeAssignment::All,
2e-5f); // x64 Linux & Windows require larger tolerance.
}

// Test creates a graph with a ReduceProd node, and checks that all
// nodes are supported by the QNN EP (cpu backend), and that the inference results match the CPU EP results.
//
// - The input and output data type is float.
// - Uses opset 13, which has "axes" as an attribute.
TEST_F(QnnCPUBackendTests, ReduceProdOpset13) {
RunReduceOpCpuTest<float>("ReduceProd",
TestInputDef<float>({2, 2}, false, -10.0f, 10.0f),
TestInputDef<float>({2, 2}, false, {-10.0f, -8.2f, 0.0f, 10.0f}),
std::vector<int64_t>{0, 1},
true, // keepdims
13,
Expand Down