From 43c882f7bb41f08ac27f8b88933276d3f736d1ad Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 7 Nov 2024 03:02:03 -0800 Subject: [PATCH 01/22] refactor coreml EP creater --- .../NativeMethods.shared.cs | 2 +- .../SessionOptions.shared.cs | 8 +- .../InferenceTest.cs | 8 +- .../coreml/coreml_provider_factory.h | 4 + .../main/java/ai/onnxruntime/OrtSession.java | 12 + .../coreml/coreml_execution_provider.cc | 6 +- .../coreml/coreml_execution_provider.h | 7 +- .../coreml/coreml_provider_factory.cc | 49 +++- .../coreml/coreml_provider_factory_creator.h | 2 + .../core/session/provider_registration.cc | 16 +- .../python/onnxruntime_pybind_schema.cc | 2 +- onnxruntime/test/onnx/main.cc | 2 +- .../test/perftest/command_args_parser.cc | 34 +-- onnxruntime/test/perftest/ort_test_session.cc | 215 ++++++------------ onnxruntime/test/perftest/strings_helper.cc | 47 ++++ onnxruntime/test/perftest/strings_helper.h | 14 ++ .../ios_package_uitest_cpp_api.mm | 4 +- .../macos_package_uitest_cpp_api.mm | 3 +- .../providers/coreml/coreml_basic_test.cc | 3 +- .../providers/coreml/dynamic_input_test.cc | 15 +- onnxruntime/test/util/default_providers.cc | 5 +- 21 files changed, 248 insertions(+), 210 deletions(-) create mode 100644 onnxruntime/test/perftest/strings_helper.cc create mode 100644 onnxruntime/test/perftest/strings_helper.h diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs index be157a0419fc0..828ecaa25e6b8 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs @@ -1272,7 +1272,7 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca /// /// Append an execution provider instance to the native OrtSessionOptions instance. /// - /// 'SNPE' and 'XNNPACK' are currently supported as providerName values. + /// 'SNPE', 'XNNPACK' and 'CoreML' are currently supported as providerName values. /// /// The number of providerOptionsKeys must match the number of providerOptionsValues and equal numKeys. /// diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs index 3acd84b3016de..c6e576ca84fb9 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs @@ -430,16 +430,10 @@ public IntPtr Appender(IntPtr handle, IntPtr[] optKeys, IntPtr[] optValues, UInt /// /// Append QNN, SNPE or XNNPACK execution provider /// - /// Execution provider to add. 'QNN', 'SNPE' or 'XNNPACK' are currently supported. + /// Execution provider to add. 'QNN', 'SNPE' 'XNNPACK', 'CoreML and 'AZURE are currently supported. /// Optional key/value pairs to specify execution provider options. public void AppendExecutionProvider(string providerName, Dictionary providerOptions = null) { - if (providerName != "SNPE" && providerName != "XNNPACK" && providerName != "QNN" && providerName != "AZURE") - { - throw new NotSupportedException( - "Only QNN, SNPE, XNNPACK and AZURE execution providers can be enabled by this method."); - } - if (providerOptions == null) { providerOptions = new Dictionary(); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs index aa0e6ee62248a..1941ca72d689d 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs @@ -179,6 +179,12 @@ public void TestSessionOptions() ex = Assert.Throws(() => { opt.AppendExecutionProvider("QNN"); }); Assert.Contains("QNN execution provider is not supported in this build", ex.Message); #endif +#if USE_COREML + opt.AppendExecutionProvider("CoreML"); +#else + ex = Assert.Throws(() => { opt.AppendExecutionProvider("CoreML"); }); + Assert.Contains("CoreML execution provider is not supported in this build", ex.Message); +#endif opt.AppendExecutionProvider_CPU(1); } @@ -2041,7 +2047,7 @@ public SkipNonPackageTests() } // Test hangs on mobile. -#if !(ANDROID || IOS) +#if !(ANDROID || IOS) [Fact(DisplayName = "TestModelRunAsyncTask")] private async Task TestModelRunAsyncTask() { diff --git a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h index 98fa9e09f1ba8..79e6229e3891c 100644 --- a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h +++ b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h @@ -41,6 +41,10 @@ enum COREMLFlags { COREML_FLAG_LAST = COREML_FLAG_USE_CPU_AND_GPU, }; +// MLComputeUnits can be one of the following values: +// 'MLComputeUnitsCPUAndNeuralEngine|MLComputeUnitsCPUAndGPU|MLComputeUnitsCPUOnly|MLComputeUnitsAll' +static const char* const kCoremlProviderOption_MLComputeUnits = "MLComputeUnits"; + #ifdef __cplusplus extern "C" { #endif diff --git a/java/src/main/java/ai/onnxruntime/OrtSession.java b/java/src/main/java/ai/onnxruntime/OrtSession.java index 7280f3c88e2e8..32dc9d9f84aaa 100644 --- a/java/src/main/java/ai/onnxruntime/OrtSession.java +++ b/java/src/main/java/ai/onnxruntime/OrtSession.java @@ -1323,6 +1323,18 @@ public void addQnn(Map providerOptions) throws OrtException { addExecutionProvider(qnnProviderName, providerOptions); } + /** + * Adds CoreML as an execution backend. + * + * @param providerOptions Configuration options for the CoreML backend. Refer to the CoreML + * execution provider's documentation. + * @throws OrtException If there was an error in native code. + */ + public void addCoreML(Map providerOptions) throws OrtException { + String CoreMLProviderName = "CoreML"; + addExecutionProvider(CoreMLProviderName, providerOptions); + } + private native void setExecutionMode(long apiHandle, long nativeHandle, int mode) throws OrtException; diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index f7afbb2f98bd8..f3e8bd9b0e2af 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -23,9 +23,9 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; -CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) +CoreMLExecutionProvider::CoreMLExecutionProvider(const CoreMLOptions& options) : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, - coreml_flags_(coreml_flags), + coreml_flags_(options.coreml_flags), coreml_version_(coreml::util::CoreMLVersion()) { LOGS_DEFAULT(VERBOSE) << "CoreML version: " << coreml_version_; if (coreml_version_ < MINIMUM_COREML_VERSION) { @@ -33,7 +33,7 @@ CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) } // check if only one flag is set - if ((coreml_flags & COREML_FLAG_USE_CPU_ONLY) && (coreml_flags & COREML_FLAG_USE_CPU_AND_GPU)) { + if ((coreml_flags_ & COREML_FLAG_USE_CPU_ONLY) && (coreml_flags_ & COREML_FLAG_USE_CPU_AND_GPU)) { // multiple device options selected ORT_THROW( "Multiple device options selected, you should use at most one of the following options:" diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 24a001280eef5..d37f6bdc2732d 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -12,9 +12,14 @@ namespace coreml { class Model; } +struct CoreMLOptions { + uint32_t coreml_flags = 0; + std::string cache_path; +}; + class CoreMLExecutionProvider : public IExecutionProvider { public: - CoreMLExecutionProvider(uint32_t coreml_flags); + CoreMLExecutionProvider(const CoreMLOptions& options); virtual ~CoreMLExecutionProvider(); std::vector> diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index fcdf37c446ce7..bcb2927150713 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -9,21 +9,60 @@ using namespace onnxruntime; namespace onnxruntime { + +namespace { +CoreMLOptions ParseProviderOption(const ProviderOptions& options) { + CoreMLOptions coreml_options; + const std::unordered_map available_device_options = { + {"MLComputeUnitsCPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, + {"MLComputeUnitsCPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, + {"MLComputeUnitsCPUOnly", COREML_FLAG_USE_CPU_ONLY}, + {"MLComputeUnitsAll", COREML_FLAG_USE_NONE}, + }; + const std::unordered_map available_format_options = { + {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, + {"NeuralNetwork", COREML_FLAG_USE_NONE}, + }; + if (options.count("ComputeUnits")) { + coreml_options.coreml_flags |= available_device_options.at(options.at("ComputeUnits")); + } + if (options.count("ModelFormat")) { + coreml_options.coreml_flags |= available_format_options.at(options.at("ModelFormat")); + } + if (options.count("AllowStaticInputShapes")) { + coreml_options.coreml_flags |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; + } + if (options.count("EnableOnSubgraphs")) { + coreml_options.coreml_flags |= COREML_FLAG_ENABLE_ON_SUBGRAPH; + } + if (options.count("ModelCacheDir")) { + coreml_options.cache_path = options.at("ModelCacheDir"); + } + + return coreml_options; +} +} // namespace struct CoreMLProviderFactory : IExecutionProviderFactory { - CoreMLProviderFactory(uint32_t coreml_flags) - : coreml_flags_(coreml_flags) {} + CoreMLProviderFactory(const CoreMLOptions& options) + : options_(options) {} ~CoreMLProviderFactory() override {} std::unique_ptr CreateProvider() override; - uint32_t coreml_flags_; + CoreMLOptions options_; }; std::unique_ptr CoreMLProviderFactory::CreateProvider() { - return std::make_unique(coreml_flags_); + return std::make_unique(options_); } std::shared_ptr CoreMLProviderFactoryCreator::Create(uint32_t coreml_flags) { - return std::make_shared(coreml_flags); + CoreMLOptions coreml_options; + coreml_options.coreml_flags = coreml_flags; + return std::make_shared(coreml_options); +} + +std::shared_ptr CoreMLProviderFactoryCreator::Create(const ProviderOptions& options) { + return std::make_shared(ParseProviderOption(options)); } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory_creator.h b/onnxruntime/core/providers/coreml/coreml_provider_factory_creator.h index ba701724c4da9..93ec2af50698d 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory_creator.h +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory_creator.h @@ -5,10 +5,12 @@ #include +#include "core/framework/provider_options.h" #include "core/providers/providers.h" namespace onnxruntime { struct CoreMLProviderFactoryCreator { static std::shared_ptr Create(uint32_t coreml_flags); + static std::shared_ptr Create(const ProviderOptions& options); }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 8c512c561ea8c..4ae912b23f80b 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -155,11 +155,25 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, status = create_not_supported_status(); #endif } else if (strcmp(provider_name, "VitisAI") == 0) { +#ifdef USE_VITISAI status = OrtApis::SessionOptionsAppendExecutionProvider_VitisAI(options, provider_options_keys, provider_options_values, num_keys); +#else + status = create_not_supported_status(); +#endif + } else if (strcmp(provider_name, "CoreML") == 0) { +#if defined(USE_COREML) + std::string coreml_flags; + if (options->value.config_options.TryGetConfigEntry("coreml_flags", coreml_flags)) { + provider_options["coreml_flags"] = coreml_flags; + } + options->provider_factories.push_back(CoreMLProviderFactoryCreator::Create(provider_options)); +#else + status = create_not_supported_status(); +#endif } else { ORT_UNUSED_PARAMETER(options); status = OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, - "Unknown provider name. Currently supported values are 'OPENVINO', 'SNPE', 'XNNPACK', 'QNN', 'WEBNN' and 'AZURE'"); + "Unknown provider name. Currently supported values are 'OPENVINO', 'SNPE', 'XNNPACK', 'QNN', 'WEBNN' ,'CoreML', and 'AZURE'"); } return status; diff --git a/onnxruntime/python/onnxruntime_pybind_schema.cc b/onnxruntime/python/onnxruntime_pybind_schema.cc index 1319e8f6fe959..dcd021494c24c 100644 --- a/onnxruntime/python/onnxruntime_pybind_schema.cc +++ b/onnxruntime/python/onnxruntime_pybind_schema.cc @@ -73,7 +73,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) { onnxruntime::RknpuProviderFactoryCreator::Create(), #endif #ifdef USE_COREML - onnxruntime::CoreMLProviderFactoryCreator::Create(0), + onnxruntime::CoreMLProviderFactoryCreator::Create(ProviderOptions{}), #endif #ifdef USE_XNNPACK onnxruntime::XnnpackProviderFactoryCreator::Create(ProviderOptions{}, nullptr), diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 93a1bf9f30651..ddc453f84feb6 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -631,7 +631,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { } if (enable_coreml) { #ifdef USE_COREML - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CoreML(sf, 0)); + sf.AppendExecutionProvider("CoreML", {}); #else fprintf(stderr, "CoreML is not supported in this build"); return -1; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index e40544d950ed7..46777eda793e0 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -24,6 +24,7 @@ #include #include "test_configuration.h" +#include "strings_helper.h" namespace onnxruntime { namespace perftest { @@ -175,39 +176,6 @@ static bool ParseDimensionOverride(std::basic_string& dim_identifier, return true; } -static bool ParseSessionConfigs(const std::string& configs_string, - std::unordered_map& session_configs) { - std::istringstream ss(configs_string); - std::string token; - - while (ss >> token) { - if (token == "") { - continue; - } - - std::string_view token_sv(token); - - auto pos = token_sv.find("|"); - if (pos == std::string_view::npos || pos == 0 || pos == token_sv.length()) { - // Error: must use a '|' to separate the key and value for session configuration entries. - return false; - } - - std::string key(token_sv.substr(0, pos)); - std::string value(token_sv.substr(pos + 1)); - - auto it = session_configs.find(key); - if (it != session_configs.end()) { - // Error: specified duplicate session configuration entry: {key} - return false; - } - - session_configs.insert(std::make_pair(std::move(key), std::move(value))); - } - - return true; -} - /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) { int ch; while ((ch = getopt(argc, argv, ORT_TSTR("m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqznlR:"))) != -1) { diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 8f2e5282ede9a..85429273409de 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -17,6 +17,7 @@ #include #include "providers.h" #include "TestCase.h" +#include "strings_helper.h" #ifdef USE_OPENVINO #include "nlohmann/json.hpp" @@ -58,6 +59,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device Ort::SessionOptions session_options; provider_name_ = performance_test_config.machine_config.provider_type_name; + std::unordered_map provider_options; if (provider_name_ == onnxruntime::kDnnlExecutionProvider) { #ifdef USE_DNNL // Generate provider options @@ -72,24 +74,14 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif // defined(_MSC_VER) int num_threads = 0; - std::istringstream ss(ov_string); - std::string token; - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW( - "[ERROR] [OneDNN] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } - - auto key = token.substr(0, pos); - auto value = token.substr(pos + 1); - - if (key == "num_of_threads") { - std::stringstream sstream(value); + if (!ParseSessionConfigs(ov_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { + if (provider_option.first == "num_of_threads") { + std::stringstream sstream(provider_option.second); sstream >> num_threads; if (num_threads < 0) { ORT_THROW( @@ -144,22 +136,14 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - std::istringstream ss(ov_string); - std::string token; - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW( - "[ERROR] [CUDA] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - buffer.emplace_back(token.substr(0, pos)); - option_keys.push_back(buffer.back().c_str()); - buffer.emplace_back(token.substr(pos + 1)); - option_values.push_back(buffer.back().c_str()); + if (!ParseSessionConfigs(ov_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { + option_keys.push_back(provider_option->first.c_str()); + option_values.push_back(provider_option->first.c_str()); } Ort::Status status(api.UpdateCUDAProviderOptions(cuda_options, @@ -192,24 +176,15 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - std::istringstream ss(ov_string); - std::string token; - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW( - "[ERROR] [TensorRT] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - buffer.emplace_back(token.substr(0, pos)); - option_keys.push_back(buffer.back().c_str()); - buffer.emplace_back(token.substr(pos + 1)); - option_values.push_back(buffer.back().c_str()); + if (!ParseSessionConfigs(ov_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { + option_keys.push_back(provider_option->first.c_str()); + option_values.push_back(provider_option->first.c_str()); } - Ort::Status status(api.UpdateTensorRTProviderOptions(tensorrt_options, option_keys.data(), option_values.data(), option_keys.size())); if (!status.IsOK()) { @@ -239,22 +214,12 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - std::istringstream ss(option_string); - std::string token; - std::unordered_map qnn_options; - - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW("Use a '|' to separate the key and value for the run-time option you are trying to use."); - } - - std::string key(token.substr(0, pos)); - std::string value(token.substr(pos + 1)); - + if (!ParseSessionConfigs(option_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { if (key == "backend_path" || key == "profiling_file_path") { if (value.empty()) { ORT_THROW("Please provide the valid file path."); @@ -317,10 +282,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device 'qnn_saver_path', 'htp_graph_finalization_optimization_mode', 'qnn_context_priority', 'soc_model', 'htp_arch', 'device_id', 'enable_htp_fp16_precision', 'offload_graph_io_quantization'])"); } - - qnn_options[key] = value; } - session_options.AppendExecutionProvider("QNN", qnn_options); + session_options.AppendExecutionProvider("QNN", provider_options); #else ORT_THROW("QNN is not supported in this build\n"); #endif @@ -331,22 +294,12 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - std::istringstream ss(option_string); - std::string token; - std::unordered_map snpe_options; - - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW("Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - std::string key(token.substr(0, pos)); - std::string value(token.substr(pos + 1)); - + if (!ParseSessionConfigs(option_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { if (key == "runtime") { std::set supported_runtime = {"CPU", "GPU_FP32", "GPU", "GPU_FLOAT16", "DSP", "AIP_FIXED_TF"}; if (supported_runtime.find(value) == supported_runtime.end()) { @@ -368,11 +321,9 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else { ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type', 'enable_init_cache'] \n"); } - - snpe_options[key] = value; } - session_options.AppendExecutionProvider("SNPE", snpe_options); + session_options.AppendExecutionProvider("SNPE", provider_options); #else ORT_THROW("SNPE is not supported in this build\n"); #endif @@ -448,34 +399,20 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #endif } else if (provider_name_ == onnxruntime::kDmlExecutionProvider) { #ifdef USE_DML - std::unordered_map dml_options; - dml_options["performance_preference"] = "high_performance"; - dml_options["device_filter"] = "gpu"; - dml_options["disable_metacommands"] = "false"; - dml_options["enable_graph_capture"] = "false"; #ifdef _MSC_VER std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string); #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - std::istringstream ss(ov_string); - std::string token; - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW("[ERROR] [DML] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - auto key = token.substr(0, pos); - auto value = token.substr(pos + 1); - + if (!ParseSessionConfigs(ov_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { if (key == "device_filter") { std::set ov_supported_device_types = {"gpu", "npu"}; if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) { - dml_options[key] = value; } else { ORT_THROW( "[ERROR] [DML] You have selected a wrong configuration value for the key 'device_filter'. " @@ -484,7 +421,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else if (key == "performance_preference") { std::set ov_supported_values = {"default", "high_performance", "minimal_power"}; if (ov_supported_values.find(value) != ov_supported_values.end()) { - dml_options[key] = value; } else { ORT_THROW( "[ERROR] [DML] You have selected a wrong configuration value for the key 'performance_preference'. " @@ -493,7 +429,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else if (key == "disable_metacommands") { std::set ov_supported_values = {"true", "True", "false", "False"}; if (ov_supported_values.find(value) != ov_supported_values.end()) { - dml_options[key] = value; } else { ORT_THROW( "[ERROR] [DML] You have selected a wrong value for the key 'disable_metacommands'. " @@ -502,7 +437,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else if (key == "enable_graph_capture") { std::set ov_supported_values = {"true", "True", "false", "False"}; if (ov_supported_values.find(value) != ov_supported_values.end()) { - dml_options[key] = value; } else { ORT_THROW( "[ERROR] [DML] You have selected a wrong value for the key 'enable_graph_capture'. " @@ -519,7 +453,19 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } } } - session_options.AppendExecutionProvider("DML", dml_options); + if (provider_options.find("performance_preference") == provider_options.end()) { + provider_options["performance_preference"] = "high_performance"; + } + if (provider_options.find("device_filter") == provider_options.end()) { + provider_options["device_filter"] = "gpu"; + } + if (provider_options.find("disable_metacommands") == provider_options.end()) { + provider_options["disable_metacommands"] = "false"; + } + if (provider_options.find("enable_graph_capture") == provider_options.end()) { + provider_options["enable_graph_capture"] = "false"; + } + session_options.AppendExecutionProvider("DML", provider_options); #else ORT_THROW("DML is not supported in this build\n"); #endif @@ -530,21 +476,13 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif // defined(_MSC_VER) - std::istringstream ss(ov_string); - std::string token; bool enable_fast_math = false; - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW("[ERROR] [ACL] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - auto key = token.substr(0, pos); - auto value = token.substr(pos + 1); - + if (!ParseSessionConfigs(ov_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } + for (const auto& provider_option : provider_options) { if (key == "enable_fast_math") { std::set ov_supported_values = {"true", "True", "false", "False"}; if (ov_supported_values.find(value) != ov_supported_values.end()) { @@ -612,24 +550,13 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - std::istringstream ss(option_string); - std::string token; - std::unordered_map vitisai_session_options; - - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW("[ERROR] [VitisAI] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - std::string key(token.substr(0, pos)); - std::string value(token.substr(pos + 1)); - vitisai_session_options[key] = value; + if (!ParseSessionConfigs(option_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); } - session_options.AppendExecutionProvider_VitisAI(vitisai_session_options); + + session_options.AppendExecutionProvider_VitisAI(provider_options); #else ORT_THROW("VitisAI is not supported in this build\n"); #endif diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc new file mode 100644 index 0000000000000..22f682159b924 --- /dev/null +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2023 NVIDIA Corporation. +// SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates +// Licensed under the MIT License. + +#include +#include + +#include "strings_helper.h" + +namespace onnxruntime { +namespace perftest { + +bool ParseSessionConfigs(const std::string& configs_string, + std::unordered_map& session_configs) { + std::istringstream ss(configs_string); + std::string token; + + while (ss >> token) { + if (token == "") { + continue; + } + + std::string_view token_sv(token); + + auto pos = token_sv.find("|"); + if (pos == std::string_view::npos || pos == 0 || pos == token_sv.length()) { + // Error: must use a '|' to separate the key and value for session configuration entries. + return false; + } + + std::string key(token_sv.substr(0, pos)); + std::string value(token_sv.substr(pos + 1)); + + auto it = session_configs.find(key); + if (it != session_configs.end()) { + // Error: specified duplicate session configuration entry: {key} + return false; + } + + session_configs.insert(std::make_pair(std::move(key), std::move(value))); + } + + return true; +} +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h new file mode 100644 index 0000000000000..24feb90a20a61 --- /dev/null +++ b/onnxruntime/test/perftest/strings_helper.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2023 NVIDIA Corporation. +// SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates +// Licensed under the MIT License. +#include +#include + +namespace onnxruntime { +namespace perftest { + +bool ParseSessionConfigs(const std::string& configs_string, + std::unordered_map& session_configs); +} // namespace perftest +} // namespace onnxruntime diff --git a/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm b/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm index 32b4b32e299d6..9bead11109f3b 100644 --- a/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm +++ b/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm @@ -36,7 +36,9 @@ void testSigmoid(const char* modelPath, bool useCoreML = false, bool useWebGPU = #if COREML_EP_AVAILABLE if (useCoreML) { const uint32_t flags = COREML_FLAG_USE_CPU_ONLY; - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CoreML(session_options, flags)); + std::unordered_map provider_options = {{"coreml_flags", std::to_string(flags)}}; + + session_options.AppendExecutionProvider("CoreML", provider_options); } #else (void)useCoreML; diff --git a/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm b/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm index 86001b6cb50a5..a7851d078ece4 100644 --- a/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm +++ b/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm @@ -36,7 +36,8 @@ void testSigmoid(const char* modelPath, bool useCoreML = false, bool useWebGPU = #if COREML_EP_AVAILABLE if (useCoreML) { const uint32_t flags = COREML_FLAG_USE_CPU_ONLY; - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CoreML(session_options, flags)); + std::unordered_map provider_options = {{"coreml_flags", std::to_string(flags)}}; + session_options.AppendExecutionProvider("CoreML", provider_options); } #else (void)useCoreML; diff --git a/onnxruntime/test/providers/coreml/coreml_basic_test.cc b/onnxruntime/test/providers/coreml/coreml_basic_test.cc index de647d9e3aa3e..ca89b22cbc088 100644 --- a/onnxruntime/test/providers/coreml/coreml_basic_test.cc +++ b/onnxruntime/test/providers/coreml/coreml_basic_test.cc @@ -34,7 +34,8 @@ namespace test { static constexpr uint32_t s_coreml_flags = COREML_FLAG_USE_CPU_ONLY; static std::unique_ptr MakeCoreMLExecutionProvider(uint32_t flags = s_coreml_flags) { - return std::make_unique(flags); + std::unordered_map provider_options = {{"coreml_flags", std::to_string(flags)}}; + return std::make_unique(provider_options); } #if !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/test/providers/coreml/dynamic_input_test.cc b/onnxruntime/test/providers/coreml/dynamic_input_test.cc index c91ef23650040..8eecdbcce33c6 100644 --- a/onnxruntime/test/providers/coreml/dynamic_input_test.cc +++ b/onnxruntime/test/providers/coreml/dynamic_input_test.cc @@ -20,8 +20,8 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, MatMul) { auto test = [&](const size_t M) { SCOPED_TRACE(MakeString("M=", M)); - - auto coreml_ep = std::make_unique(0); + std::unordered_map options; + auto coreml_ep = std::make_unique(options); const auto ep_verification_params = EPVerificationParams{ ExpectedEPNodeAssignment::All, @@ -54,8 +54,8 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, MobileNetExcerpt) { auto test = [&](const size_t batch_size) { SCOPED_TRACE(MakeString("batch_size=", batch_size)); - - auto coreml_ep = std::make_unique(0); + std::unordered_map options; + auto coreml_ep = std::make_unique(options); const auto ep_verification_params = EPVerificationParams{ ExpectedEPNodeAssignment::All, @@ -87,6 +87,7 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, EmptyInputFails) { constexpr auto model_path = ORT_TSTR("testdata/matmul_with_dynamic_input_shape.onnx"); ModelTester tester(CurrentTestName(), model_path); + std::unordered_map options; tester.AddInput("A", {0, 2}, {}); tester.AddOutput("Y", {0, 4}, {}); @@ -94,14 +95,14 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, EmptyInputFails) { tester .Config(ModelTester::ExpectResult::kExpectFailure, "the runtime shape ({0,2}) has zero elements. This is not supported by the CoreML EP.") - .ConfigEp(std::make_unique(0)) + .ConfigEp(std::make_unique(options)) .RunWithConfig(); } TEST(CoreMLExecutionProviderDynamicInputShapeTest, OnlyAllowStaticInputShapes) { constexpr auto model_path = ORT_TSTR("testdata/matmul_with_dynamic_input_shape.onnx"); - - auto coreml_ep = std::make_unique(COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES); + std::unordered_map options = {{"coreml_flags", std::to_string(COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES)}}; + auto coreml_ep = std::make_unique(options); TestModelLoad(model_path, std::move(coreml_ep), // expect no supported nodes because we disable dynamic input shape support diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 62bdedd833025..32a0560a6bc87 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -259,8 +259,9 @@ std::unique_ptr DefaultCoreMLExecutionProvider(bool use_mlpr if (use_mlprogram) { coreml_flags |= COREML_FLAG_CREATE_MLPROGRAM; } - - return CoreMLProviderFactoryCreator::Create(coreml_flags)->CreateProvider(); + auto option = ProviderOptions(); + option["coreml_flags"] = std::to_string(coreml_flags); + return CoreMLProviderFactoryCreator::Create(option)->CreateProvider(); #else ORT_UNUSED_PARAMETER(use_mlprogram); return nullptr; From a78de112824b9d4cc20ee08512b870e1daf3e445 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 7 Nov 2024 03:20:56 -0800 Subject: [PATCH 02/22] refactor coreml test option --- .../test/perftest/command_args_parser.cc | 8 +++- onnxruntime/test/perftest/ort_test_session.cc | 39 +++++++++++-------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 46777eda793e0..0da55a9ee676d 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -130,8 +130,12 @@ namespace perftest { "\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n" "\t [Example] [For NNAPI EP] -e nnapi -i \"NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED\"\n" "\n" - "\t [CoreML only] [COREML_FLAG_CREATE_MLPROGRAM COREML_FLAG_USE_CPU_ONLY COREML_FLAG_USE_CPU_AND_GPU]: Create an ML Program model instead of Neural Network.\n" - "\t [Example] [For CoreML EP] -e coreml -i \"COREML_FLAG_CREATE_MLPROGRAM\"\n" + "\t [CoreML only] [ModelFormat]:[MLProgram, NeuralNetwork] Create an ML Program model or Neural Network.\n" + "\t [CoreML only] [ComputeUnits]:[MLComputeUnitsCPUAndNeuralEngine MLComputeUnitsCPUAndGPU MLComputeUnitsCPUAndGPU MLComputeUnitsCPUOnly] the backend device to run model.\n" + "\t [CoreML only] [AllowStaticInputShapes]:[0 1].\n" + "\t [CoreML only] [EnableOnSubgraphs]:[0 1].\n" + "\t [CoreML only] [ModelCacheDir]: a path to cached compiled coreml model.\n" + "\t [Example] [For CoreML EP] -e coreml -i \"ModelFormat|MLProgram ComputeUnits|MLComputeUnitsCPUAndGPU\"\n" "\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" "\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 85429273409de..f6041e7ae194b 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -367,30 +367,37 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else if (provider_name_ == onnxruntime::kCoreMLExecutionProvider) { #ifdef __APPLE__ #ifdef USE_COREML - uint32_t coreml_flags = 0; std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; - std::istringstream ss(ov_string); + if (!ParseSessionConfigs(ov_string, provider_options)) { + ORT_THROW( + "[ERROR] Use a '|' to separate the key and value for the " + "run-time option you are trying to use.\n"); + } - std::string key; - while (ss >> key) { - if (key == "COREML_FLAG_CREATE_MLPROGRAM") { - coreml_flags |= COREML_FLAG_CREATE_MLPROGRAM; - std::cout << "Enabling ML Program.\n"; - } else if (key == "COREML_FLAG_USE_CPU_ONLY") { - coreml_flags |= COREML_FLAG_USE_CPU_ONLY; - std::cout << "CoreML enabled COREML_FLAG_USE_CPU_ONLY.\n"; - } else if (key == "COREML_FLAG_USE_CPU_AND_GPU") { - coreml_flags |= COREML_FLAG_USE_CPU_AND_GPU; - std::cout << "CoreML enabled COREML_FLAG_USE_CPU_AND_GPU.\n"; - } else if (key.empty()) { + std::unordered_map available_options = { + {"MLComputeUnitsCPUAndNeuralEngine", "1"}, + {"MLComputeUnitsCPUAndGPU", "1"}, + {"MLComputeUnitsCPUOnly", "1"}, + {"MLComputeUnitsAll", "1"}, + }; + for (const auto& provider_option : provider_options) { + if (provider_option.first == kCoremlProviderOption_MLComputeUnits && + available_options.find(provider_option.second) != available_options.end()) { + } else if (provider_option.first == "ModelFormat" && + (provider_option.second == "MLProgram" || provider_option.second == "NeuralNetwork")) { + } else if (provider_option.first == "AllowStaticInputShapes" && + (provider_option.second == "1" || provider_option.second == "0")) { + } else if (provider_option.first == "EnableOnSubgraphs" && + (provider_option.second == "0" || provider_option.second == "1")) { } else { ORT_THROW( "[ERROR] [CoreML] wrong key type entered. Choose from the following runtime key options " - "that are available for CoreML. ['COREML_FLAG_CREATE_MLPROGRAM'] \n"); + "that are available for CoreML. " + "['MLComputeUnits', 'ModelFormat', 'AllowStaticInputShapes', 'EnableOnSubgraphs'] \n"); } } // COREML_FLAG_CREATE_MLPROGRAM - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CoreML(session_options, coreml_flags)); + session_options.AppendExecutionProvider("CoreML", provider_options); #else ORT_THROW("CoreML is not supported in this build\n"); #endif From f25b18f6a19c471a8cb525bbfd663b1bf56f36db Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 7 Nov 2024 21:45:14 -0800 Subject: [PATCH 03/22] define constant name to replace raw string --- .../coreml/coreml_provider_factory.h | 4 ++++ .../coreml/coreml_provider_factory.cc | 20 +++++++++---------- onnxruntime/test/perftest/ort_test_session.cc | 6 +++--- .../providers/coreml/coreml_basic_test.cc | 17 ++++++++-------- .../providers/coreml/dynamic_input_test.cc | 13 +++++++----- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h index 79e6229e3891c..bf190f96fe7b2 100644 --- a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h +++ b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h @@ -44,6 +44,10 @@ enum COREMLFlags { // MLComputeUnits can be one of the following values: // 'MLComputeUnitsCPUAndNeuralEngine|MLComputeUnitsCPUAndGPU|MLComputeUnitsCPUOnly|MLComputeUnitsAll' static const char* const kCoremlProviderOption_MLComputeUnits = "MLComputeUnits"; +static const char* const kCoremlProviderOption_MLModelFormat = "MLModelFormat"; +static const char* const kCoremlProviderOption_MLAllowStaticInputShapes = "MLAllowStaticInputShapes"; +static const char* const kCoremlProviderOption_MLEnableOnSubgraphs = "MLEnableOnSubgraphs"; +static const char* const kCoremlProviderOption_MLModelCacheDir = "MLModelCacheDir"; #ifdef __cplusplus extern "C" { diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index bcb2927150713..6e73bdfaaee04 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -13,30 +13,30 @@ namespace onnxruntime { namespace { CoreMLOptions ParseProviderOption(const ProviderOptions& options) { CoreMLOptions coreml_options; - const std::unordered_map available_device_options = { + const std::unordered_map available_computeunits_options = { {"MLComputeUnitsCPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, {"MLComputeUnitsCPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, {"MLComputeUnitsCPUOnly", COREML_FLAG_USE_CPU_ONLY}, {"MLComputeUnitsAll", COREML_FLAG_USE_NONE}, }; - const std::unordered_map available_format_options = { + const std::unordered_map available_modelformat_options = { {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, {"NeuralNetwork", COREML_FLAG_USE_NONE}, }; - if (options.count("ComputeUnits")) { - coreml_options.coreml_flags |= available_device_options.at(options.at("ComputeUnits")); + if (options.count(kCoremlProviderOption_MLComputeUnits)) { + coreml_options.coreml_flags |= available_computeunits_options.at(options.at(kCoremlProviderOption_MLComputeUnits)); } - if (options.count("ModelFormat")) { - coreml_options.coreml_flags |= available_format_options.at(options.at("ModelFormat")); + if (options.count(kCoremlProviderOption_MLModelFormat)) { + coreml_options.coreml_flags |= available_modelformat_options.at(options.at(kCoremlProviderOption_MLModelFormat)); } - if (options.count("AllowStaticInputShapes")) { + if (options.count(kCoremlProviderOption_MLAllowStaticInputShapes)) { coreml_options.coreml_flags |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; } - if (options.count("EnableOnSubgraphs")) { + if (options.count(kCoremlProviderOption_MLEnableOnSubgraphs)) { coreml_options.coreml_flags |= COREML_FLAG_ENABLE_ON_SUBGRAPH; } - if (options.count("ModelCacheDir")) { - coreml_options.cache_path = options.at("ModelCacheDir"); + if (options.count(kCoremlProviderOption_MLModelCacheDir)) { + coreml_options.cache_path = options.at(kCoremlProviderOption_MLModelCacheDir); } return coreml_options; diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index f6041e7ae194b..7292b570e1895 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -383,11 +383,11 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); for (const auto& provider_option : provider_options) { if (provider_option.first == kCoremlProviderOption_MLComputeUnits && available_options.find(provider_option.second) != available_options.end()) { - } else if (provider_option.first == "ModelFormat" && + } else if (provider_option.first == kCoremlProviderOption_MLModelFormat && (provider_option.second == "MLProgram" || provider_option.second == "NeuralNetwork")) { - } else if (provider_option.first == "AllowStaticInputShapes" && + } else if (provider_option.first == kCoremlProviderOption_MLAllowStaticInputShapes && (provider_option.second == "1" || provider_option.second == "0")) { - } else if (provider_option.first == "EnableOnSubgraphs" && + } else if (provider_option.first == kCoremlProviderOption_MLEnableOnSubgraphs && (provider_option.second == "0" || provider_option.second == "1")) { } else { ORT_THROW( diff --git a/onnxruntime/test/providers/coreml/coreml_basic_test.cc b/onnxruntime/test/providers/coreml/coreml_basic_test.cc index ca89b22cbc088..acedfd1241021 100644 --- a/onnxruntime/test/providers/coreml/coreml_basic_test.cc +++ b/onnxruntime/test/providers/coreml/coreml_basic_test.cc @@ -4,7 +4,7 @@ #include "core/common/logging/logging.h" #include "core/graph/graph.h" #include "core/graph/graph_viewer.h" -#include "core/providers/coreml/coreml_execution_provider.h" +#include "core/providers/coreml/coreml_provider_factory_creator.h" #include "core/providers/coreml/coreml_provider_factory.h" #include "core/session/inference_session.h" #include "test/common/tensor_op_test_utils.h" @@ -30,12 +30,11 @@ using namespace ::onnxruntime::logging; namespace onnxruntime { namespace test { -// We want to run UT on CPU only to get output value without losing precision to pass the verification -static constexpr uint32_t s_coreml_flags = COREML_FLAG_USE_CPU_ONLY; - -static std::unique_ptr MakeCoreMLExecutionProvider(uint32_t flags = s_coreml_flags) { - std::unordered_map provider_options = {{"coreml_flags", std::to_string(flags)}}; - return std::make_unique(provider_options); +static std::unique_ptr MakeCoreMLExecutionProvider( + std::string ModelFormat = "NeuralNetwork", std::string ComputeUnits = "MLComputeUnitsCPUOnly") { + std::unordered_map provider_options = {{kCoremlProviderOption_MLComputeUnits, ComputeUnits}, + {kCoremlProviderOption_MLModelFormat, ModelFormat}}; + return CoreMLProviderFactoryCreator::Create(provider_options)->CreateProvider(); } #if !defined(ORT_MINIMAL_BUILD) @@ -129,7 +128,7 @@ TEST(CoreMLExecutionProviderTest, ArgMaxCastTest) { feeds, verification_params); RunAndVerifyOutputsWithEP(model_file_name, CurrentTestName(), - MakeCoreMLExecutionProvider(COREML_FLAG_CREATE_MLPROGRAM), + MakeCoreMLExecutionProvider("MLProgram"), feeds, verification_params); #else @@ -171,7 +170,7 @@ TEST(CoreMLExecutionProviderTest, ArgMaxUnsupportedCastTest) { verification_params); RunAndVerifyOutputsWithEP(model_file_name, CurrentTestName(), - MakeCoreMLExecutionProvider(COREML_FLAG_CREATE_MLPROGRAM), + MakeCoreMLExecutionProvider("MLProgram"), feeds, verification_params); #else diff --git a/onnxruntime/test/providers/coreml/dynamic_input_test.cc b/onnxruntime/test/providers/coreml/dynamic_input_test.cc index 8eecdbcce33c6..d8adad65e1753 100644 --- a/onnxruntime/test/providers/coreml/dynamic_input_test.cc +++ b/onnxruntime/test/providers/coreml/dynamic_input_test.cc @@ -7,6 +7,7 @@ #include #include "core/providers/coreml/coreml_execution_provider.h" +#include "core/providers/coreml/coreml_provider_factory_creator.h" #include "core/providers/coreml/coreml_provider_factory.h" // for COREMLFlags #include "test/common/random_generator.h" #include "test/providers/model_tester.h" @@ -21,7 +22,7 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, MatMul) { auto test = [&](const size_t M) { SCOPED_TRACE(MakeString("M=", M)); std::unordered_map options; - auto coreml_ep = std::make_unique(options); + auto coreml_ep = CoreMLProviderFactoryCreator::Create(options)->CreateProvider(); const auto ep_verification_params = EPVerificationParams{ ExpectedEPNodeAssignment::All, @@ -55,7 +56,8 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, MobileNetExcerpt) { auto test = [&](const size_t batch_size) { SCOPED_TRACE(MakeString("batch_size=", batch_size)); std::unordered_map options; - auto coreml_ep = std::make_unique(options); + auto coreml_ep = CoreMLProviderFactoryCreator::Create(options)->CreateProvider(); + ; const auto ep_verification_params = EPVerificationParams{ ExpectedEPNodeAssignment::All, @@ -95,14 +97,15 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, EmptyInputFails) { tester .Config(ModelTester::ExpectResult::kExpectFailure, "the runtime shape ({0,2}) has zero elements. This is not supported by the CoreML EP.") - .ConfigEp(std::make_unique(options)) + .ConfigEp(CoreMLProviderFactoryCreator::Create(options)->CreateProvider()) .RunWithConfig(); } TEST(CoreMLExecutionProviderDynamicInputShapeTest, OnlyAllowStaticInputShapes) { constexpr auto model_path = ORT_TSTR("testdata/matmul_with_dynamic_input_shape.onnx"); - std::unordered_map options = {{"coreml_flags", std::to_string(COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES)}}; - auto coreml_ep = std::make_unique(options); + std::unordered_map options = {{kCoremlProviderOption_MLAllowStaticInputShapes, "1"}}; + auto coreml_ep = CoreMLProviderFactoryCreator::Create(options)->CreateProvider(); + ; TestModelLoad(model_path, std::move(coreml_ep), // expect no supported nodes because we disable dynamic input shape support From 4ee1cee75f67ff83f9bc54fae2dc289384cc3075 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 8 Nov 2024 01:09:55 -0800 Subject: [PATCH 04/22] fix coreml_flag --- onnxruntime/core/session/provider_registration.cc | 4 ---- onnxruntime/test/perftest/ort_test_session.cc | 8 ++++++-- .../ios_package_uitest_cpp_api.mm | 5 ++--- .../macos_package_uitest_cpp_api.mm | 4 ++-- onnxruntime/test/util/default_providers.cc | 9 ++++----- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 4ae912b23f80b..11ebea9f7f738 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -162,10 +162,6 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, #endif } else if (strcmp(provider_name, "CoreML") == 0) { #if defined(USE_COREML) - std::string coreml_flags; - if (options->value.config_options.TryGetConfigEntry("coreml_flags", coreml_flags)) { - provider_options["coreml_flags"] = coreml_flags; - } options->provider_factories.push_back(CoreMLProviderFactoryCreator::Create(provider_options)); #else status = create_not_supported_status(); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 7292b570e1895..ed6201c7d6ba4 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -142,8 +142,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device "run-time option you are trying to use.\n"); } for (const auto& provider_option : provider_options) { - option_keys.push_back(provider_option->first.c_str()); - option_values.push_back(provider_option->first.c_str()); + option_keys.push_back(provider_option.first.c_str()); + option_values.push_back(provider_option.second.c_str()); } Ort::Status status(api.UpdateCUDAProviderOptions(cuda_options, @@ -220,6 +220,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device "run-time option you are trying to use.\n"); } for (const auto& provider_option : provider_options) { + const std::string& key = provider_option.first; + const std::string& value = provider_option.second; if (key == "backend_path" || key == "profiling_file_path") { if (value.empty()) { ORT_THROW("Please provide the valid file path."); @@ -417,6 +419,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); "run-time option you are trying to use.\n"); } for (const auto& provider_option : provider_options) { + const std::string& key = provider_option.first; + const std::string& value = provider_option.second; if (key == "device_filter") { std::set ov_supported_device_types = {"gpu", "npu"}; if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) { diff --git a/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm b/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm index 9bead11109f3b..8fe382b8fe246 100644 --- a/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm +++ b/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm @@ -35,9 +35,8 @@ void testSigmoid(const char* modelPath, bool useCoreML = false, bool useWebGPU = #if COREML_EP_AVAILABLE if (useCoreML) { - const uint32_t flags = COREML_FLAG_USE_CPU_ONLY; - std::unordered_map provider_options = {{"coreml_flags", std::to_string(flags)}}; - + std::unordered_map provider_options = { + {kCoremlProviderOption_MLComputeUnits, "MLComputeUnitsCPUOnly"}}; session_options.AppendExecutionProvider("CoreML", provider_options); } #else diff --git a/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm b/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm index a7851d078ece4..8a877bc4c9f7a 100644 --- a/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm +++ b/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm @@ -35,8 +35,8 @@ void testSigmoid(const char* modelPath, bool useCoreML = false, bool useWebGPU = #if COREML_EP_AVAILABLE if (useCoreML) { - const uint32_t flags = COREML_FLAG_USE_CPU_ONLY; - std::unordered_map provider_options = {{"coreml_flags", std::to_string(flags)}}; + std::unordered_map provider_options = { + {kCoremlProviderOption_MLComputeUnits, "MLComputeUnitsCPUOnly"}}; session_options.AppendExecutionProvider("CoreML", provider_options); } #else diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 32a0560a6bc87..2b435c7cdcf67 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -253,14 +253,13 @@ std::unique_ptr DefaultCoreMLExecutionProvider(bool use_mlpr // The test will create a model but execution of it will obviously fail. #if defined(USE_COREML) && defined(__APPLE__) // We want to run UT on CPU only to get output value without losing precision - uint32_t coreml_flags = 0; - coreml_flags |= COREML_FLAG_USE_CPU_ONLY; + auto option = ProviderOptions(); + option[kCoremlProviderOption_MLComputeUnits] = "MLComputeUnitsCPUOnly"; if (use_mlprogram) { - coreml_flags |= COREML_FLAG_CREATE_MLPROGRAM; + option[kCoremlProviderOption_MLModelFormat] = "MLProgram"; } - auto option = ProviderOptions(); - option["coreml_flags"] = std::to_string(coreml_flags); + return CoreMLProviderFactoryCreator::Create(option)->CreateProvider(); #else ORT_UNUSED_PARAMETER(use_mlprogram); From 8c129321b9d5337b0b13bf74011afe147db077d1 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 8 Nov 2024 04:25:15 -0800 Subject: [PATCH 05/22] fix --- onnxruntime/test/perftest/ort_test_session.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index ed6201c7d6ba4..b715e32cad3d0 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -182,8 +182,8 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device "run-time option you are trying to use.\n"); } for (const auto& provider_option : provider_options) { - option_keys.push_back(provider_option->first.c_str()); - option_values.push_back(provider_option->first.c_str()); + option_keys.push_back(provider_option.first.c_str()); + option_values.push_back(provider_option.second.c_str()); } Ort::Status status(api.UpdateTensorRTProviderOptions(tensorrt_options, option_keys.data(), option_values.data(), option_keys.size())); From f710c1454b1c4c8b26a48f6488395bed2f3773c7 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Mon, 18 Nov 2024 19:35:06 +0800 Subject: [PATCH 06/22] add python new api --- .../coreml/coreml_provider_factory.cc | 45 +++++++++++++------ .../python/onnxruntime_pybind_state.cc | 3 ++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index 6e73bdfaaee04..72cb8e2fa12f3 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -23,20 +23,37 @@ CoreMLOptions ParseProviderOption(const ProviderOptions& options) { {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, {"NeuralNetwork", COREML_FLAG_USE_NONE}, }; - if (options.count(kCoremlProviderOption_MLComputeUnits)) { - coreml_options.coreml_flags |= available_computeunits_options.at(options.at(kCoremlProviderOption_MLComputeUnits)); - } - if (options.count(kCoremlProviderOption_MLModelFormat)) { - coreml_options.coreml_flags |= available_modelformat_options.at(options.at(kCoremlProviderOption_MLModelFormat)); - } - if (options.count(kCoremlProviderOption_MLAllowStaticInputShapes)) { - coreml_options.coreml_flags |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; - } - if (options.count(kCoremlProviderOption_MLEnableOnSubgraphs)) { - coreml_options.coreml_flags |= COREML_FLAG_ENABLE_ON_SUBGRAPH; - } - if (options.count(kCoremlProviderOption_MLModelCacheDir)) { - coreml_options.cache_path = options.at(kCoremlProviderOption_MLModelCacheDir); + std::unordered_set valid_options = { + kCoremlProviderOption_MLComputeUnits, + kCoremlProviderOption_MLModelFormat, + kCoremlProviderOption_MLAllowStaticInputShapes, + kCoremlProviderOption_MLEnableOnSubgraphs, + kCoremlProviderOption_MLModelCacheDir, + }; + // Validate the options + for (const auto& option : options) { + if (valid_options.find(option.first) == valid_options.end()) { + ORT_THROW("Unknown option: ", option.first); + } + if (kCoremlProviderOption_MLComputeUnits == option.first) { + if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { + ORT_THROW("Invalid value for option ", option.first, ": ", option.second); + }else { + coreml_options.coreml_flags |= available_computeunits_options.at(option.second); + } + } else if (kCoremlProviderOption_MLModelFormat == option.first) { + if (available_modelformat_options.find(option.second) == available_modelformat_options.end()) { + ORT_THROW("Invalid value for option ", option.first, ": ", option.second); + } else { + coreml_options.coreml_flags |= available_modelformat_options.at(option.second); + } + } else if (okCoremlProviderOption_MLAllowStaticInputShapes == option.first) { + coreml_options.coreml_flags |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; + } else if (okCoremlProviderOption_MLEnableOnSubgraphs == option.first) { + coreml_options.coreml_flags |= COREML_FLAG_ENABLE_ON_SUBGRAPH; + } else if (okCoremlProviderOption_MLModelCacheDir == option.first) { + coreml_options.cache_path = option.second; + } } return coreml_options; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 4d9583be0ef0f..2b5a7c450f3a9 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1222,6 +1222,9 @@ std::unique_ptr CreateExecutionProviderInstance( if (flags_str.find("COREML_FLAG_CREATE_MLPROGRAM") != std::string::npos) { coreml_flags |= COREMLFlags::COREML_FLAG_CREATE_MLPROGRAM; } + } else { + // read from provider_options + return onnxruntime::CoreMLProviderFactoryCreator::Create(options)->CreateProvider(); } } From ece5e58b7accadee35d419f5364a0e9e611f8461 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Mon, 18 Nov 2024 19:55:49 -0800 Subject: [PATCH 07/22] add object-c api --- .../include/ort_coreml_execution_provider.h | 12 +++++++++- objectivec/ort_coreml_execution_provider.mm | 15 ++++++++++++ objectivec/test/ort_session_test.mm | 23 +++++++++++++++++++ .../coreml/coreml_provider_factory.cc | 6 ++--- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/objectivec/include/ort_coreml_execution_provider.h b/objectivec/include/ort_coreml_execution_provider.h index d7d873f5eb0e0..427e78cd539c8 100644 --- a/objectivec/include/ort_coreml_execution_provider.h +++ b/objectivec/include/ort_coreml_execution_provider.h @@ -70,7 +70,17 @@ NS_ASSUME_NONNULL_BEGIN */ - (BOOL)appendCoreMLExecutionProviderWithOptions:(ORTCoreMLExecutionProviderOptions*)options error:(NSError**)error; - +/** + * Enables the CoreML execution provider in the session configuration options. + * It is appended to the execution provider list which is ordered by + * decreasing priority. + * + * @param provider_options The CoreML execution provider options in dict. + * @param error Optional error information set if an error occurs. + * @return Whether the provider was enabled successfully. + */ +- (BOOL)appendCoreMLExecutionProviderWithOptions_v2:(NSDictionary*)provider_options + error:(NSError**)error; @end NS_ASSUME_NONNULL_END diff --git a/objectivec/ort_coreml_execution_provider.mm b/objectivec/ort_coreml_execution_provider.mm index 6cb5026b93521..a13f3590198fd 100644 --- a/objectivec/ort_coreml_execution_provider.mm +++ b/objectivec/ort_coreml_execution_provider.mm @@ -43,6 +43,21 @@ - (BOOL)appendCoreMLExecutionProviderWithOptions:(ORTCoreMLExecutionProviderOpti #endif } +- (BOOL)appendCoreMLExecutionProviderWithOptions_v2:(NSDictionary*)provider_options + error:(NSError**)error { +#if ORT_OBJC_API_COREML_EP_AVAILABLE + try { + return [self appendExecutionProvider:@"CoreML" providerOptions:provider_options error:error]; + } + ORT_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error); + +#else // !ORT_OBJC_API_COREML_EP_AVAILABLE + static_cast(provider_options); + ORTSaveCodeAndDescriptionToError(ORT_FAIL, "CoreML execution provider is not enabled.", error); + return NO; +#endif +} + @end NS_ASSUME_NONNULL_END diff --git a/objectivec/test/ort_session_test.mm b/objectivec/test/ort_session_test.mm index 508289f7bc748..2dde743b1a797 100644 --- a/objectivec/test/ort_session_test.mm +++ b/objectivec/test/ort_session_test.mm @@ -223,6 +223,29 @@ - (void)testAppendCoreMLEP { ORTAssertNullableResultSuccessful(session, err); } + +- (void)testAppendCoreMLEP_v2 { + NSError* err = nil; + ORTSessionOptions* sessionOptions = [ORTSessionTest makeSessionOptions]; + NSDictionary* provider_options = @{@"MLEnableOnSubgraphs" : @"1"};// set an arbitrary option + + BOOL appendResult = [sessionOptions appendCoreMLExecutionProviderWithOptions_v2:provider_options + error:&err]; + + if (!ORTIsCoreMLExecutionProviderAvailable()) { + ORTAssertBoolResultUnsuccessful(appendResult, err); + return; + } + + ORTAssertBoolResultSuccessful(appendResult, err); + + ORTSession* session = [[ORTSession alloc] initWithEnv:self.ortEnv + modelPath:[ORTSessionTest getAddModelPath] + sessionOptions:sessionOptions + error:&err]; + ORTAssertNullableResultSuccessful(session, err); +} + - (void)testAppendXnnpackEP { NSError* err = nil; ORTSessionOptions* sessionOptions = [ORTSessionTest makeSessionOptions]; diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index 72cb8e2fa12f3..19b142495a7fc 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -47,11 +47,11 @@ CoreMLOptions ParseProviderOption(const ProviderOptions& options) { } else { coreml_options.coreml_flags |= available_modelformat_options.at(option.second); } - } else if (okCoremlProviderOption_MLAllowStaticInputShapes == option.first) { + } else if (kCoremlProviderOption_MLAllowStaticInputShapes == option.first) { coreml_options.coreml_flags |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; - } else if (okCoremlProviderOption_MLEnableOnSubgraphs == option.first) { + } else if (kCoremlProviderOption_MLEnableOnSubgraphs == option.first) { coreml_options.coreml_flags |= COREML_FLAG_ENABLE_ON_SUBGRAPH; - } else if (okCoremlProviderOption_MLModelCacheDir == option.first) { + } else if (kCoremlProviderOption_MLModelCacheDir == option.first) { coreml_options.cache_path = option.second; } } From f6e2cbe86c9405ec363d5643e4cc158879eda556 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Mon, 18 Nov 2024 20:35:52 -0800 Subject: [PATCH 08/22] format --- objectivec/ort_coreml_execution_provider.mm | 2 +- objectivec/test/ort_session_test.mm | 5 ++--- onnxruntime/core/providers/coreml/coreml_provider_factory.cc | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/objectivec/ort_coreml_execution_provider.mm b/objectivec/ort_coreml_execution_provider.mm index a13f3590198fd..07e6b0e107bff 100644 --- a/objectivec/ort_coreml_execution_provider.mm +++ b/objectivec/ort_coreml_execution_provider.mm @@ -44,7 +44,7 @@ - (BOOL)appendCoreMLExecutionProviderWithOptions:(ORTCoreMLExecutionProviderOpti } - (BOOL)appendCoreMLExecutionProviderWithOptions_v2:(NSDictionary*)provider_options - error:(NSError**)error { + error:(NSError**)error { #if ORT_OBJC_API_COREML_EP_AVAILABLE try { return [self appendExecutionProvider:@"CoreML" providerOptions:provider_options error:error]; diff --git a/objectivec/test/ort_session_test.mm b/objectivec/test/ort_session_test.mm index 2dde743b1a797..8690aa5ee8183 100644 --- a/objectivec/test/ort_session_test.mm +++ b/objectivec/test/ort_session_test.mm @@ -223,14 +223,13 @@ - (void)testAppendCoreMLEP { ORTAssertNullableResultSuccessful(session, err); } - - (void)testAppendCoreMLEP_v2 { NSError* err = nil; ORTSessionOptions* sessionOptions = [ORTSessionTest makeSessionOptions]; - NSDictionary* provider_options = @{@"MLEnableOnSubgraphs" : @"1"};// set an arbitrary option + NSDictionary* provider_options = @{@"MLEnableOnSubgraphs" : @"1"}; // set an arbitrary option BOOL appendResult = [sessionOptions appendCoreMLExecutionProviderWithOptions_v2:provider_options - error:&err]; + error:&err]; if (!ORTIsCoreMLExecutionProviderAvailable()) { ORTAssertBoolResultUnsuccessful(appendResult, err); diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index 19b142495a7fc..58c84f37930ef 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -38,7 +38,7 @@ CoreMLOptions ParseProviderOption(const ProviderOptions& options) { if (kCoremlProviderOption_MLComputeUnits == option.first) { if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { ORT_THROW("Invalid value for option ", option.first, ": ", option.second); - }else { + } else { coreml_options.coreml_flags |= available_computeunits_options.at(option.second); } } else if (kCoremlProviderOption_MLModelFormat == option.first) { From 60f66249206c04675e8f1d563ca09d890da80be1 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Tue, 19 Nov 2024 03:27:47 -0800 Subject: [PATCH 09/22] add ane --- onnxruntime/core/providers/coreml/model/model.mm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onnxruntime/core/providers/coreml/model/model.mm b/onnxruntime/core/providers/coreml/model/model.mm index ff32c52f942b2..1e7aa5ecd1e60 100644 --- a/onnxruntime/core/providers/coreml/model/model.mm +++ b/onnxruntime/core/providers/coreml/model/model.mm @@ -400,6 +400,8 @@ Status Predict(const std::unordered_map& inputs, config.computeUnits = MLComputeUnitsCPUOnly; } else if (coreml_flags_ & COREML_FLAG_USE_CPU_AND_GPU) { config.computeUnits = MLComputeUnitsCPUAndGPU; + } else if (coreml_flags_ & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) { + config.computeUnits = MLComputeUnitsCPUAndNeuralEngine; // Apple Neural Engine } else { config.computeUnits = MLComputeUnitsAll; } From 373cf8f97a115dd846d723ed4c7b78808a8c0210 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 21 Nov 2024 16:19:55 +0800 Subject: [PATCH 10/22] Apply suggestions from code review Co-authored-by: Scott McKay --- objectivec/ort_coreml_execution_provider.mm | 2 +- onnxruntime/test/perftest/command_args_parser.cc | 6 +++--- onnxruntime/test/providers/coreml/dynamic_input_test.cc | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/objectivec/ort_coreml_execution_provider.mm b/objectivec/ort_coreml_execution_provider.mm index 07e6b0e107bff..53890575ffcab 100644 --- a/objectivec/ort_coreml_execution_provider.mm +++ b/objectivec/ort_coreml_execution_provider.mm @@ -43,7 +43,7 @@ - (BOOL)appendCoreMLExecutionProviderWithOptions:(ORTCoreMLExecutionProviderOpti #endif } -- (BOOL)appendCoreMLExecutionProviderWithOptions_v2:(NSDictionary*)provider_options +- (BOOL)appendCoreMLExecutionProviderWithOptionsV2:(NSDictionary*)provider_options error:(NSError**)error { #if ORT_OBJC_API_COREML_EP_AVAILABLE try { diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 0da55a9ee676d..23a1ad97e6df4 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -130,12 +130,12 @@ namespace perftest { "\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n" "\t [Example] [For NNAPI EP] -e nnapi -i \"NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED\"\n" "\n" - "\t [CoreML only] [ModelFormat]:[MLProgram, NeuralNetwork] Create an ML Program model or Neural Network.\n" - "\t [CoreML only] [ComputeUnits]:[MLComputeUnitsCPUAndNeuralEngine MLComputeUnitsCPUAndGPU MLComputeUnitsCPUAndGPU MLComputeUnitsCPUOnly] the backend device to run model.\n" + "\t [CoreML only] [ModelFormat]:[MLProgram, NeuralNetwork] Create an ML Program model or Neural Network. Default is NeuralNetwork.\n" + "\t [CoreML only] [MLComputeUnits]:[CPUAndNeuralEngine CPUAndGPU CPUAndGPU CPUOnly] Specify to limit the backend device/s used to run the model.\n" "\t [CoreML only] [AllowStaticInputShapes]:[0 1].\n" "\t [CoreML only] [EnableOnSubgraphs]:[0 1].\n" "\t [CoreML only] [ModelCacheDir]: a path to cached compiled coreml model.\n" - "\t [Example] [For CoreML EP] -e coreml -i \"ModelFormat|MLProgram ComputeUnits|MLComputeUnitsCPUAndGPU\"\n" + "\t [Example] [For CoreML EP] -e coreml -i \"ModelFormat|MLProgram MLComputeUnits|CPUAndGPU\"\n" "\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" "\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n" diff --git a/onnxruntime/test/providers/coreml/dynamic_input_test.cc b/onnxruntime/test/providers/coreml/dynamic_input_test.cc index d8adad65e1753..af81e75e5fe53 100644 --- a/onnxruntime/test/providers/coreml/dynamic_input_test.cc +++ b/onnxruntime/test/providers/coreml/dynamic_input_test.cc @@ -57,7 +57,6 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, MobileNetExcerpt) { SCOPED_TRACE(MakeString("batch_size=", batch_size)); std::unordered_map options; auto coreml_ep = CoreMLProviderFactoryCreator::Create(options)->CreateProvider(); - ; const auto ep_verification_params = EPVerificationParams{ ExpectedEPNodeAssignment::All, From b81160fc7ce99859410b0a096bb72060da93789d Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 21 Nov 2024 16:28:32 +0800 Subject: [PATCH 11/22] Update objectivec/ort_coreml_execution_provider.mm Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- objectivec/ort_coreml_execution_provider.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objectivec/ort_coreml_execution_provider.mm b/objectivec/ort_coreml_execution_provider.mm index 53890575ffcab..0c790a91fb8b9 100644 --- a/objectivec/ort_coreml_execution_provider.mm +++ b/objectivec/ort_coreml_execution_provider.mm @@ -44,7 +44,7 @@ - (BOOL)appendCoreMLExecutionProviderWithOptions:(ORTCoreMLExecutionProviderOpti } - (BOOL)appendCoreMLExecutionProviderWithOptionsV2:(NSDictionary*)provider_options - error:(NSError**)error { + error:(NSError**)error { #if ORT_OBJC_API_COREML_EP_AVAILABLE try { return [self appendExecutionProvider:@"CoreML" providerOptions:provider_options error:error]; From 6479c13b3812ea9155690042ea2162cc9bf8617d Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 21 Nov 2024 01:40:35 -0800 Subject: [PATCH 12/22] Adress comments --- .../coreml/coreml_provider_factory.h | 10 ++-- .../include/ort_coreml_execution_provider.h | 11 +++- objectivec/test/ort_session_test.mm | 6 +- .../coreml/coreml_execution_provider.cc | 45 ++++++++++++++- .../coreml/coreml_execution_provider.h | 15 ++++- .../coreml/coreml_provider_factory.cc | 55 +------------------ .../test/perftest/command_args_parser.cc | 2 +- onnxruntime/test/perftest/ort_test_session.cc | 47 +++++----------- onnxruntime/test/perftest/strings_helper.cc | 20 +++++-- onnxruntime/test/perftest/strings_helper.h | 4 +- .../ios_package_uitest_cpp_api.mm | 2 +- .../macos_package_uitest_cpp_api.mm | 2 +- .../providers/coreml/coreml_basic_test.cc | 4 +- .../providers/coreml/dynamic_input_test.cc | 2 +- onnxruntime/test/util/default_providers.cc | 4 +- 15 files changed, 119 insertions(+), 110 deletions(-) diff --git a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h index bf190f96fe7b2..9846da8f7fa7d 100644 --- a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h +++ b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h @@ -43,11 +43,13 @@ enum COREMLFlags { // MLComputeUnits can be one of the following values: // 'MLComputeUnitsCPUAndNeuralEngine|MLComputeUnitsCPUAndGPU|MLComputeUnitsCPUOnly|MLComputeUnitsAll' +// these values are intended to be used with Ort.::SessionOptions::AppendExecutionProvider(C++ API) +// /SessionOptionsAppendExecutionProvider (C API). For the old API, use COREMLFlags instead. static const char* const kCoremlProviderOption_MLComputeUnits = "MLComputeUnits"; -static const char* const kCoremlProviderOption_MLModelFormat = "MLModelFormat"; -static const char* const kCoremlProviderOption_MLAllowStaticInputShapes = "MLAllowStaticInputShapes"; -static const char* const kCoremlProviderOption_MLEnableOnSubgraphs = "MLEnableOnSubgraphs"; -static const char* const kCoremlProviderOption_MLModelCacheDir = "MLModelCacheDir"; +static const char* const kCoremlProviderOption_ModelFormat = "ModelFormat"; +static const char* const kCoremlProviderOption_RequireStaticInputShapes = "RequireStaticInputShapes"; +static const char* const kCoremlProviderOption_EnableOnSubgraphs = "EnableOnSubgraphs"; +static const char* const kCoremlProviderOption_ModelCacheDir = "ModelCacheDir"; #ifdef __cplusplus extern "C" { diff --git a/objectivec/include/ort_coreml_execution_provider.h b/objectivec/include/ort_coreml_execution_provider.h index 427e78cd539c8..417868a08972c 100644 --- a/objectivec/include/ort_coreml_execution_provider.h +++ b/objectivec/include/ort_coreml_execution_provider.h @@ -76,11 +76,18 @@ NS_ASSUME_NONNULL_BEGIN * decreasing priority. * * @param provider_options The CoreML execution provider options in dict. + * available keys-values: more detail in onnxruntime/core/providers/coreml/coreml_provider_factory.cc + * kCoremlProviderOption_MLComputeUnits: one of "MLComputeUnitsCPUAndNeuralEngine", + * "MLComputeUnitsCPUAndGPU", "MLComputeUnitsCPUOnly", "MLComputeUnitsAll" + * kCoremlProviderOption_ModelFormat: one of "MLProgram", "NeuralNetwork" + * kCoremlProviderOption_RequireStaticInputShapes: "1" or "0" + * kCoremlProviderOption_EnableOnSubgraphs: "1" or "0" + * kCoremlProviderOption_ModelCacheDir: path to the model cache directory * @param error Optional error information set if an error occurs. * @return Whether the provider was enabled successfully. */ -- (BOOL)appendCoreMLExecutionProviderWithOptions_v2:(NSDictionary*)provider_options - error:(NSError**)error; +- (BOOL)appendCoreMLExecutionProviderWithOptionsV2:(NSDictionary*)provider_options + error:(NSError**)error; @end NS_ASSUME_NONNULL_END diff --git a/objectivec/test/ort_session_test.mm b/objectivec/test/ort_session_test.mm index 8690aa5ee8183..409ee7e1584e2 100644 --- a/objectivec/test/ort_session_test.mm +++ b/objectivec/test/ort_session_test.mm @@ -226,10 +226,10 @@ - (void)testAppendCoreMLEP { - (void)testAppendCoreMLEP_v2 { NSError* err = nil; ORTSessionOptions* sessionOptions = [ORTSessionTest makeSessionOptions]; - NSDictionary* provider_options = @{@"MLEnableOnSubgraphs" : @"1"}; // set an arbitrary option + NSDictionary* provider_options = @{@"EnableOnSubgraphs" : @"1"}; // set an arbitrary option - BOOL appendResult = [sessionOptions appendCoreMLExecutionProviderWithOptions_v2:provider_options - error:&err]; + BOOL appendResult = [sessionOptions appendCoreMLExecutionProviderWithOptionsV2:provider_options + error:&err]; if (!ORTIsCoreMLExecutionProviderAvailable()) { ORTAssertBoolResultUnsuccessful(appendResult, err); diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index f3e8bd9b0e2af..7044150334a7b 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -23,9 +23,52 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; +void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& options) { + const std::unordered_map available_computeunits_options = { + {"CPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, + {"CPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, + {"CPUOnly", COREML_FLAG_USE_CPU_ONLY}, + {"ALL", COREML_FLAG_USE_NONE}, + }; + const std::unordered_map available_modelformat_options = { + {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, + {"NeuralNetwork", COREML_FLAG_USE_NONE}, + }; + std::unordered_set valid_options = { + kCoremlProviderOption_MLComputeUnits, + kCoremlProviderOption_ModelFormat, + kCoremlProviderOption_RequireStaticInputShapes, + kCoremlProviderOption_EnableOnSubgraphs, + kCoremlProviderOption_ModelCacheDir, + }; + // Validate the options + for (const auto& option : options) { + if (valid_options.find(option.first) == valid_options.end()) { + ORT_THROW("Unknown option: ", option.first); + } + if (kCoremlProviderOption_MLComputeUnits == option.first) { + if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { + ORT_THROW("Invalid value for option ", option.first, ": ", option.second); + } else { + coreml_flags_ |= available_computeunits_options.at(option.second); + } + } else if (kCoremlProviderOption_ModelFormat == option.first) { + if (available_modelformat_options.find(option.second) == available_modelformat_options.end()) { + ORT_THROW("Invalid value for option ", option.first, ": ", option.second); + } else { + coreml_flags_ |= available_modelformat_options.at(option.second); + } + } else if (kCoremlProviderOption_RequireStaticInputShapes == option.first) { + coreml_flags_ |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; + } else if (kCoremlProviderOption_EnableOnSubgraphs == option.first) { + coreml_flags_ |= COREML_FLAG_ENABLE_ON_SUBGRAPH; + } + } +} + CoreMLExecutionProvider::CoreMLExecutionProvider(const CoreMLOptions& options) : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, - coreml_flags_(options.coreml_flags), + coreml_flags_(options.CoreMLFlags()), coreml_version_(coreml::util::CoreMLVersion()) { LOGS_DEFAULT(VERBOSE) << "CoreML version: " << coreml_version_; if (coreml_version_ < MINIMUM_COREML_VERSION) { diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index d37f6bdc2732d..45679655ba429 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -12,9 +12,18 @@ namespace coreml { class Model; } -struct CoreMLOptions { - uint32_t coreml_flags = 0; - std::string cache_path; +class CoreMLOptions { + uint32_t coreml_flags_ = 0; + + public: + CoreMLOptions(uint32_t coreml_flags) : coreml_flags_(coreml_flags) {} + CoreMLOptions(const ProviderOptions& options) { + ValidateAndParseProviderOption(options); + } + uint32_t CoreMLFlags() const { return coreml_flags_; } + + private: + void ValidateAndParseProviderOption(const ProviderOptions& options); }; class CoreMLExecutionProvider : public IExecutionProvider { diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index 58c84f37930ef..bc8702d3290f6 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -10,55 +10,6 @@ using namespace onnxruntime; namespace onnxruntime { -namespace { -CoreMLOptions ParseProviderOption(const ProviderOptions& options) { - CoreMLOptions coreml_options; - const std::unordered_map available_computeunits_options = { - {"MLComputeUnitsCPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, - {"MLComputeUnitsCPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, - {"MLComputeUnitsCPUOnly", COREML_FLAG_USE_CPU_ONLY}, - {"MLComputeUnitsAll", COREML_FLAG_USE_NONE}, - }; - const std::unordered_map available_modelformat_options = { - {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, - {"NeuralNetwork", COREML_FLAG_USE_NONE}, - }; - std::unordered_set valid_options = { - kCoremlProviderOption_MLComputeUnits, - kCoremlProviderOption_MLModelFormat, - kCoremlProviderOption_MLAllowStaticInputShapes, - kCoremlProviderOption_MLEnableOnSubgraphs, - kCoremlProviderOption_MLModelCacheDir, - }; - // Validate the options - for (const auto& option : options) { - if (valid_options.find(option.first) == valid_options.end()) { - ORT_THROW("Unknown option: ", option.first); - } - if (kCoremlProviderOption_MLComputeUnits == option.first) { - if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { - ORT_THROW("Invalid value for option ", option.first, ": ", option.second); - } else { - coreml_options.coreml_flags |= available_computeunits_options.at(option.second); - } - } else if (kCoremlProviderOption_MLModelFormat == option.first) { - if (available_modelformat_options.find(option.second) == available_modelformat_options.end()) { - ORT_THROW("Invalid value for option ", option.first, ": ", option.second); - } else { - coreml_options.coreml_flags |= available_modelformat_options.at(option.second); - } - } else if (kCoremlProviderOption_MLAllowStaticInputShapes == option.first) { - coreml_options.coreml_flags |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; - } else if (kCoremlProviderOption_MLEnableOnSubgraphs == option.first) { - coreml_options.coreml_flags |= COREML_FLAG_ENABLE_ON_SUBGRAPH; - } else if (kCoremlProviderOption_MLModelCacheDir == option.first) { - coreml_options.cache_path = option.second; - } - } - - return coreml_options; -} -} // namespace struct CoreMLProviderFactory : IExecutionProviderFactory { CoreMLProviderFactory(const CoreMLOptions& options) : options_(options) {} @@ -73,13 +24,13 @@ std::unique_ptr CoreMLProviderFactory::CreateProvider() { } std::shared_ptr CoreMLProviderFactoryCreator::Create(uint32_t coreml_flags) { - CoreMLOptions coreml_options; - coreml_options.coreml_flags = coreml_flags; + CoreMLOptions coreml_options(coreml_flags); return std::make_shared(coreml_options); } std::shared_ptr CoreMLProviderFactoryCreator::Create(const ProviderOptions& options) { - return std::make_shared(ParseProviderOption(options)); + CoreMLOptions coreml_options(options); + return std::make_shared(coreml_options); } } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 23a1ad97e6df4..8f840cede63a0 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -131,7 +131,7 @@ namespace perftest { "\t [Example] [For NNAPI EP] -e nnapi -i \"NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED\"\n" "\n" "\t [CoreML only] [ModelFormat]:[MLProgram, NeuralNetwork] Create an ML Program model or Neural Network. Default is NeuralNetwork.\n" - "\t [CoreML only] [MLComputeUnits]:[CPUAndNeuralEngine CPUAndGPU CPUAndGPU CPUOnly] Specify to limit the backend device/s used to run the model.\n" + "\t [CoreML only] [MLComputeUnits]:[CPUAndNeuralEngine CPUAndGPU ALL CPUOnly] Specify to limit the backend device used to run the model.\n" "\t [CoreML only] [AllowStaticInputShapes]:[0 1].\n" "\t [CoreML only] [EnableOnSubgraphs]:[0 1].\n" "\t [CoreML only] [ModelCacheDir]: a path to cached compiled coreml model.\n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index b715e32cad3d0..a0f01eda93873 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -74,7 +74,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif // defined(_MSC_VER) int num_threads = 0; - if (!ParseSessionConfigs(ov_string, provider_options)) { + if (!ParseSessionConfigs(ov_string, provider_options, {"num_of_threads"})) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); @@ -89,10 +89,6 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device " set number of threads or use '0' for default\n"); // If the user doesnt define num_threads, auto detect threads later } - } else { - ORT_THROW( - "[ERROR] [OneDNN] wrong key type entered. " - "Choose from the following runtime key options that are available for OneDNN. ['num_of_threads']\n"); } } dnnl_options.threadpool_args = static_cast(&num_threads); @@ -214,7 +210,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(option_string, provider_options)) { + if (!ParseSessionConfigs(option_string, provider_options, {"backend_path", "profiling_file_path", "profiling_level", "rpc_control_latency", "vtcm_mb", "soc_model", "device_id", "htp_performance_mode", "qnn_saver_path", "htp_graph_finalization_optimization_mode", "qnn_context_priority", "htp_arch", "enable_htp_fp16_precision", "offload_graph_io_quantization"})) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); @@ -278,11 +274,6 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device std::string str = str_stream.str(); ORT_THROW("Wrong value for ", key, ". select from: ", str); } - } else { - ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path', -'profiling_level', 'profiling_file_path', 'rpc_control_latency', 'vtcm_mb', 'htp_performance_mode', -'qnn_saver_path', 'htp_graph_finalization_optimization_mode', 'qnn_context_priority', 'soc_model', -'htp_arch', 'device_id', 'enable_htp_fp16_precision', 'offload_graph_io_quantization'])"); } } session_options.AppendExecutionProvider("QNN", provider_options); @@ -296,7 +287,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(option_string, provider_options)) { + if (!ParseSessionConfigs(option_string, provider_options, {"runtime", "priority", "buffer_type", "enable_init_cache"})) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); @@ -320,8 +311,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); if (value != "1") { ORT_THROW("Set to 1 to enable_init_cache."); } - } else { - ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type', 'enable_init_cache'] \n"); } } @@ -370,32 +359,27 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #ifdef __APPLE__ #ifdef USE_COREML std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; - if (!ParseSessionConfigs(ov_string, provider_options)) { + if (!ParseSessionConfigs(ov_string, provider_options, {kCoremlProviderOption_MLComputeUnits, kCoremlProviderOption_ModelFormat, kCoremlProviderOption_RequireStaticInputShapes, kCoremlProviderOption_EnableOnSubgraphs})) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); } std::unordered_map available_options = { - {"MLComputeUnitsCPUAndNeuralEngine", "1"}, - {"MLComputeUnitsCPUAndGPU", "1"}, - {"MLComputeUnitsCPUOnly", "1"}, - {"MLComputeUnitsAll", "1"}, + {"CPUAndNeuralEngine", "1"}, + {"CPUAndGPU", "1"}, + {"CPUOnly", "1"}, + {"ALL", "1"}, }; for (const auto& provider_option : provider_options) { if (provider_option.first == kCoremlProviderOption_MLComputeUnits && available_options.find(provider_option.second) != available_options.end()) { - } else if (provider_option.first == kCoremlProviderOption_MLModelFormat && + } else if (provider_option.first == kCoremlProviderOption_ModelFormat && (provider_option.second == "MLProgram" || provider_option.second == "NeuralNetwork")) { - } else if (provider_option.first == kCoremlProviderOption_MLAllowStaticInputShapes && + } else if (provider_option.first == kCoremlProviderOption_RequireStaticInputShapes && (provider_option.second == "1" || provider_option.second == "0")) { - } else if (provider_option.first == kCoremlProviderOption_MLEnableOnSubgraphs && + } else if (provider_option.first == kCoremlProviderOption_EnableOnSubgraphs && (provider_option.second == "0" || provider_option.second == "1")) { - } else { - ORT_THROW( - "[ERROR] [CoreML] wrong key type entered. Choose from the following runtime key options " - "that are available for CoreML. " - "['MLComputeUnits', 'ModelFormat', 'AllowStaticInputShapes', 'EnableOnSubgraphs'] \n"); } } // COREML_FLAG_CREATE_MLPROGRAM @@ -413,7 +397,9 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(ov_string, provider_options)) { + if (!ParseSessionConfigs(ov_string, provider_options, + {"device_filter", "performance_preference", "disable_metacommands", + "enable_graph_capture", "enable_graph_serialization"})) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); @@ -488,7 +474,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif // defined(_MSC_VER) bool enable_fast_math = false; - if (!ParseSessionConfigs(ov_string, provider_options)) { + if (!ParseSessionConfigs(ov_string, provider_options, {"enable_fast_math"})) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); @@ -503,9 +489,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); "[ERROR] [ACL] You have selcted an invalid value for the key 'enable_fast_math'. " "Select from 'true' or 'false' \n"); } - } else { - ORT_THROW( - "[ERROR] [ACL] Unrecognized option: ", key); } } Ort::ThrowOnError( diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index 22f682159b924..d0ee779ff7264 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -7,12 +7,14 @@ #include #include "strings_helper.h" +#include "core/common/common.h" namespace onnxruntime { namespace perftest { bool ParseSessionConfigs(const std::string& configs_string, - std::unordered_map& session_configs) { + std::unordered_map& session_configs, + const std::unordered_set& available_keys) { std::istringstream ss(configs_string); std::string token; @@ -25,17 +27,27 @@ bool ParseSessionConfigs(const std::string& configs_string, auto pos = token_sv.find("|"); if (pos == std::string_view::npos || pos == 0 || pos == token_sv.length()) { - // Error: must use a '|' to separate the key and value for session configuration entries. - return false; + ORT_THROW("Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); } std::string key(token_sv.substr(0, pos)); std::string value(token_sv.substr(pos + 1)); + if (available_keys.empty() == false && available_keys.count(key) == 0) { + // Error: unknown option: {key} + std::string available_keys_str; + for (const auto& key : available_keys) { + available_keys_str += key; + available_keys_str += ", "; + } + ORT_THROW("[ERROR] wrong key type entered.: ", key, + " Choose from the following runtime key options that are available. ", available_keys_str); + } + auto it = session_configs.find(key); if (it != session_configs.end()) { // Error: specified duplicate session configuration entry: {key} - return false; + ORT_THROW("Specified duplicate session configuration entry: ", key); } session_configs.insert(std::make_pair(std::move(key), std::move(value))); diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h index 24feb90a20a61..b4b5704ebb2cd 100644 --- a/onnxruntime/test/perftest/strings_helper.h +++ b/onnxruntime/test/perftest/strings_helper.h @@ -4,11 +4,13 @@ // Licensed under the MIT License. #include #include +#include namespace onnxruntime { namespace perftest { bool ParseSessionConfigs(const std::string& configs_string, - std::unordered_map& session_configs); + std::unordered_map& session_configs, + const std::unordered_set& available_keys = {}); } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm b/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm index 8fe382b8fe246..fa95c1fc52b94 100644 --- a/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm +++ b/onnxruntime/test/platform/apple/apple_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm @@ -36,7 +36,7 @@ void testSigmoid(const char* modelPath, bool useCoreML = false, bool useWebGPU = #if COREML_EP_AVAILABLE if (useCoreML) { std::unordered_map provider_options = { - {kCoremlProviderOption_MLComputeUnits, "MLComputeUnitsCPUOnly"}}; + {kCoremlProviderOption_MLComputeUnits, "CPUOnly"}}; session_options.AppendExecutionProvider("CoreML", provider_options); } #else diff --git a/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm b/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm index 8a877bc4c9f7a..b53a4a2df09b4 100644 --- a/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm +++ b/onnxruntime/test/platform/apple/apple_package_test/macos_package_testUITests/macos_package_uitest_cpp_api.mm @@ -36,7 +36,7 @@ void testSigmoid(const char* modelPath, bool useCoreML = false, bool useWebGPU = #if COREML_EP_AVAILABLE if (useCoreML) { std::unordered_map provider_options = { - {kCoremlProviderOption_MLComputeUnits, "MLComputeUnitsCPUOnly"}}; + {kCoremlProviderOption_MLComputeUnits, "CPUOnly"}}; session_options.AppendExecutionProvider("CoreML", provider_options); } #else diff --git a/onnxruntime/test/providers/coreml/coreml_basic_test.cc b/onnxruntime/test/providers/coreml/coreml_basic_test.cc index acedfd1241021..a8480e7416de5 100644 --- a/onnxruntime/test/providers/coreml/coreml_basic_test.cc +++ b/onnxruntime/test/providers/coreml/coreml_basic_test.cc @@ -31,9 +31,9 @@ namespace onnxruntime { namespace test { static std::unique_ptr MakeCoreMLExecutionProvider( - std::string ModelFormat = "NeuralNetwork", std::string ComputeUnits = "MLComputeUnitsCPUOnly") { + std::string ModelFormat = "NeuralNetwork", std::string ComputeUnits = "CPUOnly") { std::unordered_map provider_options = {{kCoremlProviderOption_MLComputeUnits, ComputeUnits}, - {kCoremlProviderOption_MLModelFormat, ModelFormat}}; + {kCoremlProviderOption_ModelFormat, ModelFormat}}; return CoreMLProviderFactoryCreator::Create(provider_options)->CreateProvider(); } diff --git a/onnxruntime/test/providers/coreml/dynamic_input_test.cc b/onnxruntime/test/providers/coreml/dynamic_input_test.cc index af81e75e5fe53..8294f65745256 100644 --- a/onnxruntime/test/providers/coreml/dynamic_input_test.cc +++ b/onnxruntime/test/providers/coreml/dynamic_input_test.cc @@ -102,7 +102,7 @@ TEST(CoreMLExecutionProviderDynamicInputShapeTest, EmptyInputFails) { TEST(CoreMLExecutionProviderDynamicInputShapeTest, OnlyAllowStaticInputShapes) { constexpr auto model_path = ORT_TSTR("testdata/matmul_with_dynamic_input_shape.onnx"); - std::unordered_map options = {{kCoremlProviderOption_MLAllowStaticInputShapes, "1"}}; + std::unordered_map options = {{kCoremlProviderOption_RequireStaticInputShapes, "1"}}; auto coreml_ep = CoreMLProviderFactoryCreator::Create(options)->CreateProvider(); ; diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 2b435c7cdcf67..b27f3116911e9 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -254,10 +254,10 @@ std::unique_ptr DefaultCoreMLExecutionProvider(bool use_mlpr #if defined(USE_COREML) && defined(__APPLE__) // We want to run UT on CPU only to get output value without losing precision auto option = ProviderOptions(); - option[kCoremlProviderOption_MLComputeUnits] = "MLComputeUnitsCPUOnly"; + option[kCoremlProviderOption_MLComputeUnits] = "CPUOnly"; if (use_mlprogram) { - option[kCoremlProviderOption_MLModelFormat] = "MLProgram"; + option[kCoremlProviderOption_ModelFormat] = "MLProgram"; } return CoreMLProviderFactoryCreator::Create(option)->CreateProvider(); From cb7574129fb4a12183eb6cf5daaae5912aeeaeb2 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 21 Nov 2024 01:44:31 -0800 Subject: [PATCH 13/22] comments --- objectivec/include/ort_coreml_execution_provider.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/objectivec/include/ort_coreml_execution_provider.h b/objectivec/include/ort_coreml_execution_provider.h index 417868a08972c..d711cb42337cf 100644 --- a/objectivec/include/ort_coreml_execution_provider.h +++ b/objectivec/include/ort_coreml_execution_provider.h @@ -76,9 +76,9 @@ NS_ASSUME_NONNULL_BEGIN * decreasing priority. * * @param provider_options The CoreML execution provider options in dict. - * available keys-values: more detail in onnxruntime/core/providers/coreml/coreml_provider_factory.cc - * kCoremlProviderOption_MLComputeUnits: one of "MLComputeUnitsCPUAndNeuralEngine", - * "MLComputeUnitsCPUAndGPU", "MLComputeUnitsCPUOnly", "MLComputeUnitsAll" + * available keys-values: more detail in core/providers/coreml/coreml_execution_provider.h + * kCoremlProviderOption_MLComputeUnits: one of "CPUAndNeuralEngine", + * "CPUAndGPU", "CPUOnly", "All" * kCoremlProviderOption_ModelFormat: one of "MLProgram", "NeuralNetwork" * kCoremlProviderOption_RequireStaticInputShapes: "1" or "0" * kCoremlProviderOption_EnableOnSubgraphs: "1" or "0" From fbc166c3d8304bf23d73ab16b228e759c1177b38 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 21 Nov 2024 20:09:42 -0800 Subject: [PATCH 14/22] most likly key --- .../coreml/coreml_execution_provider.cc | 3 +- .../test/perftest/command_args_parser.cc | 1 - onnxruntime/test/perftest/ort_test_session.cc | 2 + onnxruntime/test/perftest/strings_helper.cc | 61 +++++++++++++++++-- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 7044150334a7b..19e3b55f8da38 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -39,7 +39,6 @@ void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& option kCoremlProviderOption_ModelFormat, kCoremlProviderOption_RequireStaticInputShapes, kCoremlProviderOption_EnableOnSubgraphs, - kCoremlProviderOption_ModelCacheDir, }; // Validate the options for (const auto& option : options) { @@ -48,7 +47,7 @@ void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& option } if (kCoremlProviderOption_MLComputeUnits == option.first) { if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { - ORT_THROW("Invalid value for option ", option.first, ": ", option.second); + ORT_THROW("Invalid value for option `", option.first, "`: ", option.second); } else { coreml_flags_ |= available_computeunits_options.at(option.second); } diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 8f840cede63a0..40acac8f21931 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -134,7 +134,6 @@ namespace perftest { "\t [CoreML only] [MLComputeUnits]:[CPUAndNeuralEngine CPUAndGPU ALL CPUOnly] Specify to limit the backend device used to run the model.\n" "\t [CoreML only] [AllowStaticInputShapes]:[0 1].\n" "\t [CoreML only] [EnableOnSubgraphs]:[0 1].\n" - "\t [CoreML only] [ModelCacheDir]: a path to cached compiled coreml model.\n" "\t [Example] [For CoreML EP] -e coreml -i \"ModelFormat|MLProgram MLComputeUnits|CPUAndGPU\"\n" "\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index a0f01eda93873..6fa47df5dc011 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -380,6 +380,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); (provider_option.second == "1" || provider_option.second == "0")) { } else if (provider_option.first == kCoremlProviderOption_EnableOnSubgraphs && (provider_option.second == "0" || provider_option.second == "1")) { + } else { + ORT_THROW("Invalid value for option ", provider_option.first, ": ", provider_option.second); } } // COREML_FLAG_CREATE_MLPROGRAM diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index d0ee779ff7264..a7dbaf37f4573 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -12,6 +12,59 @@ namespace onnxruntime { namespace perftest { +/** + * @brief Computes the Levenshtein distance between two strings. + * + * The Levenshtein distance is a metric for measuring the difference between two strings. + * It represents the minimum number of single-character edits required to change one string + * into the other. The possible edits are insertion, deletion, or substitution of a character. + * + * This function uses a dynamic programming approach with optimized space complexity. + * It maintains only two vectors representing the previous and current rows of the distance matrix. + * + * @param str1 The first input string. + * @param str2 The second input string. + * @return The Levenshtein distance between the two strings. + */ +static size_t LevenShtein(const std::string& str1, const std::string& str2) { + size_t m = str1.length(); + size_t n = str2.length(); + + std::vector prevRow(n + 1, 0); + std::vector currRow(n + 1, 0); + + for (size_t j = 0; j <= n; j++) { + prevRow[j] = j; + } + + for (size_t i = 1; i <= m; i++) { + currRow[0] = i; + for (size_t j = 1; j <= n; j++) { + if (str1[i - 1] == str2[j - 1]) { + currRow[j] = prevRow[j - 1]; + } else { + currRow[j] = 1 + std::min(currRow[j - 1], std::min(prevRow[j], prevRow[j - 1])); + } + } + prevRow = currRow; + } + + return currRow[n]; +} + +static std::string FindNearestKey(const std::string& key, const std::unordered_set& available_keys) { + std::string similar_key; + size_t min_distance = std::numeric_limits::max(); + for (const auto& av_key : available_keys) { + size_t distance = LevenShtein(av_key, key); + if (distance < min_distance) { + min_distance = distance; + similar_key = av_key; + } + } + return similar_key; +} + bool ParseSessionConfigs(const std::string& configs_string, std::unordered_map& session_configs, const std::unordered_set& available_keys) { @@ -36,12 +89,12 @@ bool ParseSessionConfigs(const std::string& configs_string, if (available_keys.empty() == false && available_keys.count(key) == 0) { // Error: unknown option: {key} std::string available_keys_str; - for (const auto& key : available_keys) { - available_keys_str += key; + for (const auto& av_key : available_keys) { + available_keys_str += av_key; available_keys_str += ", "; } - ORT_THROW("[ERROR] wrong key type entered.: ", key, - " Choose from the following runtime key options that are available. ", available_keys_str); + ORT_THROW("[ERROR] wrong key type entered : `", key, "`. Did you mean: `", FindNearestKey(key, available_keys), + "`?. The following runtime key options are avaible: [", available_keys_str, "]"); } auto it = session_configs.find(key); From 7ac7cfd86b0496615bf6f7da4800772e13155951 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Thu, 21 Nov 2024 21:36:31 -0800 Subject: [PATCH 15/22] remove key suggestion --- onnxruntime/test/perftest/strings_helper.cc | 57 +-------------------- 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index a7dbaf37f4573..5e6ade6b76523 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -12,59 +12,6 @@ namespace onnxruntime { namespace perftest { -/** - * @brief Computes the Levenshtein distance between two strings. - * - * The Levenshtein distance is a metric for measuring the difference between two strings. - * It represents the minimum number of single-character edits required to change one string - * into the other. The possible edits are insertion, deletion, or substitution of a character. - * - * This function uses a dynamic programming approach with optimized space complexity. - * It maintains only two vectors representing the previous and current rows of the distance matrix. - * - * @param str1 The first input string. - * @param str2 The second input string. - * @return The Levenshtein distance between the two strings. - */ -static size_t LevenShtein(const std::string& str1, const std::string& str2) { - size_t m = str1.length(); - size_t n = str2.length(); - - std::vector prevRow(n + 1, 0); - std::vector currRow(n + 1, 0); - - for (size_t j = 0; j <= n; j++) { - prevRow[j] = j; - } - - for (size_t i = 1; i <= m; i++) { - currRow[0] = i; - for (size_t j = 1; j <= n; j++) { - if (str1[i - 1] == str2[j - 1]) { - currRow[j] = prevRow[j - 1]; - } else { - currRow[j] = 1 + std::min(currRow[j - 1], std::min(prevRow[j], prevRow[j - 1])); - } - } - prevRow = currRow; - } - - return currRow[n]; -} - -static std::string FindNearestKey(const std::string& key, const std::unordered_set& available_keys) { - std::string similar_key; - size_t min_distance = std::numeric_limits::max(); - for (const auto& av_key : available_keys) { - size_t distance = LevenShtein(av_key, key); - if (distance < min_distance) { - min_distance = distance; - similar_key = av_key; - } - } - return similar_key; -} - bool ParseSessionConfigs(const std::string& configs_string, std::unordered_map& session_configs, const std::unordered_set& available_keys) { @@ -93,8 +40,8 @@ bool ParseSessionConfigs(const std::string& configs_string, available_keys_str += av_key; available_keys_str += ", "; } - ORT_THROW("[ERROR] wrong key type entered : `", key, "`. Did you mean: `", FindNearestKey(key, available_keys), - "`?. The following runtime key options are avaible: [", available_keys_str, "]"); + ORT_THROW("[ERROR] wrong key type entered : `", key, + "`. The following runtime key options are avaible: [", available_keys_str, "]"); } auto it = session_configs.find(key); From cfcc3ab7202544d66af0fca6682675425e34d7d6 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 22 Nov 2024 01:25:05 -0800 Subject: [PATCH 16/22] unify coreml options --- .../core/providers/coreml/builders/helper.cc | 13 +-- .../core/providers/coreml/builders/helper.h | 5 +- .../coreml/builders/model_builder.cc | 15 +-- .../providers/coreml/builders/model_builder.h | 8 +- .../coreml/coreml_execution_provider.cc | 94 +++++++++++-------- .../coreml/coreml_execution_provider.h | 16 +++- .../core/providers/coreml/model/model.h | 2 +- .../core/providers/coreml/model/model.mm | 12 +-- 8 files changed, 96 insertions(+), 69 deletions(-) diff --git a/onnxruntime/core/providers/coreml/builders/helper.cc b/onnxruntime/core/providers/coreml/builders/helper.cc index e1f148fa93e23..38ac629331749 100644 --- a/onnxruntime/core/providers/coreml/builders/helper.cc +++ b/onnxruntime/core/providers/coreml/builders/helper.cc @@ -24,11 +24,12 @@ namespace coreml { OpBuilderInputParams MakeOpBuilderParams(const GraphViewer& graph_viewer, int32_t coreml_version, - uint32_t coreml_flags) { + bool only_allow_static_input_shapes, + bool create_mlprogram) { return OpBuilderInputParams{graph_viewer, coreml_version, - (coreml_flags & COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES) != 0, - (coreml_flags & COREML_FLAG_CREATE_MLPROGRAM) != 0}; + only_allow_static_input_shapes, + create_mlprogram}; } const IOpBuilder* GetOpBuilder(const Node& node) { @@ -133,13 +134,13 @@ bool CheckIsConstantInitializer(const NodeArg& node_arg, const GraphViewer& grap return true; } -bool HasNeuralEngine(const logging::Logger& logger) { +bool HasNeuralEngine() { bool has_neural_engine = false; #ifdef __APPLE__ struct utsname system_info; uname(&system_info); - LOGS(logger, VERBOSE) << "Current Apple hardware info: " << system_info.machine; + LOGS_DEFAULT(VERBOSE) << "Current Apple hardware info: " << system_info.machine; #if TARGET_OS_IPHONE // utsname.machine has device identifier. For example, identifier for iPhone Xs is "iPhone11,2". @@ -163,7 +164,7 @@ bool HasNeuralEngine(const logging::Logger& logger) { #else // In this case, we are running the EP on non-apple platform, which means we are running the model // conversion with CoreML EP enabled, for this we always assume the target system has Neural Engine - LOGS(logger, INFO) << "HasNeuralEngine running on non-Apple hardware. " + LOGS_DEFAULT(INFO) << "HasNeuralEngine running on non-Apple hardware. " "Returning true to enable model conversion and local testing of CoreML EP implementation. " "No CoreML model will be compiled or run."; has_neural_engine = true; diff --git a/onnxruntime/core/providers/coreml/builders/helper.h b/onnxruntime/core/providers/coreml/builders/helper.h index 0acaa0dd8a4a3..ae7f3bdbc31a9 100644 --- a/onnxruntime/core/providers/coreml/builders/helper.h +++ b/onnxruntime/core/providers/coreml/builders/helper.h @@ -25,7 +25,8 @@ namespace coreml { OpBuilderInputParams MakeOpBuilderParams(const GraphViewer& graph_viewer, int32_t coreml_version, - uint32_t coreml_flags); + bool only_allow_static_input_shapes, + bool create_mlprogram); const IOpBuilder* GetOpBuilder(const Node& node); @@ -45,7 +46,7 @@ bool CheckIsConstantInitializer(const NodeArg& node_arg, const GraphViewer& grap // CoreML is more efficient running using Apple Neural Engine // This is to detect if the current system has Apple Neural Engine -bool HasNeuralEngine(const logging::Logger& logger); +bool HasNeuralEngine(); } // namespace coreml } // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/builders/model_builder.cc b/onnxruntime/core/providers/coreml/builders/model_builder.cc index f12e4dab5b3ec..2a02c1f4124f6 100644 --- a/onnxruntime/core/providers/coreml/builders/model_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/model_builder.cc @@ -8,6 +8,7 @@ #include "core/platform/env.h" #include "core/providers/common.h" #include "core/providers/coreml/builders/model_builder.h" +#include "core/providers/coreml/coreml_execution_provider.h" #include "core/providers/coreml/builders/helper.h" #include "core/providers/coreml/builders/op_builder_factory.h" #include "core/providers/coreml/builders/impl/builder_utils.h" @@ -401,14 +402,14 @@ std::string GetModelOutputPath(bool create_ml_program) { } // namespace ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger, - int32_t coreml_version, uint32_t coreml_flags, + int32_t coreml_version, const CoreMLOptions& coreml_options, std::vector&& onnx_input_names, std::vector&& onnx_output_names) : graph_viewer_(graph_viewer), logger_(logger), coreml_version_(coreml_version), - coreml_flags_(coreml_flags), - create_ml_program_((coreml_flags_ & COREML_FLAG_CREATE_MLPROGRAM) != 0), + coreml_compute_unit_(coreml_options.ComputeUnits()), + create_ml_program_(coreml_options.CreateMLProgram()), model_output_path_(GetModelOutputPath(create_ml_program_)), onnx_input_names_(std::move(onnx_input_names)), onnx_output_names_(std::move(onnx_output_names)), @@ -988,7 +989,7 @@ Status ModelBuilder::LoadModel(std::unique_ptr& model) { get_sanitized_io_info(std::move(input_output_info_)), std::move(scalar_outputs_), std::move(int64_outputs_), - logger_, coreml_flags_); + logger_, coreml_compute_unit_); } else #endif { @@ -998,7 +999,7 @@ Status ModelBuilder::LoadModel(std::unique_ptr& model) { std::move(input_output_info_), std::move(scalar_outputs_), std::move(int64_outputs_), - logger_, coreml_flags_); + logger_, coreml_compute_unit_); } return model->LoadModel(); // load using CoreML API, including compilation @@ -1048,11 +1049,11 @@ std::string_view ModelBuilder::AddConstant(std::string_view op_type, std::string #endif // static Status ModelBuilder::Build(const GraphViewer& graph_viewer, const logging::Logger& logger, - int32_t coreml_version, uint32_t coreml_flags, + int32_t coreml_version, const CoreMLOptions& coreml_options, std::vector&& onnx_input_names, std::vector&& onnx_output_names, std::unique_ptr& model) { - ModelBuilder builder(graph_viewer, logger, coreml_version, coreml_flags, + ModelBuilder builder(graph_viewer, logger, coreml_version, coreml_options, std::move(onnx_input_names), std::move(onnx_output_names)); ORT_RETURN_IF_ERROR(builder.CreateModel()); diff --git a/onnxruntime/core/providers/coreml/builders/model_builder.h b/onnxruntime/core/providers/coreml/builders/model_builder.h index c566dbe160b50..af47869f7e1c3 100644 --- a/onnxruntime/core/providers/coreml/builders/model_builder.h +++ b/onnxruntime/core/providers/coreml/builders/model_builder.h @@ -22,6 +22,8 @@ class StorageWriter; #endif namespace onnxruntime { +class CoreMLOptions; + namespace coreml { class IOpBuilder; @@ -29,14 +31,14 @@ class IOpBuilder; class ModelBuilder { private: ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger, - int32_t coreml_version, uint32_t coreml_flags, + int32_t coreml_version, const CoreMLOptions& coreml_options, std::vector&& onnx_input_names, std::vector&& onnx_output_names); public: // Create the CoreML model, serialize to disk, load and compile using the CoreML API and return in `model` static Status Build(const GraphViewer& graph_viewer, const logging::Logger& logger, - int32_t coreml_version, uint32_t coreml_flags, + int32_t coreml_version, const CoreMLOptions& coreml_options, std::vector&& onnx_input_names, std::vector&& onnx_output_names, std::unique_ptr& model); @@ -216,7 +218,7 @@ class ModelBuilder { const GraphViewer& graph_viewer_; const logging::Logger& logger_; const int32_t coreml_version_; - const uint32_t coreml_flags_; + const uint32_t coreml_compute_unit_; const bool create_ml_program_; // ML Program (CoreML5, iOS 15+, macOS 12+) or NeuralNetwork (old) const std::string model_output_path_; // create_ml_program_ ? dir for mlpackage : filename for mlmodel diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 19e3b55f8da38..6e3d6ee4750c8 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -23,6 +23,50 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; +CoreMLOptions::CoreMLOptions(uint32_t coreml_flags) { + // validate the flags and populate the members. should be moving code from ctor to here + require_static_shape_ = (coreml_flags & COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES) != 0; + create_mlprogram_ = (coreml_flags & COREML_FLAG_CREATE_MLPROGRAM) != 0; + enable_on_subgraph_ = (coreml_flags & COREML_FLAG_ENABLE_ON_SUBGRAPH) != 0; + +#if defined(COREML_ENABLE_MLPROGRAM) + if (coreml::util::CoreMLVersion() < MINIMUM_COREML_MLPROGRAM_VERSION && create_mlprogram_ != 0) { + LOGS_DEFAULT(WARNING) << "ML Program is not supported on this OS version. Falling back to NeuralNetwork."; + create_mlprogram_ = false; + } +#else + if (create_mlprogram_ != 0) { + LOGS_DEFAULT(WARNING) << "ML Program is not supported in this build. Falling back to NeuralNetwork."; + create_mlprogram_ = false; + } +#endif + + compute_units_ = 0; // 0 for all + + if (coreml_flags & COREML_FLAG_USE_CPU_ONLY) { + compute_units_ |= COREML_FLAG_USE_CPU_ONLY; + } + if (coreml_flags & COREML_FLAG_USE_CPU_AND_GPU) { + compute_units_ |= COREML_FLAG_USE_CPU_AND_GPU; + } + if (coreml_flags & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) { + compute_units_ |= COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE; + } + + // assure only one device option is selected + if (compute_units_ & (compute_units_ - 1)) { + // multiple device options selected + ORT_THROW( + "Multiple device options selected, you should use at most one of the following options:" + "[COREML_FLAG_USE_CPU_ONLY, COREML_FLAG_USE_CPU_AND_GPU, COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE]"); + } + + const bool has_neural_engine = coreml::HasNeuralEngine(); + if ((compute_units_ & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) && !has_neural_engine) { + ORT_THROW("The current system does not have Apple Neural Engine."); + } +} + void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& options) { const std::unordered_map available_computeunits_options = { {"CPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, @@ -49,51 +93,30 @@ void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& option if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { ORT_THROW("Invalid value for option `", option.first, "`: ", option.second); } else { - coreml_flags_ |= available_computeunits_options.at(option.second); + compute_units_ = available_computeunits_options.at(option.second); } } else if (kCoremlProviderOption_ModelFormat == option.first) { if (available_modelformat_options.find(option.second) == available_modelformat_options.end()) { ORT_THROW("Invalid value for option ", option.first, ": ", option.second); } else { - coreml_flags_ |= available_modelformat_options.at(option.second); + create_mlprogram_ = available_modelformat_options.at(option.second) & COREML_FLAG_CREATE_MLPROGRAM; } } else if (kCoremlProviderOption_RequireStaticInputShapes == option.first) { - coreml_flags_ |= COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES; + require_static_shape_ = option.second == "1"; } else if (kCoremlProviderOption_EnableOnSubgraphs == option.first) { - coreml_flags_ |= COREML_FLAG_ENABLE_ON_SUBGRAPH; + enable_on_subgraph_ = option.second == "1"; } } } CoreMLExecutionProvider::CoreMLExecutionProvider(const CoreMLOptions& options) : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, - coreml_flags_(options.CoreMLFlags()), + coreml_options_(options), coreml_version_(coreml::util::CoreMLVersion()) { LOGS_DEFAULT(VERBOSE) << "CoreML version: " << coreml_version_; if (coreml_version_ < MINIMUM_COREML_VERSION) { - LOGS_DEFAULT(ERROR) << "CoreML EP is not supported on this platform."; - } - - // check if only one flag is set - if ((coreml_flags_ & COREML_FLAG_USE_CPU_ONLY) && (coreml_flags_ & COREML_FLAG_USE_CPU_AND_GPU)) { - // multiple device options selected - ORT_THROW( - "Multiple device options selected, you should use at most one of the following options:" - "COREML_FLAG_USE_CPU_ONLY or COREML_FLAG_USE_CPU_AND_GPU or not set"); + ORT_THROW("CoreML EP is not supported on this platform."); } - -#if defined(COREML_ENABLE_MLPROGRAM) - if (coreml_version_ < MINIMUM_COREML_MLPROGRAM_VERSION && - (coreml_flags_ & COREML_FLAG_CREATE_MLPROGRAM) != 0) { - LOGS_DEFAULT(WARNING) << "ML Program is not supported on this OS version. Falling back to NeuralNetwork."; - coreml_flags_ ^= COREML_FLAG_CREATE_MLPROGRAM; - } -#else - if ((coreml_flags_ & COREML_FLAG_CREATE_MLPROGRAM) != 0) { - LOGS_DEFAULT(WARNING) << "ML Program is not supported in this build. Falling back to NeuralNetwork."; - coreml_flags_ ^= COREML_FLAG_CREATE_MLPROGRAM; - } -#endif } CoreMLExecutionProvider::~CoreMLExecutionProvider() {} @@ -103,26 +126,17 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie const IKernelLookup& /*kernel_lookup*/) const { std::vector> result; - if (coreml_version_ < MINIMUM_COREML_VERSION) { - return result; - } - const auto& logger = *GetLogger(); // We do not run CoreML EP on subgraph, instead we cover this in the control flow nodes // TODO investigate whether we want to support subgraph using CoreML EP. May simply require processing the // implicit inputs of the control flow node that contains the subgraph as inputs to the CoreML model we generate. - if (graph_viewer.IsSubgraph() && !(coreml_flags_ & COREML_FLAG_ENABLE_ON_SUBGRAPH)) { - return result; - } - - const bool has_neural_engine = coreml::HasNeuralEngine(logger); - if ((coreml_flags_ & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) && !has_neural_engine) { - LOGS(logger, WARNING) << "The current system does not have Apple Neural Engine. CoreML EP will not be used."; + if (graph_viewer.IsSubgraph() && !coreml_options_.EnableOnSubgraph()) { return result; } - const auto builder_params = coreml::MakeOpBuilderParams(graph_viewer, coreml_version_, coreml_flags_); + const auto builder_params = coreml::MakeOpBuilderParams(graph_viewer, coreml_version_, + coreml_options_.RequireStaticShape(), coreml_options_.CreateMLProgram()); const auto supported_nodes = coreml::GetSupportedNodes(graph_viewer, builder_params, logger); const auto gen_metadef_name = @@ -185,7 +199,7 @@ common::Status CoreMLExecutionProvider::Compile(const std::vector onnx_output_names = get_names(fused_node.OutputDefs()); const onnxruntime::GraphViewer& graph_viewer(fused_node_and_graph.filtered_graph); - ORT_RETURN_IF_ERROR(coreml::ModelBuilder::Build(graph_viewer, *GetLogger(), coreml_version_, coreml_flags_, + ORT_RETURN_IF_ERROR(coreml::ModelBuilder::Build(graph_viewer, *GetLogger(), coreml_version_, coreml_options_, std::move(onnx_input_names), std::move(onnx_output_names), coreml_model)); } diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 45679655ba429..40c05d5c64525 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -13,14 +13,22 @@ class Model; } class CoreMLOptions { - uint32_t coreml_flags_ = 0; + private: + bool require_static_shape_{false}; + bool create_mlprogram_{false}; + bool enable_on_subgraph_{false}; + uint32_t compute_units_{0}; public: - CoreMLOptions(uint32_t coreml_flags) : coreml_flags_(coreml_flags) {} + explicit CoreMLOptions(uint32_t coreml_flags); + CoreMLOptions(const ProviderOptions& options) { ValidateAndParseProviderOption(options); } - uint32_t CoreMLFlags() const { return coreml_flags_; } + bool RequireStaticShape() const { return require_static_shape_; } + bool CreateMLProgram() const { return create_mlprogram_; } + bool EnableOnSubgraph() const { return enable_on_subgraph_; } + uint32_t ComputeUnits(uint32_t specfic_flag=0xffffffff) const { return compute_units_&specfic_flag; } private: void ValidateAndParseProviderOption(const ProviderOptions& options); @@ -43,7 +51,7 @@ class CoreMLExecutionProvider : public IExecutionProvider { private: // The bit flags which define bool options for COREML EP, bits are defined as // COREMLFlags in include/onnxruntime/core/providers/coreml/coreml_provider_factory.h - uint32_t coreml_flags_; + CoreMLOptions coreml_options_; const int32_t coreml_version_; ModelMetadefIdGenerator metadef_id_generator_; diff --git a/onnxruntime/core/providers/coreml/model/model.h b/onnxruntime/core/providers/coreml/model/model.h index 7fdd6b25bc7db..68ecbe5fb80c4 100644 --- a/onnxruntime/core/providers/coreml/model/model.h +++ b/onnxruntime/core/providers/coreml/model/model.h @@ -53,7 +53,7 @@ class Model { std::unordered_map&& input_output_info, std::unordered_set&& scalar_outputs, std::unordered_set&& int64_outputs, - const logging::Logger& logger, uint32_t coreml_flags); + const logging::Logger& logger, uint32_t coreml_compute_unit); ~Model(); ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Model); diff --git a/onnxruntime/core/providers/coreml/model/model.mm b/onnxruntime/core/providers/coreml/model/model.mm index 1e7aa5ecd1e60..c8edb64ff55d7 100644 --- a/onnxruntime/core/providers/coreml/model/model.mm +++ b/onnxruntime/core/providers/coreml/model/model.mm @@ -320,13 +320,13 @@ Status Predict(const std::unordered_map& inputs, NSString* coreml_model_path_{nil}; NSString* compiled_model_path_{nil}; const logging::Logger& logger_; - uint32_t coreml_flags_{0}; + uint32_t coreml_compute_unit_{0}; MLModel* model_{nil}; }; -Execution::Execution(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags) +Execution::Execution(const std::string& path, const logging::Logger& logger, uint32_t coreml_compute_unit) : logger_(logger), - coreml_flags_(coreml_flags) { + coreml_compute_unit_(coreml_compute_unit) { @autoreleasepool { coreml_model_path_ = util::Utf8StringToNSString(path.c_str()); } @@ -396,11 +396,11 @@ Status Predict(const std::unordered_map& inputs, MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; - if (coreml_flags_ & COREML_FLAG_USE_CPU_ONLY) { + if (coreml_compute_unit_ & COREML_FLAG_USE_CPU_ONLY) { config.computeUnits = MLComputeUnitsCPUOnly; - } else if (coreml_flags_ & COREML_FLAG_USE_CPU_AND_GPU) { + } else if (coreml_compute_unit_ & COREML_FLAG_USE_CPU_AND_GPU) { config.computeUnits = MLComputeUnitsCPUAndGPU; - } else if (coreml_flags_ & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) { + } else if (coreml_compute_unit_ & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) { config.computeUnits = MLComputeUnitsCPUAndNeuralEngine; // Apple Neural Engine } else { config.computeUnits = MLComputeUnitsAll; From c49f250aa953c5389126bf3b1cdb9a3c1b0a27ff Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 22 Nov 2024 17:27:23 +0800 Subject: [PATCH 17/22] Apply suggestions from code review Co-authored-by: Scott McKay --- .../core/providers/coreml/coreml_provider_factory.h | 5 ++--- objectivec/include/ort_coreml_execution_provider.h | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h index 9846da8f7fa7d..3963b80de58a4 100644 --- a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h +++ b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h @@ -43,13 +43,12 @@ enum COREMLFlags { // MLComputeUnits can be one of the following values: // 'MLComputeUnitsCPUAndNeuralEngine|MLComputeUnitsCPUAndGPU|MLComputeUnitsCPUOnly|MLComputeUnitsAll' -// these values are intended to be used with Ort.::SessionOptions::AppendExecutionProvider(C++ API) -// /SessionOptionsAppendExecutionProvider (C API). For the old API, use COREMLFlags instead. +// these values are intended to be used with Ort::SessionOptions::AppendExecutionProvider (C++ API) +// and SessionOptionsAppendExecutionProvider (C API). For the old API, use COREMLFlags instead. static const char* const kCoremlProviderOption_MLComputeUnits = "MLComputeUnits"; static const char* const kCoremlProviderOption_ModelFormat = "ModelFormat"; static const char* const kCoremlProviderOption_RequireStaticInputShapes = "RequireStaticInputShapes"; static const char* const kCoremlProviderOption_EnableOnSubgraphs = "EnableOnSubgraphs"; -static const char* const kCoremlProviderOption_ModelCacheDir = "ModelCacheDir"; #ifdef __cplusplus extern "C" { diff --git a/objectivec/include/ort_coreml_execution_provider.h b/objectivec/include/ort_coreml_execution_provider.h index d711cb42337cf..41d15aa39453a 100644 --- a/objectivec/include/ort_coreml_execution_provider.h +++ b/objectivec/include/ort_coreml_execution_provider.h @@ -77,12 +77,10 @@ NS_ASSUME_NONNULL_BEGIN * * @param provider_options The CoreML execution provider options in dict. * available keys-values: more detail in core/providers/coreml/coreml_execution_provider.h - * kCoremlProviderOption_MLComputeUnits: one of "CPUAndNeuralEngine", - * "CPUAndGPU", "CPUOnly", "All" + * kCoremlProviderOption_MLComputeUnits: one of "CPUAndNeuralEngine", "CPUAndGPU", "CPUOnly", "All" * kCoremlProviderOption_ModelFormat: one of "MLProgram", "NeuralNetwork" * kCoremlProviderOption_RequireStaticInputShapes: "1" or "0" * kCoremlProviderOption_EnableOnSubgraphs: "1" or "0" - * kCoremlProviderOption_ModelCacheDir: path to the model cache directory * @param error Optional error information set if an error occurs. * @return Whether the provider was enabled successfully. */ From 07f5772bc19d7dc16bee8e57240d93116e351c45 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 22 Nov 2024 17:31:17 +0800 Subject: [PATCH 18/22] Update onnxruntime/test/perftest/ort_test_session.cc Co-authored-by: Scott McKay --- onnxruntime/test/perftest/ort_test_session.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 6fa47df5dc011..28a5fcf760a06 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -359,7 +359,11 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #ifdef __APPLE__ #ifdef USE_COREML std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; - if (!ParseSessionConfigs(ov_string, provider_options, {kCoremlProviderOption_MLComputeUnits, kCoremlProviderOption_ModelFormat, kCoremlProviderOption_RequireStaticInputShapes, kCoremlProviderOption_EnableOnSubgraphs})) { + static const std::unordered_set available_keys = {kCoremlProviderOption_MLComputeUnits, + kCoremlProviderOption_ModelFormat, + kCoremlProviderOption_RequireStaticInputShapes, + kCoremlProviderOption_EnableOnSubgraphs}; + if (!ParseSessionConfigs(ov_string, provider_options, available_keys)) { ORT_THROW( "[ERROR] Use a '|' to separate the key and value for the " "run-time option you are trying to use.\n"); From a732326007dacb10d41ff7222c35cbefffc913cd Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 22 Nov 2024 01:34:28 -0800 Subject: [PATCH 19/22] return void for ParseSessionConfigs --- .../coreml/coreml_execution_provider.h | 2 +- onnxruntime/test/perftest/ort_test_session.cc | 62 +++++-------------- onnxruntime/test/perftest/strings_helper.cc | 4 +- onnxruntime/test/perftest/strings_helper.h | 2 +- 4 files changed, 18 insertions(+), 52 deletions(-) diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 40c05d5c64525..d2436bfe64339 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -28,7 +28,7 @@ class CoreMLOptions { bool RequireStaticShape() const { return require_static_shape_; } bool CreateMLProgram() const { return create_mlprogram_; } bool EnableOnSubgraph() const { return enable_on_subgraph_; } - uint32_t ComputeUnits(uint32_t specfic_flag=0xffffffff) const { return compute_units_&specfic_flag; } + uint32_t ComputeUnits(uint32_t specfic_flag = 0xffffffff) const { return compute_units_ & specfic_flag; } private: void ValidateAndParseProviderOption(const ProviderOptions& options); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 28a5fcf760a06..02768b8c08e85 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -74,11 +74,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif // defined(_MSC_VER) int num_threads = 0; - if (!ParseSessionConfigs(ov_string, provider_options, {"num_of_threads"})) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(ov_string, provider_options, {"num_of_threads"}); for (const auto& provider_option : provider_options) { if (provider_option.first == "num_of_threads") { std::stringstream sstream(provider_option.second); @@ -132,11 +128,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(ov_string, provider_options)) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(ov_string, provider_options); for (const auto& provider_option : provider_options) { option_keys.push_back(provider_option.first.c_str()); option_values.push_back(provider_option.second.c_str()); @@ -172,11 +164,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(ov_string, provider_options)) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(ov_string, provider_options); for (const auto& provider_option : provider_options) { option_keys.push_back(provider_option.first.c_str()); option_values.push_back(provider_option.second.c_str()); @@ -210,11 +198,11 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(option_string, provider_options, {"backend_path", "profiling_file_path", "profiling_level", "rpc_control_latency", "vtcm_mb", "soc_model", "device_id", "htp_performance_mode", "qnn_saver_path", "htp_graph_finalization_optimization_mode", "qnn_context_priority", "htp_arch", "enable_htp_fp16_precision", "offload_graph_io_quantization"})) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(option_string, provider_options, + {"backend_path", "profiling_file_path", "profiling_level", "rpc_control_latency", + "vtcm_mb", "soc_model", "device_id", "htp_performance_mode", "qnn_saver_path", + "htp_graph_finalization_optimization_mode", "qnn_context_priority", "htp_arch", + "enable_htp_fp16_precision", "offload_graph_io_quantization"}); for (const auto& provider_option : provider_options) { const std::string& key = provider_option.first; const std::string& value = provider_option.second; @@ -287,11 +275,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(option_string, provider_options, {"runtime", "priority", "buffer_type", "enable_init_cache"})) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(option_string, provider_options, {"runtime", "priority", "buffer_type", "enable_init_cache"}); for (const auto& provider_option : provider_options) { if (key == "runtime") { std::set supported_runtime = {"CPU", "GPU_FP32", "GPU", "GPU_FLOAT16", "DSP", "AIP_FIXED_TF"}; @@ -363,11 +347,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); kCoremlProviderOption_ModelFormat, kCoremlProviderOption_RequireStaticInputShapes, kCoremlProviderOption_EnableOnSubgraphs}; - if (!ParseSessionConfigs(ov_string, provider_options, available_keys)) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(ov_string, provider_options, available_keys); std::unordered_map available_options = { {"CPUAndNeuralEngine", "1"}, @@ -403,13 +383,9 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(ov_string, provider_options, - {"device_filter", "performance_preference", "disable_metacommands", - "enable_graph_capture", "enable_graph_serialization"})) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(ov_string, provider_options, + {"device_filter", "performance_preference", "disable_metacommands", + "enable_graph_capture", "enable_graph_serialization"}); for (const auto& provider_option : provider_options) { const std::string& key = provider_option.first; const std::string& value = provider_option.second; @@ -480,11 +456,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; #endif // defined(_MSC_VER) bool enable_fast_math = false; - if (!ParseSessionConfigs(ov_string, provider_options, {"enable_fast_math"})) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(ov_string, provider_options, {"enable_fast_math"}); for (const auto& provider_option : provider_options) { if (key == "enable_fast_math") { std::set ov_supported_values = {"true", "True", "false", "False"}; @@ -550,11 +522,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else std::string option_string = performance_test_config.run_config.ep_runtime_config_string; #endif - if (!ParseSessionConfigs(option_string, provider_options)) { - ORT_THROW( - "[ERROR] Use a '|' to separate the key and value for the " - "run-time option you are trying to use.\n"); - } + ParseSessionConfigs(option_string, provider_options); session_options.AppendExecutionProvider_VitisAI(provider_options); #else diff --git a/onnxruntime/test/perftest/strings_helper.cc b/onnxruntime/test/perftest/strings_helper.cc index 5e6ade6b76523..e09c8fac70887 100644 --- a/onnxruntime/test/perftest/strings_helper.cc +++ b/onnxruntime/test/perftest/strings_helper.cc @@ -12,7 +12,7 @@ namespace onnxruntime { namespace perftest { -bool ParseSessionConfigs(const std::string& configs_string, +void ParseSessionConfigs(const std::string& configs_string, std::unordered_map& session_configs, const std::unordered_set& available_keys) { std::istringstream ss(configs_string); @@ -52,8 +52,6 @@ bool ParseSessionConfigs(const std::string& configs_string, session_configs.insert(std::make_pair(std::move(key), std::move(value))); } - - return true; } } // namespace perftest } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/strings_helper.h b/onnxruntime/test/perftest/strings_helper.h index b4b5704ebb2cd..0d6c56709fde6 100644 --- a/onnxruntime/test/perftest/strings_helper.h +++ b/onnxruntime/test/perftest/strings_helper.h @@ -9,7 +9,7 @@ namespace onnxruntime { namespace perftest { -bool ParseSessionConfigs(const std::string& configs_string, +void ParseSessionConfigs(const std::string& configs_string, std::unordered_map& session_configs, const std::unordered_set& available_keys = {}); } // namespace perftest From 3e5386dd93c9be3050606ad9660217cd4bbeb721 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 22 Nov 2024 01:37:31 -0800 Subject: [PATCH 20/22] not throw for first_entry_args --- onnxruntime/test/perftest/command_args_parser.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 40acac8f21931..c3d701d23c6d2 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -353,7 +353,10 @@ static bool ParseDimensionOverride(std::basic_string& dim_identifier, test_config.run_config.intra_op_thread_affinities = ToUTF8String(optarg); break; case 'C': { - if (!ParseSessionConfigs(ToUTF8String(optarg), test_config.run_config.session_config_entries)) { + ORT_TRY { + ParseSessionConfigs(ToUTF8String(optarg), test_config.run_config.session_config_entries); + } + ORT_CATCH(...) { return false; } break; From 9e7c87a4a32d1417fa61d6dc7884f21425026a20 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 22 Nov 2024 02:13:59 -0800 Subject: [PATCH 21/22] misspelling --- onnxruntime/core/providers/coreml/coreml_execution_provider.cc | 2 +- onnxruntime/core/providers/coreml/coreml_execution_provider.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 6e3d6ee4750c8..6eda0ffd551ba 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -62,7 +62,7 @@ CoreMLOptions::CoreMLOptions(uint32_t coreml_flags) { } const bool has_neural_engine = coreml::HasNeuralEngine(); - if ((compute_units_ & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) && !has_neural_engine) { + if (ComputeUnits(COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) && !has_neural_engine) { ORT_THROW("The current system does not have Apple Neural Engine."); } } diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index d2436bfe64339..afe40d537fe69 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -28,7 +28,7 @@ class CoreMLOptions { bool RequireStaticShape() const { return require_static_shape_; } bool CreateMLProgram() const { return create_mlprogram_; } bool EnableOnSubgraph() const { return enable_on_subgraph_; } - uint32_t ComputeUnits(uint32_t specfic_flag = 0xffffffff) const { return compute_units_ & specfic_flag; } + uint32_t ComputeUnits(uint32_t specific_flag = 0xffffffff) const { return compute_units_ & specific_flag; } private: void ValidateAndParseProviderOption(const ProviderOptions& options); From e2b83ca0fe4d22512322d5b506e5fc5f4e2b470a Mon Sep 17 00:00:00 2001 From: wejoncy Date: Tue, 26 Nov 2024 17:18:08 +0800 Subject: [PATCH 22/22] split coreml options in isolate file --- .../coreml/coreml_execution_provider.cc | 86 ----------------- .../coreml/coreml_execution_provider.h | 24 +---- .../core/providers/coreml/coreml_options.cc | 96 +++++++++++++++++++ .../core/providers/coreml/coreml_options.h | 32 +++++++ .../test/perftest/command_args_parser.cc | 5 +- 5 files changed, 133 insertions(+), 110 deletions(-) create mode 100644 onnxruntime/core/providers/coreml/coreml_options.cc create mode 100644 onnxruntime/core/providers/coreml/coreml_options.h diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 6eda0ffd551ba..5a2867e5524e4 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -23,92 +23,6 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; -CoreMLOptions::CoreMLOptions(uint32_t coreml_flags) { - // validate the flags and populate the members. should be moving code from ctor to here - require_static_shape_ = (coreml_flags & COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES) != 0; - create_mlprogram_ = (coreml_flags & COREML_FLAG_CREATE_MLPROGRAM) != 0; - enable_on_subgraph_ = (coreml_flags & COREML_FLAG_ENABLE_ON_SUBGRAPH) != 0; - -#if defined(COREML_ENABLE_MLPROGRAM) - if (coreml::util::CoreMLVersion() < MINIMUM_COREML_MLPROGRAM_VERSION && create_mlprogram_ != 0) { - LOGS_DEFAULT(WARNING) << "ML Program is not supported on this OS version. Falling back to NeuralNetwork."; - create_mlprogram_ = false; - } -#else - if (create_mlprogram_ != 0) { - LOGS_DEFAULT(WARNING) << "ML Program is not supported in this build. Falling back to NeuralNetwork."; - create_mlprogram_ = false; - } -#endif - - compute_units_ = 0; // 0 for all - - if (coreml_flags & COREML_FLAG_USE_CPU_ONLY) { - compute_units_ |= COREML_FLAG_USE_CPU_ONLY; - } - if (coreml_flags & COREML_FLAG_USE_CPU_AND_GPU) { - compute_units_ |= COREML_FLAG_USE_CPU_AND_GPU; - } - if (coreml_flags & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) { - compute_units_ |= COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE; - } - - // assure only one device option is selected - if (compute_units_ & (compute_units_ - 1)) { - // multiple device options selected - ORT_THROW( - "Multiple device options selected, you should use at most one of the following options:" - "[COREML_FLAG_USE_CPU_ONLY, COREML_FLAG_USE_CPU_AND_GPU, COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE]"); - } - - const bool has_neural_engine = coreml::HasNeuralEngine(); - if (ComputeUnits(COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) && !has_neural_engine) { - ORT_THROW("The current system does not have Apple Neural Engine."); - } -} - -void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& options) { - const std::unordered_map available_computeunits_options = { - {"CPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, - {"CPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, - {"CPUOnly", COREML_FLAG_USE_CPU_ONLY}, - {"ALL", COREML_FLAG_USE_NONE}, - }; - const std::unordered_map available_modelformat_options = { - {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, - {"NeuralNetwork", COREML_FLAG_USE_NONE}, - }; - std::unordered_set valid_options = { - kCoremlProviderOption_MLComputeUnits, - kCoremlProviderOption_ModelFormat, - kCoremlProviderOption_RequireStaticInputShapes, - kCoremlProviderOption_EnableOnSubgraphs, - }; - // Validate the options - for (const auto& option : options) { - if (valid_options.find(option.first) == valid_options.end()) { - ORT_THROW("Unknown option: ", option.first); - } - if (kCoremlProviderOption_MLComputeUnits == option.first) { - if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { - ORT_THROW("Invalid value for option `", option.first, "`: ", option.second); - } else { - compute_units_ = available_computeunits_options.at(option.second); - } - } else if (kCoremlProviderOption_ModelFormat == option.first) { - if (available_modelformat_options.find(option.second) == available_modelformat_options.end()) { - ORT_THROW("Invalid value for option ", option.first, ": ", option.second); - } else { - create_mlprogram_ = available_modelformat_options.at(option.second) & COREML_FLAG_CREATE_MLPROGRAM; - } - } else if (kCoremlProviderOption_RequireStaticInputShapes == option.first) { - require_static_shape_ = option.second == "1"; - } else if (kCoremlProviderOption_EnableOnSubgraphs == option.first) { - enable_on_subgraph_ = option.second == "1"; - } - } -} - CoreMLExecutionProvider::CoreMLExecutionProvider(const CoreMLOptions& options) : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider}, coreml_options_(options), diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index afe40d537fe69..650d81a4fecf7 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -3,7 +3,7 @@ #pragma once -#include "core/common/inlined_containers.h" +#include "core/providers/coreml/coreml_options.h" #include "core/framework/execution_provider.h" #include "core/framework/model_metadef_id_generator.h" @@ -12,28 +12,6 @@ namespace coreml { class Model; } -class CoreMLOptions { - private: - bool require_static_shape_{false}; - bool create_mlprogram_{false}; - bool enable_on_subgraph_{false}; - uint32_t compute_units_{0}; - - public: - explicit CoreMLOptions(uint32_t coreml_flags); - - CoreMLOptions(const ProviderOptions& options) { - ValidateAndParseProviderOption(options); - } - bool RequireStaticShape() const { return require_static_shape_; } - bool CreateMLProgram() const { return create_mlprogram_; } - bool EnableOnSubgraph() const { return enable_on_subgraph_; } - uint32_t ComputeUnits(uint32_t specific_flag = 0xffffffff) const { return compute_units_ & specific_flag; } - - private: - void ValidateAndParseProviderOption(const ProviderOptions& options); -}; - class CoreMLExecutionProvider : public IExecutionProvider { public: CoreMLExecutionProvider(const CoreMLOptions& options); diff --git a/onnxruntime/core/providers/coreml/coreml_options.cc b/onnxruntime/core/providers/coreml/coreml_options.cc new file mode 100644 index 0000000000000..df78f74383871 --- /dev/null +++ b/onnxruntime/core/providers/coreml/coreml_options.cc @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/coreml/coreml_execution_provider.h" +#include "core/providers/coreml/coreml_provider_factory.h" // defines flags +#include "core/providers/coreml/model/host_utils.h" +#include "core/providers/coreml/builders/helper.h" + +namespace onnxruntime { + +CoreMLOptions::CoreMLOptions(uint32_t coreml_flags) { + // validate the flags and populate the members. should be moving code from ctor to here + require_static_shape_ = (coreml_flags & COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES) != 0; + create_mlprogram_ = (coreml_flags & COREML_FLAG_CREATE_MLPROGRAM) != 0; + enable_on_subgraph_ = (coreml_flags & COREML_FLAG_ENABLE_ON_SUBGRAPH) != 0; + +#if defined(COREML_ENABLE_MLPROGRAM) + if (coreml::util::CoreMLVersion() < MINIMUM_COREML_MLPROGRAM_VERSION && create_mlprogram_ != 0) { + LOGS_DEFAULT(WARNING) << "ML Program is not supported on this OS version. Falling back to NeuralNetwork."; + create_mlprogram_ = false; + } +#else + if (create_mlprogram_ != 0) { + LOGS_DEFAULT(WARNING) << "ML Program is not supported in this build. Falling back to NeuralNetwork."; + create_mlprogram_ = false; + } +#endif + + compute_units_ = 0; // 0 for all + + if (coreml_flags & COREML_FLAG_USE_CPU_ONLY) { + compute_units_ |= COREML_FLAG_USE_CPU_ONLY; + } + if (coreml_flags & COREML_FLAG_USE_CPU_AND_GPU) { + compute_units_ |= COREML_FLAG_USE_CPU_AND_GPU; + } + if (coreml_flags & COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) { + compute_units_ |= COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE; + } + + // assure only one device option is selected + if (compute_units_ & (compute_units_ - 1)) { + // multiple device options selected + ORT_THROW( + "Multiple device options selected, you should use at most one of the following options:" + "[COREML_FLAG_USE_CPU_ONLY, COREML_FLAG_USE_CPU_AND_GPU, COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE]"); + } + + const bool has_neural_engine = coreml::HasNeuralEngine(); + if (ComputeUnits(COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE) && !has_neural_engine) { + ORT_THROW("The current system does not have Apple Neural Engine."); + } +} + +void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& options) { + const std::unordered_map available_computeunits_options = { + {"CPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, + {"CPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, + {"CPUOnly", COREML_FLAG_USE_CPU_ONLY}, + {"ALL", COREML_FLAG_USE_NONE}, + }; + const std::unordered_map available_modelformat_options = { + {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, + {"NeuralNetwork", COREML_FLAG_USE_NONE}, + }; + std::unordered_set valid_options = { + kCoremlProviderOption_MLComputeUnits, + kCoremlProviderOption_ModelFormat, + kCoremlProviderOption_RequireStaticInputShapes, + kCoremlProviderOption_EnableOnSubgraphs, + }; + // Validate the options + for (const auto& option : options) { + if (valid_options.find(option.first) == valid_options.end()) { + ORT_THROW("Unknown option: ", option.first); + } + if (kCoremlProviderOption_MLComputeUnits == option.first) { + if (available_computeunits_options.find(option.second) == available_computeunits_options.end()) { + ORT_THROW("Invalid value for option `", option.first, "`: ", option.second); + } else { + compute_units_ = available_computeunits_options.at(option.second); + } + } else if (kCoremlProviderOption_ModelFormat == option.first) { + if (available_modelformat_options.find(option.second) == available_modelformat_options.end()) { + ORT_THROW("Invalid value for option ", option.first, ": ", option.second); + } else { + create_mlprogram_ = available_modelformat_options.at(option.second) & COREML_FLAG_CREATE_MLPROGRAM; + } + } else if (kCoremlProviderOption_RequireStaticInputShapes == option.first) { + require_static_shape_ = option.second == "1"; + } else if (kCoremlProviderOption_EnableOnSubgraphs == option.first) { + enable_on_subgraph_ = option.second == "1"; + } + } +} +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/coreml_options.h b/onnxruntime/core/providers/coreml/coreml_options.h new file mode 100644 index 0000000000000..8bb748fcd69c9 --- /dev/null +++ b/onnxruntime/core/providers/coreml/coreml_options.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/inlined_containers.h" +#include "core/framework/execution_provider.h" + +namespace onnxruntime { + +class CoreMLOptions { + private: + bool require_static_shape_{false}; + bool create_mlprogram_{false}; + bool enable_on_subgraph_{false}; + uint32_t compute_units_{0}; + + public: + explicit CoreMLOptions(uint32_t coreml_flags); + + CoreMLOptions(const ProviderOptions& options) { + ValidateAndParseProviderOption(options); + } + bool RequireStaticShape() const { return require_static_shape_; } + bool CreateMLProgram() const { return create_mlprogram_; } + bool EnableOnSubgraph() const { return enable_on_subgraph_; } + uint32_t ComputeUnits(uint32_t specific_flag = 0xffffffff) const { return compute_units_ & specific_flag; } + + private: + void ValidateAndParseProviderOption(const ProviderOptions& options); +}; +} // namespace onnxruntime diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index c3d701d23c6d2..afd4bec49ad80 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -356,7 +356,10 @@ static bool ParseDimensionOverride(std::basic_string& dim_identifier, ORT_TRY { ParseSessionConfigs(ToUTF8String(optarg), test_config.run_config.session_config_entries); } - ORT_CATCH(...) { + ORT_CATCH(const std::exception& ex) { + ORT_HANDLE_EXCEPTION([&]() { + fprintf(stderr, "Error parsing session configuration entries: %s\n", ex.what()); + }); return false; } break;