ORT perf test support for plugin EP (microsoft#25374)

### Description
Add support for onnxruntime_perf_test to register plugin EP dll and run
plugin EP.

As support for plugin execution providers (EPs) requires additional
options and most single-character options have already been used,
multi-character options are now necessary to ensure clarity and
readability. Therefore, support for `Abseil flags` is added, which
enables multi-character options and provides cross-platform
compatibility.


**New options:**

- `--plugin_ep_libs [registration names and libraries]` Specifies a list
of plugin execution provider (EP) registration names and their
corresponding shared libraries to register.
[Usage]: `--plugin_ep_libs "plugin_ep_name_1|plugin_ep_1.dll
plugin_ep_name_2|plugin_ep_2.dll ... "`

  
- `--plugin_eps [Plugin EPs]` Specifies a semicolon-separated list of
plugin execution providers (EPs) to use.
      [Usage]: `--plugin_eps "plugin_ep_1;plugin_ep_2;... "`

- `--plugin_ep_options [EP options]` Specifies provider options for each
EP listed in --plugin_eps. Options (key-value pairs) for each EP are
separated by space and EPs are separated by semicolons.
      [Usage]:
`--plugin_ep_options "ep_1_option_1_key|ep_1_option_1_value
...;ep_2_option_1_key|ep_2_option_1_value ...;..."` or
`--plugin_ep_options ";ep_2_option_1_key|ep_2_option_1_value ...;..."`
or
`--plugin_ep_options "ep_1_option_1_key|ep_1_option_1_value
...;;ep_3_option_1_key|ep_3_option_1_value ...;..."`

- `--list_ep_devices` Prints all available device indices and their
properties (including metadata). This option makes the program exit
early without performing inference.

- ` --select_ep_devices [list of device indices]` A semicolon-separated
list of device indices to add to the session and run with.

**Usage:**

1. Use `--plugin_ep_libs` and `--list_ep_devices` to list all the
devices.

````sh
--list_ep_devices --plugin_ep_libs "TensorRTEp|C:\TensorRTEp.dll example_ep|C:\example_plugin_ep.dll"
````
   It will print the devices info
````
===== EP device id 0 ======
name: CPUExecutionProvider
vendor: Microsoft
metadata:
  version: 1.23.0

===== EP device id 1 ======
name: example_ep
vendor: Contoso
metadata:
  supported_devices: CrackGriffin 7+
  version: 0.1.0

===== EP device id 2 ======
name: TensorRTEp
vendor: Nvidia
metadata:
  gpu_type: data center
  version: 0.1.0
````

2. Use `--select_ep_devices` to select the device by index. And add
`--plugin_eps` to specify the EP name. The EP name should match the name
when ep library passes in to create the ep factory.

````sh
--plugin_ep_libs "TensorRTEp|C:\TensorRTEp.dll" --select_ep_devices 2 --plugin_eps TensorRTEp -r 1 C:\mul_op\mul_1.onnx
````

3. Or simply use `-e` to specify the EP name. ORT perf test will add all
the devices created by the plugin EP.
The EP name should match the name when ep library passes in to create
the ep factory.

````sh
--plugin_ep_libs "TensorRTEp|C:\TensorRTEp.dll" --plugin_eps TensorRTEp -r 1 C:\mul_op\mul_1.onnx
````

Author: chilo-ms

cmake/onnxruntime_unittests.cmake

@@ -1258,7 +1258,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
             onnx_test_runner_common onnxruntime_test_utils onnxruntime_common
             onnxruntime onnxruntime_flatbuffers onnx_test_data_proto
             ${onnxruntime_EXTERNAL_LIBRARIES}
-            ${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
+            absl::flags absl::flags_parse ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
       if(NOT WIN32)
         if(onnxruntime_USE_SNPE)
           list(APPEND onnxruntime_perf_test_libs onnxruntime_providers_snpe)
@@ -1278,7 +1278,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
         target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32)
       endif()
     else()
-      target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs})
+      target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common absl::flags absl::flags_parse ${onnx_test_libs})
     endif()
     set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest")
 

onnxruntime/test/perftest/command_args_parser.cc

@@ -11,7 +11,6 @@ struct PerformanceTestConfig;
 
 class CommandLineParser {
  public:
-  static void ShowUsage();
   static bool ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]);
 };
 

onnxruntime/test/perftest/command_args_parser.h

@@ -0,0 +1,95 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "test/perftest/utils.h"
+#include "test/perftest/strings_helper.h"
+#include <core/platform/path_lib.h>
+
+#include <cstdint>
+
+#include <filesystem>
+
+namespace onnxruntime {
+namespace perftest {
+namespace utils {
+
+void ListEpDevices(const Ort::Env& env) {
+  std::vector<Ort::ConstEpDevice> ep_devices = env.GetEpDevices();
+
+  for (size_t i = 0; i < ep_devices.size(); ++i) {
+    auto device = ep_devices[i];
+    std::string device_info_msg = "===== device id " + std::to_string(i) + " ======\n";
+    device_info_msg += "name: " + std::string(device.EpName()) + "\n";
+    device_info_msg += "vendor: " + std::string(device.EpVendor()) + "\n";
+
+    auto metadata = device.EpMetadata();
+    std::unordered_map<std::string, std::string> metadata_entries = metadata.GetKeyValuePairs();
+    if (!metadata_entries.empty()) {
+      device_info_msg += "metadata:\n";
+    }
+
+    for (auto& entry : metadata_entries) {
+      device_info_msg += "  " + entry.first + ": " + entry.second + "\n";
+    }
+    device_info_msg += "\n";
+    fprintf(stdout, "%s", device_info_msg.c_str());
+  }
+}
+
+void RegisterExecutionProviderLibrary(Ort::Env& env, PerformanceTestConfig& test_config) {
+  if (!test_config.plugin_ep_names_and_libs.empty()) {
+    std::unordered_map<std::string, std::string> ep_names_to_libs;
+    ParseSessionConfigs(ToUTF8String(test_config.plugin_ep_names_and_libs), ep_names_to_libs);
+    if (ep_names_to_libs.size() > 0) {
+      for (auto& pair : ep_names_to_libs) {
+        const std::filesystem::path library_path = pair.second;
+        const std::string registration_name = pair.first;
+        Ort::Status status(Ort::GetApi().RegisterExecutionProviderLibrary(env, registration_name.c_str(), ToPathString(library_path.string()).c_str()));
+        if (status.IsOK()) {
+          test_config.registered_plugin_eps.push_back(registration_name);
+        } else {
+          fprintf(stderr, "Can't register %s plugin library: %s\n", registration_name.c_str(), status.GetErrorMessage().c_str());
+        }
+      }
+    }
+  }
+}
+
+void UnregisterExecutionProviderLibrary(Ort::Env& env, PerformanceTestConfig& test_config) {
+  for (auto& registration_name : test_config.registered_plugin_eps) {
+    Ort::Status status(Ort::GetApi().UnregisterExecutionProviderLibrary(env, registration_name.c_str()));
+    if (!status.IsOK()) {
+      fprintf(stderr, "%s", status.GetErrorMessage().c_str());
+    }
+  }
+}
+
+std::vector<std::string> ConvertArgvToUtf8Strings(int argc, ORTCHAR_T* argv[]) {
+  std::vector<std::string> utf8_args;
+  utf8_args.reserve(argc);
+  for (int i = 0; i < argc; ++i) {
+    std::string utf8_string = ToUTF8String(argv[i]);
+
+    // Abseil flags doens't natively alias "-h" to "--help".
+    // We make "-h" alias to "--help" here.
+    if (utf8_string == "-h" || utf8_string == "--h") {
+      utf8_args.push_back("--help");
+    } else {
+      utf8_args.push_back(utf8_string);
+    }
+  }
+  return utf8_args;
+}
+
+std::vector<char*> CStringsFromStrings(std::vector<std::string>& utf8_args) {
+  std::vector<char*> utf8_argv;
+  utf8_argv.reserve(utf8_args.size());
+  for (auto& str : utf8_args) {
+    utf8_argv.push_back(&str[0]);
+  }
+  return utf8_argv;
+}
+
+}  // namespace utils
+}  // namespace perftest
+}  // namespace onnxruntime

onnxruntime/test/perftest/common_utils.cc

@@ -6,6 +6,8 @@
 #include <random>
 #include "command_args_parser.h"
 #include "performance_runner.h"
+#include "utils.h"
+#include "strings_helper.h"
 #include <google/protobuf/stubs/common.h>
 
 using namespace onnxruntime;
@@ -19,7 +21,7 @@ int real_main(int argc, char* argv[]) {
   g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION);
   perftest::PerformanceTestConfig test_config;
   if (!perftest::CommandLineParser::ParseArguments(test_config, argc, argv)) {
-    perftest::CommandLineParser::ShowUsage();
+    fprintf(stderr, "%s", "See 'onnxruntime_perf_test --help'.");
     return -1;
   }
   Ort::Env env{nullptr};
@@ -41,6 +43,30 @@ int real_main(int argc, char* argv[]) {
     if (failed)
       return -1;
   }
+
+  if (!test_config.plugin_ep_names_and_libs.empty()) {
+    perftest::utils::RegisterExecutionProviderLibrary(env, test_config);
+  }
+
+  // Unregister all registered plugin EP libraries before program exits.
+  // This is necessary because unregistering the plugin EP also unregisters any associated shared allocators.
+  // If we don't do this and program returns, the factories stored inside the environment will be destroyed when the environment goes out of scope.
+  // Later, when the shared allocator's deleter runs, it may cause a segmentation fault because it attempts to use the already-destroyed factory to call ReleaseAllocator.
+  // See "ep_device.ep_factory->ReleaseAllocator" in Environment::CreateSharedAllocatorImpl.
+  auto unregister_plugin_eps_at_scope_exit = gsl::finally([&]() {
+    if (!test_config.registered_plugin_eps.empty()) {
+      perftest::utils::UnregisterExecutionProviderLibrary(env, test_config);  // this won't throw
+    }
+  });
+
+  if (test_config.list_available_ep_devices) {
+    perftest::utils::ListEpDevices(env);
+    if (test_config.registered_plugin_eps.empty()) {
+      fprintf(stdout, "No plugin execution provider libraries are registered. Please specify them using \"--plugin_ep_libs\"; otherwise, only CPU may be available.\n");
+    }
+    return 0;
+  }
+
   std::random_device rd;
   perftest::PerformanceRunner perf_runner(env, test_config, rd);
 

onnxruntime/test/perftest/main.cc

@@ -62,6 +62,84 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
     : rand_engine_(rd()), input_names_(m.GetInputCount()), input_names_str_(m.GetInputCount()), input_length_(m.GetInputCount()) {
   Ort::SessionOptions session_options;
 
+  // Add EP devices if any (created by plugin EP)
+  if (!performance_test_config.registered_plugin_eps.empty()) {
+    std::vector<Ort::ConstEpDevice> ep_devices = env.GetEpDevices();
+    // EP -> associated EP devices (All OrtEpDevice instances must be from the same execution provider)
+    std::unordered_map<std::string, std::vector<Ort::ConstEpDevice>> added_ep_devices;
+    std::unordered_set<int> added_ep_device_index_set;
+
+    auto& ep_list = performance_test_config.machine_config.plugin_provider_type_list;
+    std::unordered_set<std::string> ep_set(ep_list.begin(), ep_list.end());
+
+    // Select EP devices by provided device index
+    if (!performance_test_config.selected_ep_device_indices.empty()) {
+      std::vector<int> device_list;
+      device_list.reserve(performance_test_config.selected_ep_device_indices.size());
+      ParseEpDeviceIndexList(performance_test_config.selected_ep_device_indices, device_list);
+      for (auto index : device_list) {
+        if (static_cast<size_t>(index) > (ep_devices.size() - 1)) {
+          fprintf(stderr, "%s", "The device index provided is not correct. Will skip this device id.");
+          continue;
+        }
+
+        Ort::ConstEpDevice& device = ep_devices[index];
+        if (ep_set.find(std::string(device.EpName())) != ep_set.end()) {
+          if (added_ep_device_index_set.find(index) == added_ep_device_index_set.end()) {
+            added_ep_devices[device.EpName()].push_back(device);
+            added_ep_device_index_set.insert(index);
+            fprintf(stdout, "[Plugin EP] EP Device [Index: %d, Name: %s] has been added to session.\n", index, device.EpName());
+          }
+        } else {
+          std::string err_msg = "[Plugin EP] [WARNING] : The EP device index and its corresponding OrtEpDevice is not created from " +
+                                performance_test_config.machine_config.provider_type_name + ". Will skip adding this device.\n";
+          fprintf(stderr, "%s", err_msg.c_str());
+        }
+      }
+    } else {
+      // Find and select the OrtEpDevice associated with the EP in "--plugin_eps".
+      for (size_t index = 0; index < ep_devices.size(); ++index) {
+        Ort::ConstEpDevice& device = ep_devices[index];
+        if (ep_set.find(std::string(device.EpName())) != ep_set.end()) {
+          added_ep_devices[device.EpName()].push_back(device);
+          fprintf(stdout, "EP Device [Index: %d, Name: %s] has been added to session.\n", static_cast<int>(index), device.EpName());
+        }
+      }
+    }
+
+    if (added_ep_devices.empty()) {
+      ORT_THROW("[ERROR] [Plugin EP]: No matching EP devices found.");
+    }
+
+    std::string ep_option_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
+
+    // EP's associated provider option lists
+    std::vector<std::unordered_map<std::string, std::string>> ep_options_list;
+    ParseEpOptions(ep_option_string, ep_options_list);
+
+    // If user only provide the EPs' provider option lists for the first several EPs,
+    // add empty provider option lists for the rest EPs.
+    if (ep_options_list.size() < ep_list.size()) {
+      for (size_t i = ep_options_list.size(); i < ep_list.size(); ++i) {
+        ep_options_list.emplace_back();  // Adds a new empty map
+      }
+    } else if (ep_options_list.size() > ep_list.size()) {
+      ORT_THROW("[ERROR] [Plugin EP]: Too many EP provider option lists provided.");
+    }
+
+    // EP -> associated provider options
+    std::unordered_map<std::string, std::unordered_map<std::string, std::string>> ep_options_map;
+    for (size_t i = 0; i < ep_list.size(); ++i) {
+      ep_options_map.emplace(ep_list[i], ep_options_list[i]);
+    }
+
+    for (auto& ep_and_devices : added_ep_devices) {
+      auto& ep = ep_and_devices.first;
+      auto& devices = ep_and_devices.second;
+      session_options.AppendExecutionProvider_V2(env, devices, ep_options_map[ep]);
+    }
+  }
+
   provider_name_ = performance_test_config.machine_config.provider_type_name;
   std::unordered_map<std::string, std::string> provider_options;
   if (provider_name_ == onnxruntime::kDnnlExecutionProvider) {

onnxruntime/test/perftest/ort_test_session.cc

@@ -8,6 +8,8 @@
 
 #include "strings_helper.h"
 #include "core/common/common.h"
+#include "core/common/parse_string.h"
+#include "core/common/string_utils.h"
 
 namespace onnxruntime {
 namespace perftest {
@@ -53,5 +55,40 @@ void ParseSessionConfigs(const std::string& configs_string,
     session_configs.insert(std::make_pair(std::move(key), std::move(value)));
   }
 }
+
+void ParseEpOptions(const std::string& input, std::vector<std::unordered_map<std::string, std::string>>& result) {
+  auto tokens = utils::SplitString(input, ";", true);
+
+  for (const auto& token : tokens) {
+    result.emplace_back();  // Adds a new empty map
+    if (!token.empty()) {
+      ParseSessionConfigs(std::string(token), result.back());  // only parse non-empty
+    }
+    // if token is empty, we still get an empty map in `result`
+  }
+}
+
+void ParseEpList(const std::string& input, std::vector<std::string>& result) {
+  std::stringstream ss(input);
+  std::string token;
+
+  while (std::getline(ss, token, ';')) {
+    if (!token.empty()) {
+      result.push_back(token);
+    }
+  }
+}
+
+void ParseEpDeviceIndexList(const std::string& input, std::vector<int>& result) {
+  std::stringstream ss(input);
+  std::string item;
+
+  while (std::getline(ss, item, ';')) {
+    if (!item.empty()) {
+      int value = ParseStringWithClassicLocale<int>(item);
+      result.push_back(value);
+    }
+  }
+}
 }  // namespace perftest
 }  // namespace onnxruntime

onnxruntime/test/perftest/strings_helper.cc

@@ -5,12 +5,19 @@
 #include <string_view>
 #include <unordered_map>
 #include <unordered_set>
+#include <vector>
 
 namespace onnxruntime {
 namespace perftest {
 
 void ParseSessionConfigs(const std::string& configs_string,
                          std::unordered_map<std::string, std::string>& session_configs,
                          const std::unordered_set<std::string>& available_keys = {});
+
+void ParseEpList(const std::string& input, std::vector<std::string>& result);
+
+void ParseEpOptions(const std::string& input, std::vector<std::unordered_map<std::string, std::string>>& result);
+
+void ParseEpDeviceIndexList(const std::string& input, std::vector<int>& result);
 }  // namespace perftest
 }  // namespace onnxruntime

onnxruntime/test/perftest/strings_helper.h

@@ -35,6 +35,7 @@ struct ModelInfo {
 struct MachineConfig {
   Platform platform{Platform::kWindows};
   std::string provider_type_name{onnxruntime::kCpuExecutionProvider};
+  std::vector<std::string> plugin_provider_type_list;
 };
 
 struct RunConfig {
@@ -59,8 +60,8 @@ struct RunConfig {
   bool set_denormal_as_zero{false};
   std::basic_string<ORTCHAR_T> ep_runtime_config_string;
   std::unordered_map<std::string, std::string> session_config_entries;
-  std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_name_overrides;
-  std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_denotation_overrides;
+  std::map<std::string, int64_t> free_dim_name_overrides;
+  std::map<std::string, int64_t> free_dim_denotation_overrides;
   std::string intra_op_thread_affinities;
   bool disable_spinning = false;
   bool disable_spinning_between_run = false;
@@ -74,6 +75,10 @@ struct PerformanceTestConfig {
   ModelInfo model_info;
   MachineConfig machine_config;
   RunConfig run_config;
+  std::basic_string<ORTCHAR_T> plugin_ep_names_and_libs;
+  std::vector<std::string> registered_plugin_eps;
+  std::string selected_ep_device_indices;
+  bool list_available_ep_devices = false;
 };
 
 }  // namespace perftest

onnxruntime/test/perftest/test_configuration.h

@@ -2,7 +2,8 @@
 // Licensed under the MIT License.
 
 #pragma once
-
+#include "test/perftest/test_configuration.h"
+#include <core/session/onnxruntime_cxx_api.h>
 #include <memory>
 
 namespace onnxruntime {
@@ -22,6 +23,16 @@ class ICPUUsage {
 
 std::unique_ptr<ICPUUsage> CreateICPUUsage();
 
+std::vector<std::string> ConvertArgvToUtf8Strings(int argc, ORTCHAR_T* argv[]);
+
+std::vector<char*> CStringsFromStrings(std::vector<std::string>& utf8_args);
+
+void RegisterExecutionProviderLibrary(Ort::Env& env, PerformanceTestConfig& test_config);
+
+void UnregisterExecutionProviderLibrary(Ort::Env& env, PerformanceTestConfig& test_config);
+
+void ListEpDevices(const Ort::Env& env);
+
 }  // namespace utils
 }  // namespace perftest
 }  // namespace onnxruntime