[QNN-EP] Add CastLoneQFusion to transform Cast and QNode into Convert (microsoft#25667)

qti-hungjuiw · Copilot · web-flow · commit 69e704716b73 · 2025-08-07T15:45:08.000-07:00
### Description
- Introduced `CastLoneQFusion` in QNNEP to fuse `Cast` followed by `QuantizeLinear` into a single `Convert` operation.
- Added corresponding test cases for **UINT8-to-FLOAT** `Cast` combined with `QuantizeLinear`, covering various **QuantType** scenarios.

### Motivation and Context
- To optimize the model by reducing unnecessary QDQ nodes, this fusion transformation has been implemented.

---------

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/cast_lone_q_fusion.cc b/onnxruntime/core/providers/qnn/builder/qnn_node_group/cast_lone_q_fusion.cc
@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/providers/qnn/builder/qnn_node_group/cast_lone_q_fusion.h"
+#include "core/providers/qnn/builder/qnn_model_wrapper.h"
+#include "core/providers/qnn/builder/qnn_node_group/utils.h"
+
+namespace onnxruntime {
+namespace qnn {
+
+constexpr char kOpCast[] = "Cast";
+constexpr char kOpConvert[] = "Convert";
+
+Status CreateOrValidateOnQnn(
+    QnnModelWrapper* qnn_model_wrapper,
+    gsl::span<const NodeUnit* const> node_units,
+    [[maybe_unused]] const logging::Logger& logger,
+    bool validate) {
+  const NodeUnit* cast = node_units[0];
+  const NodeUnit* quantize_linear = node_units[1];
+
+  // ProcessInputs
+  const auto& input_name = cast->Inputs()[0].node_arg.Name();
+  if (!qnn_model_wrapper->IsQnnTensorWrapperExist(input_name)) {
+    TensorInfo cast_node_input_info = {};
+    ORT_RETURN_IF_ERROR(qnn_model_wrapper->GetTensorInfo(cast->Inputs()[0], cast_node_input_info));
+    QnnTensorWrapper input_tensor_wrapper;
+    ORT_RETURN_IF_ERROR(qnn_model_wrapper->MakeTensorWrapper(cast_node_input_info, input_name, input_tensor_wrapper));
+    ORT_RETURN_IF_NOT(qnn_model_wrapper->AddTensorWrapper(std::move(input_tensor_wrapper)),
+                      "Failed to add input tensor for QNN Convert node.");
+  }
+  // ProcessAttributesAndOutputs
+  const auto& output_name = quantize_linear->Outputs()[0].node_arg.Name();
+  TensorInfo q_node_output_info = {};
+  ORT_RETURN_IF_ERROR(qnn_model_wrapper->GetTensorInfo(quantize_linear->Outputs()[0], q_node_output_info));
+  QnnTensorWrapper output_tensor_wrapper;
+  ORT_RETURN_IF_ERROR(qnn_model_wrapper->MakeTensorWrapper(q_node_output_info, output_name, output_tensor_wrapper));
+  ORT_RETURN_IF_NOT(qnn_model_wrapper->AddTensorWrapper(std::move(output_tensor_wrapper)),
+                    "Failed to add output tensor for QNN Convert node.");
+  ORT_RETURN_IF_NOT(qnn_model_wrapper->CreateQnnNode(cast->Name() + "_ort_qnn_ep_convert",
+                                                     QNN_OP_PACKAGE_NAME_QTI_AISW,
+                                                     QNN_OP_CONVERT,
+                                                     {input_name},
+                                                     {output_name},
+                                                     {},
+                                                     validate),
+                    "Failed to add fused " + std::string(kOpConvert) + " node.");
+
+  return Status::OK();
+}
+
+std::unique_ptr<IQnnNodeGroup> CastLoneQFusion::TryFusion(
+    QnnModelWrapper& qnn_model_wrapper,
+    const NodeUnit& cast_node_unit,
+    const std::unordered_map<const Node*, const NodeUnit*>& node_to_node_unit,
+    const std::unordered_map<const NodeUnit*, const IQnnNodeGroup*>& node_unit_to_qnn_node_group,
+    [[maybe_unused]] const logging::Logger& logger) {
+  if (cast_node_unit.OpType() != kOpCast || cast_node_unit.UnitType() != NodeUnit::Type::SingleNode) {
+    return nullptr;
+  }
+
+  // Transform the pattern Non-DQ Node -> Cast -> Q into Non-DQ Node -> Convert
+  const GraphViewer& graph_viewer = qnn_model_wrapper.GetGraphViewer();
+  const std::array<std::string_view, 1> child_op_types{QUANTIZE_LINEAR};
+  const NodeUnit* quantize_linear = GetOnlyChildOfType(
+      graph_viewer, cast_node_unit, child_op_types,
+      node_to_node_unit, node_unit_to_qnn_node_group);
+  const std::array<std::string_view, 1> parent_op_types{DEQUANTIZE_LINEAR};
+  const NodeUnit* dequantize_linear = GetParentOfType(
+      graph_viewer, cast_node_unit, parent_op_types,
+      node_to_node_unit, node_unit_to_qnn_node_group);
+
+  if (quantize_linear == nullptr || dequantize_linear != nullptr) {
+    return nullptr;
+  }
+
+  // Skip Constant cast
+  if (qnn_model_wrapper.IsConstantInput(cast_node_unit.Inputs()[0].node_arg.Name())) {
+    return nullptr;
+  }
+  std::array<const NodeUnit*, 2> node_unit_array{&cast_node_unit, quantize_linear};
+  auto node_units = gsl::make_span<const NodeUnit*>(node_unit_array.data(), 2);
+
+  if (CreateOrValidateOnQnn(&qnn_model_wrapper, node_units, logger, /*validate=*/true) != Status::OK()) {
+    return nullptr;
+  }
+  return std::make_unique<CastLoneQFusion>(node_units);
+}
+
+gsl::span<const NodeUnit* const> CastLoneQFusion::GetNodeUnits() const {
+  return gsl::span<const NodeUnit* const>{node_units_.data(), node_units_.size()};
+}
+
+Status CastLoneQFusion::IsSupported(
+    QnnModelWrapper& qnn_model_wrapper, [[maybe_unused]] const logging::Logger& logger) const {
+  return CreateOrValidateOnQnn(&qnn_model_wrapper, GetNodeUnits(), logger, true);
+}
+
+Status CastLoneQFusion::AddToModelBuilder(
+    QnnModelWrapper& qnn_model_wrapper, [[maybe_unused]] const logging::Logger& logger) const {
+  return CreateOrValidateOnQnn(&qnn_model_wrapper, GetNodeUnits(), logger, false);
+}
+
+}  // namespace qnn
+}  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/cast_lone_q_fusion.h b/onnxruntime/core/providers/qnn/builder/qnn_node_group/cast_lone_q_fusion.h
@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h"
+
+namespace onnxruntime {
+namespace qnn {
+/// <summary>
+/// Represents a fusion of pattern: Quantize(Cast(x)) => Convert(x)
+/// when x is not the output of Dequantize
+/// </summary>
+class CastLoneQFusion : public IQnnNodeGroup {
+ public:
+  explicit CastLoneQFusion(gsl::span<const NodeUnit* const> node_units) {
+    ORT_ENFORCE(node_units.size() == 2, "Pattern expect exactly 2 NodeUnits.");
+    node_units_[0] = node_units[0];
+    node_units_[1] = node_units[1];
+  }
+
+  ORT_DISALLOW_COPY_AND_ASSIGNMENT(CastLoneQFusion);
+
+  Status IsSupported(QnnModelWrapper& qnn_model_wrapper, const logging::Logger& logger) const override;
+  Status AddToModelBuilder(QnnModelWrapper& qnn_model_wrapper, const logging::Logger& logger) const override;
+  gsl::span<const NodeUnit* const> GetNodeUnits() const override;
+  const NodeUnit* GetTargetNodeUnit() const override { return node_units_[0]; }
+  std::string_view Type() const override { return "CastLoneQFusion"; }
+
+  /// <summary>
+  /// Traverses graph to check if the given starting NodeUnit is part of a valid Cast -> Quantize sequence.
+  /// If so, returns a IQnnNodeGroup that contains the Cast and Quantize NodeUnits.
+  /// </summary>
+  static std::unique_ptr<IQnnNodeGroup> TryFusion(
+      QnnModelWrapper& qnn_model_wrapper,
+      const NodeUnit& mul_node_unit,
+      const std::unordered_map<const Node*, const NodeUnit*>& node_to_node_unit,
+      const std::unordered_map<const NodeUnit*, const IQnnNodeGroup*>& node_unit_to_qnn_node_group,
+      const logging::Logger& logger);
+
+ private:
+  std::array<const NodeUnit*, 2> node_units_;
+};
+
+}  // namespace qnn
+}  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/qnn/builder/qnn_node_group/qnn_node_group.cc b/onnxruntime/core/providers/qnn/builder/qnn_node_group/qnn_node_group.cc
@@ -16,6 +16,7 @@
 #include "core/providers/qnn/builder/qnn_node_group/qnn_node_group.h"
 #include "core/providers/qnn/builder/qnn_node_group/reshape_gemm_fusion.h"
 #include "core/providers/qnn/builder/qnn_node_group/scale_softmax_fusion.h"
+#include "core/providers/qnn/builder/qnn_node_group/cast_lone_q_fusion.h"
 #include "core/providers/qnn/builder/qnn_node_group/channel_shuffle_fusion.h"
 #include "core/providers/qnn/builder/qnn_node_group/udo_fusion.h"
 #include "core/providers/qnn/builder/qnn_node_group/lpbqgemm_fusion.h"
@@ -80,6 +81,7 @@ static std::unordered_map<std::string, std::vector<FusionFunc>> fusions = {
     {"MatMul", {LowPowerBlockQuantizedMatMulFusion::TryFusion}},
     {"Gemm", {LowPowerBlockQuantizedGemmFusion::TryFusion, ReshapeGemmFusion::TryFusion}},
     {"Mul", {ScaleSoftmaxFusion::TryFusion}},
+    {"Cast", {CastLoneQFusion::TryFusion}},
     {"Transpose", {ChannelShuffleFusion::TryFusion}}};
 
 void registerUDO(const std::string& node_type, const std::string& op_package) {
diff --git a/onnxruntime/test/providers/qnn/qnn_basic_test.cc b/onnxruntime/test/providers/qnn/qnn_basic_test.cc
@@ -1042,10 +1042,11 @@ TEST_F(QnnHTPBackendTests, QnnContextPriorityHigh) {
 // cast_input -> Cast -> Q -> DQ ----
 //                                   |
 //             input2 -> Q -> DQ -> Add -> Q -> DQ -> output
-static GetTestModelFn BuildCastAddTestCase() {
-  return [](ModelTestBuilder& builder) {
+template <typename InputType, typename QuantType>
+static GetTestQDQModelFn<QuantType> BuildCastAddQDQTestCase() {
+  return [](ModelTestBuilder& builder, std::vector<QuantParams<QuantType>>& output_qparams) {
     // Creat Cast node int32 -> float32
-    NodeArg* cast_input = MakeTestInput(builder, TestInputDef<int32_t>({2, 3}, false, {0, 1, 0, 1, 0, 1}));
+    NodeArg* cast_input = MakeTestInput(builder, TestInputDef<InputType>({2, 3}, false, {0, 1, 0, 1, 0, 1}));
 
     auto* cast_output = builder.MakeIntermediate();
     Node& cast_node = builder.AddNode("Cast", {cast_input}, {cast_output});
@@ -1054,18 +1055,36 @@ static GetTestModelFn BuildCastAddTestCase() {
     // Create Add node
     std::vector<float> data = {0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f};
     gsl::span<float> data_range = gsl::make_span(data);
-    QuantParams<uint8_t> q_parameter = GetDataQuantParams<uint8_t>(data_range);
-    auto* add_input1_qdq = AddQDQNodePair<uint8_t>(builder, cast_output, q_parameter.scale, q_parameter.zero_point);
+    QuantParams<QuantType> q_parameter = GetDataQuantParams<QuantType>(data_range);
+    auto* add_input1_qdq = AddQDQNodePair<QuantType>(builder, cast_output, q_parameter.scale, q_parameter.zero_point);
 
     NodeArg* add_input2 = MakeTestInput(builder, TestInputDef<float>({2, 3}, false, data));
-    auto* add_input2_qdq = AddQDQNodePair<uint8_t>(builder, add_input2, q_parameter.scale, q_parameter.zero_point);
+    auto* add_input2_qdq = AddQDQNodePair<QuantType>(builder, add_input2, q_parameter.scale, q_parameter.zero_point);
 
     auto* add_output = builder.MakeIntermediate();
 
     builder.AddNode("Add", {add_input1_qdq, add_input2_qdq}, {add_output});
 
     // add_output -> Q -> DQ -> output
-    AddQDQNodePairWithOutputAsGraphOutput<uint8_t>(builder, add_output, q_parameter.scale, q_parameter.zero_point);
+    AddQDQNodePairWithOutputAsGraphOutput<QuantType>(builder, add_output, output_qparams[0].scale, output_qparams[0].zero_point);
+  };
+}
+
+template <typename InputType>
+static GetTestModelFn BuildCastAddTestCase() {
+  return [](ModelTestBuilder& builder) {
+    // Creat Cast node int32 -> float32
+    NodeArg* cast_input = MakeTestInput(builder, TestInputDef<InputType>({2, 3}, false, {0, 1, 0, 1, 0, 1}));
+
+    auto* cast_output = builder.MakeIntermediate();
+    Node& cast_node = builder.AddNode("Cast", {cast_input}, {cast_output});
+    cast_node.AddAttribute("to", static_cast<int64_t>(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT));
+
+    // Create Add node
+    NodeArg* add_input2 = MakeTestInput(builder, TestInputDef<float>({2, 3}, false, {0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f}));
+    auto* add_output = builder.MakeOutput();
+
+    builder.AddNode("Add", {cast_output, add_input2}, {add_output});
   };
 }
 
@@ -1091,19 +1110,53 @@ TEST_F(QnnHTPBackendTests, ProfilingTest) {
                   0.008f);
 }
 
-TEST_F(QnnHTPBackendTests, CastAddHTPAccuracyTest) {
+TEST_F(QnnHTPBackendTests, CastAddQDQU8) {
   ProviderOptions provider_options;
-#if defined(_WIN32)
-  provider_options["backend_path"] = "QnnHtp.dll";
-#else
-  provider_options["backend_path"] = "libQnnHtp.so";
-#endif
+  provider_options["backend_type"] = "htp";
   provider_options["offload_graph_io_quantization"] = "0";
 
-  RunQnnModelTest(BuildCastAddTestCase(),
-                  provider_options,
-                  13,  // opset
-                  ExpectedEPNodeAssignment::All);
+  TestQDQModelAccuracy<uint8_t>(BuildCastAddTestCase<uint8_t>(),
+                                BuildCastAddQDQTestCase<uint8_t, uint8_t>(),
+                                provider_options,
+                                21,
+                                ExpectedEPNodeAssignment::All);
+}
+
+TEST_F(QnnHTPBackendTests, CastAddQDQU16) {
+  ProviderOptions provider_options;
+  provider_options["backend_type"] = "htp";
+  provider_options["offload_graph_io_quantization"] = "0";
+
+  TestQDQModelAccuracy<uint16_t>(BuildCastAddTestCase<uint8_t>(),
+                                 BuildCastAddQDQTestCase<uint8_t, uint16_t>(),
+                                 provider_options,
+                                 21,
+                                 ExpectedEPNodeAssignment::All);
+}
+
+TEST_F(QnnHTPBackendTests, CastAddQDQS8) {
+  ProviderOptions provider_options;
+  provider_options["backend_type"] = "htp";
+  provider_options["offload_graph_io_quantization"] = "0";
+
+  TestQDQModelAccuracy<int8_t>(BuildCastAddTestCase<uint8_t>(),
+                               BuildCastAddQDQTestCase<uint8_t, int8_t>(),
+                               provider_options,
+                               21,
+                               ExpectedEPNodeAssignment::All);
+}
+
+TEST_F(QnnHTPBackendTests, CastAddQDQS16) {
+  ProviderOptions provider_options;
+  provider_options["backend_type"] = "htp";
+  provider_options["offload_graph_io_quantization"] = "0";
+
+  TestQDQModelAccuracy<int16_t>(BuildCastAddTestCase<uint8_t>(),
+                                BuildCastAddQDQTestCase<uint8_t, int16_t>(),
+                                provider_options,
+                                21,
+                                // QNN has not yet supported S16 Quantize/Dequantize
+                                ExpectedEPNodeAssignment::Some);
 }
 
 // Test float32 model with FP16 precision
