diff --git a/backends/qualcomm/builders/op_embedding.py b/backends/qualcomm/builders/op_embedding.py index adcf94f8f21..ea4ccc26dba 100644 --- a/backends/qualcomm/builders/op_embedding.py +++ b/backends/qualcomm/builders/op_embedding.py @@ -9,6 +9,9 @@ import numpy as np import torch +from executorch.backends.qualcomm.utils.check_qnn_version import ( + is_qnn_sdk_version_less_than, +) from executorch.backends.qualcomm.utils.constants import ( QCOM_DATA, QCOM_DTYPE, @@ -35,15 +38,19 @@ class Embedding(NodeVisitor): def __init__(self, *args) -> None: super().__init__(*args) - def define_node( + def _is_pcq_embedding(self, weight_node: torch.fx.Node) -> bool: + return ( + QCOM_QUANT_ATTRS in weight_node.meta + and weight_node.meta[QCOM_QUANT_ATTRS][QCOM_ENCODING] + in PER_CHANNEL_ENCODING + ) + + def _define_weight_and_indices( self, node: torch.fx.Node, nodes_to_wrappers: Dict[torch.fx.Node, PyQnnManager.TensorWrapper], - ) -> PyQnnManager.PyQnnOpWrapper: + ): weight_node = self.get_node(node.args[0]) - is_pcq_embedding = QCOM_QUANT_ATTRS in weight_node.meta and weight_node.meta[ - QCOM_QUANT_ATTRS - ][QCOM_ENCODING] in (PER_CHANNEL_ENCODING) weight_tensor = get_parameter(weight_node, self.edge_program) weight_tensor_wrapper = self.define_tensor( weight_node, @@ -62,6 +69,107 @@ def define_node( PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, nodes_to_wrappers, ) + return weight_node, weight_tensor, weight_tensor_wrapper, indices_tensor_wrapper + + def _act_dtype(self, node: torch.fx.Node): + act_quant_encoding, act_quant_configs = self.get_quant_encoding_conf(node, node) + act_dtype = ( + torch.uint16 + if act_quant_configs[QCOM_DTYPE] == torch.int32 + else act_quant_configs[QCOM_DTYPE] + ) + return act_quant_encoding, act_quant_configs, act_dtype + + def _build_gather_op( + self, node_name, gather_input_tensors, gather_output_tensors + ) -> PyQnnManager.PyQnnOpWrapper: + gather_op = PyQnnManager.PyQnnOpWrapper( + node_name, + QNN_OP_PACKAGE_NAME_QTI_AISW, + OpGather.op_name, + ) + gather_op.AddInputTensors(gather_input_tensors) + gather_op.AddOutputTensors(gather_output_tensors) + # For now, default axis is zero. + gather_op.AddScalarParam( + OpGather.param_axis, + PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_INT_32, + {QCOM_DATA: np.int32(0)}, + ) + return gather_op + + # Note that this pattern is only supported with QNN 2.48+. + # The weight is converted to the activation quantization before gather so + # the backend can fuse the convert into gather for better performance. + def define_node_optimize( + self, + node: torch.fx.Node, + nodes_to_wrappers: Dict[torch.fx.Node, PyQnnManager.TensorWrapper], + ) -> PyQnnManager.PyQnnOpWrapper: + op_wrapper_list = [] + ( + weight_node, + weight_tensor, + weight_tensor_wrapper, + indices_tensor_wrapper, + ) = self._define_weight_and_indices(node, nodes_to_wrappers) + + gather_input_tensors = [] + if self._is_pcq_embedding(weight_node): + act_quant_encoding, act_quant_configs, act_dtype = self._act_dtype(node) + convert_tensor_wrapper = self.define_custom_tensor_wrapper( + node_name=node.name + "_convert", + tensor_type=PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, + dtype=QNN_QUANT_TYPE_MAP[act_dtype], + quant_encoding=act_quant_encoding, + quant_configs=act_quant_configs, + dims=weight_tensor.size(), + tensor=weight_tensor, + is_fake_tensor=True, + nodes_to_wrappers=nodes_to_wrappers, + ) + convert_op = PyQnnManager.PyQnnOpWrapper( + node.name + "_convert", + QNN_OP_PACKAGE_NAME_QTI_AISW, + OpConvert.op_name, + ) + convert_op.AddInputTensors([weight_tensor_wrapper]) + convert_op.AddOutputTensors([convert_tensor_wrapper]) + op_wrapper_list.append(convert_op) + gather_input_tensors.append(convert_tensor_wrapper) + else: + gather_input_tensors.append(weight_tensor_wrapper) + gather_input_tensors.append(indices_tensor_wrapper) + + output_tensor = self.get_tensor(node, node) + output_tensor_wrapper = self.define_tensor( + node, + node, + output_tensor, + PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, + nodes_to_wrappers, + node_name=node.name, + ) + + op_wrapper_list.append( + self._build_gather_op( + node.name, gather_input_tensors, [output_tensor_wrapper] + ) + ) + return op_wrapper_list + + def define_node_legacy( + self, + node: torch.fx.Node, + nodes_to_wrappers: Dict[torch.fx.Node, PyQnnManager.TensorWrapper], + ) -> PyQnnManager.PyQnnOpWrapper: + ( + weight_node, + _, + weight_tensor_wrapper, + indices_tensor_wrapper, + ) = self._define_weight_and_indices(node, nodes_to_wrappers) + is_pcq_embedding = self._is_pcq_embedding(weight_node) gather_input_tensors = [weight_tensor_wrapper, indices_tensor_wrapper] @@ -69,24 +177,23 @@ def define_node( node_name = node.name if is_pcq_embedding: node_quant_attrs = node.meta[QCOM_QUANT_ATTRS].copy() - intermediate_quant_attrs = node.meta[QCOM_QUANT_ATTRS].copy() + weight_quant_attrs = weight_node.meta[QCOM_QUANT_ATTRS] # Based on QNN HTP quantization constraints, # we should set the scale to max of scales and per-tensor quantization for embedding op + intermediate_quant_attrs = node_quant_attrs.copy() intermediate_quant_attrs[QCOM_SCALE] = ( - weight_node.meta[QCOM_QUANT_ATTRS][QCOM_SCALES].max().item() + weight_quant_attrs[QCOM_SCALES].max().item() ) intermediate_quant_attrs[QCOM_ZERO_POINT] = ( - weight_node.meta[QCOM_QUANT_ATTRS][QCOM_ZERO_POINTS].max().item() + weight_quant_attrs[QCOM_ZERO_POINTS].max().item() ) - intermediate_quant_attrs[QCOM_DTYPE] = weight_node.meta[QCOM_QUANT_ATTRS][ - QCOM_DTYPE + intermediate_quant_attrs[QCOM_DTYPE] = weight_quant_attrs[QCOM_DTYPE] + intermediate_quant_attrs[QCOM_QUANT_MAX] = weight_quant_attrs[ + QCOM_QUANT_MAX + ] + intermediate_quant_attrs[QCOM_QUANT_MIN] = weight_quant_attrs[ + QCOM_QUANT_MIN ] - intermediate_quant_attrs[QCOM_QUANT_MAX] = weight_node.meta[ - QCOM_QUANT_ATTRS - ][QCOM_QUANT_MAX] - intermediate_quant_attrs[QCOM_QUANT_MIN] = weight_node.meta[ - QCOM_QUANT_ATTRS - ][QCOM_QUANT_MIN] node.meta[QCOM_QUANT_ATTRS] = intermediate_quant_attrs node_name += "_intermediate" output_tensor_wrapper = self.define_tensor( @@ -99,33 +206,15 @@ def define_node( ) gather_output_tensors = [output_tensor_wrapper] - gather_op = PyQnnManager.PyQnnOpWrapper( - node_name, - QNN_OP_PACKAGE_NAME_QTI_AISW, - OpGather.op_name, - ) - gather_op.AddInputTensors(gather_input_tensors) - gather_op.AddOutputTensors(gather_output_tensors) - - # For now, default axis is zero. - gather_op.AddScalarParam( - OpGather.param_axis, - PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_INT_32, - {QCOM_DATA: np.int32(0)}, - ) - - op_wrapper_list = [gather_op] + op_wrapper_list = [ + self._build_gather_op( + node_name, gather_input_tensors, gather_output_tensors + ) + ] if is_pcq_embedding: node.meta[QCOM_QUANT_ATTRS] = node_quant_attrs - act_quant_encoding, act_quant_configs = self.get_quant_encoding_conf( - node, node - ) - act_dtype = ( - torch.uint16 - if act_quant_configs[QCOM_DTYPE] == torch.int32 - else act_quant_configs[QCOM_DTYPE] - ) + act_quant_encoding, act_quant_configs, act_dtype = self._act_dtype(node) convert_tensor_wrapper = self.define_custom_tensor_wrapper( node_name=node.name, tensor_type=PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, @@ -147,3 +236,12 @@ def define_node( op_wrapper_list.append(convert_op) return op_wrapper_list + + def define_node( + self, + node: torch.fx.Node, + nodes_to_wrappers: Dict[torch.fx.Node, PyQnnManager.TensorWrapper], + ) -> PyQnnManager.PyQnnOpWrapper: + if is_qnn_sdk_version_less_than("2.48"): + return self.define_node_legacy(node, nodes_to_wrappers) + return self.define_node_optimize(node, nodes_to_wrappers) diff --git a/backends/qualcomm/export_utils.py b/backends/qualcomm/export_utils.py index bcba08ecc5a..51c30d2d186 100644 --- a/backends/qualcomm/export_utils.py +++ b/backends/qualcomm/export_utils.py @@ -37,6 +37,10 @@ QnnExecuTorchLpaiTargetEnv, QnnExecuTorchOpPackageOptions, ) +from executorch.backends.qualcomm.utils.check_qnn_version import ( + get_sdk_build_id, + is_qnn_sdk_version_less_than, +) from executorch.backends.qualcomm.utils.constants import ( HEXAGON_SDK_ROOT, HEXAGON_TOOLS_ROOT, @@ -47,10 +51,8 @@ generate_lpai_compiler_spec, generate_qnn_executorch_compiler_spec, get_qnn_context_binary_alignment, - get_sdk_build_id, get_soc_to_htp_arch_map, get_soc_to_lpai_hw_ver_map, - is_qnn_sdk_version_less_than, to_edge_transform_and_lower_to_qnn, ) from executorch.exir.capture._config import ExecutorchBackendConfig diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 49ca26d241a..05366a250aa 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -44,6 +44,10 @@ TestQNN, validate_context_binary, ) +from executorch.backends.qualcomm.utils.check_qnn_version import ( + is_qnn_sdk_version_greater_than, + is_qnn_sdk_version_less_than, +) from executorch.backends.qualcomm.utils.constants import ( QCOM_ANNOTATION, QCOM_MODULE, @@ -61,8 +65,6 @@ generate_htp_compiler_spec, generate_lpai_compiler_spec, generate_qnn_executorch_compiler_spec, - is_qnn_sdk_version_greater_than, - is_qnn_sdk_version_less_than, PyQnnManagerAdaptor, rewrite_prepared_observer, skip_annotation, @@ -3929,18 +3931,20 @@ def test_qnn_backend_embedding(self): ) self.lower_module_and_test_output(modules[i], sample_input) - # TODO: Once the accuracy issue is fixed, enable this test. - @unittest.skip("Bad accuracy for HTP") + @unittest.skipIf(is_qnn_sdk_version_less_than("2.48"), "UT pass after QNN 2.48") def test_qnn_backend_embedding_per_channel(self): module = Embedding() # noqa: F405 sample_input = (torch.Tensor([1, 2, 4, 5]).to(torch.int32),) - qdq_module = self.get_qdq_module( - module, - sample_input, - quant_dtype=QuantDtype.use_16a8w, - is_embedding_per_channel=True, - ) - self.lower_module_and_test_output(qdq_module, sample_input) + quant_dtype = [QuantDtype.use_16a8w, QuantDtype.use_16a4w] + for i, qdtype in enumerate(quant_dtype): + with self.subTest(i=i): + qdq_module = self.get_qdq_module( + module, + sample_input, + quant_dtype=qdtype, + is_embedding_per_channel=True, + ) + self.lower_module_and_test_output(qdq_module, sample_input) def test_qnn_backend_equal(self): test_comb = [ diff --git a/backends/qualcomm/utils/check_qnn_version.py b/backends/qualcomm/utils/check_qnn_version.py new file mode 100644 index 00000000000..e6cccd56984 --- /dev/null +++ b/backends/qualcomm/utils/check_qnn_version.py @@ -0,0 +1,69 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import platform +import re + +import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManagerAdaptor + + +def get_qnn_lib_name(base: str) -> str: + """Returns the platform-specific shared library filename for a QNN library.""" + if platform.system().lower() == "windows": + return f"{base}.dll" + return f"lib{base}.so" + + +def _get_qnn_host_lib_dir_name() -> str: + """Returns the QNN SDK library subdirectory name for the current x86-64 host OS.""" + if platform.system().lower() == "windows": + return "x86_64-windows-msvc" + return "x86_64-linux-clang" + + +def get_sdk_build_id(): + htp_library_path = os.path.join( + os.environ.get("QNN_SDK_ROOT", None), + "lib", + _get_qnn_host_lib_dir_name(), + get_qnn_lib_name("QnnHtp"), + ) + # The GetQnnSdkBuildId API can be used without needing to create a backend first, so it works regardless of which backend is used. + sdk_build_id = PyQnnManagerAdaptor.GetQnnSdkBuildId(htp_library_path) + return sdk_build_id + + +def is_qnn_sdk_version_less_than(target_version): + current_version = get_sdk_build_id() + + match = re.search(r"v(\d+)\.(\d+)", current_version) + if match: + current_major, current_minor = map(int, match.groups()[:2]) + else: + raise ValueError( + f"Failed to get current major and minor version from QNN SDK Build id {current_version}" + ) + + target_major, target_minor = map(int, target_version.split(".")[:2]) + + return current_major == target_major and current_minor < target_minor + + +def is_qnn_sdk_version_greater_than(target_version): + current_version = get_sdk_build_id() + + match = re.search(r"v(\d+)\.(\d+)", current_version) + if match: + current_major, current_minor = map(int, match.groups()[:2]) + else: + raise ValueError( + f"Failed to get current major and minor version from QNN SDK Build id {current_version}" + ) + + target_major, target_minor = map(int, target_version.split(".")[:2]) + + return current_major == target_major and current_minor > target_minor diff --git a/backends/qualcomm/utils/utils.py b/backends/qualcomm/utils/utils.py index 16a071f8cf0..b1c236647f9 100644 --- a/backends/qualcomm/utils/utils.py +++ b/backends/qualcomm/utils/utils.py @@ -4,9 +4,6 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import operator -import os -import platform -import re import warnings from collections import defaultdict, OrderedDict from enum import Enum @@ -57,6 +54,11 @@ flatbuffer_to_option, option_to_flatbuffer, ) +from executorch.backends.qualcomm.utils.check_qnn_version import ( + get_qnn_lib_name, + get_sdk_build_id, + is_qnn_sdk_version_less_than, +) from executorch.backends.qualcomm.utils.constants import ( QCOM_QNN_COMPILE_SPEC, QCOM_QUANTIZED_IO, @@ -78,20 +80,6 @@ from torch.library import Library -def _get_qnn_lib_name(base: str) -> str: - """Returns the platform-specific shared library filename for a QNN library.""" - if platform.system().lower() == "windows": - return f"{base}.dll" - return f"lib{base}.so" - - -def _get_qnn_host_lib_dir_name() -> str: - """Returns the QNN SDK library subdirectory name for the current x86-64 host OS.""" - if platform.system().lower() == "windows": - return "x86_64-windows-msvc" - return "x86_64-linux-clang" - - class _AnnotationSkipper(OperatorSupportBase): """ Class used to partition out unwanted graph nodes. @@ -1226,7 +1214,7 @@ def generate_qnn_executorch_compiler_spec( # noqa: C901 qnn_executorch_options.dump_intermediate_outputs = dump_intermediate_outputs if saver: - qnn_executorch_options.library_path = _get_qnn_lib_name("QnnSaver") + qnn_executorch_options.library_path = get_qnn_lib_name("QnnSaver") qnn_executorch_options.saver = True qnn_executorch_options.saver_output_dir = "saver_output" @@ -1437,49 +1425,5 @@ def rewrite_prepared_observer( setattr(graph_module, target_name, new_observer) -def get_sdk_build_id(): - htp_library_path = os.path.join( - os.environ.get("QNN_SDK_ROOT", None), - "lib", - _get_qnn_host_lib_dir_name(), - _get_qnn_lib_name("QnnHtp"), - ) - # The GetQnnSdkBuildId API can be used without needing to create a backend first, so it works regardless of which backend is used. - sdk_build_id = PyQnnManagerAdaptor.GetQnnSdkBuildId(htp_library_path) - return sdk_build_id - - -def is_qnn_sdk_version_less_than(target_version): - current_version = get_sdk_build_id() - - match = re.search(r"v(\d+)\.(\d+)", current_version) - if match: - current_major, current_minor = map(int, match.groups()[:2]) - else: - raise ValueError( - f"Failed to get current major and minor version from QNN SDK Build id {current_version}" - ) - - target_major, target_minor = map(int, target_version.split(".")[:2]) - - return current_major == target_major and current_minor < target_minor - - -def is_qnn_sdk_version_greater_than(target_version): - current_version = get_sdk_build_id() - - match = re.search(r"v(\d+)\.(\d+)", current_version) - if match: - current_major, current_minor = map(int, match.groups()[:2]) - else: - raise ValueError( - f"Failed to get current major and minor version from QNN SDK Build id {current_version}" - ) - - target_major, target_minor = map(int, target_version.split(".")[:2]) - - return current_major == target_major and current_minor > target_minor - - def get_qnn_context_binary_alignment() -> int: return PyQnnManagerAdaptor.GetQNNCtxBinAlignment() diff --git a/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py b/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py index 149a376e918..8987acbe580 100644 --- a/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py +++ b/examples/qualcomm/oss_scripts/llama/wrappers/base_component.py @@ -19,7 +19,7 @@ from executorch.backends.qualcomm.serialization.qc_schema import ( QnnExecuTorchBackendType, ) -from executorch.backends.qualcomm.utils.utils import ( +from executorch.backends.qualcomm.utils.check_qnn_version import ( get_sdk_build_id, is_qnn_sdk_version_less_than, ) diff --git a/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py b/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py index 9bab682eac8..54c63459bce 100644 --- a/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py +++ b/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py @@ -13,7 +13,7 @@ import types from functools import partial -from typing import Dict, List +from typing import Any, Dict, List import torch @@ -783,10 +783,26 @@ def activation_override(quantized_node, unquantized_node): activation_override(quantized_user, unquantized_user) def parameter_override(quantized_node, unquantized_node): - setattr( + # Some parameters need to be iterated over to retrieve attributes such as static_llama.tok_embedding.weight + def _get_attr(graph_module: torch.fx.GraphModule, target: str) -> Any: + attr: Any = graph_module + for target_atom in target.split("."): + attr = getattr(attr, target_atom) + return attr + + def _set_attr( + graph_module: torch.fx.GraphModule, target: str, replacement: Any + ) -> Any: + attr: Any = graph_module + target_list = target.split(".") + for target_atom in target_list[:-1]: + attr = getattr(attr, target_atom) + setattr(attr, target_list[-1], replacement) + + _set_attr( unquantized_model, unquantized_node.target, - getattr(quantized_model, quantized_node.target), + _get_attr(quantized_model, quantized_node.target), ) # scale / zero point are part of op's attributes if list(quantized_node.users)[0].target in ptq_target: