From c846099750843721303490f82c0ec105b165e31a Mon Sep 17 00:00:00 2001 From: Xiyue Shi Date: Thu, 8 Oct 2026 14:27:12 -0700 Subject: [PATCH 1/2] Qualcomm AI Engine Direct - Add composite lowering for aten.convolution_backward Co-authored-by: rajeswar277 Co-authored-by: quic-mohitgupta --- backends/qualcomm/_passes/i64_to_i32.py | 11 +- backends/qualcomm/_passes/layout_transform.py | 46 ++- backends/qualcomm/builders/README.md | 14 + backends/qualcomm/builders/__init__.py | 2 + .../qualcomm/builders/op_conv_backward.py | 334 ++++++++++++++++++ backends/qualcomm/partition/utils.py | 1 + backends/qualcomm/tests/models.py | 61 ++++ .../qualcomm/tests/rework/htp/op/v69/test.py | 14 + .../qualcomm/tests/rework/htp/op/v73/test.py | 14 + .../qualcomm/tests/rework/htp/op/v75/test.py | 14 + .../qualcomm/tests/rework/htp/op/v79/test.py | 14 + .../qualcomm/tests/rework/htp/op/v81/test.py | 14 + backends/qualcomm/tests/rework/src/op.py | 46 +++ backends/qualcomm/tests/test_passes.py | 100 ++++++ backends/qualcomm/tests/test_qnn_delegate.py | 5 + 15 files changed, 671 insertions(+), 19 deletions(-) create mode 100644 backends/qualcomm/builders/op_conv_backward.py diff --git a/backends/qualcomm/_passes/i64_to_i32.py b/backends/qualcomm/_passes/i64_to_i32.py index a26956b5569..6deedec3be4 100644 --- a/backends/qualcomm/_passes/i64_to_i32.py +++ b/backends/qualcomm/_passes/i64_to_i32.py @@ -73,7 +73,10 @@ def _record_original_output_dtype(self, graph_module: torch.fx.GraphModule): # Keep track of original output dtype so we ensure the dtype of the graph is consistent with nn.Module if is_graph_output(n): if isinstance(n.meta["val"], (tuple, list)): - dtype_list = [tensor.dtype for tensor in n.meta["val"]] + dtype_list = [ + tensor.dtype if tensor is not None else None + for tensor in n.meta["val"] + ] n.meta[QCOM_ORIG_DTYPE] = dtype_list else: n.meta[QCOM_ORIG_DTYPE] = n.meta["val"].dtype @@ -84,6 +87,8 @@ def _preserve_output_dtype(self, graph_module: torch.fx.GraphModule): if isinstance(n.meta["val"], (tuple, list)): for i, dtype in enumerate(n.meta[QCOM_ORIG_DTYPE]): # TODO: Enable this in future to support OP such as topK + if n.meta["val"][i] is None or dtype is None: + continue if n.meta["val"][i].dtype != dtype: raise AssertionError( "Multi output nodes currently don't support casting dtype back." @@ -115,10 +120,10 @@ def _preserve_output_dtype(self, graph_module: torch.fx.GraphModule): def _update_meta(self, node: torch.fx.node) -> None: meta_val = node.meta["val"] if isinstance(meta_val, tuple): - node.meta["val"] = ( + node.meta["val"] = tuple( ( fake_tensor.to(torch.int32) - if fake_tensor.dtype == torch.int64 + if fake_tensor is not None and fake_tensor.dtype == torch.int64 else fake_tensor ) for fake_tensor in meta_val diff --git a/backends/qualcomm/_passes/layout_transform.py b/backends/qualcomm/_passes/layout_transform.py index f2bb7a45f3c..b1ac8134ddc 100644 --- a/backends/qualcomm/_passes/layout_transform.py +++ b/backends/qualcomm/_passes/layout_transform.py @@ -47,6 +47,7 @@ class LayoutTransform(ExportPass): exir_ops.edge.aten.avg_pool2d.default, exir_ops.edge.aten.avg_pool3d.default, exir_ops.edge.aten.convolution.default, + exir_ops.edge.aten.convolution_backward.default, exir_ops.edge.aten.grid_sampler_2d.default, exir_ops.edge.aten.grid_sampler_3d.default, exir_ops.edge.aten.instance_norm.default, @@ -169,8 +170,8 @@ def mark_as_transformed(self, node: torch.fx.Node) -> None: getitem_node = list(node.users.keys())[0] if getitem_node.target.__name__ != "getitem": raise AssertionError( - "Expected node's user to be getitem, " - f"got {getitem_node.target.__name__}" + f"Expected tuple-output node {node.name} ({node.target}) to have a " + f"getitem user, got {getitem_node.target.__name__}" ) index = getitem_node.args[1] node.meta[self.transformed_tag] = self.get_axis_order( @@ -226,32 +227,45 @@ def is_edge_condition(self, node): return False - def insert_node(self, graph_module, node, revert_layout: bool) -> None: + def insert_node( + self, graph_module, node, revert_layout: bool, layout_node=None + ) -> None: if not self.insert_permute: return - with graph_module.graph.inserting_after(node): - users = node.users.copy() - if isinstance(node.meta["val"], tuple): - getitem_node = list(node.users.keys())[0] + if layout_node is None: + layout_node = node + if isinstance(node.meta["val"], tuple): + for getitem_node in node.users: if getitem_node.target.__name__ != "getitem": raise AssertionError( - f"Expected bn node's user to be getitem, got {getitem_node.target.__name__}" + f"Expected tuple-output node {node.name} ({node.target}) to have a " + f"getitem user, got {getitem_node.target.__name__}" ) - index = getitem_node.args[1] - tensor = node.meta["val"][index] - else: - tensor = node.meta["val"] + tensor = node.meta["val"][getitem_node.args[1]] + if tensor is not None: + self.insert_node( + graph_module, + getitem_node, + revert_layout, + layout_node, + ) + return + + tensor = node.meta["val"] + permute_input = node + users = node.users.copy() + with graph_module.graph.inserting_after(permute_input): permute = self.create_call_function_node( graph_module, exir_ops.edge.aten.permute_copy.default, ( - node, + permute_input, self.get_axis_order(eval_shape(tensor.shape), revert_layout), ), ) permute.meta["val"] = tensor - permute.meta[QCOM_QUANT_ATTRS] = node.meta.get(QCOM_QUANT_ATTRS) + permute.meta[QCOM_QUANT_ATTRS] = layout_node.meta.get(QCOM_QUANT_ATTRS) # we need this to check the annotation boundary permute.meta[QCOM_INSERTED_PERMUTE] = True @@ -264,14 +278,14 @@ def insert_node(self, graph_module, node, revert_layout: bool) -> None: # └--------------------------------------> qnn_premute -┙ # i.e. insert permute by condition between user and current node # if there are multiple users included - is_node_transformed = self.is_transformed_node(node) + is_node_transformed = self.is_transformed_node(layout_node) for user in users: is_user_transformed = ( self.is_transformed_node(user) or QCOM_LAYOUT_CHANGE in user.meta ) # insert permute only in exclusive condition if is_node_transformed != is_user_transformed: - user.replace_input_with(node, permute) + user.replace_input_with(permute_input, permute) def create_call_function_node( self, diff --git a/backends/qualcomm/builders/README.md b/backends/qualcomm/builders/README.md index 1f186eff9a9..94318bc40f4 100644 --- a/backends/qualcomm/builders/README.md +++ b/backends/qualcomm/builders/README.md @@ -13,6 +13,7 @@ Thank you for contributing to Qualcomm AI Engine Direct delegate for ExecuTorch. - [Implementation](#implementation) - [Quantizer Annotation](#quantizer-annotation) - [Operator Support Status](#operator-support-status) + - [Additional PyTorch Operators Supported via Composite QNN Lowering](#additional-pytorch-operators-supported-via-composite-qnn-lowering) - [Issues](#issues) - [Pull Requests](#pull-requests) @@ -540,6 +541,19 @@ The following PyTorch operators are supported through decomposition or annotatio | `aten.trunc` | `DecomposeTrunc` | | `aten.var.correction`, `aten.var.dim` | `DecomposeVar` | +## Additional PyTorch Operators Supported via Composite QNN Lowering + +The following PyTorch operators are lowered by a node visitor into multiple QNN operations. + +ConvBackward uses this path to reuse the forward Conv2d's QNN weight wrapper and derive HWOI at runtime; an ATen decomposition would otherwise require another static weight or changes to the forward Conv2d visitor. It currently supports floating-point inputs and weights only; quantized ConvBackward support is planned for a future change. + +| PyTorch Op | Builder | QNN Lowering | +|---|---|---| +| `aten.convolution_backward.default` | `ConvBackward` | `Transpose` → `TransposeConv2d` | +| `aten.copy.default` | `Copy` | `Reshape` → `Tile` | +| `aten.embedding.default` | `Embedding` | `Convert` → `Gather` or `Gather` → `Convert` | +| `aten.index_put.default` | `IndexPutVisitor` | tensor preparation → `ScatterNd` | + ## Issues Please refer to the [issue section](../README.md#issues) for more information. diff --git a/backends/qualcomm/builders/__init__.py b/backends/qualcomm/builders/__init__.py index 3dbb0d219b3..4a9df4afc45 100644 --- a/backends/qualcomm/builders/__init__.py +++ b/backends/qualcomm/builders/__init__.py @@ -29,6 +29,7 @@ op_channel_shuffle, op_clamp, op_conv, + op_conv_backward, op_copy, op_cos, op_cum_sum, @@ -148,6 +149,7 @@ op_channel_shuffle, op_clamp, op_conv, + op_conv_backward, op_copy, op_cos, op_cum_sum, diff --git a/backends/qualcomm/builders/op_conv_backward.py b/backends/qualcomm/builders/op_conv_backward.py new file mode 100644 index 00000000000..d60e84f6e75 --- /dev/null +++ b/backends/qualcomm/builders/op_conv_backward.py @@ -0,0 +1,334 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging +from typing import cast, Dict, List, Optional, Tuple + +import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManager +import numpy as np +import torch + +from .node_visitor import dq_ops, NodeVisitor +from .node_visitor_manager import register_node_visitor +from .qnn_constants import OpTranspose, OpTransposeConv2d, QNN_OP_PACKAGE_NAME_QTI_AISW +from .utils import get_parameter + + +logger = logging.getLogger(__name__) + + +@register_node_visitor +class ConvBackward(NodeVisitor): + """Lower input-gradient Conv2d to TransposeConv2d on HTP V69+. + + Reuses the forward Conv2d weight wrapper instead of materializing another + static weight. + """ + + target = ["aten.convolution_backward.default"] + + def __init__(self, *args) -> None: + super().__init__(*args) + + def _get_supported_input_nodes( + self, node: torch.fx.Node + ) -> Optional[Tuple[torch.fx.Node, torch.Tensor, torch.fx.Node]]: + output_mask = cast(List[bool], node.args[10]) + if output_mask != [True, False, False]: + logger.warning( + "ConvBackward %s is unsupported: only grad_input is supported; " + "got output_mask=%s", + node.name, + output_mask, + ) + return None + if node.args[7]: + logger.warning( + "ConvBackward %s is unsupported: transposed convolutions are not supported", + node.name, + ) + return None + + grad_output_node = node.args[0] + if grad_output_node.target in dq_ops: + logger.warning( + "ConvBackward %s is unsupported: quantized grad_output is not supported", + node.name, + ) + return None + grad_output_tensor = self.get_tensor(grad_output_node, node) + if grad_output_tensor.dim() != 4: + logger.warning( + "ConvBackward %s is unsupported: grad_output must be rank 4; got rank %s", + node.name, + grad_output_tensor.dim(), + ) + return None + + filter_node = node.args[2] + if filter_node.target in dq_ops: + logger.warning( + "ConvBackward %s is unsupported: quantized weights are not supported", + node.name, + ) + return None + return grad_output_node, grad_output_tensor, filter_node + + def _get_filter_and_grad_input_tensors( + self, node: torch.fx.Node, filter_node: torch.fx.Node + ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + filter_tensor = get_parameter(filter_node, self.edge_program) + if filter_tensor is None: + logger.warning( + "ConvBackward %s is unsupported: weight must be a static parameter", + node.name, + ) + return None + if not filter_tensor.is_floating_point(): + logger.warning( + "ConvBackward %s is unsupported: weight must be floating point; got %s", + node.name, + filter_tensor.dtype, + ) + return None + if filter_tensor.dim() != 4: + logger.warning( + "ConvBackward %s is unsupported: weight must be rank 4; got rank %s", + node.name, + filter_tensor.dim(), + ) + return None + + grad_input_tensor = self.get_tensor(node, node, 0) + if grad_input_tensor.dim() != 4: + logger.warning( + "ConvBackward %s is unsupported: grad_input must be rank 4; got rank %s", + node.name, + grad_input_tensor.dim(), + ) + return None + return filter_tensor, grad_input_tensor + + def _get_transpose_conv_params( + self, + node: torch.fx.Node, + grad_output_node: torch.fx.Node, + filter_tensor: torch.Tensor, + ) -> Optional[Tuple[List[int], List[List[int]], List[int]]]: + stride = cast(List[int], node.args[4]) + padding = cast(List[int], node.args[5]) + dilation = cast(List[int], node.args[6]) + groups = cast(int, node.args[9]) + if ( + len(stride) != 2 + or len(padding) not in (1, 2) + or len(dilation) != 2 + or any(value != 1 for value in dilation) + ): + logger.warning( + "ConvBackward %s is unsupported: requires 2D stride/padding and dilation=1; " + "got stride=%s, padding=%s, dilation=%s", + node.name, + stride, + padding, + dilation, + ) + return None + if groups != 1: + logger.warning( + "ConvBackward %s is unsupported: grouped convolutions are not supported; " + "got groups=%s", + node.name, + groups, + ) + return None + if len(padding) == 1: + padding = padding + padding + padding_2d = [[value, value] for value in padding] + + # Use unpermuted PyTorch metadata; QNN-layout tensors can place spatial + # axes differently. + _, _, kernel_height, kernel_width = filter_tensor.shape + grad_output_height, grad_output_width = grad_output_node.meta["val"].shape[-2:] + grad_input_height, grad_input_width = node.meta["val"][0].shape[-2:] + # Derive QNN's output padding from the desired grad_input shape rather + # than using the forward convolution's output_padding argument. + base_height = ( + stride[0] * (grad_output_height - 1) + kernel_height - 2 * padding[0] + ) + base_width = stride[1] * (grad_output_width - 1) + kernel_width - 2 * padding[1] + output_padding = [ + grad_input_height - base_height, + grad_input_width - base_width, + ] + if any( + value < 0 or value >= stride[index] + for index, value in enumerate(output_padding) + ): + logger.warning( + "ConvBackward %s is unsupported: derived output_padding=%s is invalid " + "for stride=%s", + node.name, + output_padding, + stride, + ) + return None + return stride, padding_2d, output_padding + + def _define_filter_hwoi( + self, + node: torch.fx.Node, + filter_node: torch.fx.Node, + filter_tensor: torch.Tensor, + nodes_to_wrappers: Dict[str, PyQnnManager.TensorWrapper], + ) -> Tuple[PyQnnManager.TensorWrapper, List[PyQnnManager.PyQnnOpWrapper]]: + # Reuse the forward Conv2d's static HWIO wrapper. HWOI is named from the + # forward Conv2d view; for this TransposeConv2d it is HWIO, produced from + # the raw OIHW weight by permute(2, 3, 0, 1). + filter_tensor_hwio = filter_tensor.permute(2, 3, 1, 0).contiguous() + filter_wrapper = self.define_tensor( + filter_node, + node, + filter_tensor_hwio, + PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_STATIC, + nodes_to_wrappers, + ) + filter_hwoi_name = f"{filter_node.name}_hwoi" + if filter_hwoi_name in nodes_to_wrappers: + return nodes_to_wrappers[filter_hwoi_name][0], [] + + filter_tensor_hwoi = filter_tensor_hwio.permute(0, 1, 3, 2) + filter_hwoi_wrapper = self.define_custom_tensor_wrapper( + node_name=filter_hwoi_name, + tensor_type=PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, + dtype=self.get_data_type(filter_tensor_hwoi, {}), + quant_encoding=PyQnnManager.Qnn_QuantizationEncoding_t.QNN_QUANTIZATION_ENCODING_UNDEFINED, + quant_configs={}, + dims=filter_tensor_hwoi.size(), + tensor=filter_tensor_hwoi, + is_fake_tensor=True, + nodes_to_wrappers=nodes_to_wrappers, + ) + weight_transpose_op = PyQnnManager.PyQnnOpWrapper( + f"{node.name}_weight_to_hwoi", + QNN_OP_PACKAGE_NAME_QTI_AISW, + OpTranspose.op_name, + ) + weight_transpose_op.AddInputTensors([filter_wrapper]) + weight_transpose_op.AddOutputTensors([filter_hwoi_wrapper]) + weight_transpose_op.AddTensorParam( + OpTranspose.param_perm, + PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_UINT_32, + 1, + [4], + np.array([0, 1, 3, 2], dtype=np.uint32), + True, + ) + return filter_hwoi_wrapper, [weight_transpose_op] + + def _define_transpose_conv_op( + self, + node: torch.fx.Node, + grad_output_wrapper: PyQnnManager.TensorWrapper, + filter_hwoi_wrapper: PyQnnManager.TensorWrapper, + grad_input_wrapper: PyQnnManager.TensorWrapper, + stride: List[int], + padding_2d: List[List[int]], + output_padding: List[int], + ) -> PyQnnManager.PyQnnOpWrapper: + transpose_conv_op = PyQnnManager.PyQnnOpWrapper( + f"{node.name}_transpose_conv", + QNN_OP_PACKAGE_NAME_QTI_AISW, + OpTransposeConv2d.op_name, + ) + transpose_conv_op.AddInputTensors([grad_output_wrapper, filter_hwoi_wrapper]) + transpose_conv_op.AddOutputTensors([grad_input_wrapper]) + transpose_conv_op.AddTensorParam( + OpTransposeConv2d.param_stride, + PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_UINT_32, + 1, + [len(stride)], + np.array(stride, dtype=np.uint32), + True, + ) + transpose_conv_op.AddTensorParam( + OpTransposeConv2d.param_pad_amount, + PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_UINT_32, + 2, + [len(padding_2d), len(padding_2d[0])], + np.array(padding_2d, dtype=np.uint32), + True, + ) + transpose_conv_op.AddTensorParam( + OpTransposeConv2d.param_output_padding, + PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_UINT_32, + 1, + [len(output_padding)], + np.array(output_padding, dtype=np.uint32), + True, + ) + return transpose_conv_op + + def define_node( + self, + node: torch.fx.Node, + nodes_to_wrappers: Dict[str, PyQnnManager.TensorWrapper], + ) -> Optional[List[PyQnnManager.PyQnnOpWrapper]]: + # convolution_backward signature: + # args[0]: grad_output + # args[1]: input + # args[2]: weight (raw parameter [Cout, Cin, kH, kW]) + # args[3]: bias_sizes + # args[4]: stride + # args[5]: padding + # args[6]: dilation + # args[7]: transposed + # args[8]: output_padding + # args[9]: groups + # args[10]: output_mask [grad_input, grad_weight, grad_bias] + supported_inputs = self._get_supported_input_nodes(node) + if supported_inputs is None: + logger.warning("ConvBackward %s is unsupported", node.name) + return None + grad_output_node, grad_output_tensor, filter_node = supported_inputs + tensors = self._get_filter_and_grad_input_tensors(node, filter_node) + if tensors is None: + return None + filter_tensor, grad_input_tensor = tensors + params = self._get_transpose_conv_params(node, grad_output_node, filter_tensor) + if params is None: + return None + stride, padding_2d, output_padding = params + + grad_output_wrapper = self.define_tensor( + grad_output_node, + node, + grad_output_tensor, + PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, + nodes_to_wrappers, + ) + filter_hwoi_wrapper, weight_transpose_ops = self._define_filter_hwoi( + node, filter_node, filter_tensor, nodes_to_wrappers + ) + + grad_input_wrapper = self.define_tensor( + node, + node, + grad_input_tensor, + PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE, + nodes_to_wrappers, + ) + + transpose_conv_op = self._define_transpose_conv_op( + node, + grad_output_wrapper, + filter_hwoi_wrapper, + grad_input_wrapper, + stride, + padding_2d, + output_padding, + ) + return weight_transpose_ops + [transpose_conv_op] diff --git a/backends/qualcomm/partition/utils.py b/backends/qualcomm/partition/utils.py index 5fa5633ac70..402bf89552e 100644 --- a/backends/qualcomm/partition/utils.py +++ b/backends/qualcomm/partition/utils.py @@ -169,6 +169,7 @@ def get_skip_decomp_table() -> List[torch._ops.OperatorBase]: torch.ops.aten.adaptive_avg_pool2d.default, torch.ops.aten.channel_shuffle.default, torch.ops.aten.col2im.default, + torch.ops.aten.convolution_backward.default, torch.ops.aten.elu.default, torch.ops.aten.floor_divide.default, torch.ops.aten.hardsigmoid.default, diff --git a/backends/qualcomm/tests/models.py b/backends/qualcomm/tests/models.py index 94d875617c7..926f5e1fb67 100644 --- a/backends/qualcomm/tests/models.py +++ b/backends/qualcomm/tests/models.py @@ -959,6 +959,67 @@ def forward(self, x): return self.second(self.first(x)) +class ConvBackward(torch.nn.Module): + def __init__( + self, + flatten_grad_input=False, + duplicate_backward=False, + return_grad_weight=False, + ): + super().__init__() + self.flatten_grad_input = flatten_grad_input + self.duplicate_backward = duplicate_backward + self.return_grad_weight = return_grad_weight + self.weight = torch.nn.Parameter(torch.randn(4, 3, 3, 3), requires_grad=False) + + def forward(self, grad_output, x): + forward_output = torch.ops.aten.convolution.default( + x, + self.weight, + None, + [2, 2], + [1, 1], + [1, 1], + False, + [0, 0], + 1, + ) + backward_outputs = torch.ops.aten.convolution_backward.default( + grad_output, + x, + self.weight, + None, + [2, 2], + [1, 1], + [1, 1], + False, + [0, 0], + 1, + [True, self.return_grad_weight, False], + ) + grad_input = backward_outputs[0] + if self.flatten_grad_input: + grad_input = torch.flatten(grad_input, 1) + if self.return_grad_weight: + return forward_output, grad_input, torch.flatten(backward_outputs[1], 1) + if not self.duplicate_backward: + return forward_output, grad_input + second_grad_input = torch.ops.aten.convolution_backward.default( + grad_output, + x, + self.weight, + None, + [2, 2], + [1, 1], + [1, 1], + False, + [0, 0], + 1, + [True, False, False], + )[0] + return forward_output, grad_input, second_grad_input + + class ConvFull(torch.nn.Module): def __init__(self, fill, full_shape): super().__init__() diff --git a/backends/qualcomm/tests/rework/htp/op/v69/test.py b/backends/qualcomm/tests/rework/htp/op/v69/test.py index e0c8fb11f5d..df0ee32625f 100644 --- a/backends/qualcomm/tests/rework/htp/op/v69/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v69/test.py @@ -501,6 +501,20 @@ def test_conv3d_transpose(request, kwargs): Conv.test_3d_transpose(request, kwargs) # noqa: F405 +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param( + {"act": None, "expected": Tolerance(rtol=1e-1)}, + id="fp", + ) + ], +) +@with_htp_context +def test_conv_backward(request, kwargs): + ConvBackward.test(request, kwargs) # noqa: F405 + + @enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)]) @with_htp_context def test_cos(request, kwargs): diff --git a/backends/qualcomm/tests/rework/htp/op/v73/test.py b/backends/qualcomm/tests/rework/htp/op/v73/test.py index 0e678a6a86b..587b2fb94d8 100644 --- a/backends/qualcomm/tests/rework/htp/op/v73/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v73/test.py @@ -489,6 +489,20 @@ def test_conv3d_transpose(request, kwargs): Conv.test_3d_transpose(request, kwargs) # noqa: F405 +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param( + {"act": None, "expected": Tolerance(rtol=1e-1)}, + id="fp", + ) + ], +) +@with_htp_context +def test_conv_backward(request, kwargs): + ConvBackward.test(request, kwargs) # noqa: F405 + + @enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)]) @with_htp_context def test_cos(request, kwargs): diff --git a/backends/qualcomm/tests/rework/htp/op/v75/test.py b/backends/qualcomm/tests/rework/htp/op/v75/test.py index 0e678a6a86b..587b2fb94d8 100644 --- a/backends/qualcomm/tests/rework/htp/op/v75/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v75/test.py @@ -489,6 +489,20 @@ def test_conv3d_transpose(request, kwargs): Conv.test_3d_transpose(request, kwargs) # noqa: F405 +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param( + {"act": None, "expected": Tolerance(rtol=1e-1)}, + id="fp", + ) + ], +) +@with_htp_context +def test_conv_backward(request, kwargs): + ConvBackward.test(request, kwargs) # noqa: F405 + + @enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)]) @with_htp_context def test_cos(request, kwargs): diff --git a/backends/qualcomm/tests/rework/htp/op/v79/test.py b/backends/qualcomm/tests/rework/htp/op/v79/test.py index 0e678a6a86b..587b2fb94d8 100644 --- a/backends/qualcomm/tests/rework/htp/op/v79/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v79/test.py @@ -489,6 +489,20 @@ def test_conv3d_transpose(request, kwargs): Conv.test_3d_transpose(request, kwargs) # noqa: F405 +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param( + {"act": None, "expected": Tolerance(rtol=1e-1)}, + id="fp", + ) + ], +) +@with_htp_context +def test_conv_backward(request, kwargs): + ConvBackward.test(request, kwargs) # noqa: F405 + + @enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)]) @with_htp_context def test_cos(request, kwargs): diff --git a/backends/qualcomm/tests/rework/htp/op/v81/test.py b/backends/qualcomm/tests/rework/htp/op/v81/test.py index 0e678a6a86b..587b2fb94d8 100644 --- a/backends/qualcomm/tests/rework/htp/op/v81/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v81/test.py @@ -489,6 +489,20 @@ def test_conv3d_transpose(request, kwargs): Conv.test_3d_transpose(request, kwargs) # noqa: F405 +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param( + {"act": None, "expected": Tolerance(rtol=1e-1)}, + id="fp", + ) + ], +) +@with_htp_context +def test_conv_backward(request, kwargs): + ConvBackward.test(request, kwargs) # noqa: F405 + + @enumerate_activation_dtype([Tolerance(), Tolerance(), Tolerance(rtol=1e-1)]) @with_htp_context def test_cos(request, kwargs): diff --git a/backends/qualcomm/tests/rework/src/op.py b/backends/qualcomm/tests/rework/src/op.py index ed526224c1c..c555c1a93b2 100644 --- a/backends/qualcomm/tests/rework/src/op.py +++ b/backends/qualcomm/tests/rework/src/op.py @@ -1439,6 +1439,52 @@ def test_3d_transpose(subtests, qnn_config, quantizer, compile_spec, expected): ) +class ConvBackward(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.randn(4, 3, 3, 3), requires_grad=False) + + def forward(self, grad_output, x): + forward_output = torch.ops.aten.convolution.default( + x, + self.weight, + None, + [2, 2], + [1, 1], + [1, 1], + False, + [0, 0], + 1, + ) + grad_input = torch.ops.aten.convolution_backward.default( + grad_output, + x, + self.weight, + None, + [2, 2], + [1, 1], + [1, 1], + False, + [0, 0], + 1, + [True, False, False], + )[0] + return forward_output, grad_input + + @staticmethod + @unpack_fixtures + def test(qnn_config, quantizer, compile_spec, expected): + with expected as metrics: + export_and_verify( + module=__class__(), + inputs=(torch.randn(1, 4, 5, 5), torch.randn(1, 3, 9, 10)), + qnn_config=qnn_config, + quantizer=quantizer, + compile_specs=compile_spec, + metrics=metrics, + ) + + class Cos(torch.nn.Module): def __init__(self): super().__init__() diff --git a/backends/qualcomm/tests/test_passes.py b/backends/qualcomm/tests/test_passes.py index 606fed01d72..f8cbb032d82 100644 --- a/backends/qualcomm/tests/test_passes.py +++ b/backends/qualcomm/tests/test_passes.py @@ -1,4 +1,5 @@ import unittest +from collections import defaultdict from unittest.mock import MagicMock, patch import torch @@ -11,6 +12,7 @@ FoldQDQ, InsertIOQDQ, InsertReshapeForReduceOps, + LayoutTransform, LiftConstantScalarOperands, RemoveRedundancy, ) @@ -22,6 +24,7 @@ to_dq_op, to_q_op, ) +from executorch.backends.qualcomm.builders.node_visitor_manager import get_node_visitors from executorch.backends.qualcomm.builders.op_custom_op import ( _resolve_qnn_data_type, CustomOp, @@ -29,6 +32,7 @@ from executorch.backends.qualcomm.builders.qnn_constants import OpContextLoader from executorch.backends.qualcomm.builders.utils import is_parameter from executorch.backends.qualcomm.partition.qnn_partitioner import QnnOperatorSupport +from executorch.backends.qualcomm.partition.utils import get_skip_decomp_table from executorch.backends.qualcomm.qnn_preprocess import QnnBackend from executorch.backends.qualcomm.quantizer.quantizer import QnnQuantizer, QuantDtype from executorch.backends.qualcomm.serialization.qc_schema import ( @@ -38,6 +42,7 @@ ) from executorch.backends.qualcomm.tests.models import ( BroadcastAndMutate, + ConvBackward, HardSigmoid, Reciprocal, TopKandIndex, @@ -63,6 +68,12 @@ class TestPasses(unittest.TestCase): + def test_skip_decomp_table_preserves_conv_backward(self): + self.assertIn( + torch.ops.aten.convolution_backward.default, + get_skip_decomp_table(), + ) + def _build_context_loader_edge_program(self, op_name, check_ir_validity=True): graph_name = "forward" custom_op = Library(OpContextLoader.namespace, "FRAGMENT") @@ -172,6 +183,95 @@ def test_context_loader_op_lowers_with_ir_validation(self): ) self.assertEqual(1, len(context_loader_nodes)) + def test_conv_backward_composite_lowering(self): + x = torch.randn(1, 3, 8, 8) + grad_output = torch.randn(1, 4, 4, 4) + edge_program = to_edge( + torch.export.export( + ConvBackward(flatten_grad_input=True, duplicate_backward=True), + (grad_output, x), + strict=True, + ) + ).exported_program() + graph_module = LayoutTransform(edge_program, insert_permute=True)( + edge_program.graph_module + ).graph_module + + conv_backwards = [ + node + for node in graph_module.graph.nodes + if node.target == exir_ops.edge.aten.convolution_backward.default + ] + self.assertEqual(2, len(conv_backwards)) + conv_backward = conv_backwards[0] + getitem = next(node for node in conv_backward.users if node.args[1] == 0) + permutes = [ + node + for node in graph_module.graph.nodes + if node.target == exir_ops.edge.aten.permute_copy.default + ] + self.assertTrue(any(node.args[0] is getitem for node in permutes)) + self.assertFalse(any(node.args[0] is conv_backward for node in permutes)) + + forward_conv = next( + node + for node in graph_module.graph.nodes + if node.target == exir_ops.edge.aten.convolution.default + ) + weight_node = forward_conv.args[1] + nodes_to_wrappers = defaultdict(dict) + visitors = get_node_visitors(edge_program) + visitors["aten.convolution.default"].define_node( + forward_conv, nodes_to_wrappers + ) + forward_weight_wrapper = nodes_to_wrappers[weight_node.name][0] + first_ops = visitors["aten.convolution_backward.default"].define_node( + conv_backwards[0], nodes_to_wrappers + ) + first_hwoi_wrapper = nodes_to_wrappers[f"{weight_node.name}_hwoi"][0] + second_ops = visitors["aten.convolution_backward.default"].define_node( + conv_backwards[1], nodes_to_wrappers + ) + second_hwoi_wrapper = nodes_to_wrappers[f"{weight_node.name}_hwoi"][0] + + self.assertEqual(2, len(first_ops)) + self.assertEqual(1, len(second_ops)) + self.assertIs(forward_weight_wrapper, nodes_to_wrappers[weight_node.name][0]) + self.assertIn(f"{weight_node.name}_hwoi", nodes_to_wrappers) + self.assertIs(first_hwoi_wrapper, second_hwoi_wrapper) + + def test_layout_transform_handles_multiple_tuple_getitems(self): + x = torch.randn(1, 3, 8, 8) + grad_output = torch.randn(1, 4, 4, 4) + edge_program = to_edge( + torch.export.export( + ConvBackward(flatten_grad_input=True, return_grad_weight=True), + (grad_output, x), + strict=True, + ) + ).exported_program() + graph_module = LayoutTransform(edge_program, insert_permute=True)( + edge_program.graph_module + ).graph_module + + conv_backward = next( + node + for node in graph_module.graph.nodes + if node.target == exir_ops.edge.aten.convolution_backward.default + ) + getitems = [ + node + for node in conv_backward.users + if node.target.__name__ == "getitem" and node.args[1] in (0, 1) + ] + self.assertEqual(2, len(getitems)) + permute_inputs = { + node.args[0] + for node in graph_module.graph.nodes + if node.target == exir_ops.edge.aten.permute_copy.default + } + self.assertTrue(all(getitem in permute_inputs for getitem in getitems)) + def _build_quantized_graph(self): """Build a quantized graph through AnnotateQuantAttrs + FoldQDQ.""" diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 221b7b4dbac..6bbf31f16e7 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -604,6 +604,11 @@ def test_qnn_backend_conv2d(self): with self.subTest(i=i): self.lower_module_and_test_output(module, sample_input) + def test_qnn_backend_conv_backward(self): + module = ConvBackward() # noqa: F405 + sample_input = (torch.randn(1, 4, 4, 4), torch.randn(1, 3, 8, 8)) + self.lower_module_and_test_output(module, sample_input) + def test_qnn_backend_conv2d_channel_last(self): modules = [ Conv2dSequential(channel_last=True), # noqa: F405 From 2fa76f4e0c76f8b99588a4fb2033b5ce372717ab Mon Sep 17 00:00:00 2001 From: Xiyue Shi Date: Tue, 27 Oct 2026 15:22:09 -0700 Subject: [PATCH 2/2] Address code review feedback --- backends/qualcomm/_passes/layout_transform.py | 7 +++- .../qualcomm/builders/op_conv_backward.py | 29 ++++++--------- .../qualcomm/partition/qnn_partitioner.py | 36 ++++++++++++++----- .../qualcomm/tests/rework/htp/op/v68/test.py | 19 ++++++++++ .../qualcomm/tests/rework/htp/op/v69/test.py | 2 +- backends/qualcomm/tests/test_passes.py | 20 ++++++++++- backends/qualcomm/tests/test_qnn_delegate.py | 6 ++++ 7 files changed, 89 insertions(+), 30 deletions(-) diff --git a/backends/qualcomm/_passes/layout_transform.py b/backends/qualcomm/_passes/layout_transform.py index b1ac8134ddc..17b8d61efb1 100644 --- a/backends/qualcomm/_passes/layout_transform.py +++ b/backends/qualcomm/_passes/layout_transform.py @@ -300,7 +300,12 @@ def create_call_function_node( ) def traverse(self, node: torch.fx.Node, graph_module: torch.fx.GraphModule) -> None: - for arg in node.args: + for index, arg in enumerate(node.args): + if ( + node.target == exir_ops.edge.aten.convolution_backward.default + and index == 1 + ): + continue if isinstance(arg, list): for arg_node in arg: self.annotate_layout(arg_node, graph_module, revert_layout=False) diff --git a/backends/qualcomm/builders/op_conv_backward.py b/backends/qualcomm/builders/op_conv_backward.py index d60e84f6e75..e9bc48f945e 100644 --- a/backends/qualcomm/builders/op_conv_backward.py +++ b/backends/qualcomm/builders/op_conv_backward.py @@ -10,6 +10,7 @@ import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManager import numpy as np import torch +from executorch.backends.qualcomm.utils.constants import QCOM_QUANT_ATTRS from .node_visitor import dq_ops, NodeVisitor from .node_visitor_manager import register_node_visitor @@ -53,7 +54,10 @@ def _get_supported_input_nodes( return None grad_output_node = node.args[0] - if grad_output_node.target in dq_ops: + if ( + grad_output_node.target in dq_ops + or grad_output_node.meta.get(QCOM_QUANT_ATTRS) is not None + ): logger.warning( "ConvBackward %s is unsupported: quantized grad_output is not supported", node.name, @@ -69,7 +73,10 @@ def _get_supported_input_nodes( return None filter_node = node.args[2] - if filter_node.target in dq_ops: + if ( + filter_node.target in dq_ops + or filter_node.meta.get(QCOM_QUANT_ATTRS) is not None + ): logger.warning( "ConvBackward %s is unsupported: quantized weights are not supported", node.name, @@ -185,9 +192,8 @@ def _define_filter_hwoi( filter_tensor: torch.Tensor, nodes_to_wrappers: Dict[str, PyQnnManager.TensorWrapper], ) -> Tuple[PyQnnManager.TensorWrapper, List[PyQnnManager.PyQnnOpWrapper]]: - # Reuse the forward Conv2d's static HWIO wrapper. HWOI is named from the - # forward Conv2d view; for this TransposeConv2d it is HWIO, produced from - # the raw OIHW weight by permute(2, 3, 0, 1). + # Reuse the forward Conv2d's static HWIO wrapper, then transpose it to + # the HWOI layout required by TransposeConv2d. filter_tensor_hwio = filter_tensor.permute(2, 3, 1, 0).contiguous() filter_wrapper = self.define_tensor( filter_node, @@ -277,21 +283,8 @@ def define_node( node: torch.fx.Node, nodes_to_wrappers: Dict[str, PyQnnManager.TensorWrapper], ) -> Optional[List[PyQnnManager.PyQnnOpWrapper]]: - # convolution_backward signature: - # args[0]: grad_output - # args[1]: input - # args[2]: weight (raw parameter [Cout, Cin, kH, kW]) - # args[3]: bias_sizes - # args[4]: stride - # args[5]: padding - # args[6]: dilation - # args[7]: transposed - # args[8]: output_padding - # args[9]: groups - # args[10]: output_mask [grad_input, grad_weight, grad_bias] supported_inputs = self._get_supported_input_nodes(node) if supported_inputs is None: - logger.warning("ConvBackward %s is unsupported", node.name) return None grad_output_node, grad_output_tensor, filter_node = supported_inputs tensors = self._get_filter_and_grad_input_tensors(node, filter_node) diff --git a/backends/qualcomm/partition/qnn_partitioner.py b/backends/qualcomm/partition/qnn_partitioner.py index 8db23a465ef..5a69b81b76a 100644 --- a/backends/qualcomm/partition/qnn_partitioner.py +++ b/backends/qualcomm/partition/qnn_partitioner.py @@ -41,6 +41,7 @@ PartitionResult, ) from executorch.exir.backend.utils import tag_constant_data, tag_mutated_buffer +from executorch.exir.dialects._ops import ops as exir_ops from torch.export.exported_program import ExportedProgram from torch.fx.passes.infra.partitioner import Partition from torch.fx.passes.operator_support import OperatorSupportBase @@ -113,19 +114,18 @@ def __init__( for soc_model in target_socs ] - def is_node_supported(self, _, node: torch.fx.Node) -> bool: - if node.op != "call_function" or node.target in not_supported_operator: - return False + def _passes_backend_capability_checks(self, node: torch.fx.Node) -> bool: + # Scoped to HTP: other backends reusing this checker have no htp_arch. + if ( + self.backend_type != QnnExecuTorchBackendType.kHtpBackend + or self.soc_info.htp_info.htp_arch >= HtpArch.V69 + ): + return True # Per-channel-group lowers to blockwise expansion (the encoding LPBQ # uses), which requires HTP >= V69. Pre-quantized weights bypass the # quantizer's validate_lpbq_support check, so gate the delegation here. - # Scoped to HTP: other backends reusing this checker have no htp_arch. - if ( - node.target in PER_CHANNEL_GROUP_ENCODING - and self.backend_type == QnnExecuTorchBackendType.kHtpBackend - and self.soc_info.htp_info.htp_arch < HtpArch.V69 - ): + if node.target in PER_CHANNEL_GROUP_ENCODING: logger.warning( f"[{self.phase}] {node.target.__name__} | Skipped: blockwise " f"(per-channel-group) encoding requires HTP arch >= V69, got " @@ -133,6 +133,24 @@ def is_node_supported(self, _, node: torch.fx.Node) -> bool: ) return False + # TransposeConv2d generated for convolution_backward is unsupported by + # the V68 compiler despite passing the QNN op-support query. + if node.target == exir_ops.edge.aten.convolution_backward.default: + logger.warning( + f"[{self.phase}] {node.target.__name__} | Skipped: requires HTP " + f"arch >= V69, got {self.soc_info.htp_info.htp_arch!r}" + ) + return False + + return True + + def is_node_supported(self, _, node: torch.fx.Node) -> bool: + if node.op != "call_function" or node.target in not_supported_operator: + return False + + if not self._passes_backend_capability_checks(node): + return False + if node.target in to_be_implemented_operator: logger.info( f"[{self.phase}] {node.target.__name__} | Skipped, this op can be supported, please report an issue in https://github.com/pytorch/executorch/issues" diff --git a/backends/qualcomm/tests/rework/htp/op/v68/test.py b/backends/qualcomm/tests/rework/htp/op/v68/test.py index 231c1c95007..506570226fd 100644 --- a/backends/qualcomm/tests/rework/htp/op/v68/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v68/test.py @@ -442,6 +442,25 @@ def test_conv2d_transpose(request, kwargs): Conv.test_2d_transpose(request, kwargs) # noqa: F405 +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param( + { + "act": None, + "expected": pytest.raises( + AssertionError, match=EXPECT_NOT_FULLY_DELEGATED + ), + }, + id="fp_fallback", + ) + ], +) +@with_htp_context +def test_conv_backward(request, kwargs): + ConvBackward.test(request, kwargs) # noqa: F405 + + @pytest.mark.parametrize( "kwargs", [ diff --git a/backends/qualcomm/tests/rework/htp/op/v69/test.py b/backends/qualcomm/tests/rework/htp/op/v69/test.py index df0ee32625f..21c02983e63 100644 --- a/backends/qualcomm/tests/rework/htp/op/v69/test.py +++ b/backends/qualcomm/tests/rework/htp/op/v69/test.py @@ -507,7 +507,7 @@ def test_conv3d_transpose(request, kwargs): pytest.param( {"act": None, "expected": Tolerance(rtol=1e-1)}, id="fp", - ) + ), ], ) @with_htp_context diff --git a/backends/qualcomm/tests/test_passes.py b/backends/qualcomm/tests/test_passes.py index f8cbb032d82..7c045384dfd 100644 --- a/backends/qualcomm/tests/test_passes.py +++ b/backends/qualcomm/tests/test_passes.py @@ -240,6 +240,25 @@ def test_conv_backward_composite_lowering(self): self.assertIn(f"{weight_node.name}_hwoi", nodes_to_wrappers) self.assertIs(first_hwoi_wrapper, second_hwoi_wrapper) + def test_conv_backward_quantized_inputs_fallback(self): + x = torch.randn(1, 3, 8, 8) + grad_output = torch.randn(1, 4, 4, 4) + edge_program = to_edge( + torch.export.export(ConvBackward(), (grad_output, x), strict=True) + ).exported_program() + conv_backward = next( + node + for node in edge_program.graph_module.graph.nodes + if node.target == exir_ops.edge.aten.convolution_backward.default + ) + visitor = get_node_visitors(edge_program)["aten.convolution_backward.default"] + + for input_index in (0, 2): + with self.subTest(input_index=input_index): + conv_backward.args[input_index].meta[QCOM_QUANT_ATTRS] = {} + self.assertIsNone(visitor.define_node(conv_backward, defaultdict(dict))) + conv_backward.args[input_index].meta.pop(QCOM_QUANT_ATTRS) + def test_layout_transform_handles_multiple_tuple_getitems(self): x = torch.randn(1, 3, 8, 8) grad_output = torch.randn(1, 4, 4, 4) @@ -253,7 +272,6 @@ def test_layout_transform_handles_multiple_tuple_getitems(self): graph_module = LayoutTransform(edge_program, insert_permute=True)( edge_program.graph_module ).graph_module - conv_backward = next( node for node in graph_module.graph.nodes diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 6bbf31f16e7..8222be64d34 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -37,6 +37,7 @@ ) from executorch.backends.qualcomm.quantizer.rules import Q_ANNOTATION_KEY from executorch.backends.qualcomm.serialization.qc_schema import ( + HtpArch, QcomChipset, QnnExecuTorchBackendType, QnnExecuTorchHtpPerformanceMode, @@ -605,6 +606,11 @@ def test_qnn_backend_conv2d(self): self.lower_module_and_test_output(module, sample_input) def test_qnn_backend_conv_backward(self): + if get_backend_type(self.backend) != QnnExecuTorchBackendType.kHtpBackend: + self.skipTest("convolution_backward is only supported on HTP") + option = flatbuffer_to_option(self.compiler_specs[0].value) + if option.soc_info.htp_info.htp_arch < HtpArch.V69: + self.skipTest("convolution_backward requires HTP V69 or later") module = ConvBackward() # noqa: F405 sample_input = (torch.randn(1, 4, 4, 4), torch.randn(1, 3, 8, 8)) self.lower_module_and_test_output(module, sample_input)