diff --git a/.ci/scripts/tests/test_wheel_vendored_packages.py b/.ci/scripts/tests/test_wheel_vendored_packages.py new file mode 100644 index 00000000000..a8a10d3e5b2 --- /dev/null +++ b/.ci/scripts/tests/test_wheel_vendored_packages.py @@ -0,0 +1,220 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Unit tests for the packages the full wheel publishes. + +The wheel used to carry the Python files and codegen scripts of every vendored third-party +checkout, because the full build passed no `packages` list and setuptools then discovered +everything under src/executorch. Those files exist to build the C++ targets, so nothing in +an installed wheel imports them. + +Asserting on the discovery result rather than on a built wheel, because the behaviour under +test is a pure function of the source tree plus the exclude patterns, and a full build takes +minutes to exercise one filter. `.ci/scripts/test_minimal_wheel.sh` already covers the +built-artifact side for the minimal wheel. + +setup.py is read rather than imported. It calls setup() at module scope, so importing it under +a test runner hands setup() the runner's own arguments and the session dies on an invalid +command name. +""" + +import ast +import functools +import unittest +from pathlib import Path +from typing import Dict, List, Tuple + +from setuptools import find_namespace_packages + +SETUP_PY = Path(__file__).resolve().parents[3] / "setup.py" +# Discovery is anchored on this file's location, not on the working directory, so the result +# does not depend on where the runner was started. +PACKAGE_ROOT = str(SETUP_PY.parent / "src") + + +def _setup_py_module() -> ast.Module: + return ast.parse(SETUP_PY.read_text()) + + +def _load_from_setup_py() -> Dict[str, object]: + """The vendored-path helpers and the package list builder, from setup.py's source. + + Only those definitions are executed, so none of setup.py's module level build logic runs. + """ + wanted = ( + "_VENDORED_DIR_NAMES", + "_vendored_prefixes", + "_is_vendored_path", + "_full_packages", + ) + + selected: List[ast.stmt] = [] + found = set() + for node in _setup_py_module().body: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in wanted: + selected.append(node) + found.add(node.name) + elif isinstance(node, ast.Assign): + names = { + target.id + for target in node.targets + if isinstance(target, ast.Name) and target.id in wanted + } + if names: + selected.append(node) + found |= names + + assert found == set( + wanted + ), f"setup.py no longer defines {sorted(set(wanted) - found)}, so this test checks nothing" + + namespace: Dict[str, object] = { + "__file__": str(SETUP_PY), + "Path": Path, + "List": List, + "Tuple": Tuple, + "functools": functools, + "find_namespace_packages": find_namespace_packages, + } + exec( + compile(ast.Module(body=selected, type_ignores=[]), str(SETUP_PY), "exec"), + namespace, + ) + return namespace + + +_NAMESPACE = _load_from_setup_py() +_VENDORED_DIR_NAMES = _NAMESPACE["_VENDORED_DIR_NAMES"] +_vendored_prefixes = _NAMESPACE["_vendored_prefixes"] +_is_vendored_path = _NAMESPACE["_is_vendored_path"] +_full_packages = _NAMESPACE["_full_packages"] + + +def _discover(exclude_vendored: bool) -> List[str]: + """Package discovery over the real tree, with and without the exclusion.""" + patterns = ( + [ + pattern + for name in sorted(_VENDORED_DIR_NAMES) + for pattern in (f"*.{name}", f"*.{name}.*") + ] + if exclude_vendored + else [] + ) + return sorted( + find_namespace_packages( + where=PACKAGE_ROOT, + include=["executorch", "executorch.*"], + exclude=patterns, + ) + ) + + +def _vendored(packages: List[str]) -> List[str]: + return [ + package for package in packages if _is_vendored_path(package.replace(".", "/")) + ] + + +class TestFullWheelPackages(unittest.TestCase): + def test_the_tree_has_vendored_packages_to_exclude(self) -> None: + """Fail rather than skip when there is nothing to exclude. + + Every other test here is vacuous on a tree with no vendored checkouts: an empty + package list contains no vendored package, so the exclusion would look correct even + if it had been deleted. Assert the premise instead of quietly passing on it. + """ + discovered = _discover(exclude_vendored=False) + self.assertNotEqual( + discovered, [], f"no packages discovered under {PACKAGE_ROOT}" + ) + self.assertNotEqual( + _vendored(discovered), + [], + "no vendored third-party packages in this tree, so the exclusion below cannot " + "be shown to do anything. Initialize the submodules before running this.", + ) + + def test_no_vendored_package_ships(self) -> None: + """No package under a vendored third-party checkout is published.""" + leaked = _vendored(_full_packages()) + # Only the count and a few names, because a regression here leaks hundreds of + # packages and the default diff would bury the message. + self.assertEqual( + len(leaked), + 0, + f"the wheel would publish {len(leaked)} vendored packages, " + f"e.g. {leaked[:3]}", + ) + + def test_the_exclusion_is_load_bearing(self) -> None: + """Discovery without the exclusion finds the packages the exclusion removes.""" + self.assertLess( + len(_discover(exclude_vendored=True)), + len(_discover(exclude_vendored=False)), + "the exclusion dropped nothing, so it is no longer doing any work", + ) + + def test_setup_passes_the_package_list(self) -> None: + """The helper is actually wired into the full build. + + Without this, every test above still passes when the assignment that hands the list + to setuptools is deleted, which is the whole of the change. The sibling wheel test + asserts its own wiring the same way and for the same reason. + """ + assigned = [ + node + for node in ast.walk(_setup_py_module()) + if isinstance(node, ast.Assign) + for target in node.targets + if isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and target.value.id == "setup_kwargs" + and isinstance(target.slice, ast.Constant) + and target.slice.value == "packages" + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "_full_packages" + ] + self.assertEqual( + len(assigned), + 1, + "setup.py does not assign _full_packages() to setup_kwargs['packages'], " + "so the full build falls back to discovering every package", + ) + + def test_is_vendored_path_matches_whole_components(self) -> None: + """The filter matches a path component, not a substring.""" + self.assertTrue( + _is_vendored_path( + "src/executorch/backends/xnnpack/third-party/XNNPACK/a.py" + ) + ) + self.assertTrue(_is_vendored_path("src/executorch/x/third_party/y.yaml")) + self.assertFalse(_is_vendored_path("src/executorch/exir/program/_program.py")) + # "third-party" as part of a longer name is a different directory. + self.assertFalse(_is_vendored_path("src/executorch/x/third-party-tools/y.py")) + + def test_submodules_outside_a_vendored_dir_are_recognized(self) -> None: + """A submodule checked out under an ordinary name is still another repository. + + These are not matched by the directory name, so they are read from .gitmodules. Their + nested copies also cannot satisfy the imports the code uses: the FACTO helper imports + facto.specdb from the top level, and the tokenizers ship as a declared dependency. + """ + prefixes = _vendored_prefixes() + self.assertIn("backends/cadence/utils/FACTO", prefixes) + self.assertIn("extension/llm/tokenizers", prefixes) + for prefix in ("backends/cadence/utils/FACTO", "extension/llm/tokenizers"): + self.assertTrue(_is_vendored_path(f"executorch/{prefix}")) + self.assertTrue(_is_vendored_path(f"src/executorch/{prefix}/setup.py")) + self.assertFalse( + _is_vendored_path("executorch/extension/llm/custom_ops/op_sdpa.py") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/nxp/BUCK b/backends/nxp/BUCK index 6dec42f04d7..81a1e04cfee 100644 --- a/backends/nxp/BUCK +++ b/backends/nxp/BUCK @@ -68,7 +68,6 @@ fbcode_target(_kind = runtime.python_library, "fbsource//third-party/pypi/neutron_converter:neutron_converter", "//caffe2:torch", "//executorch/exir:lib", - "//executorch/backends/nxp/tests:ops_aliases", ], ) diff --git a/backends/nxp/backend/edge_helper.py b/backends/nxp/backend/edge_helper.py index 408b90e264d..4e94981e776 100644 --- a/backends/nxp/backend/edge_helper.py +++ b/backends/nxp/backend/edge_helper.py @@ -7,8 +7,7 @@ import operator import torch - -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AddTensor, Amax, Amin, diff --git a/backends/nxp/backend/node_format_inference.py b/backends/nxp/backend/node_format_inference.py index 689de41f3e4..f49dbdae158 100644 --- a/backends/nxp/backend/node_format_inference.py +++ b/backends/nxp/backend/node_format_inference.py @@ -7,7 +7,6 @@ import operator import torch - from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT from executorch.backends.nxp.backend.edge_helper import ( input_rank, @@ -15,7 +14,7 @@ try_get_arg, ) from executorch.backends.nxp.backend.edge_program_converter import functions_converters -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AdaptiveAvgPool2D, Amax, Amin, diff --git a/backends/nxp/tests/ops_aliases.py b/backends/nxp/backend/ops_aliases.py similarity index 88% rename from backends/nxp/tests/ops_aliases.py rename to backends/nxp/backend/ops_aliases.py index 5ccc1d67de0..dc06fd7e4d7 100644 --- a/backends/nxp/tests/ops_aliases.py +++ b/backends/nxp/backend/ops_aliases.py @@ -3,8 +3,11 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -# This file defines ops aliases for shorter and more readable test description. List is sorted alphabetically. -# When finding a missing alias, add it at the correct place. +# This file defines ops aliases for shorter and more readable descriptions of edge operators. +# List is sorted alphabetically. When finding a missing alias, add it at the correct place. +# +# It lives in the backend package, not in tests, because the converters and passes below it +# read these aliases at import time, and production code should not depend on test modules. import operator diff --git a/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py b/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py index dafea5d259b..ffcc07dc56d 100644 --- a/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py +++ b/backends/nxp/edge_passes/move_auxiliary_operator_into_separate_qdq_cluster_pass.py @@ -6,10 +6,9 @@ import operator import torch - +from executorch.backends.nxp.backend.ops_aliases import PermuteCopy from executorch.backends.nxp.edge_passes.neutron_edge_pass import NeutronEdgePass from executorch.backends.nxp.neutron_partitioner import QDQClusterRecognizer -from executorch.backends.nxp.tests.ops_aliases import PermuteCopy # noinspection PyProtectedMember from executorch.exir.dialects._ops import ops as exir_ops diff --git a/backends/nxp/tests/BUCK b/backends/nxp/tests/BUCK index 7879e1e3db5..a3a5adb215c 100644 --- a/backends/nxp/tests/BUCK +++ b/backends/nxp/tests/BUCK @@ -4,17 +4,6 @@ load("@fbcode_macros//build_defs:python_pytest.bzl", "python_pytest") oncall("executorch") -fbcode_target(_kind = runtime.python_library, - name = "ops_aliases", - srcs = [ - "ops_aliases.py", - ], - deps = [ - "//caffe2:torch", - "//executorch/exir:lib", - ], -) - fbcode_target(_kind = runtime.python_library, name = "models", srcs = [ diff --git a/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py b/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py index b426c34c260..679e7365577 100644 --- a/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py +++ b/backends/nxp/tests/generic_tests/test_add_batch_size_for_3d_input_pool_2d_ops.py @@ -15,6 +15,13 @@ from executorch.backends.nxp.aten_passes.neutron_aten_pass_manager import ( NeutronAtenPassManager, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AdaptiveAvgPool2D, + AvgPool2D, + GetItem, + MaxPool2DWithIndices, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -28,13 +35,6 @@ MaxPool2dModule, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AdaptiveAvgPool2D, - AvgPool2D, - GetItem, - MaxPool2DWithIndices, - ViewCopy, -) @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py b/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py index 3415b79a39d..044467ca223 100644 --- a/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py +++ b/backends/nxp/tests/generic_tests/test_convert_div_to_mul.py @@ -13,6 +13,7 @@ ConvertDivToMulPass, NeutronAtenPassManager, ) +from executorch.backends.nxp.backend.ops_aliases import MulTensor from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -22,7 +23,6 @@ StaticDivLinearModel, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import MulTensor @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py b/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py index b3d68eeeb56..c68cde0c23b 100644 --- a/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py +++ b/backends/nxp/tests/generic_tests/test_convert_scalar_to_attr.py @@ -18,6 +18,7 @@ from executorch.backends.nxp.backend.edge_helper import ( try_get_tensor_constant_from_node, ) +from executorch.backends.nxp.backend.ops_aliases import AddTensor, MulTensor, SubTensor from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -31,7 +32,6 @@ AllCloseOutputComparator, lower_run_compare, ) -from executorch.backends.nxp.tests.ops_aliases import AddTensor, MulTensor, SubTensor @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py b/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py index de4e684405c..3c3c29f95b5 100644 --- a/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py +++ b/backends/nxp/tests/generic_tests/test_decompose_split_to_slices.py @@ -12,6 +12,7 @@ NeutronAtenPassManager, SplitGRUBasedOnNumLayers, ) +from executorch.backends.nxp.backend.ops_aliases import SliceCopy from executorch.backends.nxp.tests.executorch_pipeline import neutron_target_spec from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -21,7 +22,6 @@ SplitWithSize, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import SliceCopy @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/generic_tests/test_integration.py b/backends/nxp/tests/generic_tests/test_integration.py index 9916cba5bdd..f0cb25548ec 100644 --- a/backends/nxp/tests/generic_tests/test_integration.py +++ b/backends/nxp/tests/generic_tests/test_integration.py @@ -5,8 +5,8 @@ import executorch.extension.pybindings.portable_lib import executorch.kernels.quantized # noqa F401 +from executorch.backends.nxp.backend.ops_aliases import AddMM, Convolution from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.ops_aliases import AddMM, Convolution from executorch.backends.nxp.tests.use_qat import * # noqa F401 from executorch.backends.nxp.tests.executorch_pipeline import ( diff --git a/backends/nxp/tests/generic_tests/test_node_format_inference.py b/backends/nxp/tests/generic_tests/test_node_format_inference.py index 18d5f874aab..5206a029e01 100644 --- a/backends/nxp/tests/generic_tests/test_node_format_inference.py +++ b/backends/nxp/tests/generic_tests/test_node_format_inference.py @@ -13,6 +13,10 @@ NodeFormatInference, NXP_NODE_FORMAT, ) +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -21,10 +25,6 @@ MaxPool2dModule, SoftmaxModule, ) -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - MaxPool2DWithIndices, -) def test_convolution(): diff --git a/backends/nxp/tests/generic_tests/test_quantized_input_data.py b/backends/nxp/tests/generic_tests/test_quantized_input_data.py index a9f9f3e47e6..bd2a6056a0b 100644 --- a/backends/nxp/tests/generic_tests/test_quantized_input_data.py +++ b/backends/nxp/tests/generic_tests/test_quantized_input_data.py @@ -5,6 +5,7 @@ import executorch.backends.nxp.tests.nsys_testing as nsys_testing import torch +from executorch.backends.nxp.backend.ops_aliases import AvgPool2D, MulTensor from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -14,7 +15,6 @@ OUTPUTS_DIR, ReferenceModel, ) -from executorch.backends.nxp.tests.ops_aliases import AvgPool2D, MulTensor def test__single_quantized_inputs(mocker, request): diff --git a/backends/nxp/tests/graph_verifier.py b/backends/nxp/tests/graph_verifier.py index 44900b6a11b..8dad390d6ff 100644 --- a/backends/nxp/tests/graph_verifier.py +++ b/backends/nxp/tests/graph_verifier.py @@ -10,17 +10,18 @@ from dataclasses import dataclass from typing import Callable, Union -from executorch.backends.nxp.neutron_partitioner import ( - NeutronPartitioner, - NXP_DELEGATION_TAG, -) -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( DequantizePerChannel, DequantizePerTensor, QuantizePerChannel, QuantizePerTensor, ) +from executorch.backends.nxp.neutron_partitioner import ( + NeutronPartitioner, + NXP_DELEGATION_TAG, +) + from executorch.exir.dialects.edge._ops import EdgeOpOverload from pytest_mock import MockerFixture diff --git a/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py index d42ef4c6e7d..94a0aafc2a7 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_abs_converter.py @@ -8,12 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Abs from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import ( lower_run_compare, RandomDatasetCreator, ) -from executorch.backends.nxp.tests.ops_aliases import Abs from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py index 9646c04a3f2..65e2b82555c 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_adaptive_avg_pool2d_converter.py @@ -8,6 +8,10 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AdaptiveAvgPool2D, + ExecutorchDelegateCall, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -18,10 +22,6 @@ ) from executorch.backends.nxp.tests.models import AdaptiveAvgPool2dModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AdaptiveAvgPool2D, - ExecutorchDelegateCall, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py index c01d0ca818d..237d15aefbd 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_add_tensor_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import AddTensorModule, MaxPoolAddTensorModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py index 1db604d5b1e..6ebb8ac128c 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_addmm_converter.py @@ -8,6 +8,13 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddMM, + ExecutorchDelegateCall, + MM, + PermuteCopy, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -15,13 +22,6 @@ from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier, Operator from executorch.backends.nxp.tests.models import AddmmModule, LinearModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddMM, - ExecutorchDelegateCall, - MM, - PermuteCopy, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py index d348c12102e..892ea70018e 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_amax_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + Amax, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - Amax, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py index e2490d9c2c4..3ae13bafe40 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_amin_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + Amin, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - Amin, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py index 3db1158d637..c132c9e509d 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_avg_pool2d_converter.py @@ -8,17 +8,17 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AvgPool2D, + ExecutorchDelegateCall, + ViewCopy, +) from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import AvgPool2dModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AvgPool2D, - ExecutorchDelegateCall, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py index c564c024623..ee46ad94d44 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_bmm_converter.py @@ -6,6 +6,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + BMM, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.edge_passes.move_auxiliary_operator_into_separate_qdq_cluster_pass import ( ViewCopy, @@ -21,7 +26,6 @@ BatchMatMulModel, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import BMM, GetItem, MaxPool2DWithIndices from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py index b28a431e3ca..30983f4a666 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_cat_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Cat, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.executorch_pipeline import ( ModelInputSpec, @@ -16,12 +22,6 @@ from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Cat, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py index b2147a0d984..04ae8757110 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_clamp_converter.py @@ -18,6 +18,11 @@ from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOperator import ( BuiltinOperator as Ops, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + Clamp, + ExecutorchDelegateCall, +) from executorch.backends.nxp.tests.executorch_pipeline import ( ModelInputSpec, to_quantized_edge_program, @@ -25,11 +30,6 @@ from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - Clamp, - ExecutorchDelegateCall, -) from executorch.backends.nxp.tests.use_qat import * # noqa: F403 F401 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py index b4a64447aa6..730334932d7 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py @@ -12,10 +12,10 @@ from executorch.backends.nxp.backend.ir.converter.builder.model_builder import ( ModelBuilder, ) +from executorch.backends.nxp.backend.ops_aliases import ConstantPadND, Convolution from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import PadConvModule, PadModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ConstantPadND, Convolution from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py index 3d20d38bb54..f0fb0fdb7b4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py @@ -6,6 +6,11 @@ import numpy as np import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -16,11 +21,6 @@ lower_run_compare, ReferenceModel, ) -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py index b304dce2c94..9f8068f67d4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_exp_converter.py @@ -8,10 +8,10 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Exp from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Exp from executorch.backends.nxp.tests.use_qat import * # noqa F403 from executorch.backends.nxp.tests.dataset_creator import ( LinearRampDatasetCreator, diff --git a/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py index deff4e12f0c..76aaab7520e 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py @@ -8,6 +8,13 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddMM, + Convolution, + Hardswish, + PermuteCopy, + ViewCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -20,13 +27,6 @@ LinearHardswishModule, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddMM, - Convolution, - Hardswish, - PermuteCopy, - ViewCopy, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py index 66a052dba4f..1199609bb17 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_hardtanh_converter.py @@ -18,17 +18,17 @@ from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOperator import ( BuiltinOperator as Ops, ) +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + HardTanh, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import Conv2dWithActivation, HardTanhModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - HardTanh, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py index 567cf85ebe5..5176a8c61db 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_leaky_relu_converter.py @@ -8,11 +8,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import LeakyRelu from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import LeakyRelu from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py index 0b7fe88cffc..ebf6a8bfb9b 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_log_converter.py @@ -8,10 +8,10 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Log from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Log from executorch.backends.nxp.tests.use_qat import * # noqa F403 from executorch.backends.nxp.tests.dataset_creator import ( LinearRampDatasetCreator, diff --git a/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py index 55a47146bfc..7d3769c896d 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_max_pool_2d_converter.py @@ -8,17 +8,17 @@ # noinspection PyUnusedImports import pytest import torch - -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( ExecutorchDelegateCall, GetItem, MaxPool2DWithIndices, ViewCopy, ) + +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py index e25ed98fb3f..349a624242a 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_maximum_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + Maximum, + MaxPool2DWithIndices, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaximumModule, MaxPoolMaximumModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - Maximum, - MaxPool2DWithIndices, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py index 1674153540f..14f55de94d4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mean_dim_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + MeanDim, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - MeanDim, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py index 9dc2b8d77d7..7c74fd4a1c9 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_minimum_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + Minimum, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaxPoolMinimumModule, MinimumModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - Minimum, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py index 423999dc7ec..405e955a5ca 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mm_converter.py @@ -8,12 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import MM, PermuteCopy, ViewCopy from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier, Operator from executorch.backends.nxp.tests.models import LinearModule, MmModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import MM, PermuteCopy, ViewCopy from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py index 718383284be..b031ab7e47c 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_mul_tensor_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + MulTensor, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaxPoolMulTensorModule, MulTensorModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - MulTensor, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py index 691cb3bd2ca..182e7754013 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_neg_converter.py @@ -8,6 +8,7 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Convolution, Neg from executorch.backends.nxp.tests.dataset_creator import ( LinearRampDatasetCreator, @@ -15,7 +16,6 @@ ) from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Convolution, Neg from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py index 266260f9e1f..f5c0114c81e 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_pad_converter.py @@ -10,10 +10,10 @@ from executorch.backends.nxp.backend.ir.converter.builder.model_builder import ( ModelBuilder, ) +from executorch.backends.nxp.backend.ops_aliases import Convolution, Pad from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import PadConvModule, PadModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Convolution, Pad @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py index bdfd1e9da25..2317efd45d9 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_permute_copy_converter.py @@ -9,17 +9,17 @@ import pytest import torch from _pytest.mark import ParameterSet - -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( ExecutorchDelegateCall, GetItem, MaxPool2DWithIndices, PermuteCopy, ) + +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py index 884e95ec20c..358320c30c7 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_prelu_converter.py @@ -11,6 +11,17 @@ from executorch.backends.nxp.backend.edge_program_converter import ( EdgeProgramToIRConverter, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddMM, + Convolution, + ExecutorchDelegateCall, + GtScalar, + MulTensor, + PermuteCopy, + Prelu, + ViewCopy, + WhereSelf, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -25,17 +36,6 @@ ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddMM, - Convolution, - ExecutorchDelegateCall, - GtScalar, - MulTensor, - PermuteCopy, - Prelu, - ViewCopy, - WhereSelf, -) from torch.export import ExportedProgram from executorch.backends.nxp.tests.use_qat import * # noqa F403 from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program diff --git a/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py index 1f274576767..4bf9263e93f 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_relu_converter.py @@ -7,13 +7,7 @@ import pytest import torch from executorch.backends.nxp.backend.edge_program_converter import exir_ops -from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.models import Conv2dModule, LinearModule, ReLUModule -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AddMM, Convolution, DequantizePerChannel, @@ -23,6 +17,12 @@ Relu, ViewCopy, ) +from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.models import Conv2dModule, LinearModule, ReLUModule +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py index 67101410d9d..916ab5ba518 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_rsqrt_converter.py @@ -8,6 +8,7 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Rsqrt from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -15,7 +16,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Rsqrt from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py index 5c4e4f4f007..c6ea90cb5e2 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sigmoid_converter.py @@ -9,6 +9,7 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Sigmoid from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier @@ -16,7 +17,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Sigmoid from torch import nn from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py index 56d0b4bbd64..4a66c000505 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_slice_copy_tensor_converter.py @@ -8,6 +8,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + SliceCopy, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -21,11 +26,6 @@ SliceTensorModule, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - SliceCopy, -) @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py index 2ce0790fc98..8cec44b5274 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_softmax_converter.py @@ -6,6 +6,12 @@ import numpy as np import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + Convolution, + ExecutorchDelegateCall, + Softmax, + ViewCopy, +) from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -17,12 +23,6 @@ ) from executorch.backends.nxp.tests.models import SoftmaxModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - Convolution, - ExecutorchDelegateCall, - Softmax, - ViewCopy, -) @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py index 1601c1e19c2..6e88bc270e4 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sub_tensor_converter.py @@ -8,6 +8,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + SubTensor, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import ( @@ -21,12 +27,6 @@ ) from executorch.backends.nxp.tests.models import MaxPoolSubTensorModule, SubTensorModule from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - SubTensor, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py index 8b28142b63a..5726776c78a 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_sum_dim_int_list_converter.py @@ -21,6 +21,13 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.transpose_options import ( Transpose, ) +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + GetItem, + MaxPool2DWithIndices, + SumDimIntList, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops @@ -29,13 +36,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - GetItem, - MaxPool2DWithIndices, - SumDimIntList, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py index 51b7ee484a7..2795f607494 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_tanh_converter.py @@ -7,12 +7,12 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import Convolution, Tanh from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import Conv2dWithActivation from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import Convolution, Tanh from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py b/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py index 949f193b267..8513b87a1a3 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_upsample_bilinear2d.py @@ -8,6 +8,11 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + UpsampleBilinear2D, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program @@ -17,11 +22,6 @@ AllCloseOutputComparator, ) from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - UpsampleBilinear2D, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py b/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py index b3e28a7b2f8..dd3d7c9c147 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_upsample_nearest2d.py @@ -8,17 +8,17 @@ # noinspection PyUnusedImports import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import ( + AddTensor, + ExecutorchDelegateCall, + UpsampleNearest2D, +) from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( - AddTensor, - ExecutorchDelegateCall, - UpsampleNearest2D, -) from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py index 2a2d270e30a..200ffb6fc99 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_view_copy_converter.py @@ -8,16 +8,8 @@ import numpy as np import pytest import torch -from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator -from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program -from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops -from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier -from executorch.backends.nxp.tests.model_output_comparator import ( - AllCloseOutputComparator, -) -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare -from executorch.backends.nxp.tests.ops_aliases import ( +from executorch.backends.nxp.backend.ops_aliases import ( AddMM, AddTensor, AvgPool2D, @@ -28,6 +20,14 @@ Relu, ViewCopy, ) +from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator +from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program +from executorch.backends.nxp.tests.executors import graph_contains_any_of_ops +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.model_output_comparator import ( + AllCloseOutputComparator, +) +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare from torch import nn from executorch.backends.nxp.tests.use_qat import * # noqa F403 diff --git a/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py b/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py index f0489b151f7..c1850f625dd 100644 --- a/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py +++ b/backends/nxp/tests/ir/edge_passes/test_convert_reshaping_nodes_to_view.py @@ -6,6 +6,7 @@ import numpy as np import pytest import torch +from executorch.backends.nxp.backend.ops_aliases import AddTensor, ViewCopy from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier from executorch.backends.nxp.tests.models import SqueezeAddModel, UnsqueezeAddModel @@ -13,7 +14,6 @@ AllCloseOutputComparator, lower_run_compare, ) -from executorch.backends.nxp.tests.ops_aliases import AddTensor, ViewCopy @pytest.fixture(autouse=True) diff --git a/backends/nxp/tests/nsys_testing.py b/backends/nxp/tests/nsys_testing.py index a8038083c37..6ec0d821bae 100644 --- a/backends/nxp/tests/nsys_testing.py +++ b/backends/nxp/tests/nsys_testing.py @@ -24,6 +24,7 @@ from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( torch_type_to_numpy_type, ) +from executorch.backends.nxp.backend.ops_aliases import ExecutorchDelegateCall from executorch.backends.nxp.neutron_partitioner import NeutronPartitioner from executorch.backends.nxp.tests.config_importer import test_config from executorch.backends.nxp.tests.dataset_creator import ( @@ -44,7 +45,6 @@ from executorch.backends.nxp.tests.model_output_comparator import ( AllCloseOutputComparator, ) -from executorch.backends.nxp.tests.ops_aliases import ExecutorchDelegateCall from executorch.backends.nxp.tests.outputs_dir_importer import outputs_dir from executorch.backends.nxp.tests.utils import save_pte_program, store_txt_input_tensor diff --git a/pyproject.toml b/pyproject.toml index 105565862df..66176b35608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,8 +118,11 @@ license-files = ["LICENSE"] "executorch" = "src/executorch" [tool.setuptools.package-data] -# TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we -# don't need. +# Vendored third-party dirs are excluded in setup.py, so the globs below only reach the dirs +# the wheel ships. +# TODO(dbort): Prune /test[s]/ dirs and yaml files that we don't need. Test dirs cannot simply +# be dropped yet: the suites here import each other through the installed name, for example +# `from executorch.backends.arm.test import common`, so a non-editable install stops collecting. # TODO(RobertKalmar): When test[s] dirs pruned the PROJECT_DIR resolution in backends.nxp.tests_models.config.py can # avoid exporting and reading env variable. "*" = [ diff --git a/setup.py b/setup.py index 8980d1bd94a..ede5b0a3658 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ # derivative works thereof, in binary and source code form. import contextlib +import functools # Import this before distutils so that setuptools can intercept the distuils # imports. @@ -64,7 +65,7 @@ from distutils import log # type: ignore[import-not-found] from distutils.sysconfig import get_python_lib # type: ignore[import-not-found] from pathlib import Path, PurePosixPath -from typing import List, Optional +from typing import List, Optional, Tuple # Clean dynamic import using importlib _install_utils_path = Path(__file__).parent / "install_utils.py" @@ -177,6 +178,50 @@ def _minimal_cmake_flags() -> List[str]: ] +_VENDORED_DIR_NAMES = frozenset({"third-party", "third_party"}) + + +@functools.lru_cache(maxsize=None) +def _vendored_prefixes() -> Tuple[str, ...]: + """Source-tree prefixes holding code from another repository. + + Two shapes reach the wheel. Most vendored code sits in a directory named third-party, + which the name above covers wherever it appears. The rest are git submodules checked out + under an ordinary name, so they can only be recognized by asking git what they are. Both + exist to build the C++ targets, and their nested copies cannot satisfy the top level + imports the code actually uses: backends/cadence/utils/facto_util.py imports facto.specdb, + and the tokenizers are declared as pytorch-tokenizers in the dependency list. + """ + root = Path(__file__).parent + modules = root / ".gitmodules" + prefixes = [] + if modules.is_file(): + for line in modules.read_text().splitlines(): + name, separator, value = line.partition("=") + if separator and name.strip() == "path": + path = value.strip() + if not any(part in _VENDORED_DIR_NAMES for part in Path(path).parts): + prefixes.append(path) + return tuple(sorted(prefixes)) + + +def _is_vendored_path(path: str) -> bool: + """Whether a source-tree path holds code from another repository.""" + parts = Path(path).parts + if any(part in _VENDORED_DIR_NAMES for part in parts): + return True + # A submodule path is relative to the repository root, while a path here may be relative + # to src/executorch or carry a src/executorch prefix, so match on any suffix boundary. + posix = "/".join(parts) + return any( + posix == prefix + or posix.startswith(f"{prefix}/") + or posix.endswith(f"/{prefix}") + or f"/{prefix}/" in posix + for prefix in _vendored_prefixes() + ) + + def _minimal_packages() -> List[str]: return sorted( find_namespace_packages( @@ -204,6 +249,36 @@ def _minimal_packages() -> List[str]: ) +def _full_packages() -> List[str]: + """Every package the full wheel ships. + + Without an explicit list setuptools discovers all of src/executorch, which pulls in the + Python files and codegen scripts of the vendored third-party checkouts. Those exist to + build the C++ targets, so once the libraries are built no shipped module imports them. + + Test packages deliberately stay. The suites in this repository import each other through + the installed name, for example `from executorch.backends.arm.test import common`, so + dropping them from the wheel stops the suites collecting under a non-editable install. + """ + return sorted( + package + for package in find_namespace_packages( + where="src", + include=["executorch", "executorch.*"], + # A vendored dir like "third-party" is a literal path component, so it needs no + # escaping here even though it is not a valid identifier. + exclude=[ + pattern + for name in sorted(_VENDORED_DIR_NAMES) + for pattern in (f"*.{name}", f"*.{name}.*") + ], + ) + # The submodules checked out under an ordinary name are not matched by the patterns + # above, so they are filtered on the path they map to. + if not _is_vendored_path(package.replace(".", "/")) + ) + + # The published project names for the CUDA runtime components a CUDA wheel links but # does not bundle, keyed by CUDA major version. Not derivable from a suffix rule: the # CUDA 12 wheels carry a "-cu12" suffix while the CUDA 13 ones are published under @@ -1642,6 +1717,11 @@ def analyze_manifest(self): _f for _f in self.manifest_files[_pkg] if os.path.isfile(os.path.join(_root, _f)) + # A directory left out of `packages` is not simply skipped. setuptools + # walks up to the nearest listed package and records the file as that + # package's data, so a vendored *.yaml still arrives under its parent. + # Filter with the same list so the two agree. + and not _is_vendored_path(_f) ] def _copy_extra_files(self, src_to_dst, dst_root: str) -> None: @@ -2325,6 +2405,7 @@ def iter_distribution_names(self): setup_kwargs["packages"] = _minimal_packages() setup_kwargs["install_requires"] = _minimal_dependencies() else: + setup_kwargs["packages"] = _full_packages() # A CUDA wheel links the CUDA runtime but does not bundle it, so the wheels that # carry it are declared here. A CPU wheel adds nothing. setup_kwargs["install_requires"] = _base_dependencies() + _cuda_dependencies()