From 7df3d728cf779ffda531d83716f215c977650eb4 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 25 Oct 2026 23:13:54 -0700 Subject: [PATCH 1/5] Stop reusing overwritten planned inputs in the Python bindings By default, export puts each input in planned memory and lets later values reuse that memory once the input is no longer needed. A method that writes an input in place also changes that memory. So after a run, the input bytes may hold other data. Running again without setting the inputs read that data and returned wrong numbers with no error: method.set_inputs([x, y]); method.execute() # right method.execute() # wrong, silently The same happened with Module.plan_execute() and with a partial input list, which reuse inputs kept from an earlier call. The bindings already hold the inputs the caller passed, until they are replaced. Now Method installs its memory-planned inputs again from what it holds at every execute(), except when one call sets the inputs and runs at once, and Module does the same for kept inputs a call omits. What they hold is the caller's tensor or buffer, or the copy the bindings made of it when it needed converting. This changes three things on purpose: - An input is read at each run, so an in-place change the caller made to it since it was passed is now seen, as it already was for inputs that are not memory planned. - For the same reason, a change to the storage of a torch tensor or buffer input is refused with the "pass it again" error those inputs already get. - A method that writes an input in place, such as state.add_(x), used to carry that write into the next run that reused the input, as repeated Method::execute() calls in C++ still do. Now that run starts from the input as passed, the same as a call that passes it again. --- extension/pybindings/README.md | 10 ++ extension/pybindings/pybindings.cpp | 108 +++++++++++++------ extension/pybindings/test/test_pybindings.py | 29 +++++ 3 files changed, 117 insertions(+), 30 deletions(-) diff --git a/extension/pybindings/README.md b/extension/pybindings/README.md index aaf15c8c46a..94b95c28233 100644 --- a/extension/pybindings/README.md +++ b/extension/pybindings/README.md @@ -79,6 +79,16 @@ Calls on a shared method or module are serialized. A sequence of separate held by the caller around the whole sequence: `set_inputs` hides the outputs until the next `execute`, so another thread's call can land in between. +A run may reuse the memory of inputs that export planned, or write an input in +place. So each `execute` installs such inputs again from what the caller +passed, or from the copy the bindings made when an input needed converting. A +module does the same for inputs kept from an earlier call. A change to the +storage of a torch tensor or buffer input is then refused; a DLPack producer +must keep its memory in place while the bindings hold it. A result returned +with `clone_outputs=False` lives in the method's output memory, so passing it +back in as an input means each run reads what the run before it wrote there; +pass a copied result instead. + To select `ExecuTorchResult` outputs even when PyTorch is installed, set the environment variable before importing the bindings. The setting is fixed when the extension loads, so importing PyTorch later does not change the output type. diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index b2c1e069c9a..48312b4e918 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -2415,6 +2415,10 @@ struct PyModule final { py::object torch; #endif std::optional validation; + // Set for a memory-planned input. A run may give that memory to other + // values or write the input in place, so a later call that omits the + // input installs it again from here. + TensorPtr planned; }; py::object native_output_owner() { @@ -2515,9 +2519,7 @@ struct PyModule final { method_meta, i, at_tensor.scalar_type(), tensor_sizes); (void)mutable_tensor_data_ptr_no_cow(at_tensor); cpp_inputs.emplace_back(at_tensor); - if (!method_meta.input_tensor_meta(i)->is_memory_planned()) { - new_input_owners[i].validation.emplace(i, python_input); - } + new_input_owners[i].validation.emplace(i, python_input); #else torch_inputs.push_back(read_input(method_meta, i, [&] { return std::make_shared(python_input, i); @@ -2547,9 +2549,7 @@ struct PyModule final { tensor->device().type(), tensor->device().index()); cpp_inputs.emplace_back(torch::executor::Tensor(&input_tensors.back())); - if (!method_meta.input_tensor_meta(i)->is_memory_planned()) { - new_input_owners[i].validation.emplace(i, tensor->owner()); - } + new_input_owners[i].validation.emplace(i, tensor->owner()); #endif } else if (py::isinstance(python_input)) { cpp_inputs.push_back(EValue()); @@ -2609,8 +2609,7 @@ struct PyModule final { buffer->nbytes()); #endif validate_buffer_input(method_meta, i, *buffer); - if (!method_meta.input_tensor_meta(i)->is_memory_planned() && - buffer->borrows_data()) { + if (buffer->borrows_data()) { new_input_owners[i].validation.emplace(i, buffer); } #ifdef USE_ATEN_LIB @@ -2647,6 +2646,22 @@ struct PyModule final { } } + for (size_t i = 0; i < inputs_size; ++i) { + if (!cpp_inputs[i].isTensor()) { + continue; + } + const auto input_meta = method_meta.input_tensor_meta(i); + if (input_meta.ok() && input_meta->is_memory_planned()) { +#ifdef USE_ATEN_LIB + // Shares the caller's tensor, because the aliasing make_tensor_ptr + // aborts on the non-dense layouts an ATen input may have. + new_input_owners[i].planned = + std::make_shared(cpp_inputs[i].toTensor()); +#else + new_input_owners[i].planned = make_tensor_ptr(cpp_inputs[i].toTensor()); +#endif + } + } // Module keeps installed inputs for calls with an omitted or partial // input list, so their owners stay alive with it. A call that fails while // installing keeps nothing: it may have installed a prefix, but later @@ -2691,6 +2706,22 @@ struct PyModule final { static_cast(status)); } } + for (size_t i = inputs_size; i < retained.size(); ++i) { + if (!retained[i].planned) { + continue; + } + const auto status = + module_->set_input(method_name, retained[i].planned, i); + if (status != Error::Ok) { + failed_methods_.insert(method_name); + THROW_IF_ERROR( + status, + "Failed to set input %zu of method %s, error: 0x%" PRIx32, + i, + method_name.c_str(), + static_cast(status)); + } + } failed_methods_.erase(method_name); retained.resize(passed_inputs); for (size_t i = 0; i < inputs_size; ++i) { @@ -3289,6 +3320,12 @@ struct PyMethod final { std::move(state), std::move(method))) { inputs_set_ = storage_->method->inputs_size() == 0; + const auto& meta = storage_->method->method_meta(); + for (size_t i = 0; i < meta.num_inputs(); ++i) { + const auto tag = meta.input_tag(i); + has_planned_inputs_ |= tag.ok() && tag.get() == Tag::Tensor && + meta.input_tensor_meta(i)->is_memory_planned(); + } } void set_inputs(const py::object& python_inputs) { @@ -3348,11 +3385,7 @@ struct PyMethod final { i, at_tensor.scalar_type(), tensor_sizes); - const auto input_meta = - storage_->method->method_meta().input_tensor_meta(i).get(); - if (!input_meta.is_memory_planned()) { - borrowed_inputs.emplace_back(i, python_input); - } + borrowed_inputs.emplace_back(i, python_input); (void)mutable_tensor_data_ptr_no_cow(at_tensor); cpp_inputs.emplace_back(at_tensor); #else @@ -3373,13 +3406,9 @@ struct PyMethod final { .input_tensor_meta(i) ->dim_order()); }); - const auto input_meta = - storage_->method->method_meta().input_tensor_meta(i).get(); - if (!input_meta.is_memory_planned()) { - // Watch the tensor the method reads: after a layout conversion that - // is the copy, which an output aliasing the input can expose. - borrowed_inputs.emplace_back(i, view->owner()); - } + // Watch the tensor the method reads: after a layout conversion that + // is the copy, which an output aliasing the input can expose. + borrowed_inputs.emplace_back(i, view->owner()); auto tensor = for_blob(view->data(), view->sizes(), view->scalar_type()) .strides(view->strides()) .dim_order(view->dim_order()) @@ -3410,11 +3439,7 @@ struct PyMethod final { at_tensor.scalar_type(), std::vector( at_tensor.sizes().begin(), at_tensor.sizes().end())); - if (!storage_->method->method_meta() - .input_tensor_meta(i) - ->is_memory_planned()) { - borrowed_inputs.emplace_back(i, python_tensor); - } + borrowed_inputs.emplace_back(i, python_tensor); cpp_inputs.emplace_back(at_tensor); #else dlpack_inputs.push_back( @@ -3458,10 +3483,7 @@ struct PyMethod final { buffer->nbytes()); #endif validate_buffer_input(storage_->method->method_meta(), i, *buffer); - const auto input_meta = - storage_->method->method_meta().input_tensor_meta(i); - if (input_meta.ok() && !input_meta->is_memory_planned() && - buffer->borrows_data()) { + if (buffer->borrows_data()) { borrowed_inputs.emplace_back(i, buffer); } #ifdef USE_ATEN_LIB @@ -3511,6 +3533,7 @@ struct PyMethod final { } inputs_failed_ = false; inputs_set_ = true; + inputs_ = std::move(cpp_inputs); // An output can share memory with an input, by value or through memory // planning, so it would now show data no execute() produced. outputs_valid_ = false; @@ -3526,6 +3549,10 @@ struct PyMethod final { } void execute() { + execute_installed(/*inputs_just_installed=*/false); + } + + void execute_installed(bool inputs_just_installed) { ObjectLock lock(mutex_); // The output setup below reads inputs through Method::get_input(), which // marks them as set and so hides the runtime's own check for unset inputs. @@ -3547,6 +3574,21 @@ struct PyMethod final { for (const auto& input : borrowed_inputs_) { input.validate(storage_->method->method_meta().name()); } + // Each run reads the caller's inputs, like inputs that are not memory + // planned, since an earlier run may have reused or written their planned + // copy. + if (has_planned_inputs_ && !inputs_just_installed) { + const Error status = storage_->method->set_inputs( + executorch::aten::ArrayRef(inputs_.data(), inputs_.size())); + if (status != Error::Ok) { + inputs_failed_ = true; + THROW_IF_ERROR( + status, + "method->set_inputs() for method '%s' failed with error 0x%" PRIx32, + storage_->method->method_meta().name(), + static_cast(status)); + } + } const auto num_outputs = storage_->method->outputs_size(); allocate_output_storages(); std::vector> output_storage_spans(num_outputs); @@ -3611,7 +3653,7 @@ struct PyMethod final { py::list call(const py::object& inputs, bool clone_outputs = true) { ObjectLock lock(mutex_); set_inputs(inputs); - execute(); + execute_installed(/*inputs_just_installed=*/true); return get_outputs(clone_outputs); } @@ -3665,6 +3707,12 @@ struct PyMethod final { bool inputs_failed_ = false; // True only after the last execute() succeeded. bool outputs_valid_ = false; + // True if any input is memory planned, so its planned memory holds a copy + // that each run installs again from inputs_. + bool has_planned_inputs_ = false; + // What the last successful set_inputs() installed. The tensors point at + // memory the members below keep alive. + std::vector inputs_; // Borrowed results retain native storage without retaining this wrapper's // Python input owners, which could point back to those same results. struct Storage final { diff --git a/extension/pybindings/test/test_pybindings.py b/extension/pybindings/test/test_pybindings.py index bb1dd706569..6f66a72505f 100644 --- a/extension/pybindings/test/test_pybindings.py +++ b/extension/pybindings/test/test_pybindings.py @@ -2464,6 +2464,35 @@ def forward(self, x, index): output = executorch_module.forward([values, torch.tensor([3])]) torch.testing.assert_close(torch.as_tensor(np.asarray(output[0])), values[3:4]) + def test_planned_inputs_are_installed_again_at_each_run(self): + class Accumulate(torch.nn.Module): + def forward(self, x, state): + state.add_(x) + return state * 1 + + x = torch.arange(4, dtype=torch.float32) + 1 + state = torch.zeros(4) + # Each run writes the planned copy of state, so the next run has to + # install the caller's tensor again. + program = to_edge(export(Accumulate(), (x, state), strict=True)).to_executorch() + + method = self.load_prog_fn(program.buffer).load_method("forward") + method.set_inputs([x, state]) + for _ in range(2): + method.execute() + torch.testing.assert_close(method.get_outputs()[0], x) + + module = self.load_fn(program.buffer) + torch.testing.assert_close(module.forward([x, state])[0], x) + torch.testing.assert_close(module.forward([x])[0], x) + + # The inputs are read again, so a change to their storage is refused. + state.resize_(64, 64) + with self.assertRaisesRegex(ValueError, "storage changed"): + method.execute() + with self.assertRaisesRegex(ValueError, "storage changed"): + module.forward([x]) + def test_module_keeps_a_temporary_program_buffer_alive(self): exported_program, inputs = create_program(ModuleAdd()) data = bytes(bytearray(exported_program.buffer)) From 05ea9075cea8d12c21c5927be481439156f9cc5b Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 25 Oct 2026 23:17:40 -0700 Subject: [PATCH 2/5] Say why the runtime refused a call in Python binding errors When the runtime refuses an input or fails to load a method, it writes the reason to its log and returns only an error code. The Python exception then showed just the code, often with a typo: RuntimeError: method->set_inputs() for method 'forward' failed with error 0x10 RuntimeError: Failed to load method forward, error: 0x:22 The reason was already printed on stderr (for example "Expected shape (2, 2), but received (3, 2)", or that libgomp.so.1 could not be found when loading a CUDA method without PyTorch), but not in the exception, so it was easy to miss and impossible to catch. The bindings already forward runtime logs to Python. They now also keep the error lines logged on the calling thread during the current call and add them to the exception: RuntimeError: method->set_inputs() for method 'forward' failed with error 0x10: Attempted to resize a static tensor. Expected shape (2, 2), but received (3, 2).; Error resizing tensor at input 0 This also fixes the "0x:" typo in eight messages. --- extension/pybindings/pybindings.cpp | 84 ++++++++++++++++---- extension/pybindings/test/test_pybindings.py | 19 +++++ 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index 48312b4e918..d71b564b74c 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -65,14 +65,15 @@ #include #endif -/// Throws a runtime_error with the provided message if `error` is not `Ok`. +/// Throws a runtime_error with the provided message if `error` is not `Ok`, +/// followed by the errors the runtime logged during this call. #define THROW_IF_ERROR(error, message, ...) \ ({ \ if ((error) != Error::Ok) { \ char msg_buf[128]; \ snprintf(msg_buf, sizeof(msg_buf), message, ##__VA_ARGS__); \ /* pybind will convert this to a python exception. */ \ - throw std::runtime_error(msg_buf); \ + throw std::runtime_error(with_runtime_errors(msg_buf)); \ } \ }) @@ -82,7 +83,7 @@ char msg_buf[128]; \ snprintf(msg_buf, sizeof(msg_buf), message, ##__VA_ARGS__); \ /* pybind will convert this to a python exception. */ \ - throw std::out_of_range(msg_buf); \ + throw std::out_of_range(with_runtime_errors(msg_buf)); \ } \ }) @@ -126,6 +127,44 @@ namespace pybindings { namespace { +// The errors the runtime logged on this thread, which RuntimeErrorScope +// limits to the current binding call. The runtime explains a failure only in +// its log and returns a bare error code, so the exceptions raised here carry +// these lines too. Plain arrays, because a log can arrive while thread_local +// objects with destructors are already destroyed at exit. +constexpr size_t kRuntimeErrorsCapacity = 1024; +thread_local char runtime_errors[kRuntimeErrorsCapacity]; +thread_local size_t runtime_errors_size = 0; +// How many binding calls are running on this thread. Lines logged outside +// any, such as on worker threads, are not kept, so they never linger. +thread_local size_t runtime_error_scopes = 0; + +std::string with_runtime_errors(const char* message) { + std::string result(message); + if (runtime_errors_size != 0) { + result.append(": ").append(runtime_errors, runtime_errors_size); + runtime_errors_size = 0; + } + return result; +} + +// Starts every binding call with no runtime errors recorded, and gives an +// outer call its own back when a nested call returns. +struct RuntimeErrorScope final { + RuntimeErrorScope() : outer_(runtime_errors, runtime_errors_size) { + runtime_errors_size = 0; + ++runtime_error_scopes; + } + ~RuntimeErrorScope() { + --runtime_error_scopes; + std::memcpy(runtime_errors, outer_.data(), outer_.size()); + runtime_errors_size = outer_.size(); + } + + private: + std::string outer_; +}; + #ifndef USE_ATEN_LIB constexpr char kTensorOutputEnvironmentVariable[] = "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"; @@ -1777,7 +1816,7 @@ inline std::unique_ptr loader_from_file(const std::string& path) { path.c_str(), MmapDataLoader::MlockConfig::UseMlockIgnoreErrors); THROW_IF_ERROR( res.error(), - "Failed to create MmapDataLoader from file %s, error: 0x:%" PRIx32, + "Failed to create MmapDataLoader from file %s, error: 0x%" PRIx32, path.c_str(), static_cast(res.error())); @@ -3073,7 +3112,7 @@ inline std::shared_ptr load_program( Result res = Program::load(loader.get(), program_verification); THROW_IF_ERROR( res.error(), - "Failed to load program, error: 0x:%" PRIx32, + "Failed to load program, error: 0x%" PRIx32, static_cast(res.error())); // A program whose weights live outside the pte names them through a data // map. The CUDA delegate emits exactly that: a pte holding the compiled @@ -3088,7 +3127,7 @@ inline std::shared_ptr load_program( FlatTensorDataMap::load(data_map_loader.get()); THROW_IF_ERROR( map_res.error(), - "Failed to load data map from %s, error: 0x:%" PRIx32, + "Failed to load data map from %s, error: 0x%" PRIx32, data_path.value().c_str(), static_cast(map_res.error())); data_map = std::make_unique(std::move(map_res.get())); @@ -3663,7 +3702,7 @@ struct PyMethod final { storage_->method->get_attribute(name); THROW_IF_ERROR( attr.error(), - "Failed to get attribute '%s' for method '%s', error: 0x:%" PRIx32, + "Failed to get attribute '%s' for method '%s', error: 0x%" PRIx32, name.c_str(), storage_->method->method_meta().name(), static_cast(attr.error())); @@ -4200,7 +4239,7 @@ struct PyProgram final { Result res = state_->program_->get_method_name(method_index); THROW_IF_ERROR( res.error(), - "Failed get method name, error: 0x:%" PRIx32, + "Failed get method name, error: 0x%" PRIx32, static_cast(res.error())); return std::string(res.get()); } @@ -4210,7 +4249,7 @@ struct PyProgram final { state_->program_->method_meta(method_name.c_str()); THROW_IF_ERROR( meta.error(), - "Failed to get method meta for method %s, error: 0x:%" PRIx32, + "Failed to get method meta for method %s, error: 0x%" PRIx32, method_name.c_str(), static_cast(meta.error())); // Device memory is claimed here rather than at program load so that one @@ -4227,7 +4266,7 @@ struct PyProgram final { &state_->backend_options_); THROW_IF_ERROR( res.error(), - "Failed to load method %s, error: 0x:%" PRIx32, + "Failed to load method %s, error: 0x%" PRIx32, method_name.c_str(), static_cast(res.error())); return std::make_unique( @@ -4245,7 +4284,7 @@ struct PyProgram final { state_->program_->method_meta(method_name.c_str()); THROW_IF_ERROR( res.error(), - "Failed to get method meta for method %s, error: 0x:%" PRIx32, + "Failed to get method meta for method %s, error: 0x%" PRIx32, method_name.c_str(), static_cast(res.error())); return std::make_unique(state_, std::move(res.get())); @@ -4365,9 +4404,11 @@ py::dict get_option(const std::string& backend_name, const py::dict& options) { } // namespace PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { - // scoped_ostream_redirect mutates process-wide stream buffers and is not - // thread-safe when Python operations inside a binding release the GIL. - auto call_guard = py::call_guard<>(); + // Gives each call its own record of runtime errors for its exceptions. + // Runtime logs reach Python through emit_log_message below rather than + // scoped_ostream_redirect, which mutates process-wide stream buffers and is + // not thread-safe when Python operations inside a binding release the GIL. + auto call_guard = py::call_guard(); #ifdef USE_ATEN_LIB m.attr("_uses_aten") = true; @@ -4766,7 +4807,7 @@ namespace { // stream buffer. void emit_log_message( ET_UNUSED et_timestamp_t timestamp, - ET_UNUSED et_pal_log_level_t level, + et_pal_log_level_t level, const char* filename, ET_UNUSED const char* function, size_t line, @@ -4774,6 +4815,19 @@ void emit_log_message( size_t length) { const std::string formatted_message = "[" + std::string(filename) + ":" + std::to_string(line) + "] " + std::string(message, length) + "\n"; + if (level == kError && runtime_error_scopes != 0) { + // Keeps the first lines, which usually name the cause. + const char separator[] = "; "; + const size_t prefix = runtime_errors_size == 0 ? 0 : sizeof(separator) - 1; + const size_t room = kRuntimeErrorsCapacity - runtime_errors_size; + if (room > prefix) { + std::memcpy(runtime_errors + runtime_errors_size, separator, prefix); + const size_t copied = std::min(length, room - prefix); + std::memcpy( + runtime_errors + runtime_errors_size + prefix, message, copied); + runtime_errors_size += prefix + copied; + } + } if (Py_IsInitialized() && PyGILState_Check()) { try { py::gil_scoped_acquire gil; diff --git a/extension/pybindings/test/test_pybindings.py b/extension/pybindings/test/test_pybindings.py index 6f66a72505f..09140978543 100644 --- a/extension/pybindings/test/test_pybindings.py +++ b/extension/pybindings/test/test_pybindings.py @@ -398,6 +398,25 @@ def test_default_numpy_integer_dtype_error_is_descriptive(self): with self.assertRaisesRegex(ValueError, "dtype Long.*expects Float"): executorch_module(np.ones((2, 2), dtype=int)) + def test_refused_input_errors_say_why(self): + exported_program, inputs = create_program(ModuleAddScalar()) + executorch_module = self.load_fn(exported_program.buffer) + with self.assertRaises(RuntimeError) as error: + executorch_module.forward([None, inputs[1]]) + self.assertIn("type Tensor but was None", str(error.exception)) + # Only the reason is reported, not lines about other inputs. + self.assertNotIn("is not Tensor", str(error.exception)) + + # The portable runtime logs a static shape mismatch over two lines. + if self.kernel_mode == "portable": + exported_program, inputs = create_program(ModuleAdd()) + with self.assertRaisesRegex( + RuntimeError, r"received \(3, 2\)\.; Error resizing tensor at input 0" + ): + self.load_fn(exported_program.buffer).forward( + [torch.ones(3, 2), inputs[1]] + ) + def test_numpy_float_scalar_input(self): exported_program, inputs = create_program(ModuleAddScalar()) executorch_module = self.load_fn(exported_program.buffer) From 500e76b7f8b0a7505122e838dcdda7a560c56a60 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 25 Oct 2026 23:17:45 -0700 Subject: [PATCH 3/5] Keep the shape of bfloat16 results in the buffer view NumPy and the Python buffer protocol have no bfloat16 type. So far a bfloat16 result was exported as a flat run of bytes: a (2, 3) result showed up as 12 bytes, so a reader got a different shape with no error, and lost the layout of a result that is not contiguous. Other dtypes without a buffer format, such as float8 or the quantized types, could not be read through the buffer protocol at all. Now such a dtype is shown as an integer of the same width, in the result's real shape and strides: unsigned in general, and signed for qint8 and qint32, whose storage is signed. A (2, 3) bfloat16 result reads as a (2, 3) array of uint16, with each element in the right place; only the meaning of each element needs reinterpreting. DLPack still carries the real bfloat16 dtype. out = method.execute([x])[0] # a (2, 3) bfloat16 result memoryview(out).format # before "B", after "H" memoryview(out).shape # before (12,), after (2, 3) --- extension/pybindings/README.md | 15 +++++--- extension/pybindings/pybindings.cpp | 4 +- .../pybindings/pybindings_executorch_result.h | 37 +++++++++++-------- .../test/test_pybindings_cuda_dlpack.py | 4 +- .../test/test_pybindings_no_aten.py | 16 ++++---- 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/extension/pybindings/README.md b/extension/pybindings/README.md index 94b95c28233..4c3d4a9faf3 100644 --- a/extension/pybindings/README.md +++ b/extension/pybindings/README.md @@ -159,11 +159,16 @@ GIL. On an idle device a wait is short: synchronization has no C++ ATen dependency. For bfloat16 inputs, raw byte buffers use the shape declared by the exported -method. Bfloat16 results expose flat bytes through the buffer protocol because -NumPy and PEP 3118 do not define that numeric dtype. DLPack consumers receive the -original bfloat16 dtype and tensor shape. A bfloat16 result passed back in is -read through DLPack too, so it keeps its own shape and dim order instead of -taking the method's. +method. NumPy and PEP 3118 do not define bfloat16, so the buffer protocol shows +a bfloat16 result as 16-bit unsigned integers in its real shape and layout. +Other dtypes NumPy lacks, such as float8, are shown the same way, as integers of +the same width. NumPy converts that view as integers, so +`np.asarray(out, dtype=np.float32)` gives the bit patterns as numbers, not the +bfloat16 values. DLPack consumers, such as `torch.from_dlpack(out)`, receive the +original bfloat16 dtype. A bfloat16 result passed back in is read through DLPack +too, so it keeps its dtype, shape, and dim order. A bfloat16 input does not +accept 16-bit integers, so that integer data is never read as bfloat16 by +mistake; pass the result itself or raw bytes. ```python import cupy as cp diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index d71b564b74c..41c60688532 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -1618,8 +1618,8 @@ bool prefer_dlpack_input(const py::handle& value) { return true; } // A result of these bindings always takes the DLPack path, which keeps its - // real dtype, shape, and strides. A bfloat16 result exports flat bytes - // instead, which the buffer path would read in the method's layout. + // real dtype, shape, and strides. Its buffer view of a dtype such as bfloat16 + // is an unsigned integer, which a bfloat16 input would refuse. if (py::isinstance(value)) { return true; } diff --git a/extension/pybindings/pybindings_executorch_result.h b/extension/pybindings/pybindings_executorch_result.h index da1fee1f869..35e5692b3e9 100644 --- a/extension/pybindings/pybindings_executorch_result.h +++ b/extension/pybindings/pybindings_executorch_result.h @@ -50,9 +50,9 @@ inline const PyTensorDType& python_dtype( {ScalarType::ComplexFloat, "complex64", "complex64", "Zf"}, {ScalarType::ComplexDouble, "complex128", "complex128", "Zd"}, {ScalarType::Bool, "bool", "bool", "?"}, - {ScalarType::QInt8, "qint8", nullptr, nullptr}, + {ScalarType::QInt8, "qint8", nullptr, "b"}, {ScalarType::QUInt8, "quint8", nullptr, nullptr}, - {ScalarType::QInt32, "qint32", nullptr, nullptr}, + {ScalarType::QInt32, "qint32", nullptr, "i"}, {ScalarType::BFloat16, "bfloat16", nullptr, nullptr}, {ScalarType::QUInt4x2, "quint4x2", nullptr, nullptr}, {ScalarType::QUInt2x4, "quint2x4", nullptr, nullptr}, @@ -176,11 +176,6 @@ class PyExecuTorchResult final py::buffer_info buffer() { require_cpu_buffer(); - // PEP 3118 has no bfloat16 format. Expose its physical bytes without - // advertising a different numeric dtype; DLPack preserves the real dtype. - if (scalar_type_ == executorch::aten::ScalarType::BFloat16) { - return flat_buffer(); - } std::vector shape(sizes_.begin(), sizes_.end()); std::vector byte_strides; const auto itemsize = executorch::runtime::elementSize(scalar_type_); @@ -192,7 +187,7 @@ class PyExecuTorchResult final return py::buffer_info( data_, itemsize, - require_buffer_format(), + buffer_format(), ndim, std::move(shape), std::move(byte_strides), @@ -661,15 +656,27 @@ class PyExecuTorchResult final } } - const char* require_buffer_format() const { + // PEP 3118 has no format for dtypes such as bfloat16. An unsigned integer of + // the same width keeps the real shape and layout; dtype_name says how to read + // each element. + const char* buffer_format() const { const auto* format = python_dtype(scalar_type_).buffer_format; - if (format == nullptr) { - throw py::buffer_error( - std::string("ExecuTorch result dtype ") + - executorch::runtime::toString(scalar_type_) + - " cannot use the Python buffer protocol"); + if (format != nullptr) { + return format; + } + switch (executorch::runtime::elementSize(scalar_type_)) { + case 1: + return "B"; + case 2: + return "H"; + case 4: + return "I"; + default: + throw py::buffer_error( + std::string("ExecuTorch result dtype ") + + executorch::runtime::toString(scalar_type_) + + " cannot use the Python buffer protocol"); } - return format; } void* data_ = nullptr; diff --git a/extension/pybindings/test/test_pybindings_cuda_dlpack.py b/extension/pybindings/test/test_pybindings_cuda_dlpack.py index 83c772da952..a1738dd2c8f 100644 --- a/extension/pybindings/test/test_pybindings_cuda_dlpack.py +++ b/extension/pybindings/test/test_pybindings_cuda_dlpack.py @@ -747,8 +747,8 @@ def test_bfloat16_host_result_moves_to_the_device(self): on_device = method(value)[0] host = on_device.to_host() self.assertEqual(host.dtype_name, "bfloat16") - # The host export of bfloat16 is raw bytes, so passing the result itself - # is what keeps its dtype and shape. + # The host export of bfloat16 is unsigned integers, so passing the + # result itself is what keeps its dtype. back = self.bindings.ExecuTorchResult.from_host(host, self.device.index) self.assertEqual(back.dtype_name, "bfloat16") self.assertEqual(back.shape, (2, 3)) diff --git a/extension/pybindings/test/test_pybindings_no_aten.py b/extension/pybindings/test/test_pybindings_no_aten.py index c44546e00eb..509cd893769 100644 --- a/extension/pybindings/test/test_pybindings_no_aten.py +++ b/extension/pybindings/test/test_pybindings_no_aten.py @@ -354,9 +354,11 @@ def test_bfloat16_raw_buffer_input_and_output(self) -> None: for executor in (module, method): with self.subTest(executor=type(executor)): output = executor((one, one))[0] + # NumPy has no bfloat16, so the buffer shows unsigned integers + # of the same width in the real shape. raw = memoryview(output) - self.assertEqual(raw.format, "B") - self.assertEqual(raw.shape, (8,)) + self.assertEqual(raw.format, "H") + self.assertEqual(raw.shape, (2, 2)) self.assertEqual(raw.tobytes(), bytes((0x00, 0x40)) * 4) self.assertEqual(output.shape, (2, 2)) self.assertEqual(output.__dlpack_device__(), (1, 0)) @@ -383,9 +385,8 @@ def test_bfloat16_raw_buffer_input_and_output(self) -> None: def test_bfloat16_result_in_another_layout_feeds_back_correctly(self) -> None: # The method takes contiguous inputs and returns a channels-last result. - # A bfloat16 result exports flat bytes through the buffer protocol, so - # read that way it would take the method's dim order and come out - # permuted, with no error. + # Fed back, it must be read in its own layout, or its values come back + # permuted. if runtime._tensor_output != "executorch": self.skipTest("ExecuTorchResult output was not selected") with open( @@ -398,12 +399,13 @@ def test_bfloat16_result_in_another_layout_feeds_back_correctly(self) -> None: np.uint16 ) zeros = bytes(values.nbytes) - channels_last = values.reshape(1, 2, 2, 2).transpose(0, 2, 3, 1).tobytes() result = method((values.tobytes(), zeros))[0] self.assertEqual(result.strides, (16, 2, 8, 4)) + expected = values.reshape(1, 2, 2, 2) + np.testing.assert_array_equal(np.asarray(memoryview(result)), expected) fed_back = method((result, zeros))[0] - self.assertEqual(memoryview(fed_back).tobytes(), channels_last) + np.testing.assert_array_equal(np.asarray(memoryview(fed_back)), expected) def test_input_errors_name_the_input(self) -> None: class FailingProducer: From c8304797379905750dfd0cf6d6eb738e9022aaea Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 25 Oct 2026 23:17:59 -0700 Subject: [PATCH 4/5] Report a tensor's dtype by its name The Python bindings named a tensor's data type in three ways. ExecuTorchResult.dtype was a NumPy dtype and raised for types NumPy lacks, such as bfloat16, which is why ExecuTorchResult.dtype_name, a torch-style name, was added beside it. TensorInfo.dtype() was the ScalarType integer, and TensorInfo's repr printed the C++ name ("Float"). Now the result and the method metadata report a dtype by its name, the way torch spells it: "float32", "bfloat16", "int64". The names come from the one table the bindings already use to read inputs, so every ExecuTorch dtype has one, with or without torch installed. For every dtype NumPy has, the name is also NumPy's. method = program.load_method("forward") out = method.execute([x])[0] out.dtype # "bfloat16" method.metadata.output_tensor_meta(0).dtype_name() # "bfloat16" getattr(torch, out.dtype) # torch.bfloat16 - ExecuTorchResult.dtype returns the name and never raises. - ExecuTorchResult.dtype_name is removed. It was experimental and never in a release, and dtype now gives the same name. - TensorInfo.dtype_name() is new and returns the same name. TensorInfo.dtype() still returns the integer, since released code depends on it. - TensorInfo's repr prints dtype=float32, like ExecuTorchResult's. - ExecuTorchResult.__array__() now builds its array from the buffer view, so for a dtype NumPy lacks it returns the same integer view as the buffer, where it used to raise. np.asarray() already read the buffer. Error messages about input dtypes keep their existing wording. --- docs/source/runtime-python-api-reference.rst | 2 +- extension/pybindings/README.md | 6 +- extension/pybindings/pybindings.cpp | 16 ++-- extension/pybindings/pybindings.pyi | 11 ++- .../pybindings/pybindings_executorch_result.h | 88 ++++++++----------- extension/pybindings/test/test_pybindings.py | 9 +- .../test/test_pybindings_cuda_dlpack.py | 4 +- .../test/test_pybindings_no_aten.py | 12 ++- 8 files changed, 70 insertions(+), 78 deletions(-) diff --git a/docs/source/runtime-python-api-reference.rst b/docs/source/runtime-python-api-reference.rst index 45317bb661c..8df6ce7b425 100644 --- a/docs/source/runtime-python-api-reference.rst +++ b/docs/source/runtime-python-api-reference.rst @@ -21,6 +21,6 @@ For detailed information on how APIs evolve and the deprecation process, please :members: execute, metadata .. autoclass:: ExecuTorchResult - :members: shape, strides, dtype, dtype_name, nbytes, to_host, from_host + :members: shape, strides, dtype, nbytes, to_host, from_host Without PyTorch, ``Method.execute`` returns ``ExecuTorchResult`` values: read a CPU result through the buffer protocol, NumPy, or DLPack, read a CUDA result through DLPack or ``to_host()``, and place host data on a CUDA device with ``ExecuTorchResult.from_host()``. diff --git a/extension/pybindings/README.md b/extension/pybindings/README.md index 4c3d4a9faf3..43aced4398a 100644 --- a/extension/pybindings/README.md +++ b/extension/pybindings/README.md @@ -65,8 +65,10 @@ separate borrowed view. In torch output mode such a result is a view of memory torch does not own, so it cannot grow with `resize_()` and cannot be shared through `torch.multiprocessing`. Call `.clone()` for a normal torch tensor. -`dtype_name` names a result's dtype, including dtypes NumPy lacks, such as -`"bfloat16"`. +An `ExecuTorchResult`'s `dtype` is the dtype's name as torch spells it, such as +`"float32"` or `"bfloat16"`. +`method_meta().output_tensor_meta(i).dtype_name()` gives the same name before +the method runs. `Method.get_attribute()` returns a view of mutable state. A constant can sit in read-only memory, so it comes back read-only: NumPy and the buffer protocol diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index 41c60688532..25fb2abe576 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -2131,6 +2131,10 @@ struct PyTensorInfo final { info_.scalar_type()); } + const char* dtype_name() const { + return python_dtype(info_.scalar_type()).torch_name; + } + bool is_memory_planned() const { return info_.is_memory_planned(); } @@ -2164,8 +2168,7 @@ struct PyTensorInfo final { size_str.pop_back(); } size_str.append("]"); - return "TensorInfo(sizes=" + size_str + ", dtype=" + - std::string(executorch::runtime::toString(info_.scalar_type())) + + return "TensorInfo(sizes=" + size_str + ", dtype=" + dtype_name() + ", is_memory_planned=" + (info_.is_memory_planned() ? "True" : "False") + ", nbytes=" + std::to_string(info_.nbytes()) + ", device=" + device() + @@ -4438,12 +4441,8 @@ PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { .def_property_readonly( "dtype", &PyExecuTorchResult::dtype, - "NumPy dtype. Raises for dtypes NumPy lacks, such as bfloat16.") - .def_property_readonly( - "dtype_name", - &PyExecuTorchResult::torch_dtype_name, - "Name of the dtype, such as \"bfloat16\", including dtypes NumPy " - "lacks.") + "Name of the dtype, such as \"float32\" or \"bfloat16\", as " + "torch spells it.") .def_property_readonly( "nbytes", &PyExecuTorchResult::nbytes, "Size of the data in bytes.") .def_property_readonly( @@ -4703,6 +4702,7 @@ PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { py::class_(m, "TensorInfo", py::module_local()) .def("sizes", &PyTensorInfo::sizes, call_guard) .def("dtype", &PyTensorInfo::dtype, call_guard) + .def("dtype_name", &PyTensorInfo::dtype_name, call_guard) .def("is_memory_planned", &PyTensorInfo::is_memory_planned, call_guard) .def("nbytes", &PyTensorInfo::nbytes, call_guard) .def("device", &PyTensorInfo::device, call_guard) diff --git a/extension/pybindings/pybindings.pyi b/extension/pybindings/pybindings.pyi index 9bfc6728f1a..3810cc45b8f 100644 --- a/extension/pybindings/pybindings.pyi +++ b/extension/pybindings/pybindings.pyi @@ -42,10 +42,8 @@ class ExecuTorchResult: @property def strides(self) -> Tuple[int, ...]: ... @property - def dtype(self) -> Any: ... - @property - def dtype_name(self) -> str: - """The dtype's name, such as ``"bfloat16"``, including dtypes NumPy lacks.""" + def dtype(self) -> str: + """Name of the dtype, such as ``"float32"`` or ``"bfloat16"``, as torch spells it.""" ... @property @@ -199,6 +197,11 @@ class TensorInfo: for the values these integers can take.""" ... + def dtype_name(self) -> str: + """Name of the data type, such as ``"float32"`` or ``"bfloat16"``, the + same name ``ExecuTorchResult.dtype`` gives.""" + ... + def is_memory_planned(self) -> bool: """True if the tensor is already memory planned, meaning no allocation needs to be provided. False otherwise""" diff --git a/extension/pybindings/pybindings_executorch_result.h b/extension/pybindings/pybindings_executorch_result.h index 35e5692b3e9..1397c46186a 100644 --- a/extension/pybindings/pybindings_executorch_result.h +++ b/extension/pybindings/pybindings_executorch_result.h @@ -30,7 +30,6 @@ namespace py = pybind11; struct PyTensorDType final { executorch::aten::ScalarType scalar_type; const char* torch_name; - const char* numpy_name; const char* buffer_format; }; @@ -38,41 +37,41 @@ inline const PyTensorDType& python_dtype( executorch::aten::ScalarType scalar_type) { using executorch::aten::ScalarType; static constexpr PyTensorDType kDTypes[] = { - {ScalarType::Byte, "uint8", "uint8", "B"}, - {ScalarType::Char, "int8", "int8", "b"}, - {ScalarType::Short, "int16", "int16", "h"}, - {ScalarType::Int, "int32", "int32", "i"}, - {ScalarType::Long, "int64", "int64", "q"}, - {ScalarType::Half, "float16", "float16", "e"}, - {ScalarType::Float, "float32", "float32", "f"}, - {ScalarType::Double, "float64", "float64", "d"}, - {ScalarType::ComplexHalf, "complex32", nullptr, nullptr}, - {ScalarType::ComplexFloat, "complex64", "complex64", "Zf"}, - {ScalarType::ComplexDouble, "complex128", "complex128", "Zd"}, - {ScalarType::Bool, "bool", "bool", "?"}, - {ScalarType::QInt8, "qint8", nullptr, "b"}, - {ScalarType::QUInt8, "quint8", nullptr, nullptr}, - {ScalarType::QInt32, "qint32", nullptr, "i"}, - {ScalarType::BFloat16, "bfloat16", nullptr, nullptr}, - {ScalarType::QUInt4x2, "quint4x2", nullptr, nullptr}, - {ScalarType::QUInt2x4, "quint2x4", nullptr, nullptr}, - {ScalarType::Bits1x8, "bits1x8", nullptr, nullptr}, - {ScalarType::Bits2x4, "bits2x4", nullptr, nullptr}, - {ScalarType::Bits4x2, "bits4x2", nullptr, nullptr}, - {ScalarType::Bits8, "bits8", nullptr, nullptr}, - {ScalarType::Bits16, "bits16", nullptr, nullptr}, - {ScalarType::Float8_e5m2, "float8_e5m2", nullptr, nullptr}, - {ScalarType::Float8_e4m3fn, "float8_e4m3fn", nullptr, nullptr}, - {ScalarType::Float8_e5m2fnuz, "float8_e5m2fnuz", nullptr, nullptr}, - {ScalarType::Float8_e4m3fnuz, "float8_e4m3fnuz", nullptr, nullptr}, - {ScalarType::UInt16, "uint16", "uint16", "H"}, - {ScalarType::UInt32, "uint32", "uint32", "I"}, - {ScalarType::UInt64, "uint64", "uint64", "Q"}, + {ScalarType::Byte, "uint8", "B"}, + {ScalarType::Char, "int8", "b"}, + {ScalarType::Short, "int16", "h"}, + {ScalarType::Int, "int32", "i"}, + {ScalarType::Long, "int64", "q"}, + {ScalarType::Half, "float16", "e"}, + {ScalarType::Float, "float32", "f"}, + {ScalarType::Double, "float64", "d"}, + {ScalarType::ComplexHalf, "complex32", nullptr}, + {ScalarType::ComplexFloat, "complex64", "Zf"}, + {ScalarType::ComplexDouble, "complex128", "Zd"}, + {ScalarType::Bool, "bool", "?"}, + {ScalarType::QInt8, "qint8", "b"}, + {ScalarType::QUInt8, "quint8", nullptr}, + {ScalarType::QInt32, "qint32", "i"}, + {ScalarType::BFloat16, "bfloat16", nullptr}, + {ScalarType::QUInt4x2, "quint4x2", nullptr}, + {ScalarType::QUInt2x4, "quint2x4", nullptr}, + {ScalarType::Bits1x8, "bits1x8", nullptr}, + {ScalarType::Bits2x4, "bits2x4", nullptr}, + {ScalarType::Bits4x2, "bits4x2", nullptr}, + {ScalarType::Bits8, "bits8", nullptr}, + {ScalarType::Bits16, "bits16", nullptr}, + {ScalarType::Float8_e5m2, "float8_e5m2", nullptr}, + {ScalarType::Float8_e4m3fn, "float8_e4m3fn", nullptr}, + {ScalarType::Float8_e5m2fnuz, "float8_e5m2fnuz", nullptr}, + {ScalarType::Float8_e4m3fnuz, "float8_e4m3fnuz", nullptr}, + {ScalarType::UInt16, "uint16", "H"}, + {ScalarType::UInt32, "uint32", "I"}, + {ScalarType::UInt64, "uint64", "Q"}, }; const auto index = static_cast(scalar_type); if (index >= sizeof(kDTypes) / sizeof(kDTypes[0]) || kDTypes[index].scalar_type != scalar_type) { - throw std::runtime_error("Unsupported ExecuTorch result dtype"); + throw std::runtime_error("Unsupported ExecuTorch dtype"); } return kDTypes[index]; } @@ -233,21 +232,8 @@ class PyExecuTorchResult final return result; } - py::dtype dtype() const { - const auto* name = python_dtype(scalar_type_).numpy_name; - if (name == nullptr) { - throw std::runtime_error( - std::string("ExecuTorch result dtype ") + - executorch::runtime::toString(scalar_type_) + - " cannot be represented by NumPy; dtype_name names it"); - } - return py::dtype(name); - } - py::array numpy_array(const py::object& base) { - const auto array_dtype = dtype(); - auto info = buffer(); - py::array array(array_dtype, info.shape, info.strides, info.ptr, base); + py::array array(buffer(), base); if (read_only_) { array.attr("setflags")(py::arg("write") = false); } @@ -397,7 +383,7 @@ class PyExecuTorchResult final return capsule; } - const char* torch_dtype_name() const { + const char* dtype() const { return python_dtype(scalar_type_).torch_name; } @@ -497,8 +483,8 @@ class PyExecuTorchResult final if (sizes_.size() == 1) { shape += ","; } - return "ExecuTorchResult(shape=(" + shape + - "), dtype=" + torch_dtype_name() + ", device=" + device_name() + ")"; + return "ExecuTorchResult(shape=(" + shape + "), dtype=" + dtype() + + ", device=" + device_name() + ")"; } std::string device_name() const { @@ -657,8 +643,8 @@ class PyExecuTorchResult final } // PEP 3118 has no format for dtypes such as bfloat16. An unsigned integer of - // the same width keeps the real shape and layout; dtype_name says how to read - // each element. + // the same width keeps the real shape and layout; dtype says how to read each + // element. const char* buffer_format() const { const auto* format = python_dtype(scalar_type_).buffer_format; if (format != nullptr) { diff --git a/extension/pybindings/test/test_pybindings.py b/extension/pybindings/test/test_pybindings.py index 09140978543..1b661995501 100644 --- a/extension/pybindings/test/test_pybindings.py +++ b/extension/pybindings/test/test_pybindings.py @@ -1272,7 +1272,7 @@ def test_method_meta(self) -> None: self.assertEqual(meta.num_inputs(), 2) self.assertEqual(meta.num_outputs(), 1) - tensor_info = "TensorInfo(sizes=[2, 2], dtype=Float, is_memory_planned=True, nbytes=16, device=cpu)" + tensor_info = "TensorInfo(sizes=[2, 2], dtype=float32, is_memory_planned=True, nbytes=16, device=cpu)" float_dtype = 6 self.assertEqual( str(meta), @@ -1290,6 +1290,9 @@ def test_method_meta(self) -> None: del meta self.assertEqual([t.sizes() for t in input_tensors], [(2, 2), (2, 2)]) self.assertEqual([t.dtype() for t in input_tensors], [float_dtype, float_dtype]) + self.assertEqual( + [t.dtype_name() for t in input_tensors], ["float32", "float32"] + ) self.assertEqual([t.is_memory_planned() for t in input_tensors], [True, True]) self.assertEqual([t.nbytes() for t in input_tensors], [16, 16]) self.assertEqual(str(input_tensors), f"[{tensor_info}, {tensor_info}]") @@ -1902,7 +1905,7 @@ def test_program_method_meta(self) -> None: self.assertEqual(meta.num_outputs(), 1) self.assertEqual(meta.num_attributes(), 1) - tensor_info = "TensorInfo(sizes=[2, 2], dtype=Float, is_memory_planned=True, nbytes=16, device=cpu)" + tensor_info = "TensorInfo(sizes=[2, 2], dtype=float32, is_memory_planned=True, nbytes=16, device=cpu)" float_dtype = 6 self.assertEqual( @@ -1954,7 +1957,7 @@ def test_method_method_meta(self) -> None: self.assertEqual(meta.num_inputs(), 2) self.assertEqual(meta.num_outputs(), 1) - tensor_info = "TensorInfo(sizes=[2, 2], dtype=Float, is_memory_planned=True, nbytes=16, device=cpu)" + tensor_info = "TensorInfo(sizes=[2, 2], dtype=float32, is_memory_planned=True, nbytes=16, device=cpu)" float_dtype = 6 self.assertEqual( str(meta), diff --git a/extension/pybindings/test/test_pybindings_cuda_dlpack.py b/extension/pybindings/test/test_pybindings_cuda_dlpack.py index a1738dd2c8f..001d2b87108 100644 --- a/extension/pybindings/test/test_pybindings_cuda_dlpack.py +++ b/extension/pybindings/test/test_pybindings_cuda_dlpack.py @@ -746,11 +746,11 @@ def test_bfloat16_host_result_moves_to_the_device(self): method = self.bindings._load_program_from_buffer(data).load_method("forward") on_device = method(value)[0] host = on_device.to_host() - self.assertEqual(host.dtype_name, "bfloat16") + self.assertEqual(host.dtype, "bfloat16") # The host export of bfloat16 is unsigned integers, so passing the # result itself is what keeps its dtype. back = self.bindings.ExecuTorchResult.from_host(host, self.device.index) - self.assertEqual(back.dtype_name, "bfloat16") + self.assertEqual(back.dtype, "bfloat16") self.assertEqual(back.shape, (2, 3)) torch.testing.assert_close(torch.from_dlpack(back), value) with self.assertRaisesRegex(ValueError, "already on a device"): diff --git a/extension/pybindings/test/test_pybindings_no_aten.py b/extension/pybindings/test/test_pybindings_no_aten.py index 509cd893769..8488ac01e6c 100644 --- a/extension/pybindings/test/test_pybindings_no_aten.py +++ b/extension/pybindings/test/test_pybindings_no_aten.py @@ -203,8 +203,8 @@ def test_executorch_result_to_host_and_repr(self) -> None: copy = output.to_host() - self.assertEqual(output.dtype_name, "float32") - self.assertEqual(output.dtype, np.dtype(np.float32)) + self.assertIsInstance(output.dtype, str) + self.assertEqual(output.dtype, "float32") self.assertEqual( repr(copy), "ExecuTorchResult(shape=(1,), dtype=float32, device=cpu)" ) @@ -363,11 +363,9 @@ def test_bfloat16_raw_buffer_input_and_output(self) -> None: self.assertEqual(output.shape, (2, 2)) self.assertEqual(output.__dlpack_device__(), (1, 0)) if isinstance(output, runtime.ExecuTorchResult): - # NumPy has no bfloat16, so only the name says how to read - # these bytes. - self.assertEqual(output.dtype_name, "bfloat16") - with self.assertRaisesRegex(RuntimeError, "dtype_name"): - output.dtype + self.assertEqual(output.dtype, "bfloat16") + # NumPy reads the buffer first, so call __array__ directly. + self.assertEqual(output.__array__().dtype, np.uint16) for noncontiguous in ( np.arange(16, dtype=np.uint8)[::2], np.arange(8, dtype=np.uint8)[::-1], From 124aef289eda772651c1b6389a621bcc6ba94bb7 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 26 Oct 2026 03:13:25 -0700 Subject: [PATCH 5/5] Describe the inputs and outputs of the Python runtime in its docs The runtime API reference showed only torch.Tensor inputs and outputs, and its note on running without PyTorch had no example. Most users have PyTorch installed, so the torch path stays the main example. A short "Without PyTorch" section now shows the fallback: pass NumPy arrays and read the ExecuTorchResult outputs with NumPy, or use DLPack, for example with CuPy, for data on a GPU. The Method.execute docstring says the same: inputs and outputs are typically torch.Tensor objects, and without PyTorch it takes NumPy arrays, buffers, DLPack objects or ExecuTorchResult values and returns ExecuTorchResult values. The reference also lists ExecuTorchResult.device instead of the removed dtype_name. --- docs/source/runtime-python-api-reference.rst | 26 +++++++++++++++++--- runtime/__init__.py | 6 ++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/source/runtime-python-api-reference.rst b/docs/source/runtime-python-api-reference.rst index 8df6ce7b425..9345f7af53c 100644 --- a/docs/source/runtime-python-api-reference.rst +++ b/docs/source/runtime-python-api-reference.rst @@ -1,6 +1,6 @@ Runtime Python API Reference ---------------------------------- -The Python ``executorch.runtime`` module wraps the C++ ExecuTorch runtime. It can load and execute serialized ``.pte`` program files: see the `Export to ExecuTorch Tutorial `__ for how to convert a PyTorch ``nn.Module`` to an ExecuTorch ``.pte`` program file. Execution accepts and returns ``torch.Tensor`` values, making it a quick way to validate the correctness of the program. +The Python ``executorch.runtime`` module wraps the C++ ExecuTorch runtime. It can load and execute serialized ``.pte`` program files: see the `Export to ExecuTorch Tutorial `__ for how to convert a PyTorch ``nn.Module`` to an ExecuTorch ``.pte`` program file. Execution accepts and returns ``torch.Tensor`` values, making it a quick way to validate the correctness of the program. To run a program without PyTorch installed, see `Without PyTorch`_ below. For detailed information on how APIs evolve and the deprecation process, please refer to the `ExecuTorch API Life Cycle and Deprecation Policy `__. @@ -21,6 +21,26 @@ For detailed information on how APIs evolve and the deprecation process, please :members: execute, metadata .. autoclass:: ExecuTorchResult - :members: shape, strides, dtype, nbytes, to_host, from_host + :members: shape, strides, dtype, device, nbytes, to_host, from_host -Without PyTorch, ``Method.execute`` returns ``ExecuTorchResult`` values: read a CPU result through the buffer protocol, NumPy, or DLPack, read a CUDA result through DLPack or ``to_host()``, and place host data on a CUDA device with ``ExecuTorchResult.from_host()``. +Without PyTorch +~~~~~~~~~~~~~~~ + +The runtime also works when PyTorch is not installed. Then ``Method.execute`` takes NumPy arrays as inputs and returns ``ExecuTorchResult`` values, which NumPy can read. For a model that adds its two inputs: + +.. code-block:: python + + import numpy as np + from executorch.runtime import Runtime + + method = Runtime.get().load_program("model.pte").load_method("forward") + out = method.execute([np.ones((2, 2), np.float32), np.ones((2, 2), np.float32)])[0] + print(out.dtype) # float32 + result = np.asarray(out) # a NumPy view of the output, no copy + print(result.tolist()) # [[2.0, 2.0], [2.0, 2.0]] + +Other buffers, such as ``memoryview``, work as inputs too. NumPy has no bfloat16 type, so it reads a bfloat16 result as 16-bit integers that hold the raw bits. + +For data on a GPU, use DLPack, the standard way to share tensors between Python libraries. This needs ExecuTorch built with CUDA support. Pass a CuPy array to a method that takes CUDA inputs. Read a CUDA result with ``cupy.from_dlpack(out)``, or copy it to the CPU with ``out.to_host()``. ``ExecuTorchResult.from_host()`` copies CPU data to a CUDA device. + +To get ``ExecuTorchResult`` values even when PyTorch is installed, set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT=executorch`` before importing ExecuTorch. diff --git a/runtime/__init__.py b/runtime/__init__.py index c8095038719..a4ebf9a5e72 100644 --- a/runtime/__init__.py +++ b/runtime/__init__.py @@ -143,11 +143,15 @@ def execute( Args: inputs: A sequence of input values, typically torch.Tensor objects. + Without PyTorch, pass NumPy arrays, other buffers such as + memoryview, DLPack objects such as CuPy arrays, or + ExecuTorchResult values. Scalar inputs take Python numbers. clone_outputs: If False, the outputs share the method's memory and a later execution may overwrite them, which avoids a copy. Returns: - A list of output values, typically torch.Tensor objects. + A list of output values, typically torch.Tensor objects. Without + PyTorch, tensor outputs are ExecuTorchResult objects. """ return self._method(inputs, clone_outputs=clone_outputs)