diff --git a/extension/android/executorch_android/consumer-proguard-rules.pro b/extension/android/executorch_android/consumer-proguard-rules.pro index 64045ed0598..d0d9113e479 100644 --- a/extension/android/executorch_android/consumer-proguard-rules.pro +++ b/extension/android/executorch_android/consumer-proguard-rules.pro @@ -32,7 +32,8 @@ -keep class org.pytorch.executorch.EValue { *; } # Keep Tensor and its inner classes. The Tensor class has methods and fields -# accessed from JNI (dtypeJniCode, shape, getRawDataBuffer, nativeNewTensor). +# accessed from JNI (dtypeJniCode, shape, dimOrder, getRawDataBuffer, +# nativeNewTensor). -keep class org.pytorch.executorch.Tensor { *; } -keep class org.pytorch.executorch.Tensor$* { *; } diff --git a/extension/android/executorch_android/src/main/java/org/pytorch/executorch/Tensor.kt b/extension/android/executorch_android/src/main/java/org/pytorch/executorch/Tensor.kt index f2f3ebea214..fc7a9ba3481 100644 --- a/extension/android/executorch_android/src/main/java/org/pytorch/executorch/Tensor.kt +++ b/extension/android/executorch_android/src/main/java/org/pytorch/executorch/Tensor.kt @@ -41,16 +41,30 @@ import org.pytorch.executorch.annotations.Experimental * Warning: These APIs are experimental and subject to change without notice */ @Experimental -abstract class Tensor internal constructor(shape: LongArray) { +abstract class Tensor internal constructor(shape: LongArray, dimOrder: IntArray? = null) { init { for (s in shape) { require(s >= 0) { "Shape elements must be non negative" } } + if (dimOrder != null) { + require(dimOrder.size == shape.size) { "Dim order size must match shape size" } + val seen = BooleanArray(shape.size) + for (d in dimOrder) { + require(d in seen.indices && !seen[d]) { + "Dim order must be a permutation of shape dimensions" + } + seen[d] = true + } + } } @DoNotStrip @JvmField protected val shape: LongArray = shape.copyOf() + @DoNotStrip + @JvmField + protected val dimOrder: IntArray = dimOrder?.copyOf() ?: IntArray(shape.size) { it } + @DoNotStrip private var mHybridData: HybridData? = null /** Returns the number of elements in this tensor. */ @@ -59,6 +73,9 @@ abstract class Tensor internal constructor(shape: LongArray) { /** Returns the shape of this tensor. (The array is a fresh copy.) */ fun shape(): LongArray = shape.copyOf() + /** Returns the dimension order of this tensor. (The array is a fresh copy.) */ + fun dimOrder(): IntArray = dimOrder.copyOf() + abstract fun dtype(): DType // Called from native via JNI GetMethodID — must not be `internal` (name mangling breaks lookup) @@ -215,20 +232,24 @@ abstract class Tensor internal constructor(shape: LongArray) { } else -> throw IllegalArgumentException("Unknown Tensor dtype") } - val byteBuffer = ByteBuffer.allocate(1 + 1 + 4 * shape.size + dtypeSize * numel().toInt()) + val byteBuffer = ByteBuffer.allocate(1 + 1 + 5 * shape.size + dtypeSize * numel().toInt()) byteBuffer.put(dtype().jniCode.toByte()) byteBuffer.put(shape.size.toByte()) for (s in shape) { byteBuffer.putInt(s.toInt()) } + for (d in dimOrder) { + byteBuffer.put(d.toByte()) + } byteBuffer.put(tensorAsByteArray) return byteBuffer.array() } // region nested tensor types - internal class Tensor_uint8 internal constructor(private val data: ByteBuffer, shape: LongArray) : - Tensor(shape) { + internal class Tensor_uint8 + internal constructor(private val data: ByteBuffer, shape: LongArray, dimOrder: IntArray? = null) : + Tensor(shape, dimOrder) { override fun dtype(): DType = DType.UINT8 override fun getRawDataBuffer(): Buffer = data @@ -249,8 +270,9 @@ abstract class Tensor internal constructor(shape: LongArray) { override fun toString(): String = "Tensor(${Arrays.toString(shape)}, dtype=torch.uint8)" } - internal class Tensor_int8 internal constructor(private val data: ByteBuffer, shape: LongArray) : - Tensor(shape) { + internal class Tensor_int8 + internal constructor(private val data: ByteBuffer, shape: LongArray, dimOrder: IntArray? = null) : + Tensor(shape, dimOrder) { override fun dtype(): DType = DType.INT8 override fun getRawDataBuffer(): Buffer = data @@ -271,8 +293,9 @@ abstract class Tensor internal constructor(shape: LongArray) { override fun toString(): String = "Tensor(${Arrays.toString(shape)}, dtype=torch.int8)" } - internal class Tensor_int32 internal constructor(private val data: IntBuffer, shape: LongArray) : - Tensor(shape) { + internal class Tensor_int32 + internal constructor(private val data: IntBuffer, shape: LongArray, dimOrder: IntArray? = null) : + Tensor(shape, dimOrder) { override fun dtype(): DType = DType.INT32 override fun getRawDataBuffer(): Buffer = data @@ -294,7 +317,11 @@ abstract class Tensor internal constructor(shape: LongArray) { } internal class Tensor_float32 - internal constructor(private val data: FloatBuffer, shape: LongArray) : Tensor(shape) { + internal constructor( + private val data: FloatBuffer, + shape: LongArray, + dimOrder: IntArray? = null, + ) : Tensor(shape, dimOrder) { override fun dtype(): DType = DType.FLOAT override fun getRawDataBuffer(): Buffer = data @@ -316,7 +343,11 @@ abstract class Tensor internal constructor(shape: LongArray) { } internal class Tensor_float16 - internal constructor(private val data: ShortBuffer, shape: LongArray) : Tensor(shape) { + internal constructor( + private val data: ShortBuffer, + shape: LongArray, + dimOrder: IntArray? = null, + ) : Tensor(shape, dimOrder) { override fun dtype(): DType = DType.HALF override fun getRawDataBuffer(): Buffer = data @@ -386,8 +417,9 @@ abstract class Tensor internal constructor(shape: LongArray) { } } - internal class Tensor_int64 internal constructor(private val data: LongBuffer, shape: LongArray) : - Tensor(shape) { + internal class Tensor_int64 + internal constructor(private val data: LongBuffer, shape: LongArray, dimOrder: IntArray? = null) : + Tensor(shape, dimOrder) { override fun dtype(): DType = DType.INT64 override fun getRawDataBuffer(): Buffer = data @@ -409,7 +441,11 @@ abstract class Tensor internal constructor(shape: LongArray) { } internal class Tensor_float64 - internal constructor(private val data: DoubleBuffer, shape: LongArray) : Tensor(shape) { + internal constructor( + private val data: DoubleBuffer, + shape: LongArray, + dimOrder: IntArray? = null, + ) : Tensor(shape, dimOrder) { override fun dtype(): DType = DType.DOUBLE override fun getRawDataBuffer(): Buffer = data @@ -435,7 +471,8 @@ abstract class Tensor internal constructor(shape: LongArray) { private val data: ByteBuffer, shape: LongArray, private val mDtype: DType, - ) : Tensor(shape) { + dimOrder: IntArray? = null, + ) : Tensor(shape, dimOrder) { init { Log.e("ExecuTorch", "$this. Please consider re-exporting the model with a proper return type") } @@ -498,66 +535,73 @@ abstract class Tensor internal constructor(shape: LongArray) { // region fromBlob (array) @JvmStatic - fun fromBlobUnsigned(data: ByteArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlobUnsigned(data: ByteArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val byteBuffer = allocateByteBuffer(numel(shape).toInt()) byteBuffer.put(data) - return Tensor_uint8(byteBuffer, shape) + return Tensor_uint8(byteBuffer, shape, dimOrder) } @JvmStatic - fun fromBlob(data: ByteArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: ByteArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val byteBuffer = allocateByteBuffer(numel(shape).toInt()) byteBuffer.put(data) - return Tensor_int8(byteBuffer, shape) + return Tensor_int8(byteBuffer, shape, dimOrder) } @JvmStatic - fun fromBlob(data: IntArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: IntArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val intBuffer = allocateIntBuffer(numel(shape).toInt()) intBuffer.put(data) - return Tensor_int32(intBuffer, shape) + return Tensor_int32(intBuffer, shape, dimOrder) } @JvmStatic - fun fromBlob(data: FloatArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: FloatArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val floatBuffer = allocateFloatBuffer(numel(shape).toInt()) floatBuffer.put(data) - return Tensor_float32(floatBuffer, shape) + return Tensor_float32(floatBuffer, shape, dimOrder) } @JvmStatic - fun fromBlob(data: ShortArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: ShortArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val shortBuffer = allocateHalfBuffer(numel(shape).toInt()) shortBuffer.put(data) - return Tensor_float16(shortBuffer, shape) + return Tensor_float16(shortBuffer, shape, dimOrder) } @JvmStatic - fun fromBlob(data: LongArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: LongArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val longBuffer = allocateLongBuffer(numel(shape).toInt()) longBuffer.put(data) - return Tensor_int64(longBuffer, shape) + return Tensor_int64(longBuffer, shape, dimOrder) } @JvmStatic - fun fromBlob(data: DoubleArray, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: DoubleArray, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.size, shape) val doubleBuffer = allocateDoubleBuffer(numel(shape).toInt()) doubleBuffer.put(data) - return Tensor_float64(doubleBuffer, shape) + return Tensor_float64(doubleBuffer, shape, dimOrder) } // endregion fromBlob (array) @@ -565,7 +609,8 @@ abstract class Tensor internal constructor(shape: LongArray) { // region fromBlob (buffer) @JvmStatic - fun fromBlobUnsigned(data: ByteBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlobUnsigned(data: ByteBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -573,11 +618,12 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_uint8(data, shape) + return Tensor_uint8(data, shape, dimOrder) } @JvmStatic - fun fromBlob(data: ByteBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: ByteBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -585,11 +631,12 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_int8(data, shape) + return Tensor_int8(data, shape, dimOrder) } @JvmStatic - fun fromBlob(data: IntBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: IntBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -597,11 +644,12 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_int32(data, shape) + return Tensor_int32(data, shape, dimOrder) } @JvmStatic - fun fromBlob(data: FloatBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: FloatBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -609,11 +657,12 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_float32(data, shape) + return Tensor_float32(data, shape, dimOrder) } @JvmStatic - fun fromBlob(data: ShortBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: ShortBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -621,11 +670,12 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_float16(data, shape) + return Tensor_float16(data, shape, dimOrder) } @JvmStatic - fun fromBlob(data: LongBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: LongBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -633,11 +683,12 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_int64(data, shape) + return Tensor_int64(data, shape, dimOrder) } @JvmStatic - fun fromBlob(data: DoubleBuffer, shape: LongArray): Tensor { + @JvmOverloads + fun fromBlob(data: DoubleBuffer, shape: LongArray, dimOrder: IntArray? = null): Tensor { checkShape(shape) checkShapeAndDataCapacityConsistency(data.capacity(), shape) checkArgument(data.isDirect, ERROR_MSG_DATA_BUFFER_MUST_BE_DIRECT) @@ -645,7 +696,7 @@ abstract class Tensor internal constructor(shape: LongArray) { data.order() == ByteOrder.nativeOrder(), ERROR_MSG_DATA_BUFFER_MUST_HAVE_NATIVE_BYTE_ORDER, ) - return Tensor_float64(data, shape) + return Tensor_float64(data, shape, dimOrder) } // endregion fromBlob (buffer) @@ -697,19 +748,20 @@ abstract class Tensor internal constructor(shape: LongArray) { private fun nativeNewTensor( data: ByteBuffer, shape: LongArray, + dimOrder: IntArray, dtype: Int, hybridData: HybridData, ): Tensor { val tensor = when { - DType.FLOAT.jniCode == dtype -> Tensor_float32(data.asFloatBuffer(), shape) - DType.HALF.jniCode == dtype -> Tensor_float16(data.asShortBuffer(), shape) - DType.INT32.jniCode == dtype -> Tensor_int32(data.asIntBuffer(), shape) - DType.INT64.jniCode == dtype -> Tensor_int64(data.asLongBuffer(), shape) - DType.DOUBLE.jniCode == dtype -> Tensor_float64(data.asDoubleBuffer(), shape) - DType.UINT8.jniCode == dtype -> Tensor_uint8(data, shape) - DType.INT8.jniCode == dtype -> Tensor_int8(data, shape) - else -> Tensor_unsupported(data, shape, DType.fromJniCode(dtype)) + DType.FLOAT.jniCode == dtype -> Tensor_float32(data.asFloatBuffer(), shape, dimOrder) + DType.HALF.jniCode == dtype -> Tensor_float16(data.asShortBuffer(), shape, dimOrder) + DType.INT32.jniCode == dtype -> Tensor_int32(data.asIntBuffer(), shape, dimOrder) + DType.INT64.jniCode == dtype -> Tensor_int64(data.asLongBuffer(), shape, dimOrder) + DType.DOUBLE.jniCode == dtype -> Tensor_float64(data.asDoubleBuffer(), shape, dimOrder) + DType.UINT8.jniCode == dtype -> Tensor_uint8(data, shape, dimOrder) + DType.INT8.jniCode == dtype -> Tensor_int8(data, shape, dimOrder) + else -> Tensor_unsupported(data, shape, DType.fromJniCode(dtype), dimOrder) } tensor.mHybridData = hybridData return tensor @@ -731,14 +783,28 @@ abstract class Tensor internal constructor(shape: LongArray) { require(dim >= 0) { "invalid shape" } shape[i] = dim.toLong() } + val elementSize = + when (dtype.toInt()) { + DType.UINT8.jniCode, + DType.INT8.jniCode -> BYTE_SIZE_BYTES + DType.HALF.jniCode -> HALF_SIZE_BYTES + DType.INT32.jniCode -> INT_SIZE_BYTES + DType.INT64.jniCode -> LONG_SIZE_BYTES + DType.FLOAT.jniCode -> FLOAT_SIZE_BYTES + DType.DOUBLE.jniCode -> DOUBLE_SIZE_BYTES + else -> throw IllegalArgumentException("Unknown Tensor dtype") + } + // Byte arrays written before dim order was serialized have no dim order bytes. + val hasDimOrder = buffer.remaining().toLong() == shape.size + numel(shape) * elementSize + val dimOrder = if (hasDimOrder) IntArray(shape.size) { buffer.get().toInt() } else null return when (dtype.toInt()) { - DType.UINT8.jniCode -> Tensor_uint8(buffer, shape) - DType.INT8.jniCode -> Tensor_int8(buffer, shape) - DType.HALF.jniCode -> Tensor_float16(buffer.asShortBuffer(), shape) - DType.INT32.jniCode -> Tensor_int32(buffer.asIntBuffer(), shape) - DType.INT64.jniCode -> Tensor_int64(buffer.asLongBuffer(), shape) - DType.FLOAT.jniCode -> Tensor_float32(buffer.asFloatBuffer(), shape) - DType.DOUBLE.jniCode -> Tensor_float64(buffer.asDoubleBuffer(), shape) + DType.UINT8.jniCode -> Tensor_uint8(buffer, shape, dimOrder) + DType.INT8.jniCode -> Tensor_int8(buffer, shape, dimOrder) + DType.HALF.jniCode -> Tensor_float16(buffer.asShortBuffer(), shape, dimOrder) + DType.INT32.jniCode -> Tensor_int32(buffer.asIntBuffer(), shape, dimOrder) + DType.INT64.jniCode -> Tensor_int64(buffer.asLongBuffer(), shape, dimOrder) + DType.FLOAT.jniCode -> Tensor_float32(buffer.asFloatBuffer(), shape, dimOrder) + DType.DOUBLE.jniCode -> Tensor_float64(buffer.asDoubleBuffer(), shape, dimOrder) else -> throw IllegalArgumentException("Unknown Tensor dtype") } } diff --git a/extension/android/executorch_android/src/test/java/org/pytorch/executorch/TensorTest.kt b/extension/android/executorch_android/src/test/java/org/pytorch/executorch/TensorTest.kt index b9f88368255..2f6cf88f710 100644 --- a/extension/android/executorch_android/src/test/java/org/pytorch/executorch/TensorTest.kt +++ b/extension/android/executorch_android/src/test/java/org/pytorch/executorch/TensorTest.kt @@ -7,6 +7,7 @@ */ package org.pytorch.executorch +import java.nio.ByteBuffer import java.nio.ByteOrder import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.Assert.assertArrayEquals @@ -639,6 +640,59 @@ class TensorTest { .hasMessage("Inconsistent data capacity:4 and shape number elements:2 shape:[1, 2]") } + @Test + fun testDimOrder() { + val shape = longArrayOf(1, 3, 2, 2) + val dimOrder = intArrayOf(0, 2, 3, 1) + val data = FloatArray(12) { it.toFloat() } + + assertArrayEquals(dimOrder, Tensor.fromBlob(data, shape, dimOrder).dimOrder()) + + val buffer = Tensor.allocateFloatBuffer(data.size) + buffer.put(data) + assertArrayEquals(dimOrder, Tensor.fromBlob(buffer, shape, dimOrder).dimOrder()) + assertArrayEquals(intArrayOf(0, 1, 2, 3), Tensor.fromBlob(data, shape).dimOrder()) + + assertThatThrownBy { Tensor.fromBlob(data, shape, intArrayOf(0, 1, 2)) } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { Tensor.fromBlob(data, shape, intArrayOf(0, 1, 2, 2)) } + .isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun testDimOrderSerde() { + val shape = longArrayOf(1, 3, 2, 2) + val dimOrder = intArrayOf(0, 2, 3, 1) + val data = FloatArray(12) { it.toFloat() } + + val deser = Tensor.fromByteArray(Tensor.fromBlob(data, shape, dimOrder).toByteArray()) + + assertArrayEquals(shape, deser.shape()) + assertArrayEquals(dimOrder, deser.dimOrder()) + assertArrayEquals(data, deser.dataAsFloatArray, 0f) + } + + @Test + fun testSerdeWithoutDimOrder() { + val shape = longArrayOf(2, 2) + val data = intArrayOf(1, 2, 3, 4) + val buffer = ByteBuffer.allocate(1 + 1 + 4 * shape.size + 4 * data.size) + buffer.put(DType.INT32.jniCode.toByte()) + buffer.put(shape.size.toByte()) + for (s in shape) { + buffer.putInt(s.toInt()) + } + for (d in data) { + buffer.putInt(d) + } + + val deser = Tensor.fromByteArray(buffer.array()) + + assertArrayEquals(shape, deser.shape()) + assertArrayEquals(intArrayOf(0, 1), deser.dimOrder()) + assertArrayEquals(data, deser.dataAsIntArray) + } + @Test fun testLongTensorSerde() { val data = longArrayOf(1, 2, 3, 4) diff --git a/extension/android/jni/jni_layer.cpp b/extension/android/jni/jni_layer.cpp index 38ea80a9a81..4b5e6235149 100644 --- a/extension/android/jni/jni_layer.cpp +++ b/extension/android/jni/jni_layer.cpp @@ -60,8 +60,6 @@ class TensorHybrid : public facebook::jni::HybridClass { static facebook::jni::local_ref newJTensorFromTensor(const executorch::aten::Tensor& tensor) { - // Java wrapper currently only supports contiguous tensors. - const auto scalarType = tensor.scalar_type(); if (scalar_type_to_java_dtype.count(scalarType) == 0) { std::stringstream ss; @@ -83,6 +81,22 @@ class TensorHybrid : public facebook::jni::HybridClass { jTensorShape->setRegion( 0, tensor_shape_vec.size(), tensor_shape_vec.data()); + std::vector dim_order_vec; + dim_order_vec.reserve(tensor_shape_vec.size()); + const auto tensor_dim_order = tensor.dim_order(); + if (tensor_dim_order.size() == tensor_shape_vec.size()) { + for (const auto& d : tensor_dim_order) { + dim_order_vec.push_back(static_cast(d)); + } + } else { + for (size_t i = 0; i < tensor_shape_vec.size(); ++i) { + dim_order_vec.push_back(static_cast(i)); + } + } + facebook::jni::local_ref jDimOrder = + facebook::jni::make_int_array(dim_order_vec.size()); + jDimOrder->setRegion(0, dim_order_vec.size(), dim_order_vec.data()); + static auto cls = TensorHybrid::javaClassStatic(); // Note: this is safe as long as the data stored in tensor is valid; the // data won't go out of scope as long as the Method for the inference is @@ -97,10 +111,16 @@ class TensorHybrid : public facebook::jni::HybridClass { cls->getStaticMethod( facebook::jni::alias_ref, facebook::jni::alias_ref, + facebook::jni::alias_ref, jint, facebook::jni::alias_ref)>("nativeNewTensor"); return jMethodNewTensor( - cls, jTensorBuffer, jTensorShape, jdtype, makeCxxInstance(tensor)); + cls, + jTensorBuffer, + jTensorShape, + jDimOrder, + jdtype, + makeCxxInstance(tensor)); } static TensorPtr newTensorFromJTensor( @@ -112,6 +132,10 @@ class TensorHybrid : public facebook::jni::HybridClass { static const auto shapeField = cls->getField("shape"); auto jshape = jtensor->getFieldValue(shapeField); + static const auto dimOrderField = + cls->getField("dimOrder"); + auto jdim_order = jtensor->getFieldValue(dimOrderField); + static auto dataBufferMethod = cls->getMethod< facebook::jni::local_ref()>( "getRawDataBuffer"); @@ -131,6 +155,29 @@ class TensorHybrid : public facebook::jni::HybridClass { for (int i = rank - 1; i >= 0; --i) { numel *= shapeArr[i]; } + + std::vector dim_order_vec; + dim_order_vec.reserve(rank); + if (!jdim_order || jdim_order->size() != rank) { + jni_helper::throwExecutorchException( + static_cast(Error::InvalidArgument), + "Tensor dim_order size must match shape size"); + return nullptr; + } + const auto dimOrderArr = jdim_order->getRegion(0, rank); + std::vector seen_dim(rank, 0); + for (int i = 0; i < rank; i++) { + const jint d = dimOrderArr[i]; + if (d < 0 || d >= static_cast(rank) || seen_dim[d] != 0) { + jni_helper::throwExecutorchException( + static_cast(Error::InvalidArgument), + "Tensor dim_order must be a permutation of shape dimensions"); + return nullptr; + } + seen_dim[d] = 1; + dim_order_vec.push_back(static_cast(d)); + } + JNIEnv* jni = facebook::jni::Environment::current(); if (java_dtype_to_scalar_type.count(jdtype) == 0) { std::stringstream ss; @@ -163,8 +210,12 @@ class TensorHybrid : public facebook::jni::HybridClass { static_cast(Error::InvalidArgument), ss.str().c_str()); return nullptr; } - return from_blob( - jni->GetDirectBufferAddress(jbuffer.get()), shape_vec, scalar_type); + return for_blob( + jni->GetDirectBufferAddress(jbuffer.get()), + shape_vec, + scalar_type) + .dim_order(std::move(dim_order_vec)) + .make_tensor_ptr(); } private: diff --git a/runtime/core/portable_type/tensor_impl.h b/runtime/core/portable_type/tensor_impl.h index fcfe4d901f2..1c9a4e9b3c5 100644 --- a/runtime/core/portable_type/tensor_impl.h +++ b/runtime/core/portable_type/tensor_impl.h @@ -175,11 +175,17 @@ class TensorImpl { /// Returns the order the dimensions are laid out in memory. const ArrayRef dim_order() const { + if (dim_order_ == nullptr) { + return {}; + } return ArrayRef{dim_order_, static_cast(dim_)}; } /// Returns the strides of the tensor at each dimension. const ArrayRef strides() const { + if (strides_ == nullptr) { + return {}; + } return ArrayRef{strides_, static_cast(dim_)}; } diff --git a/runtime/executor/method.cpp b/runtime/executor/method.cpp index 9b8b9e34008..e0dd50b0308 100644 --- a/runtime/executor/method.cpp +++ b/runtime/executor/method.cpp @@ -755,6 +755,87 @@ Error Method::parse_values(const NamedDataMap* external_data_map) { } namespace { +/** + * Validates that t_src's physical layout matches t_dst's, which must already + * be resized to t_src's sizes. set_input() copies or shares t_src's buffer + * under t_dst's layout, so a mismatched source would be silently + * reinterpreted. A source with the default contiguous layout is only warned + * about, since callers commonly wrap data that is already in the method's + * layout in a default tensor just to pass its shape. + */ +ET_NODISCARD Error validate_input_tensor_layout( + const executorch::aten::Tensor& t_dst, + const executorch::aten::Tensor& t_src, + size_t input_idx) { + const auto dim = t_src.dim(); + if (dim == 0 || t_src.numel() == 0) { + return Error::Ok; + } + + auto src_strides = t_src.strides(); +#ifndef USE_ATEN_LIB + std::array + derived_strides{}; + if (src_strides.empty()) { + ET_CHECK_OR_RETURN_ERROR( + static_cast(dim) <= kTensorDimensionLimit, + InvalidArgument, + "Input %" ET_PRIsize_t " has unsupported rank %zd.", + input_idx, + dim); + std::array + default_dim_order{}; + for (ssize_t i = 0; i < dim; ++i) { + default_dim_order[i] = static_cast(i); + } + const auto src_dim_order = t_src.dim_order(); + ET_CHECK_OK_OR_RETURN_ERROR( + dim_order_to_stride( + t_src.sizes().data(), + src_dim_order.empty() ? default_dim_order.data() + : src_dim_order.data(), + dim, + derived_strides.data()), + "Failed to compute strides for input %" ET_PRIsize_t, + input_idx); + src_strides = {derived_strides.data(), static_cast(dim)}; + } +#endif + + const auto dst_strides = t_dst.strides(); + bool matches = true; + bool src_is_contiguous = true; + int64_t contiguous_stride = 1; + for (ssize_t i = dim - 1; i >= 0; --i) { + if (t_src.size(i) > 1) { + matches = matches && src_strides[i] == dst_strides[i]; + src_is_contiguous = + src_is_contiguous && src_strides[i] == contiguous_stride; + } + contiguous_stride *= t_src.size(i); + } + + if (matches) { + return Error::Ok; + } + if (src_is_contiguous) { + ET_LOG( + Debug, + "Input %" ET_PRIsize_t + " has the default contiguous layout but the method expects a " + "different dim order; treating its data as already in the expected " + "layout. Pass a tensor with the method's dim order to avoid this.", + input_idx); + return Error::Ok; + } + ET_LOG( + Error, + "Input %" ET_PRIsize_t + " has strides that do not match the method's expected layout.", + input_idx); + return Error::InvalidArgument; +} + /** * Private/helper method for populating operator_name from the Operator. * operator_name is a char pointer that is already allocated. The size of @@ -1345,6 +1426,10 @@ Method::set_input(const EValue& input_evalue, size_t input_idx) { resize_tensor(t_dst, t_src.sizes()), "Error resizing tensor at input %" ET_PRIsize_t, input_idx); + ET_CHECK_OK_OR_RETURN_ERROR( + validate_input_tensor_layout(t_dst, t_src, input_idx), + "Error validating tensor layout at input %" ET_PRIsize_t, + input_idx); auto tensor_meta = this->method_meta().input_tensor_meta(input_idx); if (tensor_meta->is_memory_planned()) { ET_CHECK_OK_OR_RETURN_ERROR( diff --git a/runtime/executor/test/CMakeLists.txt b/runtime/executor/test/CMakeLists.txt index af81f4acb4a..74c660dbb8b 100644 --- a/runtime/executor/test/CMakeLists.txt +++ b/runtime/executor/test/CMakeLists.txt @@ -24,7 +24,7 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") powershell ${EXECUTORCH_ROOT}/kernels/test/export_test_model.ps1 -Modules - "\"ModuleAdd,ModuleAddHalf,ModuleAddMul,ModuleDynamicCatUnallocatedIO,ModuleIndex,ModuleMultipleEntry,ModuleSimpleTrain,ModuleStateful\"" + "\"ModuleAdd,ModuleAddHalf,ModuleAddMul,ModuleChannelsLast,ModuleDynamicCatUnallocatedIO,ModuleIndex,ModuleMultipleEntry,ModuleSimpleTrain,ModuleStateful\"" -outDir "${CMAKE_CURRENT_BINARY_DIR}" -CondaEnv @@ -36,7 +36,7 @@ else() -m test.models.export_program --modules - "ModuleAdd,ModuleAddHalf,ModuleAddMul,ModuleDynamicCatUnallocatedIO,ModuleIndex,ModuleMultipleEntry,ModuleSimpleTrain,ModuleStateful" + "ModuleAdd,ModuleAddHalf,ModuleAddMul,ModuleChannelsLast,ModuleDynamicCatUnallocatedIO,ModuleIndex,ModuleMultipleEntry,ModuleSimpleTrain,ModuleStateful" --outdir "${CMAKE_CURRENT_BINARY_DIR}" ) @@ -48,6 +48,7 @@ add_custom_command( "${CMAKE_CURRENT_BINARY_DIR}/ModuleAddMul.pte" "${CMAKE_CURRENT_BINARY_DIR}/ModuleAddMulProgram.pte" "${CMAKE_CURRENT_BINARY_DIR}/ModuleAddMulProgram.ptd" + "${CMAKE_CURRENT_BINARY_DIR}/ModuleChannelsLast.pte" "${CMAKE_CURRENT_BINARY_DIR}/ModuleDynamicCatUnallocatedIO.pte" "${CMAKE_CURRENT_BINARY_DIR}/ModuleIndex.pte" "${CMAKE_CURRENT_BINARY_DIR}/ModuleMultipleEntry.pte" @@ -103,6 +104,7 @@ set(test_env "ET_MODULE_ADD_MUL_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAddMul.pte" "ET_MODULE_ADD_MUL_PROGRAM_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAddMulProgram.pte" "ET_MODULE_ADD_MUL_DATA_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAddMulProgram.ptd" + "ET_MODULE_CHANNELS_LAST_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleChannelsLast.pte" "ET_MODULE_DYNAMIC_CAT_UNALLOCATED_IO_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleDynamicCatUnallocatedIO.pte" "ET_MODULE_INDEX_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleIndex.pte" "ET_MODULE_MULTI_ENTRY_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleMultipleEntry.pte" diff --git a/runtime/executor/test/method_test.cpp b/runtime/executor/test/method_test.cpp index 0310c5d648b..73dc086e02f 100644 --- a/runtime/executor/test/method_test.cpp +++ b/runtime/executor/test/method_test.cpp @@ -82,6 +82,7 @@ class MethodTest : public ::testing::Test { load_program(std::getenv("ET_MODULE_ADD_PATH"), "add"); load_program(std::getenv("ET_MODULE_INDEX_PATH"), "index"); + load_program(std::getenv("ET_MODULE_CHANNELS_LAST_PATH"), "channels_last"); load_program( std::getenv("ET_MODULE_DYNAMIC_CAT_UNALLOCATED_IO_PATH"), "cat"); load_program(std::getenv("ET_MODULE_ADD_MUL_PATH"), "add_mul"); @@ -328,7 +329,7 @@ TEST_F(MethodTest, SetInputRejectsOverflowingSizes) { // multiplication overflows; on 64-bit the resize bounds check rejects it. int32_t sizes[2] = {2000000000, 2000000000}; uint8_t dim_order[2] = {0, 1}; - int32_t strides[2] = {1, 1}; + int32_t strides[2] = {2000000000, 1}; executorch::aten::TensorImpl impl( executorch::aten::ScalarType::Float, 2, @@ -342,6 +343,153 @@ TEST_F(MethodTest, SetInputRejectsOverflowingSizes) { EXPECT_NE(input_err, Error::Ok); } +TEST_F(MethodTest, SetInputRejectsMismatchedLayout) { + // The "cat" model expects a contiguous (2, 4) input. set_input() must not + // reinterpret a differently-laid-out buffer as that input. + ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); + Result method = programs_["cat"]->load_method("forward", &mmm.get()); + ASSERT_EQ(method.error(), Error::Ok); + + float buffer[16] = {}; + int32_t sizes[2] = {2, 4}; + + uint8_t transposed_dim_order[2] = {1, 0}; + int32_t transposed_strides[2] = {1, 2}; + executorch::aten::TensorImpl transposed_impl( + executorch::aten::ScalarType::Float, + 2, + sizes, + buffer, + transposed_dim_order, + transposed_strides); + EXPECT_EQ( + method->set_input(EValue(executorch::aten::Tensor(&transposed_impl)), 0), + Error::InvalidArgument); + + uint8_t contiguous_dim_order[2] = {0, 1}; + int32_t padded_strides[2] = {8, 1}; + executorch::aten::TensorImpl padded_impl( + executorch::aten::ScalarType::Float, + 2, + sizes, + buffer, + contiguous_dim_order, + padded_strides); + EXPECT_EQ( + method->set_input(EValue(executorch::aten::Tensor(&padded_impl)), 0), + Error::InvalidArgument); + + // A dim_order without strides still describes the buffer layout. + executorch::aten::TensorImpl dim_order_only_impl( + executorch::aten::ScalarType::Float, + 2, + sizes, + buffer, + transposed_dim_order); + EXPECT_EQ( + method->set_input( + EValue(executorch::aten::Tensor(&dim_order_only_impl)), 0), + Error::InvalidArgument); + + // TensorImpls built without dim_order/strides use the default contiguous + // layout. + executorch::aten::TensorImpl default_layout_impl( + executorch::aten::ScalarType::Float, 2, sizes, buffer); + EXPECT_EQ( + method->set_input( + EValue(executorch::aten::Tensor(&default_layout_impl)), 0), + Error::Ok); +} + +TEST_F(MethodTest, SetInputAcceptsMatchingChannelsLastLayout) { + ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); + Result method = + programs_["channels_last"]->load_method("forward", &mmm.get()); + ASSERT_EQ(method.error(), Error::Ok); + + float buffer[60]; + for (size_t i = 0; i < 60; ++i) { + buffer[i] = static_cast(i); + } + int32_t sizes[4] = {1, 3, 4, 5}; + uint8_t dim_order[4] = {0, 2, 3, 1}; + int32_t strides[4] = {60, 1, 15, 3}; + executorch::aten::TensorImpl impl( + executorch::aten::ScalarType::Float, + 4, + sizes, + buffer, + dim_order, + strides); + ASSERT_EQ( + method->set_input(EValue(executorch::aten::Tensor(&impl)), 0), Error::Ok); + ASSERT_EQ(method->execute(), Error::Ok); + + const float* out = method->get_output(0).toTensor().const_data_ptr(); + for (size_t i = 0; i < 60; ++i) { + EXPECT_FLOAT_EQ(out[i], 2.f * buffer[i]); + } +} + +TEST_F(MethodTest, SetInputRejectsMismatchedChannelsLastLayout) { + ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); + Result method = + programs_["channels_last"]->load_method("forward", &mmm.get()); + ASSERT_EQ(method.error(), Error::Ok); + + float buffer[60] = {}; + int32_t sizes[4] = {1, 3, 4, 5}; + uint8_t dim_order[4] = {0, 1, 3, 2}; + int32_t strides[4] = {60, 20, 1, 4}; + executorch::aten::TensorImpl impl( + executorch::aten::ScalarType::Float, + 4, + sizes, + buffer, + dim_order, + strides); + EXPECT_EQ( + method->set_input(EValue(executorch::aten::Tensor(&impl)), 0), + Error::InvalidArgument); +} + +TEST_F(MethodTest, SetInputAcceptsContiguousInputForChannelsLastMethod) { + // Callers may wrap data already in the method's layout in a default + // contiguous tensor; this stays accepted for backward compatibility. + ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); + Result method = + programs_["channels_last"]->load_method("forward", &mmm.get()); + ASSERT_EQ(method.error(), Error::Ok); + + float buffer[60] = {}; + int32_t sizes[4] = {1, 3, 4, 5}; + executorch::aten::TensorImpl impl( + executorch::aten::ScalarType::Float, 4, sizes, buffer); + EXPECT_EQ( + method->set_input(EValue(executorch::aten::Tensor(&impl)), 0), Error::Ok); +} + +TEST_F(MethodTest, SetInputAllowsSizeOneStrideMismatch) { + ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); + Result method = programs_["cat"]->load_method("forward", &mmm.get()); + ASSERT_EQ(method.error(), Error::Ok); + + float buffer[4] = {}; + int32_t sizes[2] = {1, 4}; + uint8_t dim_order[2] = {0, 1}; + int32_t strides[2] = {999, 1}; + executorch::aten::TensorImpl impl( + executorch::aten::ScalarType::Float, + 2, + sizes, + buffer, + dim_order, + strides); + + EXPECT_EQ( + method->set_input(EValue(executorch::aten::Tensor(&impl)), 0), Error::Ok); +} + TEST_F(MethodTest, ConstantSegmentTest) { // Execute model with constants stored in segment. ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); diff --git a/runtime/executor/test/targets.bzl b/runtime/executor/test/targets.bzl index 314eaae3c8a..63c4fbaffee 100644 --- a/runtime/executor/test/targets.bzl +++ b/runtime/executor/test/targets.bzl @@ -123,6 +123,7 @@ def define_common_targets(is_fbcode = False): # intentionally don't work in xplat (since they're host-only tools). "ET_MODULE_ADD_HALF_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleAddHalf.pte])", "ET_MODULE_ADD_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleAdd.pte])", + "ET_MODULE_CHANNELS_LAST_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleChannelsLast.pte])", "ET_MODULE_DYNAMIC_CAT_UNALLOCATED_IO_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleDynamicCatUnallocatedIO.pte])", "ET_MODULE_INDEX_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleIndex.pte])", "ET_MODULE_ADD_MUL_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleAddMul.pte])", diff --git a/test/models/export_program.py b/test/models/export_program.py index 966233e2cea..a4b08fa2ad6 100644 --- a/test/models/export_program.py +++ b/test/models/export_program.py @@ -115,6 +115,14 @@ def get_random_inputs(self): ) +class ModuleChannelsLast(nn.Module): + def forward(self, x): + return x + x + + def get_random_inputs(self): + return (torch.randn(1, 3, 4, 5).to(memory_format=torch.channels_last),) + + class ModuleDynamicCatUnallocatedIO(nn.Module): def __init__(self): super(ModuleDynamicCatUnallocatedIO, self).__init__() diff --git a/test/models/targets.bzl b/test/models/targets.bzl index bd4df80a50a..84daaffdea4 100644 --- a/test/models/targets.bzl +++ b/test/models/targets.bzl @@ -63,6 +63,7 @@ def define_common_targets(): "ModuleAddHalf", "ModuleAddMul", "ModuleBasic", + "ModuleChannelsLast", "ModuleKVCacheCachePos", "ModuleKVCacheInputPos", "ModuleMultipleEntry",