From 3df465702649962247d99d391b58fe49ec736c98 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 14 Aug 2026 14:56:01 +0200 Subject: [PATCH 1/8] Add is_like_v --- include/xsimd/utils/xsimd_type_traits.hpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/xsimd/utils/xsimd_type_traits.hpp b/include/xsimd/utils/xsimd_type_traits.hpp index 5c272469b..2313b72e8 100644 --- a/include/xsimd/utils/xsimd_type_traits.hpp +++ b/include/xsimd/utils/xsimd_type_traits.hpp @@ -122,6 +122,7 @@ namespace xsimd * Certain platforms have different types (*i.e.* not aliases) between * ``char`` and ``int8_t``, or ``long long`` and ``int{32,64}_t``, with SIMD * intrinsicts only defined for some of them. + * Similarly C++23 sized floating point type may be separate types from float/double. * Handling them requires to cast to a known predictable type. * * @tparam T arithmetic type to project from. @@ -129,6 +130,20 @@ namespace xsimd template using map_to_sized_type_t = typename detail::remap_num::type; + /** + * Check that two type are theoretically the same. + * + * @see map_to_sized_type_t + */ + template + inline constexpr bool is_like_v = std::is_same_v, map_to_sized_type_t>; + + /** + * Check that a type is theoretically the same as any in the set. + */ + template + inline constexpr bool is_like_any_v = (... || is_like_v); + /** * @ingroup type_traits * From 9db9547d2d4d19ab01e97760baae2df0246eb7b6 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 14 Aug 2026 17:39:50 +0200 Subject: [PATCH 2/8] WIP use overloads --- include/xsimd/arch/xsimd_common_fwd.hpp | 4 + include/xsimd/arch/xsimd_neon.hpp | 788 ++++++------------------ include/xsimd/arch/xsimd_neon64.hpp | 17 - include/xsimd/overload/neon.hpp | 539 ++++++++++++++++ 4 files changed, 731 insertions(+), 617 deletions(-) create mode 100644 include/xsimd/overload/neon.hpp diff --git a/include/xsimd/arch/xsimd_common_fwd.hpp b/include/xsimd/arch/xsimd_common_fwd.hpp index 139b0b282..538e73743 100644 --- a/include/xsimd/arch/xsimd_common_fwd.hpp +++ b/include/xsimd/arch/xsimd_common_fwd.hpp @@ -101,6 +101,10 @@ namespace xsimd XSIMD_INLINE void store_masked(T_out* mem, batch const& src, batch_bool_constant mask, alignment, requires_arch) noexcept; template XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept; + template + XSIMD_INLINE batch avg(batch const& x, batch const& y, requires_arch) noexcept; + template + XSIMD_INLINE batch avgr(batch const& x, batch const& y, requires_arch) noexcept; // Forward declarations for pack-level helpers namespace detail diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index 6acbe981a..fb154e740 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -12,6 +12,7 @@ #ifndef XSIMD_NEON_HPP #define XSIMD_NEON_HPP +#include "../overload/neon.hpp" #include "../types/xsimd_batch_fwd.hpp" #include "../types/xsimd_neon_register.hpp" #include "../types/xsimd_utils.hpp" @@ -32,6 +33,46 @@ namespace xsimd { using namespace types; + template + XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept; + + template + XSIMD_INLINE batch bitwise_cast(batch const& arg, batch const&, requires_arch) noexcept; + + template + XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept; + + template + XSIMD_INLINE batch bitwise_and(batch const& lhs, batch const& rhs, requires_arch) noexcept; + template + XSIMD_INLINE batch_bool bitwise_and(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch add(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch sadd(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch sub(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch ssub(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch avg(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch avgr(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch_bool eq(batch const& lhs, batch const& rhs, requires_arch) noexcept; + template + XSIMD_INLINE batch_bool eq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept; + namespace detail { /************************************** @@ -51,328 +92,76 @@ namespace xsimd * bitwise_cast * ****************/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(uint8x16_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_u8_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_u8_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_u8_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_u8_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_u8_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_u8_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_u8_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint8x16_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_u8_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_s8_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(int8x16_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_s8_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_s8_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_s8_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_s8_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_s8_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_s8_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int8x16_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_s8_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_u16_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_u16_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(uint16x8_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_u16_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_u16_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_u16_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_u16_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_u16_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint16x8_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_u16_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_s16_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_s16_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_s16_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(int16x8_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_s16_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_s16_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_s16_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_s16_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int16x8_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_s16_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_u32_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_u32_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_u32_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_u32_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(uint32x4_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_u32_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_u32_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_u32_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint32x4_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_u32_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_s32_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_s32_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_s32_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_s32_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_s32_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(int32x4_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_s32_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_s32_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int32x4_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_s32_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_u64_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_u64_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_u64_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_u64_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_u64_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_u64_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(uint64x2_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_u64_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE uint64x2_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_u64_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_s64_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_s64_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_s64_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_s64_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_s64_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_s64_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_s64_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(int64x2_t a) noexcept { return a; } - template && std::is_same_v, int> = 0> - XSIMD_INLINE int64x2_t x_vreinterpretq(float32x4_t a) noexcept { return vreinterpretq_s64_f32(a); } - - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(uint8x16_t a) noexcept { return vreinterpretq_f32_u8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(int8x16_t a) noexcept { return vreinterpretq_f32_s8(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(uint16x8_t a) noexcept { return vreinterpretq_f32_u16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(int16x8_t a) noexcept { return vreinterpretq_f32_s16(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(uint32x4_t a) noexcept { return vreinterpretq_f32_u32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(int32x4_t a) noexcept { return vreinterpretq_f32_s32(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(uint64x2_t a) noexcept { return vreinterpretq_f32_u64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(int64x2_t a) noexcept { return vreinterpretq_f32_s64(a); } - template && std::is_same_v, int> = 0> - XSIMD_INLINE float32x4_t x_vreinterpretq(float32x4_t a) noexcept { return a; } - } - template XSIMD_INLINE batch bitwise_cast(batch const& arg, batch const&, requires_arch) noexcept { - using src_register_type = typename batch::register_type; - return wrap::x_vreinterpretq, map_to_sized_type_t>(src_register_type(arg)); + if constexpr (is_like_v) + { + return arg.data; + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_u8_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_s8_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_u16_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_s16_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_u32_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_s32_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_u64_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_s64_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_f32_batch(arg); + } + else if constexpr (is_like_v) + { + return overload::vreinterpretq_f64_batch(arg); + } } /************* * broadcast * *************/ - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_u8(uint8_t(val)); - } - - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_s8(int8_t(val)); - } - - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_u16(uint16_t(val)); - } - - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_s16(int16_t(val)); - } - - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_u32(uint32_t(val)); - } - - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_s32(int32_t(val)); - } - - template = 0> - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return vdupq_n_u64(uint64_t(val)); - } - - template = 0> + template XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept { - return vdupq_n_s64(int64_t(val)); - } - - template - XSIMD_INLINE batch broadcast(float val, requires_arch) noexcept - { - return vdupq_n_f32(val); + return overload::vdupq_n_batch(val); } /************* * from_bool * *************/ - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vandq_u8(arg, vdupq_n_u8(1)); - } - - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vreinterpretq_s8_u8(vandq_u8(arg.data, vdupq_n_u8(1))); - } - - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vandq_u16(arg, vdupq_n_u16(1)); - } - - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vreinterpretq_s16_u16(vandq_u16(arg.data, vdupq_n_u16(1))); - } - - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vandq_u32(arg, vdupq_n_u32(1)); - } - - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vreinterpretq_s32_u32(vandq_u32(arg.data, vdupq_n_u32(1))); - } - - template = 0> - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vandq_u64(arg, vdupq_n_u64(1)); - } - - template = 0> + template XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept { - return vreinterpretq_s64_u64(vandq_u64(arg.data, vdupq_n_u64(1))); - } - - template - XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept - { - return vreinterpretq_f32_u32(vandq_u32(arg, vreinterpretq_u32_f32(vdupq_n_f32(1.f)))); + using uint = sized_uint_t; + auto const ones = bitwise_cast(broadcast(T(1), A {}), {}, A {}); + auto const res = batch(overload::vandq_batch(batch(arg.data), ones)); + return bitwise_cast(res, {}, A {}); } /******** @@ -763,276 +552,129 @@ namespace xsimd * neg * *******/ - template = 0> - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vreinterpretq_u8_s8(vnegq_s8(vreinterpretq_s8_u8(rhs))); - } - - template = 0> - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vnegq_s8(rhs); - } - - template = 0> - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vreinterpretq_u16_s16(vnegq_s16(vreinterpretq_s16_u16(rhs))); - } - - template = 0> - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vnegq_s16(rhs); - } - - template = 0> - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vreinterpretq_u32_s32(vnegq_s32(vreinterpretq_s32_u32(rhs))); - } - - template = 0> - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vnegq_s32(rhs); - } - - template = 0> + template XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept { - return 0 - rhs; - } - - template - XSIMD_INLINE batch neg(batch const& rhs, requires_arch) noexcept - { - return vnegq_f32(rhs); + using sint = sized_int_t; + if constexpr (std::is_signed_v && overload::vnegq_is_supported()) + { + return overload::vnegq_batch(rhs); + } + else if constexpr (overload::vnegq_is_supported()) + { + // vnegq has no unsigned overload, but the two's complement result is the same + auto const res = batch(overload::vnegq_batch(bitwise_cast(rhs, {}, A {}))); + return bitwise_cast(res, {}, A {}); + } + else + { + return sub(broadcast(0, A {}), rhs); + } } /******* * add * *******/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vaddq(uint8x16_t a, uint8x16_t b) noexcept { return vaddq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vaddq(int8x16_t a, int8x16_t b) noexcept { return vaddq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vaddq(uint16x8_t a, uint16x8_t b) noexcept { return vaddq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vaddq(int16x8_t a, int16x8_t b) noexcept { return vaddq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vaddq(uint32x4_t a, uint32x4_t b) noexcept { return vaddq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vaddq(int32x4_t a, int32x4_t b) noexcept { return vaddq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint64x2_t x_vaddq(uint64x2_t a, uint64x2_t b) noexcept { return vaddq_u64(a, b); } - template , int> = 0> - XSIMD_INLINE int64x2_t x_vaddq(int64x2_t a, int64x2_t b) noexcept { return vaddq_s64(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vaddq(float32x4_t a, float32x4_t b) noexcept { return vaddq_f32(a, b); } - } - - template = 0> + template XSIMD_INLINE batch add(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vaddq>(register_type(lhs), register_type(rhs)); + return overload::vaddq_batch(lhs, rhs); } /******* * avg * *******/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vhaddq(uint8x16_t a, uint8x16_t b) noexcept { return vhaddq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vhaddq(uint16x8_t a, uint16x8_t b) noexcept { return vhaddq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vhaddq(uint32x4_t a, uint32x4_t b) noexcept { return vhaddq_u32(a, b); } - } - - template && sizeof(T) != 8)>> + template XSIMD_INLINE batch avg(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vhaddq>(register_type(lhs), register_type(rhs)); + if constexpr (overload::vhaddq_is_supported()) + { + return overload::vhaddq_batch(lhs, rhs); + } + else + { + return kernel::avg(lhs, rhs, common {}); + } } /******** * avgr * ********/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vrhaddq(uint8x16_t a, uint8x16_t b) noexcept { return vrhaddq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vrhaddq(uint16x8_t a, uint16x8_t b) noexcept { return vrhaddq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vrhaddq(uint32x4_t a, uint32x4_t b) noexcept { return vrhaddq_u32(a, b); } - } - - template && sizeof(T) != 8)>> + template XSIMD_INLINE batch avgr(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vrhaddq>(register_type(lhs), register_type(rhs)); + if constexpr (overload::vrhaddq_is_supported()) + { + return overload::vrhaddq_batch(lhs, rhs); + } + else + { + return kernel::avgr(lhs, rhs, common {}); + } } /******** * sadd * ********/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vqaddq(uint8x16_t a, uint8x16_t b) noexcept { return vqaddq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vqaddq(int8x16_t a, int8x16_t b) noexcept { return vqaddq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vqaddq(uint16x8_t a, uint16x8_t b) noexcept { return vqaddq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vqaddq(int16x8_t a, int16x8_t b) noexcept { return vqaddq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vqaddq(uint32x4_t a, uint32x4_t b) noexcept { return vqaddq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vqaddq(int32x4_t a, int32x4_t b) noexcept { return vqaddq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint64x2_t x_vqaddq(uint64x2_t a, uint64x2_t b) noexcept { return vqaddq_u64(a, b); } - template , int> = 0> - XSIMD_INLINE int64x2_t x_vqaddq(int64x2_t a, int64x2_t b) noexcept { return vqaddq_s64(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vqaddq(float32x4_t a, float32x4_t b) noexcept { return vaddq_f32(a, b); } - } - - template = 0> + template XSIMD_INLINE batch sadd(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vqaddq>(register_type(lhs), register_type(rhs)); + if constexpr (std::is_floating_point_v) + { + return add(lhs, rhs, A {}); + } + else + { + return overload::vqaddq_batch(lhs, rhs); + } } /******* * sub * *******/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vsubq(uint8x16_t a, uint8x16_t b) noexcept { return vsubq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vsubq(int8x16_t a, int8x16_t b) noexcept { return vsubq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vsubq(uint16x8_t a, uint16x8_t b) noexcept { return vsubq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vsubq(int16x8_t a, int16x8_t b) noexcept { return vsubq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vsubq(uint32x4_t a, uint32x4_t b) noexcept { return vsubq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vsubq(int32x4_t a, int32x4_t b) noexcept { return vsubq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint64x2_t x_vsubq(uint64x2_t a, uint64x2_t b) noexcept { return vsubq_u64(a, b); } - template , int> = 0> - XSIMD_INLINE int64x2_t x_vsubq(int64x2_t a, int64x2_t b) noexcept { return vsubq_s64(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vsubq(float32x4_t a, float32x4_t b) noexcept { return vsubq_f32(a, b); } - } - - template = 0> + template XSIMD_INLINE batch sub(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vsubq>(register_type(lhs), register_type(rhs)); + return overload::vsubq_batch(lhs, rhs); } /******** * ssub * ********/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vqsubq(uint8x16_t a, uint8x16_t b) noexcept { return vqsubq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vqsubq(int8x16_t a, int8x16_t b) noexcept { return vqsubq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vqsubq(uint16x8_t a, uint16x8_t b) noexcept { return vqsubq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vqsubq(int16x8_t a, int16x8_t b) noexcept { return vqsubq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vqsubq(uint32x4_t a, uint32x4_t b) noexcept { return vqsubq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vqsubq(int32x4_t a, int32x4_t b) noexcept { return vqsubq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint64x2_t x_vqsubq(uint64x2_t a, uint64x2_t b) noexcept { return vqsubq_u64(a, b); } - template , int> = 0> - XSIMD_INLINE int64x2_t x_vqsubq(int64x2_t a, int64x2_t b) noexcept { return vqsubq_s64(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vqsubq(float32x4_t a, float32x4_t b) noexcept { return vsubq_f32(a, b); } - } - - template = 0> + template XSIMD_INLINE batch ssub(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vqsubq>(register_type(lhs), register_type(rhs)); + if constexpr (std::is_floating_point_v) + { + return sub(lhs, rhs, A {}); + } + else + { + return overload::vqsubq_batch(lhs, rhs); + } } /******* * mul * *******/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vmulq(uint8x16_t a, uint8x16_t b) noexcept { return vmulq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vmulq(int8x16_t a, int8x16_t b) noexcept { return vmulq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vmulq(uint16x8_t a, uint16x8_t b) noexcept { return vmulq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vmulq(int16x8_t a, int16x8_t b) noexcept { return vmulq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vmulq(uint32x4_t a, uint32x4_t b) noexcept { return vmulq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vmulq(int32x4_t a, int32x4_t b) noexcept { return vmulq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vmulq(float32x4_t a, float32x4_t b) noexcept { return vmulq_f32(a, b); } - } - - template = 0> + template XSIMD_INLINE batch mul(batch const& lhs, batch const& rhs, requires_arch) noexcept { - using register_type = typename batch::register_type; - return wrap::x_vmulq>(register_type(lhs), register_type(rhs)); + if constexpr (overload::vmulq_is_supported()) + { + return overload::vmulq_batch(lhs, rhs); + } + else + { + return mul(lhs, rhs, common {}); + } } /********* @@ -1181,63 +823,29 @@ namespace xsimd * eq * ******/ - namespace wrap - { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vceqq(uint8x16_t a, uint8x16_t b) noexcept { return vceqq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vceqq(int8x16_t a, int8x16_t b) noexcept { return vceqq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vceqq(uint16x8_t a, uint16x8_t b) noexcept { return vceqq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vceqq(int16x8_t a, int16x8_t b) noexcept { return vceqq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vceqq(uint32x4_t a, uint32x4_t b) noexcept { return vceqq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vceqq(int32x4_t a, int32x4_t b) noexcept { return vceqq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vceqq(float32x4_t a, float32x4_t b) noexcept { return vceqq_f32(a, b); } - } - - template = 0> - XSIMD_INLINE batch_bool eq(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - using register_type = typename batch::register_type; - return wrap::x_vceqq>(register_type(lhs), register_type(rhs)); - } - - template = 0> - XSIMD_INLINE batch_bool eq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept - { - using register_type = typename batch_bool::register_type; - return wrap::x_vceqq>(register_type(lhs), register_type(rhs)); - } - - template = 0> - XSIMD_INLINE batch_bool eq(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - auto eq32 = vceqq_u32(vreinterpretq_u32_u64(lhs.data), vreinterpretq_u32_u64(rhs.data)); - auto rev32 = vrev64q_u32(eq32); - auto eq64 = vandq_u32(eq32, rev32); - return batch_bool(vreinterpretq_u64_u32(eq64)); - } - - template = 0> + template XSIMD_INLINE batch_bool eq(batch const& lhs, batch const& rhs, requires_arch) noexcept { - auto eq32 = vceqq_u32(vreinterpretq_u32_s64(lhs.data), vreinterpretq_u32_s64(rhs.data)); - auto rev32 = vrev64q_u32(eq32); - auto eq64 = vandq_u32(eq32, rev32); - return batch_bool(vreinterpretq_u64_u32(eq64)); + if constexpr (overload::vceqq_is_supported()) + { + return overload::vceqq_batch(lhs, rhs); + } + else if constexpr (std::is_integral_v && sizeof(T) == 8) + { + auto lhs32 = bitwise_cast(lhs, {}, A {}); + auto rhs32 = bitwise_cast(rhs, {}, A {}); + auto eq32 = eq(lhs, rhs, A {}); + auto rev32 = overload::vrev64q_batch(eq32); + auto eq64 = bitwise_and(eq32, rev32, A {}); + return { bitwise_cast(eq64, {}, A {}) }; + } } - template = 0> + template XSIMD_INLINE batch_bool eq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - return eq(batch { lhs.data }, batch { rhs.data }, A {}); + using uint = sized_uint_t; + return { eq(batch(lhs.data), batch(rhs.data), A {}).data }; } /************* @@ -1457,47 +1065,27 @@ namespace xsimd * bitwise_and * ***************/ - namespace wrap + template + XSIMD_INLINE batch bitwise_and(batch const& lhs, batch const& rhs, requires_arch) noexcept { - // TODO(c++17): Make a single function with if constexpr switch - // Templating on the scalar type `T` is required because in some compilers (e.g. MSVC) - // the vector types are all aliases of the same type. - template , int> = 0> - XSIMD_INLINE uint8x16_t x_vandq(uint8x16_t a, uint8x16_t b) noexcept { return vandq_u8(a, b); } - template , int> = 0> - XSIMD_INLINE int8x16_t x_vandq(int8x16_t a, int8x16_t b) noexcept { return vandq_s8(a, b); } - template , int> = 0> - XSIMD_INLINE uint16x8_t x_vandq(uint16x8_t a, uint16x8_t b) noexcept { return vandq_u16(a, b); } - template , int> = 0> - XSIMD_INLINE int16x8_t x_vandq(int16x8_t a, int16x8_t b) noexcept { return vandq_s16(a, b); } - template , int> = 0> - XSIMD_INLINE uint32x4_t x_vandq(uint32x4_t a, uint32x4_t b) noexcept { return vandq_u32(a, b); } - template , int> = 0> - XSIMD_INLINE int32x4_t x_vandq(int32x4_t a, int32x4_t b) noexcept { return vandq_s32(a, b); } - template , int> = 0> - XSIMD_INLINE uint64x2_t x_vandq(uint64x2_t a, uint64x2_t b) noexcept { return vandq_u64(a, b); } - template , int> = 0> - XSIMD_INLINE int64x2_t x_vandq(int64x2_t a, int64x2_t b) noexcept { return vandq_s64(a, b); } - template , int> = 0> - XSIMD_INLINE float32x4_t x_vandq(float32x4_t a, float32x4_t b) noexcept + if constexpr (std::is_floating_point_v) { - return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a), - vreinterpretq_u32_f32(b))); + using uint = sized_uint_t; + auto ulhs = bitwise_cast(lhs, {}, A {}); + auto urhs = bitwise_cast(rhs, {}, A {}); + return bitwise_cast(bitwise_and(ulhs, urhs, A {}), {}, A {}); + } + else + { + return overload::vandq_batch(lhs, rhs); } } - template = 0> - XSIMD_INLINE batch bitwise_and(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - using register_type = typename batch::register_type; - return wrap::x_vandq>(register_type(lhs), register_type(rhs)); - } - - template = 0> + template XSIMD_INLINE batch_bool bitwise_and(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { - using register_type = typename batch_bool::register_type; - return wrap::x_vandq>(register_type(lhs), register_type(rhs)); + using uint = sized_uint_t; + return { bitwise_and(batch(lhs.data), batch(rhs.data), A {}).data }; } /************** diff --git a/include/xsimd/arch/xsimd_neon64.hpp b/include/xsimd/arch/xsimd_neon64.hpp index 809ef8fa5..0e067b500 100644 --- a/include/xsimd/arch/xsimd_neon64.hpp +++ b/include/xsimd/arch/xsimd_neon64.hpp @@ -109,23 +109,6 @@ namespace xsimd return any(batch_bool(vreinterpretq_u32_u64(arg)), neon64 {}); } - /************* - * broadcast * - *************/ - - // Required to avoid ambiguous call - template - XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept - { - return broadcast(val, neon {}); - } - - template - XSIMD_INLINE batch broadcast(double val, requires_arch) noexcept - { - return vdupq_n_f64(val); - } - /************* * from_bool * *************/ diff --git a/include/xsimd/overload/neon.hpp b/include/xsimd/overload/neon.hpp new file mode 100644 index 000000000..f67be969c --- /dev/null +++ b/include/xsimd/overload/neon.hpp @@ -0,0 +1,539 @@ + +/**************************************************************************** + * Copyright (c) xsimd contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_OVERLOAD_NEON_HPP +#define XSIMD_OVERLOAD_NEON_HPP + +#include "../config/xsimd_macros.hpp" +#include "../types/xsimd_batch.hpp" +#include "../utils/xsimd_type_traits.hpp" + +#include + +namespace xsimd::overload { + +template +XSIMD_INLINE constexpr bool vget_low_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vget_low_batch(batch a) { + static_assert(vget_low_is_supported(), "vget_low unsupported"); + if constexpr(is_like_v) { return vget_low_s8(a); } + else if constexpr(is_like_v) { return vget_low_u8(a); } + else if constexpr(is_like_v) { return vget_low_s16(a); } + else if constexpr(is_like_v) { return vget_low_u16(a); } + else if constexpr(is_like_v) { return vget_low_s32(a); } + else if constexpr(is_like_v) { return vget_low_u32(a); } + else if constexpr(is_like_v) { return vget_low_s64(a); } + else if constexpr(is_like_v) { return vget_low_u64(a); } + else if constexpr(is_like_v) { return vget_low_f32(a); } + else if constexpr(is_like_v) { return vget_low_f64(a); } + else { static_assert(false, "unsupported type for vget_low"); } +} + +template +XSIMD_INLINE constexpr bool vget_high_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vget_high_batch(batch a) { + static_assert(vget_high_is_supported(), "vget_high unsupported"); + if constexpr(is_like_v) { return vget_high_s8(a); } + else if constexpr(is_like_v) { return vget_high_u8(a); } + else if constexpr(is_like_v) { return vget_high_s16(a); } + else if constexpr(is_like_v) { return vget_high_u16(a); } + else if constexpr(is_like_v) { return vget_high_s32(a); } + else if constexpr(is_like_v) { return vget_high_u32(a); } + else if constexpr(is_like_v) { return vget_high_s64(a); } + else if constexpr(is_like_v) { return vget_high_u64(a); } + else if constexpr(is_like_v) { return vget_high_f32(a); } + else if constexpr(is_like_v) { return vget_high_f64(a); } + else { static_assert(false, "unsupported type for vget_high"); } +} + +template +XSIMD_INLINE constexpr bool vdupq_n_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vdupq_n_batch(T a) { + static_assert(vdupq_n_is_supported(), "vdupq_n unsupported"); + if constexpr(is_like_v) { return vdupq_n_s8(a); } + else if constexpr(is_like_v) { return vdupq_n_u8(a); } + else if constexpr(is_like_v) { return vdupq_n_s16(a); } + else if constexpr(is_like_v) { return vdupq_n_u16(a); } + else if constexpr(is_like_v) { return vdupq_n_s32(a); } + else if constexpr(is_like_v) { return vdupq_n_u32(a); } + else if constexpr(is_like_v) { return vdupq_n_s64(a); } + else if constexpr(is_like_v) { return vdupq_n_u64(a); } + else if constexpr(is_like_v) { return vdupq_n_f32(a); } + else if constexpr(is_like_v) { return vdupq_n_f64(a); } + else { static_assert(false, "unsupported type for vdupq_n"); } +} + +template +XSIMD_INLINE constexpr bool vrev64q_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vrev64q_batch(batch a) { + static_assert(vrev64q_is_supported(), "vrev64q unsupported"); + if constexpr(is_like_v) { return vrev64q_s8(a); } + else if constexpr(is_like_v) { return vrev64q_u8(a); } + else if constexpr(is_like_v) { return vrev64q_s16(a); } + else if constexpr(is_like_v) { return vrev64q_u16(a); } + else if constexpr(is_like_v) { return vrev64q_s32(a); } + else if constexpr(is_like_v) { return vrev64q_u32(a); } + else if constexpr(is_like_v) { return vrev64q_f32(a); } + else { static_assert(false, "unsupported type for vrev64q"); } +} + +template +XSIMD_INLINE constexpr bool vandq_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vandq_batch(batch a, batch b) { + static_assert(vandq_is_supported(), "vandq unsupported"); + if constexpr(is_like_v) { return vandq_s8(a, b); } + else if constexpr(is_like_v) { return vandq_u8(a, b); } + else if constexpr(is_like_v) { return vandq_s16(a, b); } + else if constexpr(is_like_v) { return vandq_u16(a, b); } + else if constexpr(is_like_v) { return vandq_s32(a, b); } + else if constexpr(is_like_v) { return vandq_u32(a, b); } + else if constexpr(is_like_v) { return vandq_s64(a, b); } + else if constexpr(is_like_v) { return vandq_u64(a, b); } + else { static_assert(false, "unsupported type for vandq"); } +} + +template +XSIMD_INLINE constexpr bool vceqq_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vceqq_batch(batch a, batch b) { + static_assert(vceqq_is_supported(), "vceqq unsupported"); + if constexpr(is_like_v) { return vceqq_s8(a, b); } + else if constexpr(is_like_v) { return vceqq_u8(a, b); } + else if constexpr(is_like_v) { return vceqq_s16(a, b); } + else if constexpr(is_like_v) { return vceqq_u16(a, b); } + else if constexpr(is_like_v) { return vceqq_s32(a, b); } + else if constexpr(is_like_v) { return vceqq_u32(a, b); } + else if constexpr(is_like_v) { return vceqq_s64(a, b); } + else if constexpr(is_like_v) { return vceqq_u64(a, b); } + else if constexpr(is_like_v) { return vceqq_f32(a, b); } + else if constexpr(is_like_v) { return vceqq_f64(a, b); } + else { static_assert(false, "unsupported type for vceqq"); } +} + +template +XSIMD_INLINE constexpr bool vaddq_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vaddq_batch(batch a, batch b) { + static_assert(vaddq_is_supported(), "vaddq unsupported"); + if constexpr(is_like_v) { return vaddq_s8(a, b); } + else if constexpr(is_like_v) { return vaddq_u8(a, b); } + else if constexpr(is_like_v) { return vaddq_s16(a, b); } + else if constexpr(is_like_v) { return vaddq_u16(a, b); } + else if constexpr(is_like_v) { return vaddq_s32(a, b); } + else if constexpr(is_like_v) { return vaddq_u32(a, b); } + else if constexpr(is_like_v) { return vaddq_s64(a, b); } + else if constexpr(is_like_v) { return vaddq_u64(a, b); } + else if constexpr(is_like_v) { return vaddq_f32(a, b); } + else if constexpr(is_like_v) { return vaddq_f64(a, b); } + else { static_assert(false, "unsupported type for vaddq"); } +} + +template +XSIMD_INLINE constexpr bool vhaddq_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vhaddq_batch(batch a, batch b) { + static_assert(vhaddq_is_supported(), "vhaddq unsupported"); + if constexpr(is_like_v) { return vhaddq_s8(a, b); } + else if constexpr(is_like_v) { return vhaddq_u8(a, b); } + else if constexpr(is_like_v) { return vhaddq_s16(a, b); } + else if constexpr(is_like_v) { return vhaddq_u16(a, b); } + else if constexpr(is_like_v) { return vhaddq_s32(a, b); } + else if constexpr(is_like_v) { return vhaddq_u32(a, b); } + else { static_assert(false, "unsupported type for vhaddq"); } +} + +template +XSIMD_INLINE constexpr bool vrhaddq_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vrhaddq_batch(batch a, batch b) { + static_assert(vrhaddq_is_supported(), "vrhaddq unsupported"); + if constexpr(is_like_v) { return vrhaddq_s8(a, b); } + else if constexpr(is_like_v) { return vrhaddq_u8(a, b); } + else if constexpr(is_like_v) { return vrhaddq_s16(a, b); } + else if constexpr(is_like_v) { return vrhaddq_u16(a, b); } + else if constexpr(is_like_v) { return vrhaddq_s32(a, b); } + else if constexpr(is_like_v) { return vrhaddq_u32(a, b); } + else { static_assert(false, "unsupported type for vrhaddq"); } +} + +template +XSIMD_INLINE constexpr bool vqaddq_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vqaddq_batch(batch a, batch b) { + static_assert(vqaddq_is_supported(), "vqaddq unsupported"); + if constexpr(is_like_v) { return vqaddq_s8(a, b); } + else if constexpr(is_like_v) { return vqaddq_u8(a, b); } + else if constexpr(is_like_v) { return vqaddq_s16(a, b); } + else if constexpr(is_like_v) { return vqaddq_u16(a, b); } + else if constexpr(is_like_v) { return vqaddq_s32(a, b); } + else if constexpr(is_like_v) { return vqaddq_u32(a, b); } + else if constexpr(is_like_v) { return vqaddq_s64(a, b); } + else if constexpr(is_like_v) { return vqaddq_u64(a, b); } + else { static_assert(false, "unsupported type for vqaddq"); } +} + +template +XSIMD_INLINE constexpr bool vnegq_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vnegq_batch(batch a) { + static_assert(vnegq_is_supported(), "vnegq unsupported"); + if constexpr(is_like_v) { return vnegq_s8(a); } + else if constexpr(is_like_v) { return vnegq_s16(a); } + else if constexpr(is_like_v) { return vnegq_s32(a); } + else if constexpr(is_like_v) { return vnegq_s64(a); } + else if constexpr(is_like_v) { return vnegq_f32(a); } + else if constexpr(is_like_v) { return vnegq_f64(a); } + else { static_assert(false, "unsupported type for vnegq"); } +} + +template +XSIMD_INLINE constexpr bool vsubq_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vsubq_batch(batch a, batch b) { + static_assert(vsubq_is_supported(), "vsubq unsupported"); + if constexpr(is_like_v) { return vsubq_s8(a, b); } + else if constexpr(is_like_v) { return vsubq_u8(a, b); } + else if constexpr(is_like_v) { return vsubq_s16(a, b); } + else if constexpr(is_like_v) { return vsubq_u16(a, b); } + else if constexpr(is_like_v) { return vsubq_s32(a, b); } + else if constexpr(is_like_v) { return vsubq_u32(a, b); } + else if constexpr(is_like_v) { return vsubq_s64(a, b); } + else if constexpr(is_like_v) { return vsubq_u64(a, b); } + else if constexpr(is_like_v) { return vsubq_f32(a, b); } + else if constexpr(is_like_v) { return vsubq_f64(a, b); } + else { static_assert(false, "unsupported type for vsubq"); } +} + +template +XSIMD_INLINE constexpr bool vqsubq_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vqsubq_batch(batch a, batch b) { + static_assert(vqsubq_is_supported(), "vqsubq unsupported"); + if constexpr(is_like_v) { return vqsubq_s8(a, b); } + else if constexpr(is_like_v) { return vqsubq_u8(a, b); } + else if constexpr(is_like_v) { return vqsubq_s16(a, b); } + else if constexpr(is_like_v) { return vqsubq_u16(a, b); } + else if constexpr(is_like_v) { return vqsubq_s32(a, b); } + else if constexpr(is_like_v) { return vqsubq_u32(a, b); } + else if constexpr(is_like_v) { return vqsubq_s64(a, b); } + else if constexpr(is_like_v) { return vqsubq_u64(a, b); } + else { static_assert(false, "unsupported type for vqsubq"); } +} + +template +XSIMD_INLINE constexpr bool vmull_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vmull_batch(batch a, batch b) { + static_assert(vmull_is_supported(), "vmull unsupported"); + if constexpr(is_like_v) { return vmull_s8(a, b); } + else if constexpr(is_like_v) { return vmull_u8(a, b); } + else if constexpr(is_like_v) { return vmull_s16(a, b); } + else if constexpr(is_like_v) { return vmull_u16(a, b); } + else if constexpr(is_like_v) { return vmull_s32(a, b); } + else if constexpr(is_like_v) { return vmull_u32(a, b); } + else { static_assert(false, "unsupported type for vmull"); } +} + +template +XSIMD_INLINE constexpr bool vmulq_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vmulq_batch(batch a, batch b) { + static_assert(vmulq_is_supported(), "vmulq unsupported"); + if constexpr(is_like_v) { return vmulq_s8(a, b); } + else if constexpr(is_like_v) { return vmulq_u8(a, b); } + else if constexpr(is_like_v) { return vmulq_s16(a, b); } + else if constexpr(is_like_v) { return vmulq_u16(a, b); } + else if constexpr(is_like_v) { return vmulq_s32(a, b); } + else if constexpr(is_like_v) { return vmulq_u32(a, b); } + else if constexpr(is_like_v) { return vmulq_f32(a, b); } + else if constexpr(is_like_v) { return vmulq_f64(a, b); } + else { static_assert(false, "unsupported type for vmulq"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_s8_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_s8_batch(batch a) { + static_assert(vreinterpretq_s8_is_supported(), "vreinterpretq_s8 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_s8_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s8_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_s8"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_u8_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_u8_batch(batch a) { + static_assert(vreinterpretq_u8_is_supported(), "vreinterpretq_u8 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_u8_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u8_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_u8"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_s16_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_s16_batch(batch a) { + static_assert(vreinterpretq_s16_is_supported(), "vreinterpretq_s16 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_s16_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s16_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_s16"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_u16_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_u16_batch(batch a) { + static_assert(vreinterpretq_u16_is_supported(), "vreinterpretq_u16 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_u16_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u16_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_u16"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_s32_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_s32_batch(batch a) { + static_assert(vreinterpretq_s32_is_supported(), "vreinterpretq_s32 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_s32_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s32_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_s32"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_u32_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_u32_batch(batch a) { + static_assert(vreinterpretq_u32_is_supported(), "vreinterpretq_u32 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_u32_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u32_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_u32"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_s64_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_s64_batch(batch a) { + static_assert(vreinterpretq_s64_is_supported(), "vreinterpretq_s64 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_s64_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_s64_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_s64"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_u64_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_u64_batch(batch a) { + static_assert(vreinterpretq_u64_is_supported(), "vreinterpretq_u64 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_u64_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_f32(a); } + else if constexpr(is_like_v) { return vreinterpretq_u64_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_u64"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_f32_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_f32_batch(batch a) { + static_assert(vreinterpretq_f32_is_supported(), "vreinterpretq_f32 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_f32_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_f32_f64(a); } + else { static_assert(false, "unsupported type for vreinterpretq_f32"); } +} + +template +XSIMD_INLINE constexpr bool vreinterpretq_f64_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vreinterpretq_f64_batch(batch a) { + static_assert(vreinterpretq_f64_is_supported(), "vreinterpretq_f64 unsupported"); + if constexpr(is_like_v) { return vreinterpretq_f64_s8(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_u8(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_s16(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_u16(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_s32(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_u32(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_s64(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_u64(a); } + else if constexpr(is_like_v) { return vreinterpretq_f64_f32(a); } + else { static_assert(false, "unsupported type for vreinterpretq_f64"); } +} + +} // namespace xsimd::overload + +#endif // XSIMD_OVERLOAD_NEON_HPP From 443e76cac45a5cab3fc87359e137fd75846d385e Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 1 Sep 2026 14:00:19 +0200 Subject: [PATCH 3/8] TMP notebook --- Overload.ipynb | 738 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 738 insertions(+) create mode 100644 Overload.ipynb diff --git a/Overload.ipynb b/Overload.ipynb new file mode 100644 index 000000000..01a216245 --- /dev/null +++ b/Overload.ipynb @@ -0,0 +1,738 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ac5b1b7a-f6ad-4e12-82d3-b7309a71ce45", + "metadata": {}, + "outputs": [], + "source": [ + "import dataclasses\n", + "import enum\n", + "import json\n", + "import re\n", + "import functools\n", + "import os\n", + "\n", + "import polars as pl" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4dd3772b-d312-4b19-b862-c3aeae08f99e", + "metadata": {}, + "outputs": [], + "source": [ + "class Arch(enum.Enum):\n", + " neon = enum.auto()\n", + " neon64 = enum.auto()\n", + "\n", + " @staticmethod\n", + " def baseline() -> \"Arch\":\n", + " return Arch.neon\n", + "\n", + " def is_baseline(self) -> bool:\n", + " return self == self.baseline()\n", + "\n", + "\n", + "FLOAT_RE = re.compile(r\"float(?P\\d+)_t\")\n", + "INT_RE = re.compile(r\"int(?P\\d+)_t\")\n", + "UINT_RE = re.compile(r\"uint(?P\\d+)_t\")\n", + "\n", + "\n", + "class Kind(enum.Enum):\n", + " signed_int = \"i\"\n", + " unsigned_int = \"u\"\n", + " float = \"f\"\n", + "\n", + " @staticmethod\n", + " def parse(s: str) -> \"Kind\":\n", + " if s.startswith(\"f\"):\n", + " return Kind.float\n", + " elif s.startswith(\"u\"):\n", + " return Kind.unsigned_int\n", + " else: # i or s notation\n", + " return Kind.signed_int\n", + "\n", + "\n", + "class PrimitiveType(enum.Enum):\n", + " i8 = enum.auto()\n", + " u8 = enum.auto()\n", + " i16 = enum.auto()\n", + " u16 = enum.auto()\n", + " i32 = enum.auto()\n", + " u32 = enum.auto()\n", + " i64 = enum.auto()\n", + " u64 = enum.auto()\n", + " f16 = enum.auto()\n", + " f32 = enum.auto()\n", + " f64 = enum.auto()\n", + "\n", + " @staticmethod\n", + " def parse(s: str) -> \"PrimitiveType\":\n", + " if m:= FLOAT_RE.search(s):\n", + " return getattr(PrimitiveType, f\"f{m[\"nbits\"]}\")\n", + " if m:= UINT_RE.search(s):\n", + " return getattr(PrimitiveType, f\"u{m[\"nbits\"]}\")\n", + " if m:= INT_RE.search(s):\n", + " return getattr(PrimitiveType, f\"i{m[\"nbits\"]}\")\n", + "\n", + " @staticmethod\n", + " def make_sized(kind: Kind, nbits: int) -> \"PrimitiveType\":\n", + " return getattr(PrimitiveType, f\"{kind.value}{nbits}\")\n", + "\n", + " @property\n", + " def kind(self) -> Kind:\n", + " match self.name[0]:\n", + " case \"i\":\n", + " return Kind.signed_int\n", + " case \"u\":\n", + " return Kind.unsigned_int\n", + " case \"f\":\n", + " return Kind.float\n", + " case c:\n", + " raise ValueError(f\"unknown type prefix {c!r} in {self.name}\")\n", + "\n", + " @property\n", + " def is_signed_int(self) -> bool:\n", + " return self.kind is Kind.signed_int\n", + "\n", + " @property\n", + " def is_unsigned_int(self) -> bool:\n", + " return self.kind is Kind.unsigned_int\n", + "\n", + " @property\n", + " def is_int(self) -> bool:\n", + " return not self.is_float\n", + "\n", + " @property\n", + " def is_float(self) -> bool:\n", + " return self.kind is Kind.float\n", + "\n", + " @property\n", + " def nbits(self) -> int:\n", + " return int(self.name[1:])\n", + "\n", + " @property\n", + " def nbytes(self) -> int:\n", + " return self.nbits // 8\n", + "\n", + " def to_signed(self) -> \"PrimitiveType\":\n", + " if self.is_unsigned_int:\n", + " return PrimitiveType[f\"i{self.nbits}\"]\n", + " return self # already signed, or a float\n", + "\n", + " def to_unsigned(self) -> \"PrimitiveType\":\n", + " if self.is_float:\n", + " raise ValueError(f\"{self.name} has no unsigned counterpart\")\n", + " if self.is_signed_int:\n", + " return PrimitiveType[f\"u{self.nbits}\"]\n", + " return self\n", + "\n", + " @property\n", + " def type(self) -> \"PrimitiveType\":\n", + " \"\"\"Meant for generic code.\"\"\"\n", + " return self\n", + "\n", + " def __lt__(self, other: \"PrimitiveType\") -> bool:\n", + " \"\"\"Order for sorting.\"\"\"\n", + " return self.value.__lt__(other.value)\n", + "\n", + "\n", + "SIMD_RE = re.compile(r\"(?P[a-z]+)(?P\\d+)x(?P\\d+)_t\")\n", + "\n", + "\n", + "@dataclasses.dataclass(frozen=True, slots=True, order=True)\n", + "class SimdType:\n", + " type: PrimitiveType\n", + " count: int\n", + "\n", + " @staticmethod\n", + " def parse(s: str) -> \"SimdType\":\n", + " m = SIMD_RE.match(s)\n", + " if m is None:\n", + " return None\n", + " return SimdType(\n", + " type=PrimitiveType.make_sized(kind=Kind.parse(m[\"type\"]), nbits=m[\"nbits\"]),\n", + " count=m[\"count\"]\n", + " )\n", + "\n", + " @property\n", + " def nbits(self) -> int:\n", + " return self.count * self.type.nbits\n", + "\n", + " @property\n", + " def nbytes(self) -> int:\n", + " return self.count * self.type.nbytes \n", + "\n", + "\n", + "@dataclasses.dataclass(frozen=True, slots=True)\n", + "class Intrinsic:\n", + " name: str\n", + " args: list[SimdType | PrimitiveType]\n", + " ret: SimdType | PrimitiveType\n", + " arch: Arch\n", + "\n", + " @property\n", + " def arity(self) -> int:\n", + " return len(self.args)\n", + "\n", + " def arg_type(self, idx: int) -> type:\n", + " return type(self.args[idx])\n", + " \n", + " def ret_type(self) -> bool:\n", + " return type(self.ret)\n", + "\n", + " def __lt__(self, other: \"Intrinsic\") -> bool:\n", + " \"\"\"Order for sorting.\"\"\"\n", + " return [a.type for a in self.args].__lt__([a.type for a in other.args])\n", + "\n", + "\n", + "SUPPORTED_PRIMITIVE_TYPE = [\n", + " PrimitiveType.u8,\n", + " PrimitiveType.i8,\n", + " PrimitiveType.u16,\n", + " PrimitiveType.i16,\n", + " PrimitiveType.u32,\n", + " PrimitiveType.i32,\n", + " PrimitiveType.u64,\n", + " PrimitiveType.i64,\n", + " PrimitiveType.f32,\n", + " PrimitiveType.f64,\n", + "]\n", + "\n", + "\n", + "@dataclasses.dataclass(frozen=True, slots=True)\n", + "class IntrinsicFamily:\n", + " pattern: re.Pattern\n", + "\n", + " @staticmethod\n", + " def parse_any_type(type: str) -> SimdType | PrimitiveType:\n", + " if (t := SimdType.parse(type)) is not None: \n", + " return t\n", + " elif (t := PrimitiveType.parse(type)) is not None: \n", + " return t\n", + " else:\n", + " raise ValueError(f\"Unrecognized type {type}\")\n", + "\n", + " def parse_instance(\n", + " self,\n", + " name: str,\n", + " args: list[str],\n", + " ret: str,\n", + " isa: str,\n", + " archs: list[str],\n", + " ) -> UnaryIntrinsic | None:\n", + " if not self.pattern.match(name):\n", + " return None \n", + " ret_p = self.parse_any_type(ret)\n", + " if ret_p.type not in SUPPORTED_PRIMITIVE_TYPE:\n", + " return None\n", + " args_p = [self.parse_any_type(a.split(\" \")[0]) for a in args]\n", + " if any(a.type not in SUPPORTED_PRIMITIVE_TYPE for a in args_p):\n", + " return None\n", + "\n", + " archs = [a.strip().lower() for a in archs]\n", + " arch = Arch.neon64\n", + " if isa.strip().lower() == \"neon\" and \"v7\" in archs:\n", + " arch = Arch.neon\n", + "\n", + " return Intrinsic(name=name, args=args_p, ret=ret_p, arch=arch) " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b46e97b3-1137-4bd0-8c0a-8e8c708e274d", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: generic multi types\n", + "FMT_IS_SUPPORTED = \"\"\"\n", + "template\n", + "XSIMD_INLINE constexpr bool {prefix}_is_supported() {{\n", + "{body}\n", + "}}\n", + "\"\"\"\n", + "\n", + "FMT_INTRINSIC = \"\"\"\n", + "template\n", + "XSIMD_INLINE auto {prefix}_batch({args}) {{\n", + "{body}\n", + "}}\n", + "\"\"\"\n", + "\n", + "@dataclasses.dataclass(frozen=True)\n", + "class XsimdIntrinsicGenerator:\n", + " intrinsics: list[Intrinsic]\n", + "\n", + " def __post_init__(self) -> None:\n", + " if len(self.intrinsics) == 0:\n", + " raise ValueError(\"Must have at least one intrinsic\")\n", + " if any(i.arity != self.arity for i in self.intrinsics):\n", + " raise ValueError(\"All intrinsics must have same arity\")\n", + " if any(i.ret_type() is not self.ret_type() for i in self.intrinsics):\n", + " raise ValueError(\"All return type must have kind of type\")\n", + " for idx in range(self.arity):\n", + " if any(i.arg_type(idx) is not self.arg_type(idx) for i in self.intrinsics):\n", + " raise ValueError(f\"All argument {idx} must have kind of type\")\n", + " \n", + " @property\n", + " def arity(self) -> int:\n", + " return self.intrinsics[0].arity\n", + "\n", + " def arg_type(self, idx: int) -> type:\n", + " return self.intrinsics[0].arg_type(idx)\n", + " \n", + " def ret_type(self) -> bool:\n", + " return self.intrinsics[0].ret_type()\n", + "\n", + " @staticmethod\n", + " def fmt_batch(type: str, arch: str | None = None, ns: str = \"\") -> str:\n", + " if len(ns) > 0 and not ns.endswith(\"::\"):\n", + " ns = f\"{ns}::\"\n", + " arch = f\", {arch}\" if arch is not None else \"\"\n", + " return f\"{ns}batch<{type}{arch}>\"\n", + "\n", + " @staticmethod\n", + " def fmt_type(type: PrimitiveType | str) -> str:\n", + " if isinstance(type, str):\n", + " return type\n", + " if type.is_float:\n", + " if type.nbits == 16:\n", + " return \"std::float16_t\" # C++23\n", + " elif type.nbits == 32:\n", + " return \"float\"\n", + " elif type.nbits == 64:\n", + " return \"double\"\n", + " prefix = \"u\" if type.is_unsigned_int else \"\"\n", + " return f\"std::{prefix}int{type.nbits}_t\"\n", + "\n", + " @staticmethod\n", + " def fmt_arch(arch: Arch | str) -> str:\n", + " if isinstance(arch, str):\n", + " return arch\n", + " return arch.name\n", + "\n", + " @classmethod\n", + " def fmt_type_equal(cls, lhs: PrimitiveType | str, rhs: PrimitiveType | str) -> str:\n", + " return f\"is_like_v<{cls.fmt_type(lhs)}, {cls.fmt_type(rhs)}>\"\n", + "\n", + " @classmethod\n", + " def fmt_type_any(cls, lhs: PrimitiveType | str, *rhs: PrimitiveType | str) -> str:\n", + " rhss = \", \".join(cls.fmt_type(r) for r in rhs)\n", + " return f\"is_like_any_v<{cls.fmt_type(lhs)}, {rhss}>\"\n", + "\n", + " @classmethod\n", + " def fmt_arch_compatible(cls, arch: Arch | str, base: Arch | str) -> str:\n", + " return f\"std::is_base_of_v<{cls.fmt_arch(base)}, {cls.fmt_arch(arch)}>\"\n", + "\n", + " @classmethod\n", + " def fmt_intrinsic_supported_case(\n", + " cls, intrinsic: Intrinsic, type: str, arch: str,\n", + " ) -> str:\n", + " t = cls.fmt_type_equal(type, intrinsic.args[0].type)\n", + " a = cls.fmt_arch_compatible(arch, intrinsic.arch)\n", + " return f\"if constexpr({t}) {{ return {a}; }}\"\n", + "\n", + " def fmt_sig_type(self, idx: int, type: str, arch: str,) -> str:\n", + " if all(isinstance(i.args[idx], PrimitiveType) for i in self.intrinsics):\n", + " return type\n", + " elif all(isinstance(i.args[idx], SimdType) for i in self.intrinsics):\n", + " return f\"batch<{type}, {arch}>\"\n", + " else:\n", + " raise ValueError(f\"Intrinsics have different parameter types in position {idx}\")\n", + "\n", + " def fmt_is_supported(self) -> str:\n", + " # Dispatching type on first arg\n", + " lines: list[str] = []\n", + " full_support = []\n", + " for intrsct in self.intrinsics:\n", + " if not intrsct.arch.is_baseline():\n", + " lines.append(self.fmt_intrinsic_supported_case(intrsct, type=\"T\", arch=\"A\"))\n", + " else:\n", + " full_support.append(intrsct.args[0].type)\n", + " # The ones for which a baseline intrinsic exist\n", + " lines.append(f\"return {self.fmt_type_any(\"T\", *full_support)};\")\n", + " body = \"\\n\".join([f\" {l}\" for l in lines])\n", + " return FMT_IS_SUPPORTED.format(prefix=self.prefix, body=body)\n", + "\n", + " def fmt_instrinsic_case(self, intrinsic: Intrinsic, type: str, arch: str, params: list[str]) -> str:\n", + " t = self.fmt_type_equal(type, intrinsic.args[0].type)\n", + " return f\"if constexpr({t}) {{ return {intrinsic.name}({', '.join(params)}); }}\"\n", + "\n", + " def fmt_instrinsic_batch(self) -> str:\n", + " params = \"abcdefghijklmnopqrstuvwxyz\"[:self.arity]\n", + " lines: list[str] = [\n", + " self.fmt_instrinsic_case(intrsct, type=\"T\", arch=\"A\", params=params)\n", + " for intrsct in self.intrinsics\n", + " ]\n", + " lines.append(f'{{ static_assert(false, \"unsupported type for {self.prefix}\"); }}')\n", + " body = \"\\n else \".join(lines)\n", + " assrt = f'static_assert({self.prefix}_is_supported(), \"{self.prefix} unsupported\");'\n", + " body = f' {assrt}\\n ' + body\n", + " args = []\n", + " for idx in range(self.arity):\n", + " t = self.fmt_sig_type(idx, type=\"T\", arch=\"A\")\n", + " p = params[idx]\n", + " args.append(f\"{t} {p}\")\n", + " return FMT_INTRINSIC.format(prefix=self.prefix, body=body, args=\", \".join(args))\n", + " \n", + " @functools.cached_property\n", + " def prefix(self) -> str:\n", + " return os.path.commonprefix([i.name for i in self.intrinsics]).strip(\"_\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "90fe85ae-1d16-4a7a-9c5d-f50713d623b2", + "metadata": {}, + "outputs": [], + "source": [ + "FILE_HEADER=\"\"\"\n", + "/****************************************************************************\n", + " * Copyright (c) xsimd contributors *\n", + " * *\n", + " * Distributed under the terms of the BSD 3-Clause License. *\n", + " * *\n", + " * The full license is in the file LICENSE, distributed with this software. *\n", + " ****************************************************************************/\n", + "\n", + "#ifndef XSIMD_OVERLOAD_{arch}_HPP\n", + "#define XSIMD_OVERLOAD_{arch}_HPP\n", + "\n", + "#include \"../config/xsimd_macros.hpp\"\n", + "#include \"../types/xsimd_batch.hpp\"\n", + "#include \"../utils/xsimd_type_traits.hpp\"\n", + "\n", + "#include \n", + "\n", + "namespace xsimd::overload {{\n", + "\"\"\"\n", + "\n", + "FILE_FOOTER=\"\"\"\n", + "}} // namespace xsimd::overload\n", + "\n", + "#endif // XSIMD_OVERLOAD_{arch}_HPP\n", + "\"\"\"\n", + "\n", + "@dataclasses.dataclass(frozen=True)\n", + "class XsimdFileGenerator:\n", + " name: str\n", + " generators: list[XsimdIntrinsicGenerator]\n", + "\n", + " def fmt(self) -> str:\n", + " lines = [FILE_HEADER.format(arch=self.name.upper())]\n", + " for gen in self.generators:\n", + " lines.append(gen.fmt_is_supported())\n", + " lines.append(gen.fmt_instrinsic_batch())\n", + " lines.append(FILE_FOOTER.format(arch=self.name.upper()))\n", + " return \"\".join(lines)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4c4cf3fd-bd88-46cd-968f-82a571a8b80a", + "metadata": {}, + "outputs": [], + "source": [ + "def load_arm_data(file: str) -> pl.DataFrame:\n", + " raw = json.load(open(file))\n", + " for r in raw:\n", + " r[\"Arguments_Preparation\"] = [\n", + " dict(arg=k, **v) for k, v in (r.get(\"Arguments_Preparation\", {})).items()\n", + " ]\n", + " df = pl.DataFrame(raw)\n", + " return df.with_columns(pl.col(\"return_type\").struct.unnest().alias(\"return_type\"))\n", + "\n", + "def get_intrinsic_generator(pattern: str, df: pl.DataFrame) -> XsimdIntrinsicGenerator:\n", + " intrinsics = []\n", + " family = IntrinsicFamily(re.compile(pattern))\n", + " matches = df.filter(pl.col(\"name\").str.contains(family.pattern.pattern))\n", + " for row in matches.iter_rows(named=True):\n", + " inst = family.parse_instance(\n", + " name=row[\"name\"],\n", + " args=row[\"arguments\"],\n", + " ret=row[\"return_type\"],\n", + " isa=row[\"SIMD_ISA\"],\n", + " archs=row[\"Architectures\"],\n", + " )\n", + " if inst is not None:\n", + " intrinsics.append(inst)\n", + "\n", + " return XsimdIntrinsicGenerator(sorted(intrinsics))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a301aeb7-eac2-47e1-a5cb-8d0489360cf5", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# TODO how to handle int8x8_t (half batch) ? Use in combinaison with vget_low?\n", + "# TODO quad _f32_u32\n", + "# TODO vshrq_n_s64 -> macro -> immediate\n", + "\n", + "df = load_arm_data(\"arm_intrinsics.json\")\n", + "gen = XsimdFileGenerator(\n", + " name=\"neon\",\n", + " generators=[\n", + " get_intrinsic_generator(r\"vget_low_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vget_high_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vdupq_n_[usf]\\d+\", df=df),\n", + "\n", + " get_intrinsic_generator(r\"vrev64q_[usf]\\d+\", df=df),\n", + " \n", + " # get_intrinsic_generator(r\"vld1q_[usf]\\d+\", df=df),\n", + " # get_intrinsic_generator(r\"vst1q_[usf]\\d+\", df=df),\n", + "\n", + " get_intrinsic_generator(r\"vandq_[usf]\\d+\", df=df),\n", + "\n", + " # Comparison\n", + " get_intrinsic_generator(r\"vceqq_[usf]\\d+\", df=df),\n", + "\n", + " # Add / Sub / Neg\n", + " get_intrinsic_generator(r\"vaddq_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vhaddq_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vrhaddq_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vqaddq_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vnegq_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vsubq_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vqsubq_[usf]\\d+\", df=df),\n", + "\n", + " get_intrinsic_generator(r\"vmull_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vmulq_[usf]\\d+\", df=df),\n", + "\n", + " # vreinterpretq\n", + " get_intrinsic_generator(r\"vreinterpretq_s8_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_u8_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_s16_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_u16_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_s32_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_u32_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_s64_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_u64_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_f32_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vreinterpretq_f64_[usf]\\d+\", df=df),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "4126b2ba-7b5e-4fc9-bfea-f63aa54ef7c1", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"include/xsimd/overload/neon.hpp\", \"w+\") as f:\n", + " f.write(gen.fmt())" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "64bdbaf2-70b9-4aec-b00b-9212b11bdc29", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (1, 7)
SIMD_ISAnameargumentsreturn_typeArguments_PreparationArchitecturesinstructions
strstrlist[str]strlist[struct[2]]list[str]list[list[str]]
"Neon""vshrq_n_s64"["int64x2_t a", "const int n"]"int64x2_t"[{"a","Vn.2D"}, {"n",null}]["v7", "A32", "A64"][["SSHR"]]
" + ], + "text/plain": [ + "shape: (1, 7)\n", + "┌──────────┬─────────────┬──────────────┬─────────────┬──────────────┬──────────────┬──────────────┐\n", + "│ SIMD_ISA ┆ name ┆ arguments ┆ return_type ┆ Arguments_Pr ┆ Architecture ┆ instructions │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ eparation ┆ s ┆ --- │\n", + "│ str ┆ str ┆ list[str] ┆ str ┆ --- ┆ --- ┆ list[list[st │\n", + "│ ┆ ┆ ┆ ┆ list[struct[ ┆ list[str] ┆ r]] │\n", + "│ ┆ ┆ ┆ ┆ 2]] ┆ ┆ │\n", + "╞══════════╪═════════════╪══════════════╪═════════════╪══════════════╪══════════════╪══════════════╡\n", + "│ Neon ┆ vshrq_n_s64 ┆ [\"int64x2_t ┆ int64x2_t ┆ [{\"a\",\"Vn.2D ┆ [\"v7\", ┆ [[\"SSHR\"]] │\n", + "│ ┆ ┆ a\", \"const ┆ ┆ \"}, ┆ \"A32\", ┆ │\n", + "│ ┆ ┆ int n\"] ┆ ┆ {\"n\",null}] ┆ \"A64\"] ┆ │\n", + "└──────────┴─────────────┴──────────────┴─────────────┴──────────────┴──────────────┴──────────────┘" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.filter(pl.col(\"name\") == \"vshrq_n_s64\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "26fbbd06-f126-44fa-9660-732df8628456", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 1)
return_type
str
"svfloat32_t"
"svfloat16_t"
"float64_t"
"svfloat64_t"
"float16_t"
"float32_t"
" + ], + "text/plain": [ + "shape: (6, 1)\n", + "┌─────────────┐\n", + "│ return_type │\n", + "│ --- │\n", + "│ str │\n", + "╞═════════════╡\n", + "│ svfloat32_t │\n", + "│ svfloat16_t │\n", + "│ float64_t │\n", + "│ svfloat64_t │\n", + "│ float16_t │\n", + "│ float32_t │\n", + "└─────────────┘" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(\n", + " df.select(\"return_type\")\n", + " .filter(pl.col(\"return_type\").str.contains(\"f\"))\n", + " .filter(~pl.col(\"return_type\").str.contains(\"x\"))\n", + " .unique()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "8dfc8187-4ff9-470d-ac42-93a709125150", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 1)
Architectures
list[str]
["A64"]
["v7", "A32", "A64"]
["A32", "A64"]
" + ], + "text/plain": [ + "shape: (3, 1)\n", + "┌──────────────────────┐\n", + "│ Architectures │\n", + "│ --- │\n", + "│ list[str] │\n", + "╞══════════════════════╡\n", + "│ [\"A64\"] │\n", + "│ [\"v7\", \"A32\", \"A64\"] │\n", + "│ [\"A32\", \"A64\"] │\n", + "└──────────────────────┘" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.select(\"Architectures\").unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "f897c931-0961-4997-9b9d-66edf816b2ca", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 1)
SIMD_ISA
str
"SVE2"
"SVE"
"Neon"
" + ], + "text/plain": [ + "shape: (3, 1)\n", + "┌──────────┐\n", + "│ SIMD_ISA │\n", + "│ --- │\n", + "│ str │\n", + "╞══════════╡\n", + "│ SVE2 │\n", + "│ SVE │\n", + "│ Neon │\n", + "└──────────┘" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.select(\"SIMD_ISA\").unique()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "170417cf-693b-4c17-838b-840fccb730de", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 22887659cf2376bc825c18d3cd8eb440d7f42d78 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 18 Sep 2026 12:08:59 +0200 Subject: [PATCH 4/8] Add vld1q_batch --- Overload.ipynb | 223 ++++++++---------------- include/xsimd/arch/xsimd_common_fwd.hpp | 4 + include/xsimd/arch/xsimd_neon.hpp | 187 ++++++-------------- include/xsimd/overload/neon.hpp | 58 ++++-- 4 files changed, 170 insertions(+), 302 deletions(-) diff --git a/Overload.ipynb b/Overload.ipynb index 01a216245..4a3648407 100644 --- a/Overload.ipynb +++ b/Overload.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "ac5b1b7a-f6ad-4e12-82d3-b7309a71ce45", "metadata": {}, "outputs": [], @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "4dd3772b-d312-4b19-b862-c3aeae08f99e", "metadata": {}, "outputs": [], @@ -164,6 +164,36 @@ "\n", " @property\n", " def nbytes(self) -> int:\n", + " return self.count * self.type.nbytes \n", + "\n", + "\n", + "PTR_RE = re.compile(r\"(?Pconst )?(?P\\w+)(?P const)? *\\*\")\n", + "\n", + "@dataclasses.dataclass(frozen=True, slots=True, order=True)\n", + "class PtrType:\n", + " type: PrimitiveType\n", + " const: bool\n", + "\n", + " @staticmethod\n", + " def parse(s: str) -> \"PtrType\":\n", + " m = PTR_RE.match(s)\n", + " if m is None:\n", + " return None\n", + " ptype = PrimitiveType.parse(m[\"type\"])\n", + " if ptype is None:\n", + " return None\n", + " \n", + " return PtrType(\n", + " type=ptype,\n", + " const=any([m[\"const_a\"], m[\"const_b\"]])\n", + " )\n", + "\n", + " @property\n", + " def nbits(self) -> int:\n", + " return self.count * self.type.nbits\n", + "\n", + " @property\n", + " def nbytes(self) -> int:\n", " return self.count * self.type.nbytes \n", "\n", "\n", @@ -208,7 +238,9 @@ " pattern: re.Pattern\n", "\n", " @staticmethod\n", - " def parse_any_type(type: str) -> SimdType | PrimitiveType:\n", + " def parse_any_type(type: str) -> SimdType | PtrType | PrimitiveType:\n", + " if \"*\" in type and (t := PtrType.parse(type)) is not None:\n", + " return t\n", " if (t := SimdType.parse(type)) is not None: \n", " return t\n", " elif (t := PrimitiveType.parse(type)) is not None: \n", @@ -229,7 +261,7 @@ " ret_p = self.parse_any_type(ret)\n", " if ret_p.type not in SUPPORTED_PRIMITIVE_TYPE:\n", " return None\n", - " args_p = [self.parse_any_type(a.split(\" \")[0]) for a in args]\n", + " args_p = [self.parse_any_type(\" \".join(a.split(\" \")[:-1])) for a in args]\n", " if any(a.type not in SUPPORTED_PRIMITIVE_TYPE for a in args_p):\n", " return None\n", "\n", @@ -243,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "b46e97b3-1137-4bd0-8c0a-8e8c708e274d", "metadata": {}, "outputs": [], @@ -296,9 +328,12 @@ " return f\"{ns}batch<{type}{arch}>\"\n", "\n", " @staticmethod\n", - " def fmt_type(type: PrimitiveType | str) -> str:\n", + " def fmt_type(type: PrimitiveType | PtrType | str) -> str:\n", " if isinstance(type, str):\n", " return type\n", + " if isinstance(type, PtrType):\n", + " s = self.fmt_type(type.type)\n", + " return f\"{s} const*\" if type.const else f\"{s}*\"\n", " if type.is_float:\n", " if type.nbits == 16:\n", " return \"std::float16_t\" # C++23\n", @@ -339,6 +374,8 @@ " def fmt_sig_type(self, idx: int, type: str, arch: str,) -> str:\n", " if all(isinstance(i.args[idx], PrimitiveType) for i in self.intrinsics):\n", " return type\n", + " if all(isinstance(i.args[idx], PtrType) for i in self.intrinsics):\n", + " return f\"{type}*\" # MSVC behave badly with const\n", " elif all(isinstance(i.args[idx], SimdType) for i in self.intrinsics):\n", " return f\"batch<{type}, {arch}>\"\n", " else:\n", @@ -386,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "90fe85ae-1d16-4a7a-9c5d-f50713d623b2", "metadata": {}, "outputs": [], @@ -434,7 +471,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "4c4cf3fd-bd88-46cd-968f-82a571a8b80a", "metadata": {}, "outputs": [], @@ -468,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "a301aeb7-eac2-47e1-a5cb-8d0489360cf5", "metadata": { "scrolled": true @@ -483,18 +520,20 @@ "gen = XsimdFileGenerator(\n", " name=\"neon\",\n", " generators=[\n", + " # Mem\n", + " get_intrinsic_generator(r\"vld1q_[usf]\\d+$\", df=df),\n", + " # get_intrinsic_generator(r\"vst1q_[usf]\\d+$\", df=df),\n", + "\n", + " # Utils\n", " get_intrinsic_generator(r\"vget_low_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vget_high_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vdupq_n_[usf]\\d+\", df=df),\n", "\n", - " get_intrinsic_generator(r\"vrev64q_[usf]\\d+\", df=df),\n", - " \n", - " # get_intrinsic_generator(r\"vld1q_[usf]\\d+\", df=df),\n", - " # get_intrinsic_generator(r\"vst1q_[usf]\\d+\", df=df),\n", - "\n", + " # Bit\n", " get_intrinsic_generator(r\"vandq_[usf]\\d+\", df=df),\n", "\n", " # Comparison\n", + " get_intrinsic_generator(r\"vrev64q_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vceqq_[usf]\\d+\", df=df),\n", "\n", " # Add / Sub / Neg\n", @@ -506,6 +545,7 @@ " get_intrinsic_generator(r\"vsubq_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vqsubq_[usf]\\d+\", df=df),\n", "\n", + " # Mul\n", " get_intrinsic_generator(r\"vmull_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vmulq_[usf]\\d+\", df=df),\n", "\n", @@ -526,7 +566,17 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, + "id": "599d7015-850a-421e-8241-65ad8966af43", + "metadata": {}, + "outputs": [], + "source": [ + "get_intrinsic_generator(r\"vld1q_[usf]\\d+$\", df=df).intrinsics[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "4126b2ba-7b5e-4fc9-bfea-f63aa54ef7c1", "metadata": {}, "outputs": [], @@ -537,85 +587,20 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "64bdbaf2-70b9-4aec-b00b-9212b11bdc29", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (1, 7)
SIMD_ISAnameargumentsreturn_typeArguments_PreparationArchitecturesinstructions
strstrlist[str]strlist[struct[2]]list[str]list[list[str]]
"Neon""vshrq_n_s64"["int64x2_t a", "const int n"]"int64x2_t"[{"a","Vn.2D"}, {"n",null}]["v7", "A32", "A64"][["SSHR"]]
" - ], - "text/plain": [ - "shape: (1, 7)\n", - "┌──────────┬─────────────┬──────────────┬─────────────┬──────────────┬──────────────┬──────────────┐\n", - "│ SIMD_ISA ┆ name ┆ arguments ┆ return_type ┆ Arguments_Pr ┆ Architecture ┆ instructions │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ eparation ┆ s ┆ --- │\n", - "│ str ┆ str ┆ list[str] ┆ str ┆ --- ┆ --- ┆ list[list[st │\n", - "│ ┆ ┆ ┆ ┆ list[struct[ ┆ list[str] ┆ r]] │\n", - "│ ┆ ┆ ┆ ┆ 2]] ┆ ┆ │\n", - "╞══════════╪═════════════╪══════════════╪═════════════╪══════════════╪══════════════╪══════════════╡\n", - "│ Neon ┆ vshrq_n_s64 ┆ [\"int64x2_t ┆ int64x2_t ┆ [{\"a\",\"Vn.2D ┆ [\"v7\", ┆ [[\"SSHR\"]] │\n", - "│ ┆ ┆ a\", \"const ┆ ┆ \"}, ┆ \"A32\", ┆ │\n", - "│ ┆ ┆ int n\"] ┆ ┆ {\"n\",null}] ┆ \"A64\"] ┆ │\n", - "└──────────┴─────────────┴──────────────┴─────────────┴──────────────┴──────────────┴──────────────┘" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "df.filter(pl.col(\"name\") == \"vshrq_n_s64\")" + "df.filter(pl.col(\"name\").str.starts_with(\"vld1q_\"))" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "26fbbd06-f126-44fa-9660-732df8628456", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 1)
return_type
str
"svfloat32_t"
"svfloat16_t"
"float64_t"
"svfloat64_t"
"float16_t"
"float32_t"
" - ], - "text/plain": [ - "shape: (6, 1)\n", - "┌─────────────┐\n", - "│ return_type │\n", - "│ --- │\n", - "│ str │\n", - "╞═════════════╡\n", - "│ svfloat32_t │\n", - "│ svfloat16_t │\n", - "│ float64_t │\n", - "│ svfloat64_t │\n", - "│ float16_t │\n", - "│ float32_t │\n", - "└─────────────┘" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " df.select(\"return_type\")\n", @@ -627,80 +612,20 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "8dfc8187-4ff9-470d-ac42-93a709125150", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 1)
Architectures
list[str]
["A64"]
["v7", "A32", "A64"]
["A32", "A64"]
" - ], - "text/plain": [ - "shape: (3, 1)\n", - "┌──────────────────────┐\n", - "│ Architectures │\n", - "│ --- │\n", - "│ list[str] │\n", - "╞══════════════════════╡\n", - "│ [\"A64\"] │\n", - "│ [\"v7\", \"A32\", \"A64\"] │\n", - "│ [\"A32\", \"A64\"] │\n", - "└──────────────────────┘" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.select(\"Architectures\").unique()" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "f897c931-0961-4997-9b9d-66edf816b2ca", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 1)
SIMD_ISA
str
"SVE2"
"SVE"
"Neon"
" - ], - "text/plain": [ - "shape: (3, 1)\n", - "┌──────────┐\n", - "│ SIMD_ISA │\n", - "│ --- │\n", - "│ str │\n", - "╞══════════╡\n", - "│ SVE2 │\n", - "│ SVE │\n", - "│ Neon │\n", - "└──────────┘" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.select(\"SIMD_ISA\").unique()" ] diff --git a/include/xsimd/arch/xsimd_common_fwd.hpp b/include/xsimd/arch/xsimd_common_fwd.hpp index 538e73743..e4913a8bb 100644 --- a/include/xsimd/arch/xsimd_common_fwd.hpp +++ b/include/xsimd/arch/xsimd_common_fwd.hpp @@ -93,6 +93,10 @@ namespace xsimd XSIMD_INLINE batch load(T const* mem, aligned_mode, requires_arch) noexcept; template XSIMD_INLINE batch load(T const* mem, unaligned_mode, requires_arch) noexcept; + template + XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool b, requires_arch) noexcept; + template + XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool b, requires_arch) noexcept; template XSIMD_INLINE batch load_masked(T_in const* mem, batch_bool_constant mask, convert, alignment, requires_arch) noexcept; template diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index fb154e740..4815ebbdc 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -33,6 +33,16 @@ namespace xsimd { using namespace types; + template + XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept; + template + XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool t, requires_arch) noexcept; + template + + XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept; + template + XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept; + template XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept; @@ -168,152 +178,59 @@ namespace xsimd * load * ********/ - // It is not possible to use a call to A::alignment() here, so use an - // immediate instead. -#if defined(__clang__) || defined(__GNUC__) -#define xsimd_aligned_load(inst, type, expr) inst((type)__builtin_assume_aligned(expr, 16)) -#else -#define xsimd_aligned_load(inst, type, expr) inst((type)expr) -#endif - - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_u8, uint8_t*, src); - } - - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_s8, int8_t*, src); - } - - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_u16, uint16_t*, src); - } - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_s16, int16_t*, src); - } - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_u32, uint32_t*, src); - } - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_s32, int32_t*, src); - } - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_u64, uint64_t*, src); - } - template = 0> - XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_s64, int64_t*, src); - } - - template - XSIMD_INLINE batch load_aligned(float const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_f32, float*, src); - } - -#undef xsimd_aligned_load - - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_u8((uint8_t*)src); - } - - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_s8((int8_t*)src); - } - - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_u16((uint16_t*)src); - } - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_s16((int16_t*)src); - } - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_u32((uint32_t*)src); - } - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_s32((int32_t*)src); - } - template = 0> - XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept - { - return vld1q_u64((uint64_t*)src); - } - template = 0> + template XSIMD_INLINE batch load_unaligned(T const* src, convert, requires_arch) noexcept { - return vld1q_s64((int64_t*)src); + // Pointer type must match exactly sized integer types + using type = map_to_sized_type_t; + return overload::vld1q_batch((type*)src); } - template - XSIMD_INLINE batch load_unaligned(float const* src, convert, requires_arch) noexcept + template + XSIMD_INLINE batch load_aligned(T const* src, convert, requires_arch) noexcept { - return vld1q_f32(src); + // Pointer type must match exactly sized integer types + using type = map_to_sized_type_t; +#if defined(__clang__) || defined(__GNUC__) + // It is not possible to use a call to A::alignment() here, so use an + // immediate instead. + return overload::vld1q_batch((type*)__builtin_assume_aligned(src, 16)); +#else + return overload::vld1q_batch((type*)src); +#endif } /* batch bool version */ - template = 0> - XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool, requires_arch) noexcept - { - auto vmem = load_unaligned((unsigned char const*)mem, convert {}, A {}); - auto const zero = batch { 0 }; - return { (zero - vmem).data }; - } - template = 0> - XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept - { - return load_unaligned(mem, t, r); - } - - template = 0> - XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool, requires_arch) noexcept - { - auto const vmem = batch(vmovl_u8(vld1_u8((unsigned char const*)mem))); - auto const zero = batch { 0 }; - return { (zero - vmem).data }; - } - - template = 0> - XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept - { - return load_unaligned(mem, t, r); - } - - template = 0> - XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool, requires_arch) noexcept + template + XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool t, requires_arch) noexcept { - uint8x8_t tmp = vreinterpret_u8_u32(vset_lane_u32(*(unsigned int*)mem, vdup_n_u32(0), 0)); - auto const vmem = batch(vmovl_u16(vget_low_u16(vmovl_u8(tmp)))); - auto const zero = batch { 0 }; - return { (zero - vmem).data }; + if constexpr (sizeof(T) == 8) + { + return load_unaligned(mem, t, common {}); + } + else + { + using uint = sized_uint_t; + batch const zero = batch { 0 }; + batch vmem; + if constexpr (sizeof(T) == 1) + { + vmem = load_unaligned((uint const*)mem, convert {}, A {}); + } + else if constexpr (sizeof(T) == 2) + { + vmem = vmovl_u8(vld1_u8((std::uint8_t*)mem)); + } + else if constexpr (sizeof(T) == 4) + { + auto tmp = vreinterpret_u8_u32(vset_lane_u32(*(uint*)mem, vdup_n_u32(0), 0)); + vmem = vmovl_u16(vget_low_u16(vmovl_u8(tmp))); + } + return { (zero - vmem).data }; + } } - template = 0> + template XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept { return load_unaligned(mem, t, r); diff --git a/include/xsimd/overload/neon.hpp b/include/xsimd/overload/neon.hpp index f67be969c..6eba2c714 100644 --- a/include/xsimd/overload/neon.hpp +++ b/include/xsimd/overload/neon.hpp @@ -18,6 +18,28 @@ namespace xsimd::overload { +template +XSIMD_INLINE constexpr bool vld1q_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vld1q_batch(T* a) { + static_assert(vld1q_is_supported(), "vld1q unsupported"); + if constexpr(is_like_v) { return vld1q_s8(a); } + else if constexpr(is_like_v) { return vld1q_u8(a); } + else if constexpr(is_like_v) { return vld1q_s16(a); } + else if constexpr(is_like_v) { return vld1q_u16(a); } + else if constexpr(is_like_v) { return vld1q_s32(a); } + else if constexpr(is_like_v) { return vld1q_u32(a); } + else if constexpr(is_like_v) { return vld1q_s64(a); } + else if constexpr(is_like_v) { return vld1q_u64(a); } + else if constexpr(is_like_v) { return vld1q_f32(a); } + else if constexpr(is_like_v) { return vld1q_f64(a); } + else { static_assert(false, "unsupported type for vld1q"); } +} + template XSIMD_INLINE constexpr bool vget_low_is_supported() { if constexpr(is_like_v) { return std::is_base_of_v; } @@ -84,24 +106,6 @@ XSIMD_INLINE auto vdupq_n_batch(T a) { else { static_assert(false, "unsupported type for vdupq_n"); } } -template -XSIMD_INLINE constexpr bool vrev64q_is_supported() { - return is_like_any_v; -} - -template -XSIMD_INLINE auto vrev64q_batch(batch a) { - static_assert(vrev64q_is_supported(), "vrev64q unsupported"); - if constexpr(is_like_v) { return vrev64q_s8(a); } - else if constexpr(is_like_v) { return vrev64q_u8(a); } - else if constexpr(is_like_v) { return vrev64q_s16(a); } - else if constexpr(is_like_v) { return vrev64q_u16(a); } - else if constexpr(is_like_v) { return vrev64q_s32(a); } - else if constexpr(is_like_v) { return vrev64q_u32(a); } - else if constexpr(is_like_v) { return vrev64q_f32(a); } - else { static_assert(false, "unsupported type for vrev64q"); } -} - template XSIMD_INLINE constexpr bool vandq_is_supported() { return is_like_any_v; @@ -121,6 +125,24 @@ XSIMD_INLINE auto vandq_batch(batch a, batch b) { else { static_assert(false, "unsupported type for vandq"); } } +template +XSIMD_INLINE constexpr bool vrev64q_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vrev64q_batch(batch a) { + static_assert(vrev64q_is_supported(), "vrev64q unsupported"); + if constexpr(is_like_v) { return vrev64q_s8(a); } + else if constexpr(is_like_v) { return vrev64q_u8(a); } + else if constexpr(is_like_v) { return vrev64q_s16(a); } + else if constexpr(is_like_v) { return vrev64q_u16(a); } + else if constexpr(is_like_v) { return vrev64q_s32(a); } + else if constexpr(is_like_v) { return vrev64q_u32(a); } + else if constexpr(is_like_v) { return vrev64q_f32(a); } + else { static_assert(false, "unsupported type for vrev64q"); } +} + template XSIMD_INLINE constexpr bool vceqq_is_supported() { if constexpr(is_like_v) { return std::is_base_of_v; } From dc7e9f7cfe649384da045c271022ed21e33bd463 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 18 Sep 2026 12:27:07 +0200 Subject: [PATCH 5/8] Add vst1q_batch --- Overload.ipynb | 33 ++++++++--------- include/xsimd/arch/xsimd_neon.hpp | 60 ++++++------------------------- include/xsimd/overload/neon.hpp | 22 ++++++++++++ 3 files changed, 49 insertions(+), 66 deletions(-) diff --git a/Overload.ipynb b/Overload.ipynb index 4a3648407..aacc3603a 100644 --- a/Overload.ipynb +++ b/Overload.ipynb @@ -197,11 +197,14 @@ " return self.count * self.type.nbytes \n", "\n", "\n", + "class VoidType:\n", + " ...\n", + "\n", "@dataclasses.dataclass(frozen=True, slots=True)\n", "class Intrinsic:\n", " name: str\n", - " args: list[SimdType | PrimitiveType]\n", - " ret: SimdType | PrimitiveType\n", + " args: list[SimdType | PrimitiveType | PtrType]\n", + " ret: SimdType | PrimitiveType | VoidType\n", " arch: Arch\n", "\n", " @property\n", @@ -238,7 +241,9 @@ " pattern: re.Pattern\n", "\n", " @staticmethod\n", - " def parse_any_type(type: str) -> SimdType | PtrType | PrimitiveType:\n", + " def parse_any_type(type: str) -> SimdType | PtrType | PrimitiveType | VoidType:\n", + " if type == \"void\":\n", + " return VoidType()\n", " if \"*\" in type and (t := PtrType.parse(type)) is not None:\n", " return t\n", " if (t := SimdType.parse(type)) is not None: \n", @@ -259,7 +264,9 @@ " if not self.pattern.match(name):\n", " return None \n", " ret_p = self.parse_any_type(ret)\n", - " if ret_p.type not in SUPPORTED_PRIMITIVE_TYPE:\n", + " if isinstance(ret_p, PrimitiveType) and ret_p not in SUPPORTED_PRIMITIVE_TYPE:\n", + " return None\n", + " if isinstance(ret_p, SimdType) and ret_p.type not in SUPPORTED_PRIMITIVE_TYPE:\n", " return None\n", " args_p = [self.parse_any_type(\" \".join(a.split(\" \")[:-1])) for a in args]\n", " if any(a.type not in SUPPORTED_PRIMITIVE_TYPE for a in args_p):\n", @@ -328,7 +335,9 @@ " return f\"{ns}batch<{type}{arch}>\"\n", "\n", " @staticmethod\n", - " def fmt_type(type: PrimitiveType | PtrType | str) -> str:\n", + " def fmt_type(type: PrimitiveType | VoidType | PtrType | str) -> str:\n", + " if isinstance(type, VoidType):\n", + " return void\n", " if isinstance(type, str):\n", " return type\n", " if isinstance(type, PtrType):\n", @@ -372,6 +381,8 @@ " return f\"if constexpr({t}) {{ return {a}; }}\"\n", "\n", " def fmt_sig_type(self, idx: int, type: str, arch: str,) -> str:\n", + " if all(isinstance(i.args[idx], VoidType) for i in self.intrinsics):\n", + " return \"void\"\n", " if all(isinstance(i.args[idx], PrimitiveType) for i in self.intrinsics):\n", " return type\n", " if all(isinstance(i.args[idx], PtrType) for i in self.intrinsics):\n", @@ -522,7 +533,7 @@ " generators=[\n", " # Mem\n", " get_intrinsic_generator(r\"vld1q_[usf]\\d+$\", df=df),\n", - " # get_intrinsic_generator(r\"vst1q_[usf]\\d+$\", df=df),\n", + " get_intrinsic_generator(r\"vst1q_[usf]\\d+$\", df=df),\n", "\n", " # Utils\n", " get_intrinsic_generator(r\"vget_low_[usf]\\d+\", df=df),\n", @@ -564,16 +575,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "599d7015-850a-421e-8241-65ad8966af43", - "metadata": {}, - "outputs": [], - "source": [ - "get_intrinsic_generator(r\"vld1q_[usf]\\d+$\", df=df).intrinsics[0]" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index 4815ebbdc..d33b75b35 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -43,6 +43,12 @@ namespace xsimd template XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept; + template + XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept; + + template + XSIMD_INLINE void store_unaligned(T* dst, batch const& src, requires_arch) noexcept; + template XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept; @@ -276,58 +282,12 @@ namespace xsimd * store * *********/ - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_u8((uint8_t*)dst, src); - } - - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_s8((int8_t*)dst, src); - } - - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_u16((uint16_t*)dst, src); - } - - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_s16((int16_t*)dst, src); - } - - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_u32((uint32_t*)dst, src); - } - - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_s32((int32_t*)dst, src); - } - - template = 0> - XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept - { - vst1q_u64((uint64_t*)dst, src); - } - - template = 0> + template XSIMD_INLINE void store_aligned(T* dst, batch const& src, requires_arch) noexcept { - vst1q_s64((int64_t*)dst, src); - } - - template - XSIMD_INLINE void store_aligned(float* dst, batch const& src, requires_arch) noexcept - { - vst1q_f32(dst, src); + // Pointer type must match exactly sized integer types + using type = map_to_sized_type_t; + return overload::vst1q_batch((type*)dst, src.data); } template diff --git a/include/xsimd/overload/neon.hpp b/include/xsimd/overload/neon.hpp index 6eba2c714..ca8e3ffc0 100644 --- a/include/xsimd/overload/neon.hpp +++ b/include/xsimd/overload/neon.hpp @@ -40,6 +40,28 @@ XSIMD_INLINE auto vld1q_batch(T* a) { else { static_assert(false, "unsupported type for vld1q"); } } +template +XSIMD_INLINE constexpr bool vst1q_is_supported() { + if constexpr(is_like_v) { return std::is_base_of_v; } + return is_like_any_v; +} + +template +XSIMD_INLINE auto vst1q_batch(T* a, batch b) { + static_assert(vst1q_is_supported(), "vst1q unsupported"); + if constexpr(is_like_v) { return vst1q_s8(a, b); } + else if constexpr(is_like_v) { return vst1q_u8(a, b); } + else if constexpr(is_like_v) { return vst1q_s16(a, b); } + else if constexpr(is_like_v) { return vst1q_u16(a, b); } + else if constexpr(is_like_v) { return vst1q_s32(a, b); } + else if constexpr(is_like_v) { return vst1q_u32(a, b); } + else if constexpr(is_like_v) { return vst1q_s64(a, b); } + else if constexpr(is_like_v) { return vst1q_u64(a, b); } + else if constexpr(is_like_v) { return vst1q_f32(a, b); } + else if constexpr(is_like_v) { return vst1q_f64(a, b); } + else { static_assert(false, "unsupported type for vst1q"); } +} + template XSIMD_INLINE constexpr bool vget_low_is_supported() { if constexpr(is_like_v) { return std::is_base_of_v; } From 13f9921bebbb4e505827ee38574d24c41a24ee94 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 18 Sep 2026 15:31:06 +0200 Subject: [PATCH 6/8] Store bools --- include/xsimd/arch/xsimd_neon.hpp | 171 +++++++++++++++------------- include/xsimd/arch/xsimd_neon64.hpp | 10 -- 2 files changed, 90 insertions(+), 91 deletions(-) diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index d33b75b35..d2f3667e7 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -49,6 +49,9 @@ namespace xsimd template XSIMD_INLINE void store_unaligned(T* dst, batch const& src, requires_arch) noexcept; + template + XSIMD_INLINE void store(batch_bool x, bool* mem, requires_arch) noexcept; + template XSIMD_INLINE batch broadcast(T val, requires_arch) noexcept; @@ -206,43 +209,10 @@ namespace xsimd #endif } - /* batch bool version */ - template - XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool t, requires_arch) noexcept - { - if constexpr (sizeof(T) == 8) - { - return load_unaligned(mem, t, common {}); - } - else - { - using uint = sized_uint_t; - batch const zero = batch { 0 }; - batch vmem; - if constexpr (sizeof(T) == 1) - { - vmem = load_unaligned((uint const*)mem, convert {}, A {}); - } - else if constexpr (sizeof(T) == 2) - { - vmem = vmovl_u8(vld1_u8((std::uint8_t*)mem)); - } - else if constexpr (sizeof(T) == 4) - { - auto tmp = vreinterpret_u8_u32(vset_lane_u32(*(uint*)mem, vdup_n_u32(0), 0)); - vmem = vmovl_u16(vget_low_u16(vmovl_u8(tmp))); - } - return { (zero - vmem).data }; - } - } - - template - XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept - { - return load_unaligned(mem, t, r); - } + /*************** + * load masked * + ***************/ - /* masked version */ namespace detail { template @@ -278,6 +248,45 @@ namespace xsimd return detail::load_masked::template apply<0>(mem, batch(T(0))); } + /************* + * load bool * + *************/ + + template + XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool t, requires_arch) noexcept + { + if constexpr (sizeof(T) == 8) + { + return load_unaligned(mem, t, common {}); + } + else + { + using uint = sized_uint_t; + batch const zero = batch { 0 }; + batch vmem; + if constexpr (sizeof(T) == 1) + { + vmem = load_unaligned((uint const*)mem, convert {}, A {}); + } + else if constexpr (sizeof(T) == 2) + { + vmem = vmovl_u8(vld1_u8((std::uint8_t*)mem)); + } + else if constexpr (sizeof(T) == 4) + { + auto tmp = vreinterpret_u8_u32(vset_lane_u32(*(uint*)mem, vdup_n_u32(0), 0)); + vmem = vmovl_u16(vget_low_u16(vmovl_u8(tmp))); + } + return { (zero - vmem).data }; + } + } + + template + XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept + { + return load_unaligned(mem, t, r); + } + /********* * store * *********/ @@ -296,6 +305,51 @@ namespace xsimd store_aligned(dst, src, A {}); } + /************** + * store bool * + **************/ + + template + XSIMD_INLINE void store(batch_bool x, bool* mem, requires_arch) noexcept + { + if constexpr (std::is_floating_point_v) + { + using uint = sized_uint_t; + return store(batch_bool(x.data), mem, A {}); + } + + constexpr std::size_t buffer_size = std::max(batch_bool::size, 8u); + alignas(A::alignment()) std::uint8_t buffer[buffer_size]; + + if constexpr (sizeof(T) == 1) + { + uint8x16_t val = vshrq_n_u8(x.data, 7); + vst1q_u8(buffer, val); + } + else if constexpr (sizeof(T) == 2) + { + uint16x8_t c = x.data; + uint8x8_t val = vshr_n_u8(vqmovn_u16(c), 7); + vst1_u8(buffer, val); + } + else if constexpr (sizeof(T) == 4) + { + uint32x4_t b = x.data; + uint16x8_t c = vcombine_u16(vqmovn_u32(b), vdup_n_u16(0)); + uint8x8_t val = vshr_n_u8(vqmovn_u16(c), 7); + vst1_u8(buffer, val); + } + else if constexpr (sizeof(T) == 8) + { + uint64x2_t a = x.data; + uint32x4_t b = vcombine_u32(vqmovn_u64(a), vdup_n_u32(0)); + uint16x8_t c = vcombine_u16(vqmovn_u32(b), vdup_n_u16(0)); + uint8x8_t val = vshr_n_u8(vqmovn_u16(c), 7); + vst1_u8(buffer, val); + } + std::memcpy(mem, buffer, batch_bool::size); + } + /**************** * load_complex * ****************/ @@ -337,51 +391,6 @@ namespace xsimd store_complex_aligned(dst, src, A {}); } - /********************* - * store * - *********************/ - template = 0> - XSIMD_INLINE void store(batch_bool b, bool* mem, requires_arch) noexcept - { - uint8x16_t val = vshrq_n_u8(b.data, 7); - alignas(A::alignment()) uint8_t buffer[batch_bool::size]; - vst1q_u8(buffer, val); - memcpy(mem, buffer, sizeof(buffer)); - } - - template = 0> - XSIMD_INLINE void store(batch_bool b, bool* mem, requires_arch) noexcept - { - uint8x8_t val = vshr_n_u8(vqmovn_u16(b.data), 7); - alignas(A::alignment()) uint8_t buffer[batch_bool::size]; - vst1_u8(buffer, val); - memcpy(mem, buffer, sizeof(buffer)); - } - - template = 0> - XSIMD_INLINE void store(batch_bool b, bool* mem, requires_arch) noexcept - { - uint8x8_t val = vshr_n_u8(vqmovn_u16(vcombine_u16(vqmovn_u32(b.data), vdup_n_u16(0))), 7); - alignas(A::alignment()) uint8_t buffer[8]; - vst1_u8(buffer, val); - memcpy(mem, buffer, batch_bool::size); - } - - template = 0> - XSIMD_INLINE void store(batch_bool b, bool* mem, requires_arch) noexcept - { - uint8x8_t val = vshr_n_u8(vqmovn_u16(vcombine_u16(vqmovn_u32(vcombine_u32(vqmovn_u64(b.data), vdup_n_u32(0))), vdup_n_u16(0))), 7); - alignas(A::alignment()) uint8_t buffer[8]; - vst1_u8(buffer, val); - memcpy(mem, buffer, batch_bool::size); - } - - template - XSIMD_INLINE void store(batch_bool b, bool* mem, requires_arch) noexcept - { - store(batch_bool(b.data), mem, A {}); - } - /******* * set * *******/ diff --git a/include/xsimd/arch/xsimd_neon64.hpp b/include/xsimd/arch/xsimd_neon64.hpp index 0e067b500..15b9c132c 100644 --- a/include/xsimd/arch/xsimd_neon64.hpp +++ b/include/xsimd/arch/xsimd_neon64.hpp @@ -240,16 +240,6 @@ namespace xsimd } #endif - /********************* - * store * - *********************/ - - template - XSIMD_INLINE void store(batch_bool b, bool* mem, requires_arch) noexcept - { - store(batch_bool(b.data), mem, A {}); - } - /**************** * load_complex * ****************/ From a34c6486d12a96b5493041f145b8333174fb8cb5 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 18 Sep 2026 15:31:35 +0200 Subject: [PATCH 7/8] Clea neon64 load stores --- include/xsimd/arch/xsimd_neon64.hpp | 37 ----------------------------- 1 file changed, 37 deletions(-) diff --git a/include/xsimd/arch/xsimd_neon64.hpp b/include/xsimd/arch/xsimd_neon64.hpp index 15b9c132c..c319d67be 100644 --- a/include/xsimd/arch/xsimd_neon64.hpp +++ b/include/xsimd/arch/xsimd_neon64.hpp @@ -119,43 +119,6 @@ namespace xsimd return vreinterpretq_f64_u64(vandq_u64(arg, vreinterpretq_u64_f64(vdupq_n_f64(1.)))); } - /******** - * load * - ********/ -#if defined(__clang__) || defined(__GNUC__) -#define xsimd_aligned_load(inst, type, expr) inst((type)__builtin_assume_aligned(expr, 16)) -#else -#define xsimd_aligned_load(inst, type, expr) inst((type)expr) -#endif - - template - XSIMD_INLINE batch load_aligned(double const* src, convert, requires_arch) noexcept - { - return xsimd_aligned_load(vld1q_f64, double*, src); - } - - template - XSIMD_INLINE batch load_unaligned(double const* src, convert, requires_arch) noexcept - { - return vld1q_f64(src); - } -#undef xsimd_aligned_load - - /********* - * store * - *********/ - - template - XSIMD_INLINE void store_aligned(double* dst, batch const& src, requires_arch) noexcept - { - vst1q_f64(dst, src); - } - - template - XSIMD_INLINE void store_unaligned(double* dst, batch const& src, requires_arch) noexcept - { - return store_aligned(dst, src, A {}); - } /**************** * store_stream * From 4c456113c6740bfb9613b2f03e27c2d5d323370c Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 18 Sep 2026 17:30:36 +0200 Subject: [PATCH 8/8] Factor mu_hilo --- Overload.ipynb | 11 +- include/xsimd/arch/xsimd_neon.hpp | 186 ++++++++++++++-------------- include/xsimd/arch/xsimd_neon64.hpp | 10 -- include/xsimd/overload/neon.hpp | 34 ++--- 4 files changed, 115 insertions(+), 126 deletions(-) diff --git a/Overload.ipynb b/Overload.ipynb index aacc3603a..5b3f77381 100644 --- a/Overload.ipynb +++ b/Overload.ipynb @@ -241,7 +241,7 @@ " pattern: re.Pattern\n", "\n", " @staticmethod\n", - " def parse_any_type(type: str) -> SimdType | PtrType | PrimitiveType | VoidType:\n", + " def parse_any_type(type: str, strict: bool = True) -> SimdType | PtrType | PrimitiveType | VoidType | str:\n", " if type == \"void\":\n", " return VoidType()\n", " if \"*\" in type and (t := PtrType.parse(type)) is not None:\n", @@ -250,8 +250,10 @@ " return t\n", " elif (t := PrimitiveType.parse(type)) is not None: \n", " return t\n", - " else:\n", + " elif strict:\n", " raise ValueError(f\"Unrecognized type {type}\")\n", + " else:\n", + " return type\n", "\n", " def parse_instance(\n", " self,\n", @@ -263,7 +265,7 @@ " ) -> UnaryIntrinsic | None:\n", " if not self.pattern.match(name):\n", " return None \n", - " ret_p = self.parse_any_type(ret)\n", + " ret_p = self.parse_any_type(ret, strict=False)\n", " if isinstance(ret_p, PrimitiveType) and ret_p not in SUPPORTED_PRIMITIVE_TYPE:\n", " return None\n", " if isinstance(ret_p, SimdType) and ret_p.type not in SUPPORTED_PRIMITIVE_TYPE:\n", @@ -539,6 +541,7 @@ " get_intrinsic_generator(r\"vget_low_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vget_high_[usf]\\d+\", df=df),\n", " get_intrinsic_generator(r\"vdupq_n_[usf]\\d+\", df=df),\n", + " get_intrinsic_generator(r\"vuzpq_[usf]\\d+\", df=df),\n", "\n", " # Bit\n", " get_intrinsic_generator(r\"vandq_[usf]\\d+\", df=df),\n", @@ -557,7 +560,7 @@ " get_intrinsic_generator(r\"vqsubq_[usf]\\d+\", df=df),\n", "\n", " # Mul\n", - " get_intrinsic_generator(r\"vmull_[usf]\\d+\", df=df),\n", + " # get_intrinsic_generator(r\"vmull_[usf]\\d+\", df=df), # half batch\n", " get_intrinsic_generator(r\"vmulq_[usf]\\d+\", df=df),\n", "\n", " # vreinterpretq\n", diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index d2f3667e7..58448d582 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -92,6 +92,16 @@ namespace xsimd template XSIMD_INLINE batch_bool eq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept; + template + XSIMD_INLINE batch mul(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE std::pair, batch> + mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept; + + template + XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept; + namespace detail { /************************************** @@ -563,111 +573,95 @@ namespace xsimd } } - /********* - * mul_hi * - *********/ - - template - XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - int16x8_t lo = vmull_s8(vget_low_s8(lhs), vget_low_s8(rhs)); - int16x8_t hi = vmull_s8(vget_high_s8(lhs), vget_high_s8(rhs)); - return vuzpq_s8(vreinterpretq_s8_s16(lo), vreinterpretq_s8_s16(hi)).val[1]; - } - template - XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - uint16x8_t lo = vmull_u8(vget_low_u8(lhs), vget_low_u8(rhs)); - uint16x8_t hi = vmull_u8(vget_high_u8(lhs), vget_high_u8(rhs)); - return vuzpq_u8(vreinterpretq_u8_u16(lo), vreinterpretq_u8_u16(hi)).val[1]; - } - template - XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - int32x4_t lo = vmull_s16(vget_low_s16(lhs), vget_low_s16(rhs)); - int32x4_t hi = vmull_s16(vget_high_s16(lhs), vget_high_s16(rhs)); - return vuzpq_s16(vreinterpretq_s16_s32(lo), vreinterpretq_s16_s32(hi)).val[1]; - } - template - XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - uint32x4_t lo = vmull_u16(vget_low_u16(lhs), vget_low_u16(rhs)); - uint32x4_t hi = vmull_u16(vget_high_u16(lhs), vget_high_u16(rhs)); - return vuzpq_u16(vreinterpretq_u16_u32(lo), vreinterpretq_u16_u32(hi)).val[1]; - } - template - XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - int64x2_t lo = vmull_s32(vget_low_s32(lhs), vget_low_s32(rhs)); - int64x2_t hi = vmull_s32(vget_high_s32(lhs), vget_high_s32(rhs)); - return vuzpq_s32(vreinterpretq_s32_s64(lo), vreinterpretq_s32_s64(hi)).val[1]; - } - template - XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - uint64x2_t lo = vmull_u32(vget_low_u32(lhs), vget_low_u32(rhs)); - uint64x2_t hi = vmull_u32(vget_high_u32(lhs), vget_high_u32(rhs)); - return vuzpq_u32(vreinterpretq_u32_u64(lo), vreinterpretq_u32_u64(hi)).val[1]; - } - // 64-bit intentionally falls through to the common scalar fallback - /************ * mul_hilo * ************/ - template - XSIMD_INLINE std::pair, batch> - mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - int16x8_t lo = vmull_s8(vget_low_s8(lhs), vget_low_s8(rhs)); - int16x8_t hi = vmull_s8(vget_high_s8(lhs), vget_high_s8(rhs)); - int8x16x2_t uzp = vuzpq_s8(vreinterpretq_s8_s16(lo), vreinterpretq_s8_s16(hi)); - return { batch(uzp.val[1]), batch(uzp.val[0]) }; - } - template - XSIMD_INLINE std::pair, batch> - mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - uint16x8_t lo = vmull_u8(vget_low_u8(lhs), vget_low_u8(rhs)); - uint16x8_t hi = vmull_u8(vget_high_u8(lhs), vget_high_u8(rhs)); - uint8x16x2_t uzp = vuzpq_u8(vreinterpretq_u8_u16(lo), vreinterpretq_u8_u16(hi)); - return { batch(uzp.val[1]), batch(uzp.val[0]) }; - } - template - XSIMD_INLINE std::pair, batch> - mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - int32x4_t lo = vmull_s16(vget_low_s16(lhs), vget_low_s16(rhs)); - int32x4_t hi = vmull_s16(vget_high_s16(lhs), vget_high_s16(rhs)); - int16x8x2_t uzp = vuzpq_s16(vreinterpretq_s16_s32(lo), vreinterpretq_s16_s32(hi)); - return { batch(uzp.val[1]), batch(uzp.val[0]) }; - } - template - XSIMD_INLINE std::pair, batch> - mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept + // TODO manually move to intrinsct + namespace detail { - uint32x4_t lo = vmull_u16(vget_low_u16(lhs), vget_low_u16(rhs)); - uint32x4_t hi = vmull_u16(vget_high_u16(lhs), vget_high_u16(rhs)); - uint16x8x2_t uzp = vuzpq_u16(vreinterpretq_u16_u32(lo), vreinterpretq_u16_u32(hi)); - return { batch(uzp.val[1]), batch(uzp.val[0]) }; + template + XSIMD_INLINE constexpr bool vmull_is_supported() + { + return is_like_any_v; + } + + template + XSIMD_INLINE auto vmull_batch(U a, U b) + { + static_assert(vmull_is_supported(), "vmull unsupported"); + if constexpr (is_like_v) + { + return vmull_s8(a, b); + } + else if constexpr (is_like_v) + { + return vmull_u8(a, b); + } + else if constexpr (is_like_v) + { + return vmull_s16(a, b); + } + else if constexpr (is_like_v) + { + return vmull_u16(a, b); + } + else if constexpr (is_like_v) + { + return vmull_s32(a, b); + } + else if constexpr (is_like_v) + { + return vmull_u32(a, b); + } + else + { + static_assert(false, "unsupported type for vmull"); + } + } } - template - XSIMD_INLINE std::pair, batch> - mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept + + template + XSIMD_INLINE std::pair, batch> mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept { - int64x2_t lo = vmull_s32(vget_low_s32(lhs), vget_low_s32(rhs)); - int64x2_t hi = vmull_s32(vget_high_s32(lhs), vget_high_s32(rhs)); - int32x4x2_t uzp = vuzpq_s32(vreinterpretq_s32_s64(lo), vreinterpretq_s32_s64(hi)); - return { batch(uzp.val[1]), batch(uzp.val[0]) }; + if constexpr (sizeof(T) == 8) + { + return mul_hilo(lhs, rhs, common {}); + } + else + { + const auto lo_lhs = overload::vget_low_batch(lhs); + const auto lo_rhs = overload::vget_low_batch(rhs); + const auto lo_mul = detail::vmull_batch(lo_lhs, lo_rhs); + const batch lo = bitwise_cast, T>(lo_mul, {}, A {}); + + const auto hi_lhs = overload::vget_high_batch(lhs); + const auto hi_rhs = overload::vget_high_batch(rhs); + const auto hi_mul = detail::vmull_batch(hi_lhs, hi_rhs); + const batch hi = bitwise_cast, T>(hi_mul, {}, A {}); + + const auto uzp = overload::vuzpq_batch(lo, hi); + return { batch(uzp.val[1]), batch(uzp.val[0]) }; + } } - template - XSIMD_INLINE std::pair, batch> - mul_hilo(batch const& lhs, batch const& rhs, requires_arch) noexcept + + /********* + * mul_hi * + *********/ + + template + XSIMD_INLINE batch mul_hi(batch const& lhs, batch const& rhs, requires_arch a) noexcept { - uint64x2_t lo = vmull_u32(vget_low_u32(lhs), vget_low_u32(rhs)); - uint64x2_t hi = vmull_u32(vget_high_u32(lhs), vget_high_u32(rhs)); - uint32x4x2_t uzp = vuzpq_u32(vreinterpretq_u32_u64(lo), vreinterpretq_u32_u64(hi)); - return { batch(uzp.val[1]), batch(uzp.val[0]) }; + if constexpr (sizeof(T) == 8) + { + // common mul_hilo is itself implemented in terms of mul_hi, so the 64 bit + // path must not go back through the neon mul_hilo fallback. + return mul_hi(lhs, rhs, common {}); + } + else + { + return mul_hilo(lhs, rhs, a).first; + } } /******* diff --git a/include/xsimd/arch/xsimd_neon64.hpp b/include/xsimd/arch/xsimd_neon64.hpp index c319d67be..453628464 100644 --- a/include/xsimd/arch/xsimd_neon64.hpp +++ b/include/xsimd/arch/xsimd_neon64.hpp @@ -328,16 +328,6 @@ namespace xsimd return sub(lhs, rhs, neon64 {}); } - /******* - * mul * - *******/ - - template - XSIMD_INLINE batch mul(batch const& lhs, batch const& rhs, requires_arch) noexcept - { - return vmulq_f64(lhs, rhs); - } - /******* * div * *******/ diff --git a/include/xsimd/overload/neon.hpp b/include/xsimd/overload/neon.hpp index ca8e3ffc0..6cf741107 100644 --- a/include/xsimd/overload/neon.hpp +++ b/include/xsimd/overload/neon.hpp @@ -128,6 +128,24 @@ XSIMD_INLINE auto vdupq_n_batch(T a) { else { static_assert(false, "unsupported type for vdupq_n"); } } +template +XSIMD_INLINE constexpr bool vuzpq_is_supported() { + return is_like_any_v; +} + +template +XSIMD_INLINE auto vuzpq_batch(batch a, batch b) { + static_assert(vuzpq_is_supported(), "vuzpq unsupported"); + if constexpr(is_like_v) { return vuzpq_s8(a, b); } + else if constexpr(is_like_v) { return vuzpq_u8(a, b); } + else if constexpr(is_like_v) { return vuzpq_s16(a, b); } + else if constexpr(is_like_v) { return vuzpq_u16(a, b); } + else if constexpr(is_like_v) { return vuzpq_s32(a, b); } + else if constexpr(is_like_v) { return vuzpq_u32(a, b); } + else if constexpr(is_like_v) { return vuzpq_f32(a, b); } + else { static_assert(false, "unsupported type for vuzpq"); } +} + template XSIMD_INLINE constexpr bool vandq_is_supported() { return is_like_any_v; @@ -324,22 +342,6 @@ XSIMD_INLINE auto vqsubq_batch(batch a, batch b) { else { static_assert(false, "unsupported type for vqsubq"); } } -template -XSIMD_INLINE constexpr bool vmull_is_supported() { - return is_like_any_v; -} - -template -XSIMD_INLINE auto vmull_batch(batch a, batch b) { - static_assert(vmull_is_supported(), "vmull unsupported"); - if constexpr(is_like_v) { return vmull_s8(a, b); } - else if constexpr(is_like_v) { return vmull_u8(a, b); } - else if constexpr(is_like_v) { return vmull_s16(a, b); } - else if constexpr(is_like_v) { return vmull_u16(a, b); } - else if constexpr(is_like_v) { return vmull_s32(a, b); } - else if constexpr(is_like_v) { return vmull_u32(a, b); } - else { static_assert(false, "unsupported type for vmull"); } -} template XSIMD_INLINE constexpr bool vmulq_is_supported() {